├── src └── jcci │ ├── __init__.py │ ├── config.py │ ├── constant.py │ ├── diff_parse.py │ ├── sql.py │ ├── mapper_parse.py │ ├── database.py │ ├── graph.py │ ├── analyze.py │ └── java_parse.py ├── images ├── wechat.jpg ├── donation.png ├── jcci_dingding.jpg └── cii-result-tree.png ├── requirements.txt ├── .gitignore ├── .github ├── ISSUE_TEMPLATE │ └── bug_report.md └── workflows │ └── python-publish.yml ├── pyproject.toml ├── README.pypi.md ├── jcci-result.html ├── README.md ├── README.en.md └── LICENSE /src/jcci/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /images/wechat.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/baikaishuipp/jcci/HEAD/images/wechat.jpg -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/baikaishuipp/jcci/HEAD/requirements.txt -------------------------------------------------------------------------------- /images/donation.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/baikaishuipp/jcci/HEAD/images/donation.png -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | dist 2 | tests 3 | src/jcci.egg-info 4 | .idea 5 | src/jcci/__pycache__/ 6 | test/ 7 | -------------------------------------------------------------------------------- /images/jcci_dingding.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/baikaishuipp/jcci/HEAD/images/jcci_dingding.jpg -------------------------------------------------------------------------------- /images/cii-result-tree.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/baikaishuipp/jcci/HEAD/images/cii-result-tree.png -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/bug_report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug report 3 | about: Create a report to help us improve 4 | title: '' 5 | labels: '' 6 | assignees: baikaishuipp 7 | 8 | --- 9 | 10 | ** 描述问题 ** 11 | 12 | ** 提供信息 ** 13 | 1. JCCI的版本/分支 14 | 2. python 的版本 15 | 3. A->B链路断了反馈：当前A节点截图、类名(com.XXX.XX.AClass)，调用A节点的B方法截图（包含调用A地方)、类名(com.XXX.XX.BClass)、B方法节点的method_invocation_map：（select method_invocation_map from methods where method_name='B') 16 | 4. 控制台日志文件 17 | -------------------------------------------------------------------------------- /src/jcci/config.py: -------------------------------------------------------------------------------- 1 | # -*- coding: UTF-8 -*- 2 | import os 3 | 4 | # sqlite3 path 5 | db_path = os.path.dirname(os.path.abspath(__file__)) 6 | # git project clone file path 7 | project_path = os.path.dirname(os.path.abspath(__file__)) 8 | # ignore file pattern 9 | ignore_file = ['*/pom.xml', '*/test/*', '*.sh', '*.md', '*/checkstyle.xml', '*.yml', '.git/*'] 10 | # project package startswith 11 | package_prefix = ['com.', 'cn.', 'net.'] 12 | # Whether to reparse the class when there is class data in the database 13 | reparse_class = True 14 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | requires = ["setuptools>=61.0"] 3 | build-backend = "setuptools.build_meta" 4 | 5 | [project] 6 | name = "jcci" 7 | version = "0.2.1" 8 | authors = [ 9 | { name="Oliver Li", email="441640312@qq.com" }, 10 | ] 11 | description = "Java code commit impact, java code change impact analysis, java代码改动影响范围分析工具" 12 | readme = "README.pypi.md" 13 | license = { file="LICENSE" } 14 | requires-python = ">=3.9" 15 | classifiers = [ 16 | "Programming Language :: Python :: 3", 17 | "License :: OSI Approved :: Apache Software License", 18 | "Operating System :: OS Independent", 19 | ] 20 | dependencies = [ 21 | "javalang >= 0.13.0", 22 | "unidiff >= 0.7.4" 23 | ] 24 | 25 | [project.urls] 26 | "Homepage" = "https://github.com/baikaishuipp/jcci" 27 | "Bug Tracker" = "https://github.com/baikaishuipp/jcci/issues" -------------------------------------------------------------------------------- /.github/workflows/python-publish.yml: -------------------------------------------------------------------------------- 1 | # This workflow will upload a Python Package using Twine when a release is created 2 | # For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python#publishing-to-package-registries 3 | 4 | # This workflow uses actions that are not certified by GitHub. 5 | # They are provided by a third-party and are governed by 6 | # separate terms of service, privacy policy, and support 7 | # documentation. 8 | 9 | name: Upload Python Package 10 | 11 | on: 12 | release: 13 | types: [published] 14 | 15 | permissions: 16 | contents: read 17 | 18 | jobs: 19 | deploy: 20 | 21 | runs-on: ubuntu-latest 22 | 23 | steps: 24 | - uses: actions/checkout@v4 25 | - name: Set up Python 26 | uses: actions/setup-python@v3 27 | with: 28 | python-version: '3.9' 29 | - name: Install dependencies 30 | run: | 31 | python -m pip install --upgrade pip 32 | pip install build 33 | - name: Build package 34 | run: python -m build 35 | - name: Publish package 36 | uses: pypa/gh-action-pypi-publish@27b31702a0e7fc50959f5ad993c78deac1bdfc29 37 | with: 38 | user: __token__ 39 | password: ${{ secrets.PYPI_API_TOKEN }} 40 | -------------------------------------------------------------------------------- /README.pypi.md: -------------------------------------------------------------------------------- 1 | #### Description 2 | Java code commit impact analysis, is a pure python library that analyzes the impact of two git submissions of Java projects on the project and generates tree chart data. 3 | 4 | Github: [jcci](https://github.com/baikaishuipp/jcci) 5 | 6 | #### Achieve Effect 7 | ![效果图](./images/cii-result-tree.png) 8 | 9 | #### Instructions 10 | Start a new python project, add a new python file, code example: 11 | 12 | ``` 13 | from jcci.analyze import JCCI 14 | 15 | # Compare different commits on the same branch 16 | commit_analyze = JCCI('git@xxxx.git', 'username1') 17 | commit_analyze.analyze_two_commit('master','commit_id1','commit_id2') 18 | 19 | # To analyze the impact of methods in a class, use the analyze_class_method method. The last parameter is the line number(s) of the method(s) you want to analyze. If multiple methods are specified, separate their line numbers with commas. If left blank, it will analyze the impact of the entire class. 20 | class_analyze = JCCI('git@xxxx.git', 'username1') 21 | class_analyze.analyze_class_method('master','commit_id1', 'package\src\main\java\ClassA.java', '20,81') 22 | 23 | # Compare different branches 24 | branch_analyze = JCCI('git@xxxx.git', 'username1') 25 | branch_analyze.analyze_two_branch('branch_new','branch_old') 26 | ``` 27 | 28 | At the same time, the project will be cloned in the directory and then analyzed to generate a file with the suffix format xxxx.cci, 29 | which contains the tree diagram data generated by the analysis results, download [jcci-result.html](https://github.com/baikaishuipp/jcci/blob/main/jcci-result.html) , 30 | upload analyze result file end with .cci, then can be displayed through the view. 31 | 32 | -------------------------------------------------------------------------------- /src/jcci/constant.py: -------------------------------------------------------------------------------- 1 | 2 | ENTITY = 'entity' 3 | RETURN_TYPE = 'return_type' 4 | PARAMETERS = 'parameters' 5 | BODY = 'body' 6 | METHODS = 'methods' 7 | FIELDS = 'fields' 8 | 9 | PARAMETER_TYPE_METHOD_INVOCATION_UNKNOWN = 'unknown' 10 | # 11 | 12 | # 13 | NODE_TYPE_CLASS = 'class' 14 | NODE_TYPE_METHOD = 'method' 15 | NODE_TYPE_FIELD = 'field' 16 | NODE_TYPE_MAPPER = 'mapper' 17 | NODE_TYPE_MAPPER_SQL = 'sql' 18 | NODE_TYPE_MAPPER_RESULT_MAP = 'resultMap' 19 | NODE_TYPE_MAPPER_STATEMENT = 'statement' 20 | 21 | DIFF_TYPE_CHANGED = 'changed' 22 | DIFF_TYPE_IMPACTED = 'impacted' 23 | 24 | JAVA_BASIC_TYPE = ['string', 'int', 'integer', 'boolean', 'long', 'byte', 'short', 'float', 'double', 'char'] 25 | JAVA_BASIC_TYPE_SWITCH = ['int', 'integer', 'boolean', 'long', 'byte', 'short', 'float', 'double', 'char'] 26 | JAVA_UTIL_TYPE = [ 27 | 'ArrayList', 'Base64', 'Calendar', 'Collection', 'Collections', 'Comparators', 'Date', 'Dictionary', 28 | 'EnumMap', 'EnumSet', 'EventListener', 'EventObject', 'Formatter', 29 | 'HashMap', 'HashSet', 'Hashtable', 'Iterator', 'LinkedHashMap', 'LinkedHashSet', 'LinkedList', 30 | 'List', 'ListIterator', 'Locale', 'Map', 'NavigableMap', 'NavigableSet', 'Objects', 31 | 'Optional', 'OptionalDouble', 'OptionalInt', 'OptionalLong', 'Properties', 'Queue', 'Random', 32 | 'RegularEnumSet', 'ResourceBundle', 'Scanner', 'ServiceLoader', 'Set', 'SimpleTimeZone', 33 | 'SortedMap', 'SortedSet', 'Spliterator', 'Spliterators', 'SplittableRandom', 'Stack', 'StringJoiner', 34 | 'StringTokenizer', 'TaskQueue', 'Timer', 'TimerTask', 'TimerThread', 'TimeZone', 'TimSort', 'TreeMap', 35 | 'TreeSet', 'Tripwire', 'UUID', 'Vector', 'WeakHashMap'] 36 | MAPPING_LIST = ['PostMapping', 'GetMapping', 'DeleteMapping', 'PutMapping', 'PatchMapping', 'RequestMapping'] 37 | -------------------------------------------------------------------------------- /src/jcci/diff_parse.py: -------------------------------------------------------------------------------- 1 | import unidiff 2 | import os 3 | 4 | 5 | def _diff_patch_lines(patch): 6 | line_num_added = [] 7 | line_num_removed = [] 8 | line_content_added = [] 9 | line_content_removed = [] 10 | for hunk in patch: 11 | if hunk.added > 0: 12 | targets = hunk.target 13 | target_start = hunk.target_start 14 | for i in range(0, len(targets)): 15 | if targets[i].startswith('+') and not targets[i][1:].strip().startswith(('*', '//', 'import ')) and targets[i][1:].strip(): 16 | line_num_added.append(target_start + i) 17 | line_content_added.append(targets[i][1:]) 18 | if hunk.removed > 0: 19 | sources = hunk.source 20 | source_start = hunk.source_start 21 | for i in range(0, len(sources)): 22 | if sources[i].startswith('-') and not sources[i][1:].strip().startswith(('*', '//', 'import ')) and sources[i][1:].strip(): 23 | line_num_removed.append(source_start + i) 24 | line_content_removed.append(sources[i][1:]) 25 | return line_num_added, line_content_added, line_num_removed, line_content_removed 26 | 27 | 28 | def get_diff_info(file_path): 29 | patch_results = {} 30 | with open(file_path, encoding='UTF-8') as f: 31 | diff_text = f.read() 32 | patch_set = unidiff.PatchSet(diff_text) 33 | for patch in patch_set: 34 | if '.git' in patch.path or os.path.join('src', 'test') in patch.path or (not patch.path.endswith(('.java', '.xml'))): 35 | continue 36 | line_num_added, line_content_added, line_num_removed, line_content_removed = _diff_patch_lines(patch) 37 | java_file_path = patch.path 38 | patch_results[java_file_path] = { 39 | 'line_num_added': line_num_added, 40 | 'line_content_added': line_content_added, 41 | 'line_num_removed': line_num_removed, 42 | 'line_content_removed': line_content_removed 43 | } 44 | return patch_results 45 | 46 | 47 | if __name__ == '__main__': 48 | print('jcci') 49 | 50 | -------------------------------------------------------------------------------- /src/jcci/sql.py: -------------------------------------------------------------------------------- 1 | 2 | create_tables = ''' 3 | CREATE TABLE project ( 4 | project_id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, 5 | project_name TEXT NOT NULL, 6 | git_url TEXT NOT NULL, 7 | branch TEXT NOT NULL, 8 | commit_or_branch_new TEXT NOT NULL, 9 | commit_or_branch_old TEXT, 10 | create_at TIMESTAMP NOT NULL DEFAULT (datetime('now','localtime')) 11 | ); 12 | 13 | CREATE TABLE class ( 14 | class_id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, 15 | filepath TEXT, 16 | access_modifier TEXT, 17 | class_type TEXT NOT NULL, 18 | class_name TEXT NOT NULL, 19 | package_name TEXT NOT NULL, 20 | extends_class TEXT, 21 | project_id INTEGER NOT NULL, 22 | implements TEXT, 23 | annotations TEXT, 24 | documentation TEXT, 25 | is_controller REAL, 26 | controller_base_url TEXT, 27 | commit_or_branch TEXT, 28 | create_at TIMESTAMP NOT NULL DEFAULT (datetime('now','localtime')) 29 | ); 30 | 31 | CREATE TABLE import ( 32 | import_id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, 33 | class_id INTEGER NOT NULL, 34 | project_id INTEGER NOT NULL, 35 | start_line INTEGER, 36 | end_line INTEGER, 37 | import_path TEXT, 38 | is_static REAL, 39 | is_wildcard REAL, 40 | create_at TIMESTAMP NOT NULL DEFAULT (datetime('now','localtime')) 41 | ); 42 | 43 | CREATE TABLE field ( 44 | field_id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, 45 | class_id INTEGER, 46 | project_id INTEGER NOT NULL, 47 | annotations TEXT, 48 | access_modifier TEXT, 49 | field_type TEXT, 50 | field_name TEXT, 51 | is_static REAL, 52 | start_line INTEGER, 53 | end_line INTEGER, 54 | documentation TEXT, 55 | create_at TIMESTAMP NOT NULL DEFAULT (datetime('now','localtime')) 56 | ); 57 | 58 | 59 | CREATE TABLE methods ( 60 | method_id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, 61 | class_id INTEGER NOT NULL, 62 | project_id INTEGER NOT NULL, 63 | annotations TEXT, 64 | access_modifier TEXT, 65 | return_type TEXT, 66 | method_name TEXT NOT NULL, 67 | parameters TEXT, 68 | body TEXT, 69 | method_invocation_map TEXT, 70 | is_static REAL, 71 | is_abstract REAL, 72 | is_api REAL, 73 | api_path TEXT, 74 | start_line INTEGER NOT NULL, 75 | end_line INTEGER NOT NULL, 76 | documentation TEXT, 77 | create_at TIMESTAMP NOT NULL DEFAULT (datetime('now','localtime')) 78 | ); 79 | ''' 80 | 81 | -------------------------------------------------------------------------------- /jcci-result.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | JCCI Result 6 | 7 | 8 | 9 | 10 |
Choose CCI Result File 11 | 12 | 13 | 14 | 15 | 16 | 17 |
18 |
19 | 83 | 84 | 85 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ![Visitors](https://api.visitorbadge.io/api/visitors?path=https%3A%2F%2Fgithub.com%2Fbaikaishuipp%2Fjcci&countColor=%23263759) 2 | ![PyPI Downloads](https://static.pepy.tech/badge/jcci) 3 | 4 | ### [English Doc](https://github.com/baikaishuipp/jcci/blob/main/README.en.md) 5 | 6 | # jcci 7 | 8 | ### 介绍 9 | Java代码提交影响分析，是一个纯python库，分析Java项目的两次git提交差异对项目的影响，并生成树形图数据。 10 | 11 | PYPI: [jcci](https://pypi.org/project/jcci/) （会落后github几个版本） 12 | 13 | ### 实现效果 14 | ![效果图](./images/cii-result-tree.png) 15 | 16 | ### 软件架构 17 | 大致原理同Idea的Find Usage一致，通过代码改动定位代码影响，并不断遍历受影响的类和方法直至找到最上层的controller层 18 | 19 | 代码主要由python编写，主要涉及2个库： 20 | 21 | * javalang java文件语法解析库 22 | * unidiff git diff信息解析库 23 | 24 | 通过javalang语法解析获取每个Java文件的import class extends implements declarators methods 等信息 25 | 26 | 通过unidiff 解析git diff信息（diff file, added_line_num, removed_lin_num) 27 | 28 | 然后根据文件增删的代码行去判断影响了哪些类和方法，不断遍历受影响的类和方法直至找到最上层的controller层 29 | 30 | 通过传入项目git地址 分支 两次的commit id，即可分析出两次commit id之间代码改动所带来的影响，并生成树图数据方便展示影响链路。 31 | 32 | ### 安装教程 33 | 要求python >= 3.9 , sqlite3 >= 3.38 34 | 35 | ### 使用说明 36 | 项目克隆下来后，新建python文件，引入jcci项目src目录下的jcci 37 | ``` 38 | from path.to.jcci.src.jcci.analyze import JCCI 39 | 40 | # 同一分支不同commit分析 41 | commit_analyze = JCCI('git@xxxx.git', 'username1') 42 | commit_analyze.analyze_two_commit('master','commit_id1','commit_id2') 43 | 44 | # 分析一个类的方法影响, analyze_class_method方法最后参数为方法所在行数，不同方法行数用逗号分割，不填则分析完整类影响 45 | class_analyze = JCCI('git@xxxx.git', 'username1') 46 | class_analyze.analyze_class_method('master','commit_id1', 'package\src\main\java\ClassA.java', '20,81') 47 | 48 | # 不同分支分析 49 | branch_analyze = JCCI('git@xxxx.git', 'username1') 50 | branch_analyze.analyze_two_branch('branch_new','branch_old') 51 | 52 | # 多项目联合分析，上述三种方法都支持，以analyze_two_commit方法举例 53 | dependents = [ 54 | { 55 | 'git_url': 'git@xxxx.git', 56 | 'branch': 'master', # default master when empty 57 | 'commit_id': 'HEAD' # default HEAD when empty 58 | } 59 | ] 60 | commit_analyze = JCCI('git@xxxx.git', 'username1') 61 | commit_analyze.analyze_two_commit('master','commit_id1','commit_id2', dependents=dependents) 62 | 63 | ``` 64 | #### 参数说明： 65 | * project_git_url - 项目git地址，代码使用本机git配置clone代码，确保本机git权限或通过用户名密码/token的方式拼接url来clone代码。示例：https://userName:password@github.com/xxx.git 或 https://token@github.com/xxx.git 66 | * username1 - 随便输入，为了避免并发分析同一项目导致结果错误，用户1分析项目A时，用户B需要等待，所以设置了该参数 67 | 68 | 运行时，会将项目克隆到目录中，然后进行分析，生成后缀格式为.cci的文件，其中包含分析结果生成的关系图数据，下载[jcci-result.html](https://github.com/baikaishuipp/jcci/blob/main/jcci-result.html) ，选择分析结果的.cci文件，即可可通过视图显示。 69 | 70 | ### 安全性 71 | 项目分析都是基于本地环境执行，无任何代码收集和日志上报，源码可查，请放心使用。 72 | 73 | ### 开源不易，如本工具对您有帮助，请点一下右上角 star ⭐ 74 | 75 | ### 谁在使用 76 | 如果您在使用JCCI，请让我们知道，您的使用对我们非常重要：[登记链接](#33) （按登记顺序排列） 77 | 78 | 79 | 83 | 87 | 91 | 92 |
80 | 81 | Hand-china 82 | 84 | 85 | sf-tech 86 | 88 | 89 | dazhenyun 90 |
93 | 94 | ### 沟通交流 95 | 扫码加微信，备注：JCCI微信群交流，或者扫码加入钉钉交流群 96 | 97 | ![微信交流群](./images/wechat.jpg) ![钉钉交流群](./images/jcci_dingding.jpg) 98 | 99 | ### 鸣谢 100 | 感谢一下同学请作者喝咖啡、提供意见或反馈Bug, 排名不分先后 101 | [zouchengli](https://github.com/zouchengli) 102 | 103 | ### Star History 104 | 105 | [![Star History Chart](https://api.star-history.com/svg?repos=baikaishuipp/jcci&type=Date)](https://star-history.com/#baikaishuipp/jcci&Date) 106 | -------------------------------------------------------------------------------- /README.en.md: -------------------------------------------------------------------------------- 1 | ![Visitors](https://api.visitorbadge.io/api/visitors?path=https%3A%2F%2Fgithub.com%2Fbaikaishuipp%2Fjcci&countColor=%23263759) 2 | 3 | ### [中文说明](https://github.com/baikaishuipp/jcci/blob/main/README.md) 4 | 5 | # jcci 6 | 7 | ### Description 8 | Java code commit impact analysis, is a pure python library that analyzes the impact of two git submissions of Java projects on the project and generates tree chart data. 9 | 10 | PYPI: [jcci](https://pypi.org/project/jcci/) (It will be several versions behind github) 11 | 12 | ### Achieve Effect 13 | ![效果图](./images/cii-result-tree.png) 14 | 15 | ### Software Architecture 16 | The general principle is the same as Find Usage of Idea, locate the impact of the code through code changes, and continuously traverse the affected classes and methods until the top controller layer is found 17 | 18 | The code is mainly written by python and mainly involves 2 libraries: 19 | 20 | * javalang java file syntax parsing library 21 | * unidiff git diff information parsing library 22 | 23 | Obtain information such as import class extends implements declarators methods of each Java file through javalang syntax analysis 24 | 25 | Parse git diff information through unidiff (diff file, added_line_num, removed_lin_num) 26 | 27 | Then judge which classes and methods are affected according to the code lines added or deleted in the file, and continuously traverse the affected classes and methods until you find the top controller layer 28 | 29 | By passing in the commit id of the project git address branch twice, the impact of code changes between the two commit ids can be analyzed, and the tree diagram data can be generated to display the affected links. 30 | 31 | ### Installation 32 | Require python >= 3.9 , sqlite3 >= 3.38 33 | 34 | ### Instructions 35 | After the project is cloned, create a new python file and introduce jcci in the src directory of the jcci project. 36 | ``` 37 | from path.to.jcci.src.jcci.analyze import JCCI 38 | 39 | # Compare different commits on the same branch 40 | commit_analyze = JCCI('git@xxxx.git', 'username1') 41 | commit_analyze.analyze_two_commit('master','commit_id1','commit_id2') 42 | 43 | # To analyze the impact of methods in a class, use the analyze_class_method method. The last parameter is the line number(s) of the method(s) you want to analyze. If multiple methods are specified, separate their line numbers with commas. If left blank, it will analyze the impact of the entire class. 44 | class_analyze = JCCI('git@xxxx.git', 'username1') 45 | class_analyze.analyze_class_method('master','commit_id1', 'package\src\main\java\ClassA.java', '20,81') 46 | 47 | # Compare different branches 48 | branch_analyze = JCCI('git@xxxx.git', 'username1') 49 | branch_analyze.analyze_two_branch('branch_new','branch_old') 50 | 51 | 52 | # Multi-project joint analysis is supported by the above three methods. Take the analyze_two_commit method as an example. 53 | dependents = [ 54 | { 55 | 'git_url': 'git@xxxx.git', 56 | 'branch': 'master', # default master when empty 57 | 'commit_id': 'HEAD' # default HEAD when empty 58 | } 59 | ] 60 | commit_analyze = JCCI('git@xxxx.git', 'username1') 61 | commit_analyze.analyze_two_commit('master','commit_id1','commit_id2', dependents=dependents) 62 | 63 | ``` 64 | 65 | #### Parameter Description: 66 | * project_git_url - project git address, the code uses the native git configuration to clone the code, ensure the native git permissions or splice the url through username, password/token to clone the code. Example: https://userName:password@github.com/xxx.git or https://token@github.com/xxx.git 67 | * username1 - Enter whatever you want. In order to avoid incorrect results caused by concurrent analysis of the same project, when user 1 analyzes project A, user B needs to wait, so this parameter is set. 68 | 69 | At the same time, the project will be cloned in the directory and then analyzed to generate a file with the suffix format xxx.cci, which contains the tree diagram data generated by the analysis results, download [jcci-result.html](https://github.com/baikaishuipp/jcci/blob/main/jcci-result.html) , upload analyze result file end with .cci, then can be displayed through the view. 70 | 71 | ### Security 72 | Project analysis is performed based on the local environment. There is no code collection or log reporting. The source code can be checked, so please feel free to use it. 73 | 74 | ### If this tool is helpful to you, please click star in the upper right corner ⭐ 75 | 76 | ### Communication 77 | Scan QR code via WeChat app, and comment：JCCI communication 78 | 79 | ![communicate via Wechat](./images/wechat.jpg) 80 | -------------------------------------------------------------------------------- /src/jcci/mapper_parse.py: -------------------------------------------------------------------------------- 1 | # -*- coding: UTF-8 -*- 2 | import xml.etree.ElementTree as ET 3 | import re 4 | 5 | class Mapper(object): 6 | def __init__(self, namespace, result_maps, sqls, statements): 7 | self.namespace = namespace 8 | self.result_maps = result_maps 9 | self.sqls = sqls 10 | self.statements = statements 11 | 12 | 13 | class MapperElement(object): 14 | def __init__(self, id, type, start, end, content): 15 | self.id = id 16 | self.name = id 17 | self.type = type 18 | self.start = start 19 | self.end = end 20 | self.content = content 21 | self.diff_impact = None 22 | 23 | 24 | class MapperStatement(MapperElement): 25 | def __init__(self, id, type, start_line, end_line, content, statement_tag, result_map, include_sql): 26 | super(MapperStatement, self).__init__(id, type, start_line, end_line, content) 27 | self.statement_tag = statement_tag 28 | self.result_map = result_map 29 | self.include_sql = include_sql 30 | 31 | def extract_value(string, tag): 32 | pattern = tag + r'\s*=\s*[\'"](\w+)[\'"]' 33 | match = re.search(pattern, string) 34 | if match: 35 | value = match.group(1) 36 | return value 37 | else: 38 | return None 39 | 40 | 41 | def check_string(tag, id_str, string): 42 | pattern = r'^' + tag + '.*?id\s*=\s*[\'"]' + id_str + '[\'"]' 43 | match = re.search(pattern, string) 44 | return bool(match) 45 | 46 | def parse(filepath): 47 | # 读取XML文件内容 48 | try: 49 | with open(filepath, "r", encoding="utf-8") as file: 50 | xml_content = file.read() 51 | except: 52 | return None 53 | 54 | # 解析XML文件 55 | tree = ET.ElementTree(ET.fromstring(xml_content)) 56 | root = tree.getroot() 57 | 58 | # 获取namespace 59 | try: 60 | namespace = root.attrib["namespace"] 61 | if namespace is None: 62 | return None 63 | except: 64 | return None 65 | # 存储resultMap和每条语句的id以及对应的起始行号和截止行号 66 | result_map_info = [] 67 | sql_info = [] 68 | statement_info = [] 69 | 70 | # 获取resultMap的id以及起始行号和截止行号 71 | for element in root.findall(".//resultMap"): 72 | result_map_id = element.attrib["id"] 73 | start_line = 0 74 | end_line = 0 75 | for i, line in enumerate(xml_content.splitlines(), start=1): 76 | if check_string('' in line and start_line != 0: 79 | end_line = i 80 | break 81 | content = xml_content.splitlines()[start_line - 1: end_line] 82 | result_map_info.append(MapperElement(result_map_id, 'resultMap', start_line, end_line, content)) 83 | 84 | # 获取resultMap的id以及起始行号和截止行号 85 | for sql_element in root.findall(".//sql"): 86 | sql_id = sql_element.attrib["id"] 87 | start_line = 0 88 | end_line = 0 89 | for i, line in enumerate(xml_content.splitlines(), start=1): 90 | if check_string('' in line and start_line != 0: 93 | end_line = i 94 | break 95 | content = xml_content.splitlines()[start_line - 1: end_line] 96 | sql_info.append(MapperElement(sql_id, 'sql', start_line, end_line, content)) 97 | 98 | # 获取每条语句的id以及起始行号和截止行号 99 | statements = root.findall(".//select") + root.findall(".//insert") + root.findall(".//update") + root.findall(".//delete") 100 | for statement_element in statements: 101 | statement_id = statement_element.attrib["id"] 102 | start_line = 0 103 | end_line = 0 104 | result_map = None 105 | include_sql = None 106 | for i, line in enumerate(xml_content.splitlines(), start=1): 107 | if check_string('<' + statement_element.tag, statement_id, line.strip()): 108 | start_line = i 109 | if f'resultMap' in line and start_line != 0: 110 | result_map = extract_value(line, 'resultMap') 111 | if line.strip().startswith('' in line and start_line != 0: 114 | end_line = i 115 | break 116 | content = xml_content.splitlines()[start_line - 1: end_line] 117 | statement_info.append(MapperStatement(statement_id, 'statement', start_line, end_line, content, statement_element.tag, result_map, include_sql)) 118 | 119 | return Mapper(namespace, result_map_info, sql_info, statement_info) 120 | -------------------------------------------------------------------------------- /src/jcci/database.py: -------------------------------------------------------------------------------- 1 | # -*- coding: UTF-8 -*- 2 | import sqlite3 3 | import time 4 | import logging 5 | import os 6 | from .sql import create_tables 7 | 8 | logging.basicConfig(format='%(asctime)s %(message)s', level=logging.DEBUG) 9 | 10 | class SqliteHelper(object): 11 | def __init__(self, db_path): 12 | self.db_path = db_path 13 | self.sql_result_map = {} 14 | 15 | def connect(self): 16 | try: 17 | if not os.path.exists(os.path.dirname(self.db_path)): 18 | os.makedirs(os.path.dirname(self.db_path)) 19 | if not os.path.exists(self.db_path): 20 | db_file = open(self.db_path, "x") 21 | db_file.close() 22 | conn = sqlite3.connect(self.db_path) 23 | # 执行创建表的SQL语句 24 | try: 25 | conn.cursor().executescript(create_tables) 26 | logging.info("Table created successfully") 27 | except sqlite3.Error as e: 28 | logging.error(f"Error creating table: {e}") 29 | else: 30 | conn = sqlite3.connect(self.db_path) 31 | except Exception as e: 32 | logging.error(f'connect fail {e}') 33 | time.sleep(1) 34 | conn = sqlite3.connect(self.db_path) 35 | return conn 36 | 37 | def add_project(self, project_name, git_url, branch, commit_or_branch_new, commit_or_branch_old): 38 | try: 39 | projects = self.select_data(f'SELECT * FROM project where project_name="{project_name}" and git_url="{git_url}" ' 40 | f'and branch="{branch}" AND commit_or_branch_new="{commit_or_branch_new}" and commit_or_branch_old="{commit_or_branch_old}"') 41 | if projects: 42 | return projects[0]['project_id'] 43 | conn = self.connect() 44 | c = conn.cursor() 45 | c.execute(f'INSERT INTO project ' 46 | f'(project_name, git_url, branch, commit_or_branch_new, commit_or_branch_old) ' 47 | f'VALUES("{project_name}", "{git_url}", "{branch}", "{commit_or_branch_new}", "{commit_or_branch_old}")') 48 | project_id = c.lastrowid 49 | conn.commit() 50 | conn.close() 51 | return project_id 52 | except Exception as e: 53 | logging.error(f'add_project fail') 54 | 55 | def add_class(self, filepath, access_modifier, class_type, class_name, package_name, extends_class, project_id, implements, annotations, documentation, is_controller, controller_base_url, commit_or_branch): 56 | try: 57 | class_list = self.select_data(f'SELECT * FROM class WHERE project_id={project_id} and package_name="{package_name}" and class_name="{class_name}" and commit_or_branch="{commit_or_branch}"') 58 | if class_list: 59 | return class_list[0]['class_id'], False 60 | conn = self.connect() 61 | c = conn.cursor() 62 | c.execute('INSERT INTO class (filepath, access_modifier, class_type, class_name, package_name, extends_class, project_id, implements, annotations, documentation, is_controller, controller_base_url, commit_or_branch) ' 63 | 'VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?)', (filepath, access_modifier, class_type, class_name, package_name, extends_class, project_id, implements, annotations, documentation, is_controller, controller_base_url, commit_or_branch)) 64 | class_id = c.lastrowid 65 | conn.commit() 66 | conn.close() 67 | return class_id, True 68 | except Exception as e: 69 | logging.error(e) 70 | logging.error(f'add_class fail') 71 | 72 | def select_data(self, sql): 73 | try: 74 | if sql in self.sql_result_map.keys(): 75 | return self.sql_result_map.get(sql) 76 | conn = self.connect() 77 | c = conn.cursor() 78 | cursor = c.execute(sql) 79 | res = cursor.fetchall() 80 | columns = cursor.description 81 | field = [column_name[0] for column_name in columns] 82 | zip_data = [dict(zip(field, item)) for item in res] 83 | conn.close() 84 | self.sql_result_map[sql] = zip_data 85 | return zip_data 86 | except Exception as e: 87 | logging.error(f'select_data fail') 88 | raise e 89 | 90 | def update_data(self, sql): 91 | try: 92 | conn = self.connect() 93 | c = conn.cursor() 94 | c.execute(sql) 95 | conn.commit() 96 | conn.close() 97 | except Exception as e: 98 | logging.error(f'select_data fail') 99 | 100 | def insert_data(self, table_name: str, data) -> bool: 101 | try: 102 | conn = self.connect() 103 | c = conn.cursor() 104 | if isinstance(data, list): 105 | for item in data: 106 | keys = ",".join(list(item.keys())) 107 | values = ",".join([f'''"{x.replace('"', '""').replace("'", "''")}"''' if isinstance(x, str) else f"'{x}'" for x in list(item.values())]) 108 | sql = f"INSERT INTO {table_name} ({keys}) VALUES ({values});" 109 | c.execute(sql) 110 | elif isinstance(data, dict): 111 | keys = ",".join(list(data.keys())) 112 | values = ",".join([f"'{x}'" for x in list(data.values())]) 113 | sql = f"INSERT INTO {table_name} ({keys}) VALUES ({values});" 114 | c.execute(sql) 115 | conn.commit() 116 | conn.close() 117 | return True 118 | except Exception as ex: 119 | logging.error(f"insert data error {ex}") 120 | return False 121 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright 2024 baikaishuipp 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. -------------------------------------------------------------------------------- /src/jcci/graph.py: -------------------------------------------------------------------------------- 1 | from . import constant as constant 2 | from collections import deque 3 | 4 | def max_relationship_length(relationships): 5 | if not relationships: 6 | return {} 7 | # 构建邻接列表 8 | graph = {} 9 | for relationship in relationships: 10 | source = relationship['source'] 11 | target = relationship['target'] 12 | if source == target: 13 | continue 14 | if source not in graph: 15 | graph[source] = set() 16 | if target not in graph: 17 | graph[target] = set() 18 | graph[source].add(target) 19 | 20 | # BFS遍历计算每个节点到起点的最长路径长度 21 | longest_paths = {node: 0 for node in graph.keys()} 22 | unvisited_nodes = set(graph.keys()) 23 | while unvisited_nodes: 24 | start_node = unvisited_nodes.pop() 25 | queue = deque([(start_node, 0)]) 26 | visited_queue_node_list = [start_node] 27 | while queue: 28 | node, path_length = queue.popleft() 29 | if node not in visited_queue_node_list: 30 | visited_queue_node_list.append(node) 31 | unvisited_nodes.discard(node) 32 | for neighbor in graph.get(node, set()): 33 | if path_length + 1 > longest_paths[neighbor]: 34 | longest_paths[neighbor] = path_length + 1 35 | if neighbor not in visited_queue_node_list: 36 | queue.append((neighbor, path_length + 1)) 37 | return longest_paths 38 | 39 | 40 | class Graph(object): 41 | 42 | def __init__(self): 43 | self.nodes = [] 44 | self.links = [] 45 | self.categories = [] 46 | self.node_index_init = 0 47 | 48 | def create_node_category(self, class_or_xml, name, type, diff_type, diff_content, file_path, documentation, body, extend_dict: dict): 49 | category = { 50 | 'name': class_or_xml 51 | } 52 | if category not in self.categories: 53 | self.categories.append(category) 54 | if class_or_xml == name: 55 | return 56 | category_id = self.categories.index(category) 57 | node_create = { 58 | 'category': category_id, 59 | 'id': str(self.node_index_init), 60 | 'name': class_or_xml + '.' + name, 61 | 'type': type, 62 | 'diff_type': [diff_type], 63 | 'file_path': file_path, 64 | } 65 | if diff_type == constant.DIFF_TYPE_CHANGED: 66 | node_create.update({ 67 | 'diff_content': diff_content, 68 | 'documentation': documentation, 69 | 'body': body, 70 | }) 71 | node_exist = [node for node in self.nodes if node['name'] == node_create['name'] and node['type'] == type and node['file_path'] == file_path] 72 | if node_exist: 73 | node_create: dict = node_exist[0] 74 | self.nodes.remove(node_create) 75 | if diff_type not in node_create['diff_type']: 76 | node_create['diff_type'].append(diff_type) 77 | node_create.update(extend_dict) 78 | self.nodes.append(node_create) 79 | else: 80 | node_create.update(extend_dict) 81 | self.nodes.append(node_create) 82 | self.node_index_init += 1 83 | return node_create['id'] 84 | 85 | def create_node_link(self, source_node_id, target_node_id): 86 | if source_node_id is None or target_node_id is None: 87 | return 88 | if source_node_id == target_node_id: 89 | return 90 | link = { 91 | 'source': source_node_id, 92 | 'target': target_node_id 93 | } 94 | reverse_link = { 95 | 'source': target_node_id, 96 | 'target': source_node_id 97 | } 98 | if link not in self.links and reverse_link not in self.links: 99 | self.links.append(link) 100 | 101 | def draw_graph(self, canvas_width, canvas_height): 102 | # 每个类别区域划分的行数 103 | all_node = [] 104 | result = max_relationship_length(self.links) 105 | changed_nodes = [node for node in self.nodes if 'changed' in node['diff_type']] 106 | impacted_nodes = [node for node in self.nodes if 'changed' not in node['diff_type']] 107 | for changed_node in changed_nodes: 108 | changed_node['x'] = 100 109 | changed_node['y'] = (changed_nodes.index(changed_node) + 1) * (canvas_height / len(changed_nodes)) 110 | changed_node['symbolSize'] = 20 111 | changed_node['label'] = { 112 | 'show': True, 113 | 'formatter': changed_node["name"].split("(")[0] 114 | } 115 | tooltip = f'{changed_node["name"].split("(")[0]}
[Changed]{changed_node.get("diff_content", "")}' 116 | if changed_node.get('is_api'): 117 | tooltip = tooltip + f'
[API]{changed_node.get("api_path")}' 118 | changed_node['tooltip'] = { 119 | 'show': True, 120 | 'position': 'right', 121 | 'formatter': tooltip 122 | } 123 | all_node.append(changed_node) 124 | max_link_count = max([value for key, value in result.items()]) if result else 1 125 | count_node_result = {} 126 | for key, value in result.items(): 127 | value = str(value) 128 | if value not in count_node_result: 129 | count_node_result[value] = [] 130 | count_node_result[value].append(key) 131 | for impacted_node in impacted_nodes: 132 | path_level = result.get(impacted_node['id'], 0) 133 | level_node_list = count_node_result.get(str(path_level), [impacted_node['id']]) 134 | level_node_index = level_node_list.index(impacted_node['id']) if impacted_node['id'] in level_node_list else 1 135 | impacted_node['x'] = 100 + ((canvas_width - 100) / max_link_count) * (path_level + 1) 136 | impacted_node['y'] = (canvas_height / len(count_node_result.get(str(path_level), [1]))) * level_node_index 137 | impacted_node['label'] = { 138 | 'show': True, 139 | 'formatter': impacted_node["name"].split("(")[0] 140 | } 141 | if impacted_node.get('is_api'): 142 | tooltip = f'{impacted_node["name"].split("(")[0]}
[API]{impacted_node.get("api_path")}' 143 | impacted_node['tooltip'] = { 144 | 'show': True, 145 | 'position': 'right', 146 | 'formatter': tooltip 147 | } 148 | all_node.append(impacted_node) 149 | self.nodes = all_node 150 | 151 | if __name__ == '__main__': 152 | # relationships = [{'source': '0', 'target': '9'}, {'source': '0', 'target': '2'}, {'source': '0', 'target': '10'}, {'source': '0', 'target': '1'}, {'source': '0', 'target': '11'}, {'source': '0', 'target': '12'}, {'source': '1', 'target': '13'}, {'source': '2', 'target': '13'}, {'source': '7', 'target': '14'}, {'source': '8', 'target': '15'}, {'source': '9', 'target': '13'}, {'source': '10', 'target': '13'}, {'source': '11', 'target': '13'}, {'source': '12', 'target': '13'}, {'source': '13', 'target': '16'}, {'source': '13', 'target': '17'}, {'source': '13', 'target': '18'}, {'source': '13', 'target': '19'}, {'source': '13', 'target': '20'}, {'source': '13', 'target': '21'}, {'source': '13', 'target': '22'}, {'source': '13', 'target': '23'}, {'source': '13', 'target': '24'}, {'source': '13', 'target': '25'}, {'source': '13', 'target': '26'}, {'source': '13', 'target': '27'}, {'source': '13', 'target': '28'}, {'source': '13', 'target': '7'}, {'source': '13', 'target': '8'}, {'source': '13', 'target': '29'}, {'source': '13', 'target': '30'}, {'source': '13', 'target': '31'}, {'source': '13', 'target': '32'}, {'source': '13', 'target': '33'}, {'source': '13', 'target': '34'}, {'source': '13', 'target': '35'}, {'source': '13', 'target': '36'}, {'source': '13', 'target': '37'}, {'source': '13', 'target': '38'}, {'source': '13', 'target': '39'}, {'source': '13', 'target': '40'}, {'source': '13', 'target': '41'}, {'source': '13', 'target': '42'}, {'source': '13', 'target': '43'}, {'source': '13', 'target': '44'}, {'source': '13', 'target': '45'}, {'source': '13', 'target': '46'}, {'source': '13', 'target': '47'}, {'source': '13', 'target': '48'}, {'source': '13', 'target': '49'}, {'source': '13', 'target': '50'}, {'source': '16', 'target': '51'}, {'source': '16', 'target': '52'}, {'source': '16', 'target': '53'}, {'source': '16', 'target': '54'}, {'source': '16', 'target': '55'}, {'source': '16', 'target': '56'}, {'source': '16', 'target': '57'}, {'source': '16', 'target': '58'}, {'source': '16', 'target': '59'}, {'source': '17', 'target': '60'}, {'source': '18', 'target': '61'}, {'source': '19', 'target': '62'}, {'source': '20', 'target': '63'}, {'source': '21', 'target': '64'}, {'source': '22', 'target': '65'}, {'source': '23', 'target': '66'}, {'source': '24', 'target': '67'}, {'source': '25', 'target': '68'}, {'source': '26', 'target': '69'}, {'source': '27', 'target': '70'}, {'source': '28', 'target': '71'}, {'source': '29', 'target': '72'}, {'source': '30', 'target': '73'}, {'source': '31', 'target': '74'}, {'source': '32', 'target': '75'}, {'source': '33', 'target': '76'}, {'source': '34', 'target': '77'}, {'source': '35', 'target': '78'}, {'source': '36', 'target': '79'}, {'source': '36', 'target': '80'}, {'source': '36', 'target': '81'}, {'source': '36', 'target': '82'}, {'source': '37', 'target': '83'}, {'source': '38', 'target': '84'}, {'source': '39', 'target': '85'}, {'source': '40', 'target': '86'}, {'source': '41', 'target': '87'}, {'source': '42', 'target': '83'}, {'source': '43', 'target': '88'}, {'source': '44', 'target': '89'}, {'source': '45', 'target': '90'}, {'source': '46', 'target': '91'}, {'source': '47', 'target': '92'}, {'source': '48', 'target': '93'}, {'source': '49', 'target': '94'}, {'source': '50', 'target': '95'}, {'source': '51', 'target': '96'}, {'source': '52', 'target': '97'}, {'source': '53', 'target': '52'}, {'source': '53', 'target': '98'}, {'source': '54', 'target': '99'}, {'source': '55', 'target': '100'}, {'source': '56', 'target': '101'}, {'source': '57', 'target': '98'}, {'source': '57', 'target': '102'}, {'source': '58', 'target': '103'}, {'source': '59', 'target': '104'}, {'source': '60', 'target': '105'}, {'source': '61', 'target': '105'}, {'source': '62', 'target': '105'}, {'source': '63', 'target': '105'}, {'source': '64', 'target': '105'}, {'source': '65', 'target': '106'}, {'source': '66', 'target': '105'}, {'source': '67', 'target': '105'}, {'source': '68', 'target': '105'}, {'source': '69', 'target': '105'}, {'source': '75', 'target': '107'}, {'source': '75', 'target': '108'}, {'source': '75', 'target': '109'}, {'source': '79', 'target': '110'}, {'source': '80', 'target': '111'}, {'source': '81', 'target': '105'}, {'source': '82', 'target': '112'}, {'source': '83', 'target': '113'}, {'source': '85', 'target': '114'}, {'source': '87', 'target': '115'}, {'source': '88', 'target': '105'}, {'source': '98', 'target': '116'}, {'source': '106', 'target': '105'}, {'source': '107', 'target': '117'}, {'source': '107', 'target': '118'}, {'source': '107', 'target': '119'}, {'source': '107', 'target': '120'}, {'source': '107', 'target': '121'}, {'source': '107', 'target': '122'}, {'source': '107', 'target': '123'}, {'source': '107', 'target': '124'}, {'source': '107', 'target': '125'}, {'source': '109', 'target': '126'}, {'source': '110', 'target': '13'}, {'source': '111', 'target': '13'}, {'source': '112', 'target': '105'}, {'source': '114', 'target': '127'}, {'source': '115', 'target': '105'}, {'source': '118', 'target': '128'}, {'source': '119', 'target': '129'}, {'source': '120', 'target': '130'}, {'source': '121', 'target': '131'}, {'source': '122', 'target': '132'}, {'source': '123', 'target': '133'}, {'source': '124', 'target': '134'}, {'source': '125', 'target': '135'}, {'source': '126', 'target': '136'}, {'source': '127', 'target': '105'}, {'source': '128', 'target': '105'}, {'source': '129', 'target': '105'}, {'source': '130', 'target': '105'}, {'source': '131', 'target': '105'}, {'source': '132', 'target': '105'}, {'source': '133', 'target': '105'}, {'source': '134', 'target': '105'}, {'source': '135', 'target': '105'}] 153 | relationships = [{'source': 'A', 'target': 'B'},{'source': 'B', 'target': 'C'},{'source': 'C', 'target': 'D'},{'source': 'B', 'target': 'D'}] 154 | bb = max_relationship_length(relationships) 155 | print(bb) 156 | -------------------------------------------------------------------------------- /src/jcci/analyze.py: -------------------------------------------------------------------------------- 1 | import json 2 | import os 3 | import sys 4 | import time 5 | import atexit 6 | import logging 7 | import datetime 8 | import fnmatch 9 | from . import config as config 10 | from .database import SqliteHelper 11 | from .java_parse import JavaParse, calculate_similar_score_method_params 12 | from . import mapper_parse as mapper_parse 13 | from . import diff_parse as diff_parse 14 | from . import graph as graph 15 | from . import constant as constant 16 | 17 | logging.basicConfig(format='%(asctime)s %(message)s', level=logging.DEBUG) 18 | 19 | 20 | class JCCI(object): 21 | def __init__(self, git_url, username): 22 | self.git_url = git_url 23 | self.username: str = username 24 | self.branch_name: str = '' 25 | self.commit_or_branch_new: str = '' 26 | self.commit_or_branch_old: str = '' 27 | self.project_id: int = -1 28 | self.cci_filepath: str = '' 29 | self.project_name: str = '' 30 | self.file_path: str = '' 31 | self.sqlite = SqliteHelper(config.db_path + '/' + username + '_jcci.db') 32 | self.view = graph.Graph() 33 | self.t1 = datetime.datetime.now() 34 | self.need_analyze_obj_list = [] 35 | self.analyzed_obj_set = [] 36 | self.diff_parse_map = {} 37 | self.xml_parse_results_new = {} 38 | self.xml_parse_results_old = {} 39 | 40 | # Step 1.1 41 | def _can_analyze(self, filepath, cci_file_path): 42 | # 已有分析结果 43 | if os.path.exists(cci_file_path): 44 | logging.info('Has analyze result, skip!') 45 | with open(cci_file_path, 'r') as read: 46 | result = read.read() 47 | result_json = json.loads(result) 48 | print(result, flush=True) 49 | print(f'Impacted api list: {result_json["impacted_api_list"]}', flush=True) 50 | sys.exit(0) 51 | 52 | # 正在分析 53 | wait_index = 0 54 | occupy_filepath = os.path.join(filepath, 'Occupy.ing') 55 | atexit.register(self._clean_occupy, occupy_filepath) 56 | while os.path.exists(occupy_filepath) and wait_index < 30: 57 | logging.info(f'Analyzing by others, waiting or clean occupying file manually at: {occupy_filepath} to continue') 58 | time.sleep(3) 59 | wait_index += 1 60 | if os.path.exists(occupy_filepath): 61 | logging.info(f'Analyzing by others, waiting timeout') 62 | sys.exit(0) 63 | 64 | # Step 1.2 65 | def _clean_occupy(self, occupy_path): 66 | if os.path.exists(occupy_path): 67 | os.remove(occupy_path) 68 | 69 | # Step 1.3 70 | def _occupy_project(self): 71 | # 占住项目分析 72 | logging.info('Start occupying project, and others can not analyze until released') 73 | occupy_filepath = os.path.join(self.file_path, 'Occupy.ing') 74 | with open(occupy_filepath, 'w') as ow: 75 | ow.write(f'Occupy by {self.username}') 76 | time.sleep(1) 77 | 78 | def _clone_dependents_project(self, dependents): 79 | for dependent in dependents: 80 | dependent_git_url = dependent.get('git_url') 81 | if not dependent_git_url: 82 | continue 83 | dependent_branch = dependent.get('branch', 'master') 84 | dependent_commit_id = dependent.get('commit_id', 'HEAD') 85 | dependent_project_name = dependent_git_url.split('/')[-1].split('.git')[0] 86 | dependent_file_path = os.path.join(self.file_path, dependent_project_name) 87 | if not os.path.exists(dependent_file_path): 88 | logging.info(f'Cloning dependent project: {dependent_git_url}') 89 | os.system(f'git clone -b {dependent_branch} {dependent_git_url} {dependent_file_path} && cd {dependent_file_path} && git reset --hard {dependent_commit_id}') 90 | else: 91 | os.system(f'cd {dependent_file_path} && git fetch --all && git checkout -b {dependent_branch} origin/{dependent_branch} && git reset --hard {dependent_commit_id}') 92 | os.system(f'cd {dependent_file_path} && git checkout -b {dependent_branch} && git reset --hard {dependent_commit_id}') 93 | 94 | # Step 2 95 | def _get_diff_parse_map(self, filepath, branch, commit_first, commit_second): 96 | logging.info('Git pull project to HEAD') 97 | os.system(f'cd {filepath} && git checkout {branch} && git pull') 98 | time.sleep(1) 99 | logging.info(f'Git diff between {commit_first} and {commit_second}') 100 | diff_base = f'cd {self.file_path} && git diff {commit_second}..{commit_first} > diff_{commit_second}..{commit_first}.txt' 101 | os.system(diff_base) 102 | diff_txt = os.path.join(self.file_path, f'diff_{commit_second}..{commit_first}.txt') 103 | logging.info(f'Analyzing diff file, location: {diff_txt}') 104 | return diff_parse.get_diff_info(diff_txt) 105 | 106 | # Step 2 107 | def _get_branch_diff_parse_map(self, filepath, branch_first, branch_second): 108 | logging.info('Git pull project to HEAD') 109 | os.system(f'cd {filepath} && git fetch --all && git checkout -b {branch_second} origin/{branch_second} && git checkout {branch_second} && git pull') 110 | time.sleep(1) 111 | os.system(f'cd {filepath} && git fetch --all && git checkout -b {branch_first} origin/{branch_first} && git checkout {branch_first} && git pull') 112 | time.sleep(1) 113 | logging.info(f'Git diff between {branch_first} and {branch_second}') 114 | diff_base = f'cd {self.file_path} && git diff {branch_second}..{branch_first} > diff_{branch_second.replace("/", "#")}..{branch_first.replace("/", "#")}.txt' 115 | os.system(diff_base) 116 | diff_txt = os.path.join(self.file_path, f'diff_{branch_second.replace("/", "#")}..{branch_first.replace("/", "#")}.txt') 117 | logging.info(f'Analyzing diff file, location: {diff_txt}') 118 | return diff_parse.get_diff_info(diff_txt) 119 | 120 | # Step 3 121 | def _parse_project(self, project_dir, new_commit_or_branch, old_commit_or_branch): 122 | # 解析最新的项目文件 123 | os.system(f'cd {project_dir} && git reset --hard {new_commit_or_branch}') 124 | time.sleep(2) 125 | file_path_list = self._get_project_files(project_dir) 126 | diff_xml_file_path = [key for key in file_path_list if key.endswith('.xml') and any(key.endswith(diff_path) for diff_path in self.diff_parse_map.keys())] 127 | java_parse = JavaParse(self.sqlite.db_path, self.project_id) 128 | java_parse.parse_java_file_list(file_path_list, new_commit_or_branch) 129 | xml_parse_result_new = self._parse_xml_file(diff_xml_file_path) 130 | xml_parse_result_old = {} 131 | if not old_commit_or_branch: 132 | return xml_parse_result_new, xml_parse_result_old 133 | # 解析旧版本有差异的文件 134 | os.system(f'cd {project_dir} && git reset --hard {old_commit_or_branch}') 135 | time.sleep(2) 136 | xml_parse_result_old = self._parse_xml_file(diff_xml_file_path) 137 | for key in self.diff_parse_map.keys(): 138 | matched_file_path_list = [filepath for filepath in file_path_list if filepath.endswith(key)] 139 | if not matched_file_path_list: 140 | continue 141 | matched_file_path = matched_file_path_list[0] 142 | java_parse.parse_java_file(matched_file_path, old_commit_or_branch, parse_import_first=False) 143 | return xml_parse_result_new, xml_parse_result_old 144 | 145 | # Step 3 146 | def _parse_branch_project(self, project_dir, new_branch, old_branch): 147 | # 解析最新的项目文件 148 | os.system(f'cd {project_dir} && git checkout {new_branch}') 149 | time.sleep(2) 150 | file_path_list = self._get_project_files(project_dir) 151 | diff_xml_file_path = [key for key in file_path_list if key.endswith('.xml') and any(key.endswith(diff_path) for diff_path in self.diff_parse_map.keys())] 152 | java_parse = JavaParse(self.sqlite.db_path, self.project_id) 153 | java_parse.parse_java_file_list(file_path_list, new_branch) 154 | xml_parse_result_new = self._parse_xml_file(diff_xml_file_path) 155 | # 解析旧版本有差异的文件 156 | os.system(f'cd {project_dir} && git checkout {old_branch}') 157 | time.sleep(2) 158 | xml_parse_result_old = self._parse_xml_file(diff_xml_file_path) 159 | for key in self.diff_parse_map.keys(): 160 | matched_file_path_list = [filepath for filepath in file_path_list if filepath.endswith(key)] 161 | if not matched_file_path_list: 162 | continue 163 | matched_file_path = matched_file_path_list[0] 164 | java_parse.parse_java_file(matched_file_path, old_branch, parse_import_first=False) 165 | return xml_parse_result_new, xml_parse_result_old 166 | 167 | # Step 3.1 get all java files 168 | def _get_project_files(self, project_dir): 169 | file_lists = [] 170 | for root, dirs, files in os.walk(project_dir): 171 | if '.git' in root or os.path.join('src', 'test') in root: 172 | continue 173 | for file in files: 174 | ignore = False 175 | filepath = os.path.join(root, file) 176 | for pattern in config.ignore_file: 177 | if fnmatch.fnmatch(filepath, pattern): 178 | ignore = True 179 | break 180 | if ignore: 181 | continue 182 | filepath = filepath.replace('\\', '/') 183 | file_lists.append(filepath) 184 | return file_lists 185 | 186 | # Step 3.3 187 | def _parse_xml_file(self, file_path_list): 188 | xml_parse_results = {} 189 | for filepath in file_path_list: 190 | if filepath.endswith('.xml'): 191 | xml_parse_result = mapper_parse.parse(filepath) 192 | if xml_parse_result: 193 | xml_parse_results[filepath] = xml_parse_result 194 | return xml_parse_results 195 | 196 | # Step 4 197 | def _diff_analyze(self, patch_filepath: str, diff_parse_obj: dict): 198 | is_xml_file = patch_filepath.endswith('.xml') 199 | if is_xml_file: 200 | self._xml_diff_analyze(patch_filepath, diff_parse_obj) 201 | else: 202 | self._java_diff_analyze(patch_filepath, diff_parse_obj) 203 | 204 | # Step 4.1 205 | def _xml_diff_analyze(self, patch_filepath, diff_parse_obj: dict): 206 | xml_file_path_list = [filepath for filepath in self.xml_parse_results_new.keys() if filepath.endswith(patch_filepath)] 207 | if not xml_file_path_list: 208 | return 209 | xml_file_path = xml_file_path_list[0] 210 | xml_name = xml_file_path.split('/')[-1] 211 | xml_parse_result_new: mapper_parse.Mapper = self.xml_parse_results_new.get(xml_file_path) 212 | if xml_parse_result_new: 213 | methods = xml_parse_result_new.result_maps + xml_parse_result_new.sqls + xml_parse_result_new.statements 214 | self._xml_method_diff_analyze(methods, diff_parse_obj['line_num_added'], diff_parse_obj['line_content_added'], xml_parse_result_new, xml_name, xml_file_path, self.commit_or_branch_new) 215 | xml_parse_result_old = self.xml_parse_results_old.get(xml_file_path) 216 | if xml_parse_result_old: 217 | methods = xml_parse_result_old.result_maps + xml_parse_result_old.sqls + xml_parse_result_old.statements 218 | self._xml_method_diff_analyze(methods, diff_parse_obj['line_num_removed'], diff_parse_obj['line_content_removed'], xml_parse_result_old, xml_name, xml_file_path, self.commit_or_branch_old) 219 | 220 | # Step 4.1.1 221 | def _xml_method_diff_analyze(self, methods: list, line_num_list: list, line_content_list: list, xml_parse_result, xml_name, xml_file_path, commit_or_branch): 222 | namespace = xml_parse_result.namespace 223 | mapper_extend_dict = { 224 | 'mapper_file_name': xml_name, 225 | 'mapper_filepath': xml_file_path 226 | } 227 | for line_num in line_num_list: 228 | method_changed = [method for method in methods if self._is_line_num_in_xml_method_range(method, line_num)] 229 | method_changed_name = [method.name for method in method_changed] 230 | for method in method_changed: 231 | diff_content = line_content_list[line_num_list.index(line_num)] 232 | method_node_id = self.view.create_node_category(xml_name, method.name, method.type, constant.DIFF_TYPE_CHANGED, diff_content, xml_file_path, '', method.content, {}) 233 | if method.type == constant.NODE_TYPE_MAPPER_STATEMENT: 234 | mapper_extend_dict['method_node_id'] = method_node_id 235 | self._add_to_need_analyze_obj_list('xml', namespace, None, method.name, commit_or_branch, mapper_extend_dict) 236 | continue 237 | for statement in xml_parse_result.statements: 238 | if statement.result_map in method_changed_name or statement.include_sql in method_changed_name: 239 | statement_node_id = self.view.create_node_category(xml_name, statement.name, statement.type, constant.DIFF_TYPE_IMPACTED, '', xml_file_path, '', statement.content, {}) 240 | self.view.create_node_link(method_node_id, statement_node_id) 241 | mapper_extend_dict['method_node_id'] = statement_node_id 242 | self._add_to_need_analyze_obj_list('xml', namespace, None, statement.name, commit_or_branch, mapper_extend_dict) 243 | 244 | # Step 4.1.1.1 245 | def _is_line_num_in_xml_method_range(self, method, line_num): 246 | line_num_in_method = False 247 | if method.start <= line_num <= method.end: 248 | line_num_in_method = True 249 | return line_num_in_method 250 | 251 | # Step 4.2 252 | def _java_diff_analyze(self, patch_filepath: str, diff_parse_obj: dict): 253 | # new branch or commit 254 | class_db = self.sqlite.select_data(f'''SELECT * FROM class WHERE project_id = {self.project_id} and commit_or_branch = "{self.commit_or_branch_new}" and filepath LIKE "%{patch_filepath}"''') 255 | for class_db_obj in class_db: 256 | self._java_field_method_diff_analyze(class_db_obj, diff_parse_obj['line_num_added'], diff_parse_obj['line_content_added'], self.commit_or_branch_new) 257 | # old branch or commit 258 | if not self.commit_or_branch_old: 259 | return 260 | class_db = self.sqlite.select_data(f'SELECT * FROM class WHERE project_id = {self.project_id} and commit_or_branch = "{self.commit_or_branch_old}" and filepath LIKE "%{patch_filepath}"') 261 | for class_db_obj in class_db: 262 | self._java_field_method_diff_analyze(class_db_obj, diff_parse_obj['line_num_removed'], diff_parse_obj['line_content_removed'], self.commit_or_branch_old) 263 | 264 | # Step 4.2.1 265 | def _java_field_method_diff_analyze(self, class_db: dict, line_num_list: list, line_content_list: list, commit_or_branch: str or None): 266 | if not commit_or_branch: 267 | return 268 | class_name = class_db['class_name'] 269 | class_filepath = class_db['filepath'] 270 | is_controller = class_db['is_controller'] 271 | data_in_annotation = [annotation for annotation in json.loads(class_db['annotations']) if annotation['name'] in ['Data', 'Getter', 'Setter', 'Builder', 'NoArgsConstructor', 'AllArgsConstructor']] 272 | for line_num in line_num_list: 273 | diff_content = line_content_list[line_num_list.index(line_num)] 274 | fields_list = self.sqlite.select_data(f'SELECT field_id, class_id, field_type, field_name, documentation, is_static FROM field WHERE class_id = {class_db["class_id"]} AND start_line <={line_num} AND end_line >= {line_num} order by start_line asc limit 1') 275 | methods_list = self.sqlite.select_data(f'SELECT method_id, class_id, method_name, parameters, return_type, is_api, api_path, documentation, body FROM methods WHERE class_id = {class_db["class_id"]} AND start_line <={line_num} AND end_line >= {line_num} order by start_line asc limit 1') 276 | class_node_id = None 277 | if fields_list: 278 | is_not_static_fields = [field for field in fields_list if field.get('is_static') == 'False'] 279 | if is_not_static_fields and data_in_annotation: 280 | self._add_to_need_analyze_obj_list('java', f'{class_db["package_name"]}.{class_name}', None, None, commit_or_branch, class_db) 281 | class_node_id = self.view.create_node_category(class_name, 'entity', constant.NODE_TYPE_CLASS, constant.DIFF_TYPE_CHANGED, '', self.file_path, '', '', {}) 282 | elif is_not_static_fields and not data_in_annotation: 283 | field_method_name = [] 284 | for field in is_not_static_fields: 285 | field_name = field['field_name'] 286 | field_name_capitalize = field_name[0].upper() + field_name[1:] 287 | field_method_name += ['get' + field_name_capitalize, 'set' + field_name_capitalize, 'is' + field_name_capitalize] 288 | field_method_name_str = '"' + '","'.join(field_method_name) + '"' 289 | field_method_db = self.sqlite.select_data(f'SELECT method_id FROM methods WHERE class_id = {class_db["class_id"]} AND method_name in ({field_method_name_str})') 290 | if field_method_db: 291 | self._add_to_need_analyze_obj_list('java', f'{class_db["package_name"]}.{class_name}', None, None, commit_or_branch, class_db) 292 | class_node_id = self.view.create_node_category(class_name, 'entity', constant.NODE_TYPE_CLASS, constant.DIFF_TYPE_CHANGED, '', self.file_path, '', '', {}) 293 | for field_db in fields_list: 294 | node_id = self.view.create_node_category(class_name, field_db['field_name'], constant.NODE_TYPE_FIELD, constant.DIFF_TYPE_CHANGED, diff_content, class_filepath, field_db['documentation'], '', {}) 295 | field_db['field_node_id'] = node_id 296 | self._add_to_need_analyze_obj_list('java', f'{class_db["package_name"]}.{class_name}', field_db['field_name'], None, commit_or_branch, field_db) 297 | if class_node_id: 298 | self.view.create_node_link(node_id, class_node_id) 299 | for method_db in methods_list: 300 | node_extend_dict = {'is_api': False} 301 | if is_controller and method_db['is_api']: 302 | node_extend_dict = { 303 | 'is_api': True, 304 | 'api_path': method_db['api_path'] 305 | } 306 | method_name_param = f'{method_db["method_name"]}({",".join([param["parameter_type"] for param in json.loads(method_db["parameters"])])})' 307 | node_id = self.view.create_node_category(class_name, method_name_param, constant.NODE_TYPE_METHOD, constant.DIFF_TYPE_CHANGED, diff_content, class_filepath, method_db.get('documentation'), method_db.get('body'), node_extend_dict) 308 | method_db['method_node_id'] = node_id 309 | self._add_to_need_analyze_obj_list('java', f'{class_db["package_name"]}.{class_name}', None, method_name_param, commit_or_branch, method_db) 310 | 311 | # Step 5 312 | def _impacted_analyze(self, need_analyze_obj: dict): 313 | file_type = need_analyze_obj['file_type'] 314 | package_class = need_analyze_obj['package_class'] 315 | commit_or_branch = need_analyze_obj['commit_or_branch'] 316 | package_name = '.'.join(package_class.split('.')[0: -1]) 317 | class_name = package_class.split('.')[-1] 318 | class_db_list = self.sqlite.select_data(f'SELECT class_id, filepath, commit_or_branch, is_controller, annotations, extends_class, implements ' 319 | f' FROM class WHERE project_id = {self.project_id} and class_name="{class_name}" and package_name="{package_name}"') 320 | class_entity = self._get_right_class_entity(class_db_list, commit_or_branch) 321 | if not class_entity: 322 | return 323 | class_filepath = class_entity['filepath'] 324 | class_id = class_entity["class_id"] 325 | # gengxin 326 | commit_or_branch = class_entity['commit_or_branch'] 327 | is_controller = class_entity['is_controller'] 328 | # todo 粗查，待细化 329 | if file_type == 'xml': 330 | method_name = need_analyze_obj['method_param'] 331 | mapper_method_node_id = need_analyze_obj['method_node_id'] 332 | impacted_methods = self.sqlite.select_data(f'SELECT method_id, class_id, method_name, parameters, return_type, is_api, api_path, documentation, body ' 333 | f'FROM methods WHERE class_id={class_id} and method_name="{method_name}"') 334 | if not impacted_methods: 335 | return 336 | for impacted_method in impacted_methods: 337 | node_extend_dict = {'is_api': False} 338 | if is_controller and impacted_method['is_api']: 339 | node_extend_dict = { 340 | 'is_api': True, 341 | 'api_path': impacted_method['api_path'] 342 | } 343 | method_name_param = f'{impacted_method["method_name"]}({",".join([param["parameter_type"] for param in json.loads(impacted_method["parameters"])])})' 344 | impacted_method_node_id = self.view.create_node_category(class_name, method_name_param, 345 | constant.NODE_TYPE_METHOD, constant.DIFF_TYPE_IMPACTED, 346 | impacted_method.get('body'), class_filepath, impacted_method.get('documentation'), 347 | impacted_method.get('body'), node_extend_dict) 348 | self.view.create_node_link(mapper_method_node_id, impacted_method_node_id) 349 | extend_dict = {'method_node_id': impacted_method_node_id} 350 | extend_dict.update(impacted_method) 351 | self._add_to_need_analyze_obj_list('java', package_class, None, self._get_method_param_string(impacted_method), commit_or_branch, extend_dict) 352 | else: 353 | # analyze entity use 354 | entity_impacted_methods = [] 355 | entity_impacted_fields = [] 356 | source_node_id = None 357 | if not need_analyze_obj.get('field_name') and not need_analyze_obj.get('method_param'): 358 | class_node_id = self.view.create_node_category(class_name, 'entity', constant.NODE_TYPE_CLASS, constant.DIFF_TYPE_IMPACTED, '', self.file_path, '', '', {}) 359 | entity_impacted_methods = self._get_entity_invocation_in_methods_table(package_class) 360 | entity_impacted_fields = self._get_entity_invocation_in_field_table(package_class) 361 | source_node_id = class_node_id 362 | elif need_analyze_obj.get('field_name'): 363 | annotations: list = json.loads(class_entity['annotations']) 364 | entity_impacted_methods = self._get_field_invocation_in_methods_table(package_class, need_analyze_obj, annotations, commit_or_branch, class_id) 365 | source_node_id = need_analyze_obj.get('field_node_id') 366 | elif need_analyze_obj.get('method_param'): 367 | method_param = need_analyze_obj.get('method_param') 368 | method_name: str = method_param.split('(')[0] 369 | method_node_id = need_analyze_obj.get('method_node_id') 370 | source_node_id = method_node_id 371 | entity_impacted_methods = self._get_method_invocation_in_methods_table(package_class, method_param, commit_or_branch) 372 | method_db = self.sqlite.select_data(f'SELECT annotations FROM methods WHERE method_id = {need_analyze_obj.get("method_id")}')[0] 373 | is_override_method = 'Override' in method_db['annotations'] 374 | if is_override_method: 375 | 376 | if class_entity['extends_class']: 377 | abstract_package_class, method_params = self._is_method_param_in_extends_package_class(method_param, class_entity['extends_class'], 'True', commit_or_branch) 378 | if abstract_package_class: 379 | extends_methods = self._get_method_invocation_in_methods_table(abstract_package_class, method_params, commit_or_branch) 380 | # for method in extends_methods: 381 | # method['class_id'] = class_id 382 | entity_impacted_methods += extends_methods 383 | 384 | if class_entity['implements']: 385 | class_implements = class_entity['implements'].split(',') 386 | class_implements_obj = self.sqlite.select_data(f'''select c.package_name , c.class_name from methods m left join class c on c.class_id = m.class_id 387 | where c.project_id = {self.project_id} and m.method_name = '{method_name}' and c.class_name in ("{'","'.join(class_implements)}")''') 388 | if class_implements_obj: 389 | implements_package_class = class_implements_obj[0].get('package_name') + '.' + class_implements_obj[0].get('class_name') 390 | implements_package_class, method_params = self._is_method_param_in_extends_package_class(method_param, implements_package_class, 'False', commit_or_branch) 391 | if implements_package_class: 392 | implements_methods = self._get_method_invocation_in_methods_table(implements_package_class, method_params, commit_or_branch) 393 | # implements_methods = self._get_method_invocation_in_methods_table(implements_package_class, method_param, commit_or_branch) 394 | entity_impacted_methods += implements_methods 395 | else: 396 | class_method_db = self.sqlite.select_data(f'SELECT method_id FROM methods WHERE class_id = {class_id} and method_name = "{method_name}"') 397 | if not class_method_db: 398 | extends_package_class, method_params = self._is_method_param_in_extends_package_class(method_param, class_entity['extends_class'], 'False', commit_or_branch) 399 | if extends_package_class: 400 | extends_methods = self._get_method_invocation_in_methods_table(extends_package_class, method_params, commit_or_branch) 401 | entity_impacted_methods += extends_methods 402 | self._handle_impacted_methods(entity_impacted_methods, source_node_id) 403 | self._handle_impacted_fields(entity_impacted_fields, source_node_id) 404 | 405 | def _is_method_param_in_extends_package_class(self, method_param, extends_package_class, is_abstract, commit_or_branch): 406 | if not extends_package_class: 407 | return None, None 408 | method_name: str = method_param.split('(')[0] 409 | method_arguments = method_param.split('(')[1].split(')')[0].split(',') 410 | method_arguments = [ma for ma in method_arguments if ma] 411 | extends_package = '.'.join(extends_package_class.split('.')[0: -1]) 412 | extends_class_name = extends_package_class.split('.')[-1] 413 | extends_class_db = self.sqlite.select_data(f'SELECT class_id, extends_class FROM class WHERE package_name = "{extends_package}" and class_name = "{extends_class_name}" and project_id = {self.project_id} and commit_or_branch = "{commit_or_branch}"') 414 | if not extends_class_db: 415 | extends_class_db = self.sqlite.select_data(f'SELECT class_id, extends_class FROM class WHERE package_name = "{extends_package}" and class_name = "{extends_class_name}" and project_id = {self.project_id}') 416 | if not extends_class_db: 417 | return None, None 418 | extends_class_id = extends_class_db[0]['class_id'] 419 | methods_db_list = self.sqlite.select_data(f'SELECT * FROM methods WHERE class_id = {extends_class_id} and method_name = "{method_name}" and is_abstract = "{is_abstract}"') 420 | filter_methods = [method for method in methods_db_list if len(json.loads(method.get('parameters', '[]'))) == len(method_arguments)] 421 | if not filter_methods: 422 | if extends_class_db[0]['extends_class']: 423 | return self._is_method_param_in_extends_package_class(method_param, extends_class_db[0]['extends_class'], is_abstract, commit_or_branch) 424 | else: 425 | return None, None 426 | if len(filter_methods) == 1: 427 | method_db = filter_methods[0] 428 | method_params = f'{method_db.get("method_name", method_name)}({",".join([param["parameter_type"] for param in json.loads(method_db.get("parameters", "[]"))])})' 429 | return extends_package_class, method_params 430 | else: 431 | max_score = -float('inf') 432 | max_score_method = None 433 | for method_db in filter_methods: 434 | method_db_params = [param["parameter_type"] for param in json.loads(method_db.get("parameters", "[]"))] 435 | score = calculate_similar_score_method_params(method_arguments, method_db_params) 436 | if score > max_score: 437 | max_score = score 438 | max_score_method = method_db 439 | if max_score_method is None: 440 | max_score_method = filter_methods[0] 441 | method_params = f'{max_score_method.get("method_name", method_name)}({",".join([param["parameter_type"] for param in json.loads(max_score_method.get("parameters", "[]"))])})' 442 | return extends_package_class, method_params 443 | 444 | def _get_extends_package_class(self, package_class): 445 | extends_package_class_list = [] 446 | extends_package_class_db = self.sqlite.select_data(f'SELECT package_name, class_name FROM class WHERE project_id = {self.project_id} AND extends_class="{package_class}"') 447 | if extends_package_class_db: 448 | extends_package_class_list = [f'{class_item["package_name"]}.{class_item["class_name"]}' for class_item in extends_package_class_db] 449 | for extends_package_class in extends_package_class_list: 450 | extends_package_class_list += self._get_extends_package_class(extends_package_class) 451 | return extends_package_class_list 452 | 453 | # Step 5.1 454 | def _get_right_class_entity(self, class_db_list, commit_or_branch): 455 | right_class_entity = next((item for item in class_db_list if item.get("commit_or_branch") == commit_or_branch), None) 456 | if right_class_entity is None: 457 | right_class_entity = next((item for item in class_db_list if item.get("commit_or_branch") == self.commit_or_branch_new), None) 458 | return right_class_entity 459 | 460 | # Step 5.2 461 | def _get_entity_invocation_in_methods_table(self, package_class: str): 462 | return self.sqlite.select_data(f'''SELECT method_id, class_id, method_name, parameters, return_type, is_api, api_path, documentation, body FROM methods WHERE project_id = {self.project_id} AND json_extract(method_invocation_map, '$."{package_class}".entity') IS NOT NULL''') 463 | 464 | # Step 5.2 465 | def _get_entity_invocation_in_field_table(self, package_class: str): 466 | return self.sqlite.select_data(f'''SELECT field_id, class_id, annotations, field_type, field_name, is_static, documentation FROM field WHERE project_id = {self.project_id} AND field_type = "{package_class}"''') 467 | 468 | # Step 5.3 469 | def _get_field_invocation_in_methods_table(self, package_class, field_obj, annotations, commit_or_branch, class_id): 470 | is_static = field_obj['is_static'] 471 | field_name = field_obj['field_name'] 472 | field_name_capitalize = field_name[0].upper() + field_name[1:] 473 | if not is_static: 474 | # todo static maybe has bug 475 | field_methods_set = set() 476 | for annotation in annotations: 477 | annotation_name = annotation.get('name') 478 | if annotation_name == 'Data': 479 | field_methods_set.add(f'get{field_name_capitalize}(') 480 | field_methods_set.add(f'set{field_name_capitalize}(') 481 | elif annotation_name == 'Getter': 482 | field_methods_set.add(f'get{field_name_capitalize}(') 483 | elif annotation_name == 'Setter': 484 | field_methods_set.add(f'set{field_name_capitalize}(') 485 | else: 486 | continue 487 | if not field_methods_set: 488 | return [] 489 | json_extract_sql_list = [] 490 | for field_method in field_methods_set: 491 | sql_part = f'''json_extract(method_invocation_map, '$."{package_class}".methods.keys(@.startsWith("{field_method}"))') IS NOT NULL''' 492 | json_extract_sql_list.append(sql_part) 493 | sql = f'SELECT method_id, class_id, method_name, parameters, return_type, is_api, api_path, documentation, body FROM methods WHERE project_id = {self.project_id} AND (' + ' OR '.join(json_extract_sql_list) + ')' 494 | else: 495 | sql = f'''SELECT method_id, class_id, method_name, parameters, return_type, is_api, api_path, documentation, body FROM methods 496 | WHERE project_id = {self.project_id} AND 497 | (json_extract(method_invocation_map, '$."{package_class}".fields.{field_name}') IS NOT NULL OR json_extract(method_invocation_map, '$."{package_class}.{field_name}"') IS NOT NULL)''' 498 | methods = self.sqlite.select_data(sql) 499 | if not methods: 500 | field_method_name_list = ['get' + field_name_capitalize, 'set' + field_name_capitalize, 'is' + field_name_capitalize] 501 | field_method_name_str = '"' + '","'.join(field_method_name_list) + '"' 502 | methods = self.sqlite.select_data(f'SELECT method_id, class_id, method_name, parameters, return_type, is_api, api_path, documentation, body FROM methods WHERE class_id = {class_id} AND method_name in ({field_method_name_str})') 503 | class_ids = [str(method['class_id']) for method in methods] 504 | class_sql = f'SELECT class_id FROM class WHERE class_id in ({", ".join(class_ids)}) and commit_or_branch ="{commit_or_branch}"' 505 | class_db = self.sqlite.select_data(class_sql) 506 | class_db_id = [class_item['class_id'] for class_item in class_db] 507 | return [method for method in methods if method['class_id'] in class_db_id] 508 | 509 | # Step 5.4 510 | def _get_method_invocation_in_methods_table(self, package_class, method_param, commit_or_branch): 511 | all_possible_method_param_type_list = self._gen_all_possible_method_param_list(method_param) 512 | json_extract_sql_list = [] 513 | for param_type in all_possible_method_param_type_list: 514 | sql_part = f'''json_extract(method_invocation_map, '$."{package_class}".methods."{param_type}"') IS NOT NULL''' 515 | json_extract_sql_list.append(sql_part) 516 | if len(json_extract_sql_list) > 1000: 517 | json_extract_sql_list = json_extract_sql_list[0: 995] 518 | sql = f'SELECT method_id, class_id, method_name, parameters, return_type, is_api, api_path, documentation, body FROM methods WHERE project_id = {self.project_id} AND (' + ' OR '.join(json_extract_sql_list) + ')' 519 | logging.info(f'{package_class} {method_param} invocation sql: {sql}') 520 | methods = self.sqlite.select_data(sql) 521 | class_ids = [str(method['class_id']) for method in methods] 522 | class_sql = f'SELECT class_id FROM class WHERE class_id in ({", ".join(class_ids)}) and commit_or_branch ="{commit_or_branch}"' 523 | class_db = self.sqlite.select_data(class_sql) 524 | if not class_db: 525 | class_sql = f'SELECT class_id FROM class WHERE class_id in ({", ".join(class_ids)})' 526 | class_db = self.sqlite.select_data(class_sql) 527 | class_db_id = [class_item['class_id'] for class_item in class_db] 528 | return [method for method in methods if method['class_id'] in class_db_id] 529 | 530 | # Step 5.4.1 531 | def _gen_all_possible_method_param_list(self, method_param): 532 | method_param_list = [] 533 | method_name = method_param.split('(')[0] 534 | param_type_str = method_param.split('(')[1].split(')')[0] 535 | param_type_list = param_type_str.split(',') 536 | if not param_type_list: 537 | return method_param_list 538 | all_possible_method_param_list = self._replace_with_null_unknown(param_type_list) 539 | for param_type_list in all_possible_method_param_list: 540 | method_param_list.append(f'{method_name}({",".join(param_type_list)})') 541 | return method_param_list 542 | 543 | def _replace_extends_class(self, new_lst, results): 544 | for i in range(0, len(new_lst)): 545 | if new_lst[i].lower() in constant.JAVA_BASIC_TYPE \ 546 | or new_lst[i] == constant.PARAMETER_TYPE_METHOD_INVOCATION_UNKNOWN \ 547 | or new_lst[i] == 'null': 548 | continue 549 | extends_package_class_list = self._get_extends_package_class(new_lst[i]) 550 | for extends_package_class in extends_package_class_list: 551 | result_item = [item for item in new_lst] 552 | result_item[i] = extends_package_class 553 | results.append(result_item) 554 | 555 | # Step 5.4.1.1 556 | def _replace_with_null_unknown(self, lst: list): 557 | need_replace_list = [] 558 | replaced_list = [] 559 | results = set() 560 | self._replace_params_with_unknown(lst, results, 0, need_replace_list) 561 | for item in need_replace_list: 562 | if len(results) > 1000: 563 | break 564 | if item not in replaced_list: 565 | replaced_list.append(item) 566 | self._replace_params_with_unknown(item['list'], results, item['index'], need_replace_list) 567 | return list(results) 568 | 569 | def _replace_param_switch(self, param: str): 570 | if 'int' in param.lower(): 571 | if param == 'int': 572 | param = 'Integer' 573 | else: 574 | param = 'int' 575 | else: 576 | if param[0].isupper(): 577 | param = param[0].lower() + param[1:] 578 | else: 579 | param = param[0].upper() + param[1:] 580 | return param 581 | 582 | def _replace_params_with_unknown(self, lst: list, results: set, idx: int, need_replace_list: list): 583 | # data = [item.split('<')[0].replace('<', '').replace('>', '') for item in data] 584 | for i in range(idx, len(lst)): 585 | new_lst = lst[:] 586 | results.add(tuple(new_lst)) 587 | new_lst2 = new_lst[:] 588 | if new_lst[i].lower() not in constant.JAVA_BASIC_TYPE: 589 | if new_lst[i].startswith('List'): 590 | new_lst2[i] = 'ArrayList' 591 | elif new_lst[i].startswith('Map'): 592 | new_lst2[i] = 'HashMap' 593 | elif new_lst[i].startswith('Set'): 594 | new_lst2[i] = 'HashSet' 595 | else: 596 | if new_lst[i].lower() in constant.JAVA_BASIC_TYPE_SWITCH: 597 | new_lst2 = new_lst[:] 598 | param = self._replace_param_switch(new_lst[i]) 599 | new_lst2[i] = param 600 | if tuple(new_lst2) not in results: 601 | results.add(tuple(new_lst2)) 602 | if {'list': new_lst2, 'index': idx} not in need_replace_list: 603 | need_replace_list.append({'list': new_lst2, 'index': idx}) 604 | 605 | for el in ['null', 'unknown']: 606 | new_lst_tmp = new_lst[:] 607 | new_lst_tmp[i] = el 608 | if tuple(new_lst_tmp) not in results: 609 | results.add(tuple(new_lst_tmp)) 610 | if {'list': new_lst_tmp, 'index': min(idx, len(new_lst) - 1)} not in need_replace_list: 611 | need_replace_list.append({'list': new_lst_tmp, 'index': min(idx, len(new_lst) - 1)}) 612 | 613 | # Step 5.5 614 | def _get_method_param_string(self, method_db: dict): 615 | method_name: str = method_db['method_name'] 616 | params: list = json.loads(method_db['parameters']) 617 | params_type_list = [param['parameter_type'] for param in params] 618 | return f'{method_name}({",".join(params_type_list)})' 619 | 620 | def _handle_impacted_fields(self, impacted_fields: list, source_node_id): 621 | for impacted_field in impacted_fields: 622 | class_id = impacted_field['class_id'] 623 | class_entity = self.sqlite.select_data(f'SELECT package_name, class_name, commit_or_branch, filepath FROM class WHERE class_id={class_id}')[0] 624 | class_name = class_entity['class_name'] 625 | package_name = class_entity['package_name'] 626 | package_class = f'{package_name}.{class_name}' 627 | commit_or_branch = class_entity['commit_or_branch'] 628 | class_filepath = class_entity['filepath'] 629 | impacted_field_node_id = self.view.create_node_category(class_name, impacted_field['field_name'], constant.NODE_TYPE_FIELD, constant.DIFF_TYPE_IMPACTED, None, class_filepath, impacted_field['documentation'], '', {}) 630 | self.view.create_node_link(source_node_id, impacted_field_node_id) 631 | extend_dict = {'field_node_id': impacted_field_node_id, 'class_filepath': class_filepath} 632 | extend_dict.update(impacted_field) 633 | self._add_to_need_analyze_obj_list('java', package_class, impacted_field['field_name'], None, commit_or_branch, extend_dict) 634 | if impacted_field['is_static'] == 'False': 635 | self._add_to_need_analyze_obj_list('java', package_class, None, None, commit_or_branch, class_entity) 636 | class_node_id = self.view.create_node_category(class_name, 'entity', constant.NODE_TYPE_CLASS, constant.DIFF_TYPE_IMPACTED, '', self.file_path, '', '', {}) 637 | self.view.create_node_link(impacted_field_node_id, class_node_id) 638 | 639 | # Step 5.9 640 | def _handle_impacted_methods(self, impacted_methods: list, source_node_id): 641 | for impacted_method in impacted_methods: 642 | node_extend_dict = {'is_api': False} 643 | if impacted_method.get('is_api') == 'True': 644 | node_extend_dict = { 645 | 'is_api': True, 646 | 'api_path': impacted_method['api_path'] 647 | } 648 | class_id = impacted_method['class_id'] 649 | class_entity = self.sqlite.select_data(f'SELECT package_name, class_name, commit_or_branch, filepath FROM class WHERE class_id={class_id}')[0] 650 | class_name = class_entity['class_name'] 651 | package_name = class_entity['package_name'] 652 | package_class = f'{package_name}.{class_name}' 653 | commit_or_branch = class_entity['commit_or_branch'] 654 | class_filepath = class_entity['filepath'] 655 | method_name_param = f'{impacted_method["method_name"]}({",".join([param["parameter_type"] for param in json.loads(impacted_method["parameters"])])})' 656 | impacted_method_node_id = self.view.create_node_category(class_name, method_name_param, constant.NODE_TYPE_METHOD, constant.DIFF_TYPE_IMPACTED, impacted_method.get('body'), class_filepath, impacted_method.get('documentation'), impacted_method.get('body'), node_extend_dict) 657 | self.view.create_node_link(source_node_id, impacted_method_node_id) 658 | extend_dict = {'method_node_id': impacted_method_node_id, 'class_filepath': class_filepath} 659 | extend_dict.update(impacted_method) 660 | self._add_to_need_analyze_obj_list('java', package_class, None, self._get_method_param_string(impacted_method), commit_or_branch, extend_dict) 661 | 662 | def _add_to_need_analyze_obj_list(self, file_type: str, package_class: str, field_name: str or None, method_param: str or None, commit_or_branch: str, mapper_extend_dict: dict): 663 | need_analyze_entity: dict = { 664 | 'file_type': file_type, 665 | 'package_class': package_class, 666 | 'field_name': field_name, 667 | 'method_param': method_param, 668 | 'commit_or_branch': commit_or_branch 669 | } 670 | is_exist = [obj for obj in self.need_analyze_obj_list if self.check_dict_keys_equal_values(need_analyze_entity, obj)] 671 | if not is_exist: 672 | need_analyze_entity.update(mapper_extend_dict) 673 | self.need_analyze_obj_list.append(need_analyze_entity) 674 | 675 | def check_dict_keys_equal_values(self, dict1, dict2): 676 | for key in dict1: 677 | if key in dict2 and dict1[key] != dict2[key]: 678 | return False 679 | return True 680 | 681 | def _draw_and_write_result(self): 682 | if self.view.nodes: 683 | self.view.draw_graph(1200, 600) 684 | logging.info(f'Analyze success, generating cci result file......') 685 | result = { 686 | 'nodes': self.view.nodes, 687 | 'links': self.view.links, 688 | 'categories': self.view.categories, 689 | 'impacted_api_list': [node['api_path'] for node in self.view.nodes if node.get('is_api')] 690 | } 691 | print(json.dumps(result), flush=True) 692 | print(f'Impacted api list: {result["impacted_api_list"]}', flush=True) 693 | with open(self.cci_filepath, 'w') as w: 694 | w.write(json.dumps(result, ensure_ascii=False)) 695 | logging.info(f'Generating cci result file success, location: {self.cci_filepath}') 696 | 697 | def _start_analysis_diff_and_impact(self): 698 | for patch_path, patch_obj in self.diff_parse_map.items(): 699 | self._diff_analyze(patch_path, patch_obj) 700 | 701 | # 遍历列表 702 | for obj in self.need_analyze_obj_list: 703 | if obj not in self.analyzed_obj_set: # 判断对象是否已分析过 704 | self.analyzed_obj_set.append(obj) # 标记为已分析过 705 | self._impacted_analyze(obj) # 处理对象,返回新增对象列表 706 | 707 | self._draw_and_write_result() 708 | t2 = datetime.datetime.now() 709 | try: 710 | logging.info(f'Analyze done, remove occupy, others can analyze now') 711 | os.remove(os.path.join(self.file_path, 'Occupy.ing')) 712 | finally: 713 | pass 714 | logging.info(f'Analyze done, spend: {t2 - self.t1}') 715 | 716 | def analyze_two_branch(self, branch_first, branch_second, **kwargs): 717 | logging.info('*' * 10 + 'Analyze start' + '*' * 10) 718 | self.commit_or_branch_new = branch_first 719 | self.commit_or_branch_old = branch_second 720 | self.branch_name = branch_first 721 | self.project_name = self.git_url.split('/')[-1].split('.git')[0] 722 | self.file_path = os.path.join(config.project_path, self.project_name) 723 | self.project_id = self.sqlite.add_project(self.project_name, self.git_url, self.branch_name, branch_first, branch_second) 724 | # 已有分析结果 725 | self.cci_filepath = os.path.join(self.file_path, f'{branch_second.replace("/", "#")}..{branch_first.replace("/", "#")}.cci') 726 | self._can_analyze(self.file_path, self.cci_filepath) 727 | # 无此项目, 先clone项目 728 | if not os.path.exists(self.file_path): 729 | logging.info(f'Cloning project: {self.git_url}') 730 | os.system(f'git clone -b {branch_first} {self.git_url} {self.file_path}') 731 | 732 | dependents: list[dict] = kwargs.get('dependents', []) 733 | self._clone_dependents_project(dependents) 734 | 735 | self._occupy_project() 736 | self.diff_parse_map = self._get_branch_diff_parse_map(self.file_path, branch_first, branch_second) 737 | self.xml_parse_results_new, self.xml_parse_results_old = self._parse_branch_project(self.file_path, branch_first, branch_second) 738 | self._start_analysis_diff_and_impact() 739 | 740 | def analyze_two_commit(self, branch, commit_first, commit_second, **kwargs): 741 | logging.info('*' * 10 + 'Analyze start' + '*' * 10) 742 | self.branch_name = branch 743 | self.commit_or_branch_new = commit_first[0: 7] if len(commit_first) > 7 else commit_first 744 | self.commit_or_branch_old = commit_second[0: 7] if len(commit_second) > 7 else commit_second 745 | 746 | self.project_name = self.git_url.split('/')[-1].split('.git')[0] 747 | self.file_path = os.path.join(config.project_path, self.project_name) 748 | 749 | self.project_id = self.sqlite.add_project(self.project_name, self.git_url, self.branch_name, self.commit_or_branch_new, self.commit_or_branch_old) 750 | # 已有分析结果 751 | self.cci_filepath = os.path.join(self.file_path, f'{self.commit_or_branch_old}..{self.commit_or_branch_new}.cci') 752 | self._can_analyze(self.file_path, self.cci_filepath) 753 | 754 | # 无此项目, 先clone项目 755 | if not os.path.exists(self.file_path): 756 | logging.info(f'Cloning project: {self.git_url}') 757 | os.system(f'git clone -b {self.branch_name} {self.git_url} {self.file_path}') 758 | 759 | dependents: list[dict] = kwargs.get('dependents', []) 760 | self._clone_dependents_project(dependents) 761 | 762 | self._occupy_project() 763 | self.diff_parse_map = self._get_diff_parse_map(self.file_path, self.branch_name, self.commit_or_branch_new, self.commit_or_branch_old) 764 | 765 | self.xml_parse_results_new, self.xml_parse_results_old = self._parse_project(self.file_path, self.commit_or_branch_new, self.commit_or_branch_old) 766 | 767 | self._start_analysis_diff_and_impact() 768 | 769 | def analyze_class_method(self, branch, commit_id, package_class, method_nums, **kwargs): 770 | logging.info('*' * 10 + 'Analyze start' + '*' * 10) 771 | package_class = package_class.replace("\\", "/") 772 | self.branch_name = branch 773 | self.commit_or_branch_new = commit_id 774 | self.commit_or_branch_new = self.commit_or_branch_new[0: 7] if len(self.commit_or_branch_new) > 7 else self.commit_or_branch_new 775 | self.project_name = self.git_url.split('/')[-1].split('.git')[0] 776 | self.file_path = os.path.join(config.project_path, self.project_name) 777 | 778 | self.project_id = self.sqlite.add_project(self.project_name, self.git_url, self.branch_name, self.commit_or_branch_new, f'{package_class}.{method_nums}') 779 | class_name = package_class.split("/")[-1].replace('.java', '') 780 | cci_path = f'{branch.replace("/", "#")}_{commit_id}_{class_name}_{method_nums}.cci' 781 | self.cci_filepath = os.path.join(self.file_path, cci_path) 782 | self._can_analyze(self.file_path, self.cci_filepath) 783 | 784 | # 无此项目, 先clone项目 785 | if not os.path.exists(self.file_path): 786 | logging.info(f'Cloning project: {self.git_url}') 787 | os.system(f'git clone -b {self.branch_name} {self.git_url} {self.file_path}') 788 | 789 | dependents: list[dict] = kwargs.get('dependents', []) 790 | self._clone_dependents_project(dependents) 791 | 792 | self._occupy_project() 793 | 794 | logging.info('Git pull project to HEAD') 795 | os.system(f'cd {self.file_path} && git checkout {branch} && git pull') 796 | time.sleep(1) 797 | 798 | if not method_nums: 799 | method_nums_all = [] 800 | # todo 801 | class_db = self.sqlite.select_data('SELECT * FROM class WHERE project_id = ' + str(self.project_id) + ' and filepath LIKE "%' + package_class + '"') 802 | if not class_db: 803 | logging.error(f'Can not find {package_class} in db') 804 | class_id = class_db[0]['class_id'] 805 | field_db = self.sqlite.select_data(f'SELECT * FROM field WHERE class_id = {class_id}') 806 | method_nums_all += [field['start_line'] for field in field_db] 807 | method_db = self.sqlite.select_data(f'SELECT * FROM methods WHERE class_id = {class_id}') 808 | method_nums_all += [method['start_line'] for method in method_db] 809 | else: 810 | method_nums_all = [int(num) for num in method_nums.split(',')] 811 | 812 | self.diff_parse_map[package_class] = { 813 | 'line_num_added': method_nums_all, 814 | 'line_content_added': method_nums_all, 815 | 'line_num_removed': [], 816 | 'line_content_removed': [] 817 | } 818 | 819 | self.xml_parse_results_new, self.xml_parse_results_old = self._parse_project(self.file_path, self.commit_or_branch_new, None) 820 | 821 | self._start_analysis_diff_and_impact() 822 | -------------------------------------------------------------------------------- /src/jcci/java_parse.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | import logging 4 | import json 5 | import javalang 6 | from concurrent.futures import ThreadPoolExecutor, as_completed 7 | from .database import SqliteHelper 8 | from .constant import ENTITY, RETURN_TYPE, PARAMETERS, BODY, METHODS, FIELDS, \ 9 | PARAMETER_TYPE_METHOD_INVOCATION_UNKNOWN, JAVA_BASIC_TYPE, MAPPING_LIST, JAVA_UTIL_TYPE 10 | from . import config as config 11 | 12 | sys.setrecursionlimit(10000) 13 | logging.basicConfig(format='%(asctime)s %(message)s', level=logging.DEBUG) 14 | 15 | 16 | def calculate_similar_score_method_params(except_method_param_list, method_param_list): 17 | score = 0 18 | positions = {} 19 | 20 | # 记录list1中每个元素的位置 21 | for i, item in enumerate(except_method_param_list): 22 | positions[item] = i 23 | 24 | # 遍历list2,计算分数 25 | for i, item in enumerate(method_param_list): 26 | if item in positions: 27 | score += 1 28 | score -= abs(i - positions[item]) 29 | 30 | return score 31 | 32 | 33 | class JavaParse(object): 34 | def __init__(self, db_path, project_id): 35 | self.project_id = project_id 36 | self.sqlite = SqliteHelper(db_path) 37 | self.sibling_dirs = [] 38 | self.parsed_filepath = [] 39 | 40 | def _handle_extends(self, extends, import_list: list, package_name): 41 | if isinstance(extends, list): 42 | extends_package_class_list = [] 43 | extends_class = extends[0].name 44 | extends_package_class = self._get_extends_class_full_package(extends_class, import_list, package_name) 45 | extends_package_class_list.append(extends_package_class) 46 | if 'arguments' in extends[0].attrs and extends[0].arguments: 47 | extends_arguments = extends[0].arguments 48 | extends_argument_classes = [] 49 | for extends_argument in extends_arguments: 50 | if type(extends_argument) == javalang.tree.TypeArgument: 51 | extends_argument_class = extends_argument.type.name 52 | else: 53 | extends_argument_class = extends_argument.name 54 | extends_argument_package_class = self._get_extends_class_full_package(extends_argument_class, import_list, package_name) 55 | extends_argument_classes.append(extends_argument_package_class) 56 | extends_package_class_list += extends_argument_classes 57 | return extends_package_class + '<' + ','.join(extends_argument_classes) + '>', extends_package_class_list 58 | else: 59 | return extends_package_class, [extends_package_class] 60 | else: 61 | extends_class = self._get_extends_class_full_package(extends.name, import_list, package_name) 62 | return extends_class, [extends_class] 63 | 64 | def _get_extends_class_full_package(self, extends_class, import_list, package_name): 65 | extends_in_imports = [import_obj for import_obj in import_list if extends_class in import_obj['import_path']] 66 | return extends_in_imports[0]['import_path'] if extends_in_imports else package_name + '.' + extends_class 67 | 68 | def _parse_class(self, node, filepath: str, package_name: str, import_list: list, commit_or_branch: str, parse_import_first): 69 | # 处理class信息 70 | documentation = node.documentation 71 | class_name = node.name 72 | package_class = package_name + '.' + node.name 73 | class_type = type(node).__name__.replace('Declaration', '') 74 | access_modifier = [m for m in list(node.modifiers) if m.startswith('p')][0] if list([m for m in list(node.modifiers) if m.startswith('p')]) else 'public' 75 | annotations_json = json.dumps(node.annotations, default=lambda obj: obj.__dict__) 76 | is_controller, controller_base_url = self._judge_is_controller(node.annotations) 77 | extends_package_class = None 78 | if 'extends' in node.attrs and node.extends: 79 | extends_package_class, extends_package_class_list = self._handle_extends(node.extends, import_list, package_name) 80 | package_path = package_class.replace('.', '/') + '.java' 81 | base_filepath = filepath.replace(package_path, '') 82 | for extends_package_class_item in extends_package_class_list: 83 | if extends_package_class_item == package_class: 84 | continue 85 | extends_class_filepath = base_filepath + extends_package_class_item.replace('.', '/') + '.java' 86 | self.parse_java_file(extends_class_filepath, commit_or_branch, parse_import_first=parse_import_first) 87 | implements = ','.join([implement.name for implement in node.implements]) if 'implements' in node.attrs and node.implements else None 88 | class_id, new_add = self.sqlite.add_class(filepath.replace('\\', '/'), access_modifier, class_type, class_name, package_name, extends_package_class, self.project_id, implements, annotations_json, documentation, is_controller, controller_base_url, commit_or_branch) 89 | return class_id, new_add 90 | 91 | def _parse_imports(self, imports): 92 | import_list = [] 93 | for import_decl in imports: 94 | import_obj = { 95 | 'import_path': import_decl.path, 96 | 'is_static': import_decl.static, 97 | 'is_wildcard': import_decl.wildcard, 98 | 'start_line': import_decl.position.line, 99 | 'end_line': import_decl.position.line 100 | } 101 | import_list.append(import_obj) 102 | return import_list 103 | 104 | def _parse_fields(self, fields, package_name, class_name, class_id, import_map, filepath): 105 | field_list = [] 106 | package_class = package_name + "." + class_name 107 | for field_obj in fields: 108 | field_annotations = json.dumps(field_obj.annotations, default=lambda obj: obj.__dict__) 109 | access_modifier = next((m for m in list(field_obj.modifiers) if m.startswith('p')), 'public') 110 | field_name = field_obj.declarators[0].name 111 | field_type: str = field_obj.type.name 112 | if field_type.lower() in JAVA_BASIC_TYPE: 113 | pass 114 | elif field_type in JAVA_UTIL_TYPE and ('java.util' in import_map.values() or 'java.util.' + field_type in import_map.values()): 115 | var_declarator_type_arguments = self._deal_arguments_type(field_obj.type.arguments, FIELDS, {}, {}, {}, import_map, {}, package_name, filepath, [], {}, class_id) 116 | if var_declarator_type_arguments: 117 | field_type = field_type + '<' + '#'.join(var_declarator_type_arguments) + '>' 118 | elif field_type in import_map.keys(): 119 | field_type = import_map.get(field_type) 120 | else: 121 | in_import = False 122 | for key in import_map.keys(): 123 | if key[0].isupper(): 124 | continue 125 | field_type_db = self.sqlite.select_data(f'select class_id from class where project_id={self.project_id} and package_name = "{import_map.get(key)}" and class_name = "{field_type}" limit 1') 126 | if field_type_db: 127 | field_type = f'{import_map.get(key)}.{field_type}' 128 | in_import = True 129 | break 130 | if not in_import: 131 | field_type_db = self.sqlite.select_data(f'select class_id from class where project_id={self.project_id} and package_name = "{package_class}" and class_name = "{field_type}" limit 1') 132 | if field_type_db: 133 | field_type = f'{package_class}.{field_type}' 134 | else: 135 | field_type = package_name + '.' + field_type 136 | import_map[field_obj.type.name] = field_type 137 | else: 138 | import_map[field_obj.type.name] = field_type 139 | is_static = 'static' in list(field_obj.modifiers) 140 | documentation = field_obj.documentation 141 | start_line = field_obj.position.line if not field_obj.annotations else field_obj.annotations[0].position.line 142 | end_line = self._get_method_end_line(field_obj) 143 | field_obj = { 144 | 'class_id': class_id, 145 | 'project_id': self.project_id, 146 | 'annotations': field_annotations, 147 | 'access_modifier': access_modifier, 148 | 'field_type': field_type, 149 | 'field_name': field_name, 150 | 'is_static': is_static, 151 | 'documentation': documentation, 152 | 'start_line': start_line, 153 | 'end_line': end_line 154 | } 155 | field_list.append(field_obj) 156 | self.sqlite.update_data(f'DELETE FROM field where class_id={class_id}') 157 | self.sqlite.insert_data('field', field_list) 158 | return field_list 159 | 160 | def _parse_method_body_variable(self, node, parameters_map, variable_map, field_map, import_map, method_invocation, package_name, filepath, methods, method_name_entity_map, class_id): 161 | var_declarator = node.declarators[0].name 162 | var_declarator_type = self._deal_declarator_type(node.type, BODY, parameters_map, variable_map, field_map, import_map, method_invocation, package_name, filepath, methods, method_name_entity_map, class_id) 163 | variable_map[var_declarator] = var_declarator_type 164 | initializer = node.declarators[0].initializer 165 | if self._is_valid_prefix(var_declarator_type): 166 | self._add_entity_used_to_method_invocation(method_invocation, var_declarator_type, BODY) 167 | if not initializer: 168 | return var_declarator_type 169 | for init_path, init_node in initializer.filter(javalang.tree.MemberReference): 170 | self._deal_member_reference(init_node, parameters_map, variable_map, field_map, import_map, method_invocation, BODY, package_name, filepath) 171 | return var_declarator_type 172 | 173 | def _parse_method_body_class_creator(self, node, parameters_map, variable_map, field_map, import_map, method_invocation, package_name, filepath, methods, method_name_entity_map, class_id): 174 | qualifier = node.type.name 175 | node_line = node.position.line if node.position else None 176 | qualifier_type = self._get_var_type(qualifier, parameters_map, variable_map, field_map, import_map, method_invocation, BODY, package_name, filepath) 177 | node_arguments = self._deal_var_type(node.arguments, BODY, parameters_map, variable_map, field_map, import_map, method_invocation, package_name, filepath, methods, method_name_entity_map, class_id) 178 | if node.selectors is None or not node_arguments: 179 | self._add_entity_used_to_method_invocation(method_invocation, qualifier_type, BODY) 180 | else: 181 | if node_arguments: 182 | qualifier_package_class, method_params, method_db = self._find_method_in_package_class(qualifier_type, qualifier, node_arguments) 183 | if not method_db: 184 | return qualifier_type 185 | self._add_method_used_to_method_invocation(method_invocation, qualifier_type, method_params, [node_line]) 186 | self._parse_node_selectors(node.selectors, qualifier_type, parameters_map, variable_map, field_map, import_map, method_invocation, package_name, filepath, methods, method_name_entity_map, class_id) 187 | if self._is_valid_prefix(qualifier_type): 188 | self._add_entity_used_to_method_invocation(method_invocation, qualifier_type, BODY) 189 | return qualifier_type 190 | 191 | def _parse_method_body_method_invocation(self, node, parameters_map, variable_map, field_map, import_map, method_invocation, package_name, filepath, methods, method_name_entity_map, class_id): 192 | qualifier = node.qualifier 193 | member = node.member 194 | return_type = PARAMETER_TYPE_METHOD_INVOCATION_UNKNOWN 195 | # 类静态方法调用 196 | if not qualifier and not member[0].islower(): 197 | qualifier_type = self._get_var_type(member, parameters_map, variable_map, field_map, import_map, method_invocation, BODY, package_name, filepath) 198 | # todo a.b.c 199 | qualifier_type = self._parse_node_selectors(node.selectors, qualifier_type, parameters_map, variable_map, field_map, import_map, method_invocation, package_name, filepath, methods, method_name_entity_map, class_id) 200 | return_type = qualifier_type 201 | elif qualifier: 202 | qualifier_type = self._get_var_type(qualifier, parameters_map, variable_map, field_map, import_map, method_invocation, BODY, package_name, filepath) 203 | node_arguments = self._deal_var_type(node.arguments, BODY, parameters_map, variable_map, field_map, import_map, method_invocation, package_name, filepath, methods, method_name_entity_map, class_id) 204 | node_line = node.position.line 205 | node_arguments = [n for n in node_arguments if n] 206 | node_method = f'{member}({",".join(node_arguments)})' 207 | self._add_method_used_to_method_invocation(method_invocation, qualifier_type, node_method, [node_line]) 208 | if self._is_valid_prefix(qualifier_type): 209 | qualifier_package_class, method_params, method_db = self._find_method_in_package_class(qualifier_type, member, node_arguments) 210 | if not method_db: 211 | return qualifier_type 212 | if method_params != node_method: 213 | self._add_method_used_to_method_invocation(method_invocation, qualifier_type, method_params, [node_line]) 214 | method_db_type = method_db.get("return_type", method_db.get("field_type")) 215 | elif qualifier_type.startswith('Map<') and member == 'get': 216 | method_db_type = qualifier_type.split('#')[1].split('>')[0] 217 | else: 218 | method_db_type = qualifier_type 219 | method_db_type = self._parse_node_selectors(node.selectors, method_db_type, parameters_map, variable_map, field_map, import_map, method_invocation, package_name, filepath, methods, method_name_entity_map, class_id) 220 | return_type = method_db_type 221 | # 在一个类的方法或父类方法 222 | elif member: 223 | class_db = self.sqlite.select_data(f'SELECT package_name, class_name, extends_class FROM class where project_id = {self.project_id} and class_id={class_id} limit 1')[0] 224 | package_class = class_db['package_name'] + '.' + class_db['class_name'] 225 | node_line = node.position.line 226 | node_arguments = self._deal_var_type(node.arguments, BODY, parameters_map, variable_map, field_map, import_map, method_invocation, package_name, filepath, methods, method_name_entity_map, class_id) 227 | # todo 同级方法, 判断参数长度，不精确 228 | if method_name_entity_map.get(member): 229 | same_class_method = None 230 | max_score = -float('inf') 231 | for method_item in methods: 232 | if method_item.name != member or len(node.arguments) != len(method_item.parameters): 233 | continue 234 | method_item_param_types = [self._deal_declarator_type(parameter.type, PARAMETERS, parameters_map, variable_map, field_map, import_map, method_invocation, package_name, filepath, methods, method_name_entity_map, class_id) for parameter in method_item.parameters] 235 | score = calculate_similar_score_method_params(node_arguments, method_item_param_types) 236 | if score > max_score: 237 | max_score = score 238 | same_class_method = method_item 239 | if same_class_method: 240 | node_arguments = self._deal_var_type(same_class_method.parameters, BODY, parameters_map, variable_map, field_map, import_map, method_invocation, package_name, filepath, methods, method_name_entity_map, class_id) 241 | node_method = f'{member}({",".join(node_arguments)})' 242 | self._add_method_used_to_method_invocation(method_invocation, package_class, node_method, [node_line]) 243 | return_type = self._deal_declarator_type(same_class_method.return_type, BODY, parameters_map, variable_map, field_map, import_map, method_invocation, package_name, filepath, methods, method_name_entity_map, class_id) 244 | # todo 继承方法 245 | elif class_db['extends_class']: 246 | extends_package_class, method_params, method_db = self._find_method_in_package_class(class_db['extends_class'], member, node_arguments) 247 | if extends_package_class: 248 | self._add_method_used_to_method_invocation(method_invocation, extends_package_class, method_params, [node_line]) 249 | return_type = method_db.get("return_type", method_db.get("field_type")) 250 | return return_type 251 | 252 | def _parse_node_selectors(self, selectors, qualifier_type, parameters_map, variable_map, field_map, import_map, method_invocation, package_name, filepath, methods, method_name_entity_map, class_id): 253 | if not selectors: 254 | return qualifier_type 255 | selector_qualifier_type = qualifier_type 256 | for selector in selectors: 257 | if type(selector) == javalang.tree.ArraySelector: 258 | continue 259 | selector_member = selector.member 260 | if type(selector) == javalang.tree.MethodInvocation: 261 | self._parse_method_body_method_invocation(selector, parameters_map, variable_map, field_map, import_map, method_invocation, package_name, filepath, methods, method_name_entity_map, class_id) 262 | selector_arguments = self._deal_var_type(selector.arguments, BODY, parameters_map, variable_map, field_map, import_map, method_invocation, package_name, filepath, methods, method_name_entity_map, class_id) 263 | selector_line = selector.position.line 264 | selector_method = f'{selector_member}({",".join(selector_arguments)})' 265 | if self._is_valid_prefix(selector_qualifier_type): 266 | self._add_method_used_to_method_invocation(method_invocation, selector_qualifier_type, selector_method, [selector_line]) 267 | selector_package_class, method_params, method_db = self._find_method_in_package_class(selector_qualifier_type, selector_member, selector_arguments) 268 | if not method_db: 269 | continue 270 | method_db_type = method_db.get("return_type", method_db.get("field_type")) 271 | selector_qualifier_type = method_db_type 272 | elif type(selector) == javalang.tree.MemberReference: 273 | self._deal_member_reference(selector, parameters_map, variable_map, field_map, import_map, method_invocation, BODY, package_name, filepath) 274 | selector_qualifier_type = self._get_var_type(selector_member, parameters_map, variable_map, field_map, import_map, method_invocation, BODY, package_name, filepath) 275 | if self._is_valid_prefix(selector_qualifier_type): 276 | self._add_field_used_to_method_invocation(method_invocation, selector_qualifier_type, selector_member, [None]) 277 | return selector_qualifier_type 278 | 279 | def _parse_enum(self, enum_body, lines, class_id, import_map, field_map, package_name, filepath): 280 | constants = enum_body.constants 281 | field_list = [] 282 | init_line = 0 283 | for constant in constants: 284 | constant_type = 'ENUM' 285 | constant_name = constant.name 286 | arguments = constant.arguments 287 | start_text = constant_name if not arguments else constant_name + '(' 288 | start_lines = [lines.index(line) for line in lines if line.strip().startswith(start_text)] 289 | if start_lines: 290 | start_line = start_lines[0] + 1 291 | init_line = start_line 292 | else: 293 | start_line = init_line 294 | end_line = start_line 295 | field_obj = { 296 | 'class_id': class_id, 297 | 'project_id': self.project_id, 298 | 'annotations': None, 299 | 'access_modifier': 'public', 300 | 'field_type': constant_type, 301 | 'field_name': constant_name, 302 | 'is_static': True, 303 | 'documentation': None, 304 | 'start_line': start_line, 305 | 'end_line': end_line 306 | } 307 | field_list.append(field_obj) 308 | self.sqlite.insert_data('field', field_list) 309 | 310 | def _parse_constructors(self, constructors, lines, class_id, import_map, field_map, package_name, filepath): 311 | all_method = [] 312 | for constructor in constructors: 313 | method_invocation = {} 314 | cs_name = constructor.name 315 | annotations = json.dumps(constructor.annotations, default=lambda obj: obj.__dict__) # annotations 316 | 317 | access_modifier = [m for m in list(constructor.modifiers) if m.startswith('p')][0] if list([m for m in list(constructor.modifiers) if m.startswith('p')]) else 'public' 318 | parameters = [] 319 | parameters_map = {} 320 | for parameter in constructor.parameters: 321 | parameter_obj = { 322 | 'parameter_type': self._deal_declarator_type(parameter.type, PARAMETERS, parameters_map, {}, field_map, import_map, method_invocation, package_name, filepath, [], {}, class_id), 323 | 'parameter_name': parameter.name, 324 | 'parameter_varargs': parameter.varargs 325 | } 326 | parameters.append(parameter_obj) 327 | parameters_map = {parameter['parameter_name']: parameter['parameter_type'] for parameter in parameters} 328 | return_type = package_name + '.' + cs_name 329 | start_line = constructor.position.line 330 | if constructor.annotations: 331 | start_line = constructor.annotations[0].position.line 332 | end_line = self._get_method_end_line(constructor) 333 | cs_body = lines[start_line - 1: end_line + 1] 334 | for body in constructor.body: 335 | for path, node in body.filter(javalang.tree.This): 336 | self._parse_node_selectors(node.selectors, None, {}, {}, field_map, import_map, method_invocation, package_name, filepath, [], {}, class_id) 337 | 338 | method_db = { 339 | 'class_id': class_id, 340 | 'project_id': self.project_id, 341 | 'annotations': annotations, 342 | 'access_modifier': access_modifier, 343 | 'return_type': return_type, 344 | 'method_name': cs_name, 345 | 'parameters': json.dumps(parameters), 346 | 'body': json.dumps(cs_body), 347 | 'method_invocation_map': json.dumps(method_invocation), 348 | 'is_static': False, 349 | 'is_abstract': False, 350 | 'is_api': False, 351 | 'api_path': None, 352 | 'start_line': start_line, 353 | 'end_line': end_line, 354 | 'documentation': constructor.documentation 355 | } 356 | all_method.append(method_db) 357 | self.sqlite.insert_data('methods', all_method) 358 | 359 | def _parse_method(self, methods, lines, class_id, import_map, field_map, package_name, filepath): 360 | # 处理 methods 361 | all_method = [] 362 | class_db = self.sqlite.select_data(f'SELECT controller_base_url, implements FROM class WHERE project_id = {self.project_id} and class_id = {class_id}')[0] 363 | base_url = class_db['controller_base_url'] if class_db['controller_base_url'] else '' 364 | class_implements = class_db['implements'] 365 | method_name_entity_map = {method.name: method for method in methods} 366 | for method_obj in methods: 367 | method_invocation = {} 368 | variable_map = {} 369 | method_name = method_obj.name 370 | documentation = method_obj.documentation # document 371 | annotations = json.dumps(method_obj.annotations, default=lambda obj: obj.__dict__) # annotations 372 | is_override_method = 'Override' in annotations 373 | is_api, api_path = self._judge_is_api(method_obj.annotations, base_url, method_name) 374 | if not is_api and class_implements and is_override_method: 375 | class_implements_list = class_implements.split(',') 376 | class_implements_obj = self.sqlite.select_data(f'''select m.is_api, m.api_path from methods m left join class c on c.class_id = m.class_id 377 | where c.project_id = {self.project_id} and m.method_name = '{method_name}' and c.class_name in ("{'","'.join(class_implements_list)}")''') 378 | if class_implements_obj: 379 | is_api = class_implements_obj[0]['is_api'] 380 | api_path = class_implements_obj[0]['api_path'] 381 | access_modifier = [m for m in list(method_obj.modifiers) if m.startswith('p')][0] if list([m for m in list(method_obj.modifiers) if m.startswith('p')]) else 'public' 382 | is_static = 'static' in list(method_obj.modifiers) 383 | is_abstract = 'abstract' in list(method_obj.modifiers) 384 | parameters = [] 385 | parameters_map = {} 386 | type_parameters = method_obj.type_parameters if method_obj.type_parameters else [] 387 | for type_parameter in type_parameters: 388 | type_parameter_name = type_parameter.name 389 | type_parameter_extends_name = type_parameter.extends[0].name if type_parameter.extends else None 390 | if type_parameter_extends_name: 391 | type_parameter_extends_name_type = self._deal_declarator_type(type_parameter.extends[0], PARAMETERS, parameters_map, variable_map, field_map, import_map, method_invocation, package_name, filepath, methods, method_name_entity_map, class_id) 392 | else: 393 | type_parameter_extends_name_type = type_parameter_name 394 | parameters_map[type_parameter_name] = type_parameter_extends_name_type 395 | for parameter in method_obj.parameters: 396 | parameter_obj = { 397 | 'parameter_type': self._deal_declarator_type(parameter.type, PARAMETERS, parameters_map, variable_map, field_map, import_map, method_invocation, package_name, filepath, methods, method_name_entity_map, class_id), 398 | 'parameter_name': parameter.name, 399 | 'parameter_varargs': parameter.varargs 400 | } 401 | parameters.append(parameter_obj) 402 | parameters_map.update({parameter['parameter_name']: parameter['parameter_type'] for parameter in parameters}) 403 | # 处理返回对象 404 | return_type = self._deal_declarator_type(method_obj.return_type, RETURN_TYPE, parameters_map, variable_map, field_map, import_map, method_invocation, package_name, filepath, methods, method_name_entity_map, class_id) 405 | if self._is_valid_prefix(return_type): 406 | self._add_entity_used_to_method_invocation(method_invocation, return_type, RETURN_TYPE) 407 | method_start_line = method_obj.position.line 408 | if method_obj.annotations: 409 | method_start_line = method_obj.annotations[0].position.line 410 | method_end_line = self._get_method_end_line(method_obj) 411 | method_body = lines[method_start_line - 1: method_end_line + 1] 412 | 413 | # 处理方法体 414 | if not method_obj.body: 415 | method_obj.body = [] 416 | for body in method_obj.body: 417 | for path, node in body.filter(javalang.tree.VariableDeclaration): 418 | self._parse_method_body_variable(node, parameters_map, variable_map, field_map, import_map, method_invocation, package_name, filepath, methods, method_name_entity_map, class_id) 419 | for path, node in body.filter(javalang.tree.ClassCreator): 420 | self._parse_method_body_class_creator(node, parameters_map, variable_map, field_map, import_map, method_invocation, package_name, filepath, methods, method_name_entity_map, class_id) 421 | for path, node in body.filter(javalang.tree.This): 422 | self._parse_node_selectors(node.selectors, None, parameters_map, variable_map, field_map, import_map, method_invocation, package_name, filepath, methods, method_name_entity_map, class_id) 423 | for path, node in body.filter(javalang.tree.MethodInvocation): 424 | self._parse_method_body_method_invocation(node, parameters_map, variable_map, field_map, import_map, method_invocation, package_name, filepath, methods, method_name_entity_map, class_id) 425 | method_db = { 426 | 'class_id': class_id, 427 | 'project_id': self.project_id, 428 | 'annotations': annotations, 429 | 'access_modifier': access_modifier, 430 | 'return_type': return_type, 431 | 'method_name': method_name, 432 | 'parameters': json.dumps(parameters), 433 | 'body': json.dumps(method_body), 434 | 'method_invocation_map': json.dumps(method_invocation), 435 | 'is_static': is_static, 436 | 'is_abstract': is_abstract, 437 | 'is_api': is_api, 438 | 'api_path': json.dumps(api_path) if is_api else None, 439 | 'start_line': method_start_line, 440 | 'end_line': method_end_line, 441 | 'documentation': documentation 442 | } 443 | all_method.append(method_db) 444 | self.sqlite.update_data(f'DELETE FROM methods where class_id={class_id}') 445 | self.sqlite.insert_data('methods', all_method) 446 | 447 | def _find_method_in_package_class(self, package_class: str, method_name: str, method_arguments): 448 | if not package_class or not self._is_valid_prefix(package_class): 449 | return None, None, None 450 | # 查表有没有记录 451 | extend_package = '.'.join(package_class.split('.')[0: -1]) 452 | extend_class = package_class.split('.')[-1] 453 | extend_class_db = self.sqlite.select_data(f'SELECT class_id, package_name, class_name, extends_class, annotations ' 454 | f'FROM class WHERE package_name="{extend_package}" ' 455 | f'AND class_name="{extend_class}" ' 456 | f'AND project_id={self.project_id} limit 1') 457 | 458 | if not extend_class_db: 459 | return None, None, None 460 | extend_class_entity = extend_class_db[0] 461 | extend_class_id = extend_class_entity['class_id'] 462 | methods_db_list = self.sqlite.select_data(f'SELECT method_name, parameters, return_type FROM methods WHERE project_id = {self.project_id} and class_id={extend_class_id} and method_name = "{method_name}"') 463 | data_in_annotation = [annotation for annotation in json.loads(extend_class_entity['annotations']) if annotation['name'] in ['Data', 'Getter', 'Setter', 'Builder', 'NoArgsConstructor', 'AllArgsConstructor']] 464 | if not methods_db_list and data_in_annotation and (method_name.startswith('get') or method_name.startswith('set')) and method_name[3:]: 465 | field_name = method_name[3:] 466 | field_name = field_name[0].lower() + field_name[1:] if len(field_name) > 1 else field_name[0].lower() 467 | methods_db_list = self.sqlite.select_data(f'SELECT field_name, field_type FROM field WHERE project_id = {self.project_id} and class_id={extend_class_id} and field_name = "{field_name}"') 468 | if not methods_db_list and not extend_class_entity['extends_class']: 469 | return None, None, None 470 | if not methods_db_list: 471 | return self._find_method_in_package_class(extend_class_entity['extends_class'], method_name, method_arguments) 472 | else: 473 | filter_methods = [method for method in methods_db_list if len(json.loads(method.get('parameters', '[]'))) == len(method_arguments)] 474 | if not filter_methods: 475 | return self._find_method_in_package_class(extend_class_entity['extends_class'], method_name, method_arguments) 476 | # package_class = extend_class_entity['package_name'] + '.' + extend_class_entity['class_name'] 477 | if len(filter_methods) == 1: 478 | method_db = filter_methods[0] 479 | method_params = f'{method_db.get("method_name", method_name)}({",".join([param["parameter_type"] for param in json.loads(method_db.get("parameters", "[]"))])})' 480 | return package_class, method_params, method_db 481 | else: 482 | max_score = -float('inf') 483 | max_score_method = None 484 | for method_db in filter_methods: 485 | method_db_params = [param["parameter_type"] for param in json.loads(method_db.get("parameters", "[]"))] 486 | score = calculate_similar_score_method_params(method_arguments, method_db_params) 487 | if score > max_score: 488 | max_score = score 489 | max_score_method = method_db 490 | if max_score_method is None: 491 | max_score_method = filter_methods[0] 492 | method_params = f'{max_score_method.get("method_name", method_name)}({",".join([param["parameter_type"] for param in json.loads(max_score_method.get("parameters", "[]"))])})' 493 | return package_class, method_params, max_score_method 494 | 495 | def _get_method_end_line(self, method_obj): 496 | method_end_line = method_obj.position.line 497 | while True: 498 | if isinstance(method_obj, list): 499 | method_obj = [obj for obj in method_obj if obj and not isinstance(obj, str)] 500 | if len(method_obj) == 0: 501 | break 502 | length = len(method_obj) 503 | for i in range(0, length): 504 | temp = method_obj[length - 1 - i] 505 | if temp is not None: 506 | method_obj = temp 507 | break 508 | if method_obj is None: 509 | break 510 | if isinstance(method_obj, list): 511 | continue 512 | if hasattr(method_obj, 'position') \ 513 | and method_obj.position is not None \ 514 | and method_obj.position.line > method_end_line: 515 | method_end_line = method_obj.position.line 516 | if hasattr(method_obj, 'children'): 517 | method_obj = method_obj.children 518 | else: 519 | break 520 | return method_end_line 521 | 522 | def _get_element_value(self, method_element): 523 | method_api_path = [] 524 | if type(method_element).__name__ == 'BinaryOperation': 525 | operandl = method_element.operandl 526 | operandr = method_element.operandr 527 | operandl_str = self._get_api_part_route(operandl) 528 | operandr_str = self._get_api_part_route(operandr) 529 | method_api_path = [operandl_str + operandr_str] 530 | elif type(method_element).__name__ == 'MemberReference': 531 | method_api_path = [method_element.member.replace('"', '')] 532 | elif type(method_element).__name__ == 'ElementArrayValue': 533 | method_api_path = self._get_element_with_values(method_element) 534 | elif method_element.value is not None: 535 | method_api_path = [method_element.value.replace('"', '')] 536 | return method_api_path 537 | 538 | def _get_element_with_values(self, method_api_path_obj): 539 | result = [] 540 | for method_api_value in method_api_path_obj.values: 541 | result += self._get_element_value(method_api_value) 542 | return result 543 | 544 | def _get_api_part_route(self, part): 545 | part_class = type(part).__name__ 546 | if part_class == 'MemberReference': 547 | return part.member.replace('"', '') 548 | elif part_class == 'Literal': 549 | return part.value.replace('"', '') 550 | else: 551 | return '' 552 | 553 | def _judge_is_controller(self, annotation_list): 554 | is_controller = any('Controller' in annotation.name for annotation in annotation_list) 555 | base_request = '' 556 | if not is_controller: 557 | return is_controller, base_request 558 | for annotation in annotation_list: 559 | if 'RequestMapping' != annotation.name: 560 | continue 561 | if annotation.element is None: 562 | continue 563 | if isinstance(annotation.element, list): 564 | base_request_list = [] 565 | for annotation_element in annotation.element: 566 | if annotation_element.name != 'value' and annotation_element.name != 'path': 567 | continue 568 | if 'values' in annotation_element.value.attrs: 569 | base_request_list += self._get_element_with_values(annotation_element.value) 570 | else: 571 | base_request_list += self._get_element_value(annotation_element.value) 572 | if len(base_request_list) > 0: 573 | base_request = base_request_list[0] 574 | else: 575 | if 'value' in annotation.element.attrs: 576 | base_request = annotation.element.value.replace('"', '') 577 | elif 'values' in annotation.element.attrs: 578 | base_request = ' || '.join([literal.value for literal in annotation.element.values]) 579 | if is_controller and not base_request.endswith('/'): 580 | base_request += '/' 581 | return is_controller, base_request 582 | 583 | def _judge_is_api(self, method_annotations, base_request, method_name): 584 | api_path_list = [] 585 | req_method_list = [] 586 | method_api_path = [] 587 | is_api = False 588 | for method_annotation in method_annotations: 589 | if method_annotation.name not in MAPPING_LIST: 590 | continue 591 | is_api = True 592 | if method_annotation.name != 'RequestMapping': 593 | req_method_list.append(method_annotation.name.replace('Mapping', '')) 594 | else: 595 | if not method_annotation.element: 596 | continue 597 | for method_annotation_element in method_annotation.element: 598 | if type(method_annotation_element) == tuple: 599 | req_method_list = ['ALL'] 600 | break 601 | if 'name' not in method_annotation_element.attrs or method_annotation_element.name != 'method': 602 | continue 603 | method_annotation_element_value = method_annotation_element.value 604 | if 'member' in method_annotation_element_value.attrs: 605 | req_method_list.append(method_annotation_element_value.member) 606 | elif 'values' in method_annotation_element_value.attrs: 607 | method_annotation_element_values = method_annotation_element_value.values 608 | req_method_list += [method_annotation_element_temp.member for 609 | method_annotation_element_temp in 610 | method_annotation_element_values 611 | if 'member' in method_annotation_element_temp.attrs] 612 | if not isinstance(method_annotation.element, list): 613 | if method_annotation.element is None: 614 | continue 615 | method_api_path += self._get_element_value(method_annotation.element) 616 | else: 617 | method_api_path_list = [method_annotation_element.value for method_annotation_element in method_annotation.element 618 | if method_annotation_element.name == 'path' or method_annotation_element.name == 'value'] 619 | if len(method_api_path_list) == 0: 620 | continue 621 | method_api_path_obj = method_api_path_list[0] 622 | if 'value' in method_api_path_obj.attrs: 623 | method_api_path += [method_api_path_obj.value.replace('"', '')] 624 | else: 625 | if 'values' in method_api_path_obj.attrs: 626 | for method_api_value in method_api_path_obj.values: 627 | method_api_path += self._get_element_value(method_api_value) 628 | else: 629 | method_api_path += [method_name + '/cci-unknown'] 630 | if len(method_api_path) == 0: 631 | method_api_path = ['/'] 632 | for method_api_path_obj in method_api_path: 633 | if method_api_path_obj.startswith('/'): 634 | method_api_path_obj = method_api_path_obj[1:] 635 | api_path = base_request + method_api_path_obj 636 | if not api_path: 637 | continue 638 | if api_path.endswith('/'): 639 | api_path = api_path[0:-1] 640 | if len(req_method_list) > 0: 641 | api_path_list += ['[' + req_method_temp + ']' + api_path for req_method_temp in req_method_list] 642 | else: 643 | api_path_list.append('[ALL]' + api_path) 644 | return is_api, api_path_list 645 | 646 | def _add_entity_used_to_method_invocation(self, method_invocation, package_class, section): 647 | if package_class not in method_invocation.keys(): 648 | method_invocation[package_class] = {ENTITY: {section: True}} 649 | elif ENTITY not in method_invocation[package_class].keys(): 650 | method_invocation[package_class][ENTITY] = {section: True} 651 | elif section not in method_invocation[package_class][ENTITY].keys(): 652 | method_invocation[package_class][ENTITY][section] = True 653 | 654 | def _add_method_used_to_method_invocation(self, method_invocation, package_class, method, lines): 655 | if package_class not in method_invocation.keys(): 656 | method_invocation[package_class] = {METHODS: {method: lines}} 657 | elif METHODS not in method_invocation[package_class].keys(): 658 | method_invocation[package_class][METHODS] = {method: lines} 659 | elif method not in method_invocation[package_class][METHODS].keys(): 660 | method_invocation[package_class][METHODS][method] = lines 661 | else: 662 | method_invocation[package_class][METHODS][method] += lines 663 | 664 | def _add_field_used_to_method_invocation(self, method_invocation, package_class, field, lines): 665 | if package_class not in method_invocation.keys(): 666 | method_invocation[package_class] = {FIELDS: {field: lines}} 667 | elif FIELDS not in method_invocation[package_class].keys(): 668 | method_invocation[package_class][FIELDS] = {field: lines} 669 | elif field not in method_invocation[package_class][FIELDS].keys(): 670 | method_invocation[package_class][FIELDS][field] = lines 671 | else: 672 | method_invocation[package_class][FIELDS][field] += lines 673 | 674 | def _deal_declarator_type(self, node_type, section, parameters_map, variable_map, field_map, import_map, method_invocation, package_name, filepath, methods, method_name_entity_map, class_id): 675 | if node_type is None: 676 | return node_type 677 | if type(node_type) == javalang.tree.BasicType: 678 | node_name = node_type.name 679 | if node_name != 'int': 680 | node_name = node_name[0].upper() + node_name[1:] 681 | return node_name 682 | var_declarator_type = self._parse_sub_type(node_type) 683 | var_declarator_type = self._get_var_type(var_declarator_type, parameters_map, variable_map, field_map, import_map, method_invocation, section, package_name, filepath) 684 | var_declarator_type_arguments = self._deal_arguments_type(node_type.arguments, section, parameters_map, variable_map, field_map, import_map, method_invocation, package_name, filepath, methods, method_name_entity_map, class_id) 685 | if var_declarator_type_arguments: 686 | var_declarator_type = var_declarator_type + '<' + '#'.join(var_declarator_type_arguments) + '>' 687 | return var_declarator_type 688 | 689 | def _parse_sub_type(self, type_obj): 690 | type_name = type_obj.name 691 | if 'sub_type' in type_obj.attrs and type_obj.sub_type: 692 | type_name = type_name + '.' + self._parse_sub_type(type_obj.sub_type) 693 | return type_name 694 | 695 | def _deal_arguments_type(self, arguments, section, parameters_map, variable_map, field_map, import_map, method_invocation, package_name, filepath, methods, method_name_entity_map, class_id): 696 | var_declarator_type_arguments_new = [] 697 | if not arguments: 698 | return var_declarator_type_arguments_new 699 | var_declarator_type_arguments = [] 700 | for argument in arguments: 701 | argument_type = type(argument) 702 | if argument_type == javalang.tree.MethodInvocation: 703 | var_declarator_type_argument = self._parse_method_body_method_invocation(argument, parameters_map, variable_map, field_map, import_map, method_invocation, package_name, filepath, methods, method_name_entity_map, class_id) 704 | elif argument_type == javalang.tree.This: 705 | var_declarator_type_argument = self._parse_node_selectors(argument.selectors, None, parameters_map, variable_map, field_map, import_map, method_invocation, package_name, filepath, methods, method_name_entity_map, class_id) 706 | else: 707 | var_declarator_type_argument = self._deal_type(argument) 708 | var_declarator_type_argument = self._get_var_type(var_declarator_type_argument, parameters_map, variable_map, field_map, import_map, method_invocation, section, package_name, filepath) 709 | if self._is_valid_prefix(var_declarator_type_argument): 710 | self._add_entity_used_to_method_invocation(method_invocation, var_declarator_type_argument, section) 711 | var_declarator_type_arguments.append(var_declarator_type_argument) 712 | return var_declarator_type_arguments 713 | 714 | def _deal_member_reference(self, member_reference, parameters_map, variable_map, field_map, import_map, method_invocation, section, package_name, filepath): 715 | member = member_reference.member 716 | qualifier: str = member_reference.qualifier 717 | if not qualifier: 718 | qualifier_type = self._get_var_type(member, parameters_map, variable_map, field_map, import_map, method_invocation, section, package_name, filepath) 719 | else: 720 | qualifier_type = self._get_var_type(qualifier, parameters_map, variable_map, field_map, import_map, method_invocation, section, package_name, filepath) 721 | if self._is_valid_prefix(qualifier_type): 722 | self._add_field_used_to_method_invocation(method_invocation, qualifier_type, member, [None]) 723 | return qualifier_type 724 | 725 | def _deal_type(self, argument): 726 | if not argument: 727 | return None 728 | argument_type = type(argument) 729 | if argument_type == javalang.tree.MemberReference: 730 | var_declarator_type_argument = argument.member 731 | elif argument_type == javalang.tree.ClassCreator: 732 | var_declarator_type_argument = argument.type.name 733 | elif argument_type == javalang.tree.Literal: 734 | var_declarator_type_argument = self._deal_literal_type(argument.value) 735 | elif argument_type == javalang.tree.LambdaExpression: 736 | var_declarator_type_argument = PARAMETER_TYPE_METHOD_INVOCATION_UNKNOWN 737 | elif argument_type == javalang.tree.BinaryOperation: 738 | # todo BinaryOperation temp set string 739 | var_declarator_type_argument = 'String' 740 | elif argument_type == javalang.tree.MethodReference or argument_type == javalang.tree.TernaryExpression: 741 | # todo MethodReference temp set unknown 742 | var_declarator_type_argument = PARAMETER_TYPE_METHOD_INVOCATION_UNKNOWN 743 | elif argument_type == javalang.tree.SuperMethodInvocation: 744 | logging.info(argument_type) 745 | var_declarator_type_argument = PARAMETER_TYPE_METHOD_INVOCATION_UNKNOWN 746 | elif argument_type == javalang.tree.Assignment: 747 | var_declarator_type_argument = self._deal_type(argument.value) 748 | elif argument_type == javalang.tree.Cast: 749 | var_declarator_type_argument = argument.type.name 750 | # todo 751 | elif argument_type == javalang.tree.SuperMemberReference: 752 | var_declarator_type_argument = 'String' 753 | elif 'type' in argument.attrs and argument.type is not None: 754 | var_declarator_type_argument = argument.type.name 755 | else: 756 | logging.info(f'argument type is None：{argument}') 757 | var_declarator_type_argument = PARAMETER_TYPE_METHOD_INVOCATION_UNKNOWN 758 | return var_declarator_type_argument 759 | 760 | def _deal_literal_type(self, text): 761 | if 'true' == text or 'false' == text: 762 | return 'Boolean' 763 | if text.isdigit(): 764 | return 'Int' 765 | return 'String' 766 | 767 | def _deal_var_type(self, arguments, section, parameters_map, variable_map, field_map, import_map, method_invocation, package_name, filepath, methods, method_name_entity_map, class_id): 768 | var_declarator_type_arguments_new = [] 769 | if not arguments: 770 | return var_declarator_type_arguments_new 771 | var_declarator_type_arguments = [] 772 | for argument in arguments: 773 | argument_type = type(argument) 774 | if argument_type == javalang.tree.MethodInvocation: 775 | var_declarator_type_argument = self._parse_method_body_method_invocation(argument, parameters_map, variable_map, field_map, import_map, method_invocation, package_name, filepath, methods, method_name_entity_map, class_id) 776 | elif argument_type == javalang.tree.MemberReference: 777 | var_declarator_type_argument = self._deal_member_reference(argument, parameters_map, variable_map, field_map, import_map, method_invocation, section, package_name, filepath) 778 | elif argument_type == javalang.tree.This: 779 | var_declarator_type_argument = self._parse_node_selectors(argument.selectors, None, parameters_map, variable_map, field_map, import_map, method_invocation, package_name, filepath, methods, method_name_entity_map, class_id) 780 | if var_declarator_type_argument is None: 781 | var_declarator_type_argument = PARAMETER_TYPE_METHOD_INVOCATION_UNKNOWN 782 | else: 783 | var_declarator_type_argument = self._deal_type(argument) 784 | var_declarator_type_argument = self._get_var_type(var_declarator_type_argument, parameters_map, variable_map, field_map, import_map, method_invocation, section, package_name, filepath) 785 | type_arguments = self._deal_arguments_type(argument.type.arguments, section, parameters_map, variable_map, field_map, import_map, method_invocation, package_name, filepath, methods, method_name_entity_map, class_id) \ 786 | if 'type' in argument.attrs \ 787 | and not isinstance(argument.type, str) \ 788 | and 'arguments' in argument.type.attrs \ 789 | and argument.type.arguments \ 790 | else [] 791 | if type_arguments: 792 | var_declarator_type_argument = var_declarator_type_argument + '<' + '#'.join(type_arguments) + '>' 793 | var_declarator_type_arguments.append(var_declarator_type_argument) 794 | return var_declarator_type_arguments 795 | 796 | def _get_var_type(self, var, parameters_map, variable_map, field_map, import_map, method_invocation, section, package_name, filepath): 797 | if not var: 798 | return var 799 | if var.lower() in JAVA_BASIC_TYPE or var in JAVA_UTIL_TYPE: 800 | return var 801 | var_path = "/".join(filepath.split("/")[0: -1]) + "/" + var + ".java" 802 | if var in parameters_map.keys(): 803 | return parameters_map.get(var) 804 | elif var in variable_map.keys(): 805 | return variable_map.get(var) 806 | elif var in field_map.keys(): 807 | field_type = field_map.get(var)['field_type'] 808 | package_class = field_map.get(var)['package_class'] 809 | start_line = field_map.get(var)['start_line'] 810 | self._add_field_used_to_method_invocation(method_invocation, package_class, var, [start_line]) 811 | return field_type 812 | elif var in import_map.keys(): 813 | if '.' in var: 814 | return self._parse_layer_call_var_type(var, import_map, method_invocation) 815 | var_type = import_map.get(var) 816 | return var_type 817 | elif os.path.exists(var_path): 818 | var_type = f'{package_name}.{var}' 819 | return var_type 820 | if '.' not in var: 821 | sql = "select package_name, class_name from class where project_id = {} and class_name=\"{}\"".format(self.project_id, var) 822 | var_class_db = self.sqlite.select_data(sql) 823 | if len(var_class_db) == 1: 824 | return var_class_db[0]['package_name'] + '.' + var_class_db[0]['class_name'] 825 | import_values = import_map.values() 826 | var_class_db_matched = [vcd for vcd in var_class_db if vcd['package_name'] in import_values] 827 | if var_class_db_matched: 828 | return var_class_db_matched[0]['package_name'] + '.' + var_class_db_matched[0]['class_name'] 829 | return self._parse_layer_call_var_type(var, import_map, method_invocation) 830 | 831 | def _parse_layer_call_var_type(self, var, import_map, method_invocation): 832 | ## 判断是否内部类 833 | var_split = var.split('.') 834 | var_class = var_split[-1] 835 | if var_class.lower() in JAVA_BASIC_TYPE or var_class in JAVA_UTIL_TYPE: 836 | return var 837 | elif len(var_split) > 1: 838 | var_field = var_split[-1] 839 | var_class = var_split[-2] 840 | if var_class in import_map.keys(): 841 | var_type = import_map.get(var_class) 842 | var_type_package = '.'.join(var_type.split('.')[0: -1]) 843 | var_field_db = self.sqlite.select_data(f'select field_type from field where project_id={self.project_id} and field_name="{var_field}" ' 844 | f'and class_id in (select class_id from class where project_id={self.project_id} and class_name="{var_class}" and package_name="{var_type_package}")') 845 | if var_field_db: 846 | self._add_field_used_to_method_invocation(method_invocation, var_type, var_field, [None]) 847 | field_type = var_field_db[0]['field_type'] 848 | if field_type == 'ENUM': 849 | return var_type 850 | return field_type 851 | var_package_end = '.'.join(var_split[0: -1]) 852 | sql = "select package_name, class_name from class where project_id = {} and class_name=\"{}\" and package_name like \"%{}\"".format(self.project_id, var_field, var_package_end) 853 | var_class_db = self.sqlite.select_data(sql) 854 | if var_class_db: 855 | var_type = var_class_db[0]['package_name'] + '.' + var_class_db[0]['class_name'] 856 | return var_type 857 | elif var != PARAMETER_TYPE_METHOD_INVOCATION_UNKNOWN: 858 | return var[0].upper() + var[1:] 859 | return var 860 | 861 | def _get_extends_class_fields_map(self, class_id: int): 862 | class_db = self.sqlite.select_data(f'SELECT extends_class FROM class WHERE project_id = {self.project_id} and class_id = {class_id}')[0] 863 | extend_package_class = class_db['extends_class'] 864 | if not extend_package_class: 865 | return {} 866 | extend_package = '.'.join(extend_package_class.split('.')[0: -1]) 867 | extend_class = extend_package_class.split('.')[-1] 868 | extend_class_db = self.sqlite.select_data(f'SELECT class_id, extends_class FROM class WHERE package_name="{extend_package}" ' 869 | f'AND class_name="{extend_class}" ' 870 | f'AND project_id={self.project_id} limit 1') 871 | if not extend_class_db: 872 | return {} 873 | extend_class_entity = extend_class_db[0] 874 | extend_class_id = extend_class_entity['class_id'] 875 | extend_class_fields = self.sqlite.select_data(f'SELECT field_name, field_type, start_line FROM field WHERE project_id = {self.project_id} and class_id = {extend_class_id}') 876 | extend_class_fields_map = {field_obj['field_name']: {'field_type': field_obj['field_type'], 'package_class': extend_package_class, 'start_line': field_obj['start_line']} for field_obj in extend_class_fields} 877 | if not extend_class_entity['extends_class']: 878 | return extend_class_fields_map 879 | else: 880 | extend_new_map = self._get_extends_class_fields_map(extend_class_id) 881 | extend_new_map.update(extend_class_fields_map) 882 | return extend_new_map 883 | 884 | def _is_valid_prefix(self, import_str): 885 | for prefix in config.package_prefix: 886 | if import_str and import_str.startswith(prefix): 887 | return True 888 | return False 889 | 890 | def _get_sibling_dirs(self, path): 891 | parent_dir = os.path.abspath(os.path.join(path, os.pardir)) 892 | dirs = [os.path.join(parent_dir, d).replace('\\', '/') for d in os.listdir(parent_dir) if os.path.isdir(os.path.join(parent_dir, d)) and not d.startswith('.')] 893 | return dirs 894 | 895 | def _list_files(self, directory): 896 | # 使用 os.listdir() 获取目录下所有文件和文件夹名 897 | all_contents = os.listdir(directory) 898 | 899 | # 完整的文件路径列表 900 | full_paths = [os.path.join(directory, f) for f in all_contents] 901 | 902 | # 筛选出是文件的路径 903 | only_files = [f.replace('\\', '/') for f in full_paths if os.path.isfile(f)] 904 | 905 | return only_files 906 | 907 | def _parse_import_file(self, imports, commit_or_branch, parse_import_first): 908 | for import_decl in imports: 909 | import_path = import_decl.path 910 | is_static = import_decl.static 911 | is_wildcard = import_decl.wildcard 912 | if not self._is_valid_prefix(import_path): 913 | continue 914 | if is_static: 915 | import_path = '.'.join(import_path.split('.')[0:-1]) 916 | java_files = [] 917 | if is_wildcard: 918 | import_filepaths = [file_path + '/src/main/java/' + import_path.replace('.', '/') for file_path in self.sibling_dirs] 919 | for import_filepath in import_filepaths: 920 | if not os.path.exists(import_filepath): 921 | continue 922 | java_files += self._list_files(import_filepath) 923 | else: 924 | java_files = [file_path + '/src/main/java/' + import_path.replace('.', '/') + '.java' for file_path in self.sibling_dirs] 925 | for import_filepath in java_files: 926 | if not os.path.exists(import_filepath): 927 | continue 928 | self.parse_java_file(import_filepath, commit_or_branch, parse_import_first=parse_import_first) 929 | 930 | def _parse_tree_class(self, class_declaration, filepath, tree_imports, package_name, commit_or_branch, lines, parse_import_first): 931 | class_name = class_declaration.name 932 | package_class = package_name + '.' + class_name 933 | import_list = self._parse_imports(tree_imports) 934 | import_map = {import_obj['import_path'].split('.')[-1]: import_obj['import_path'] for import_obj in import_list} 935 | 936 | # 处理 class 信息 937 | class_type = type(class_declaration).__name__.replace('Declaration', '') 938 | class_id, new_add = self._parse_class(class_declaration, filepath, package_name, import_list, commit_or_branch, parse_import_first) 939 | # 已经处理过了，返回 940 | if not new_add and not config.reparse_class: 941 | return 942 | # 导入import 943 | imports = [dict(import_obj, class_id=class_id, project_id=self.project_id) for import_obj in import_list] 944 | self.sqlite.update_data(f'DELETE FROM import WHERE class_id={class_id}') 945 | self.sqlite.insert_data('import', imports) 946 | 947 | # 处理 inner class 948 | inner_class_declarations = [inner_class for inner_class in class_declaration.body 949 | if type(inner_class) == javalang.tree.ClassDeclaration 950 | or type(inner_class) == javalang.tree.InterfaceDeclaration] 951 | for inner_class_obj in inner_class_declarations: 952 | self._parse_tree_class(inner_class_obj, filepath, tree_imports, package_class, commit_or_branch, lines, parse_import_first) 953 | 954 | # 处理 field 信息 955 | field_list = self._parse_fields(class_declaration.fields, package_name, class_name, class_id, import_map, filepath) 956 | field_map = {field_obj['field_name']: {'field_type': field_obj['field_type'], 'package_class': package_class, 'start_line': field_obj['start_line']} for field_obj in field_list} 957 | import_map = dict((k, v) for k, v in import_map.items() if self._is_valid_prefix(v)) 958 | 959 | # 将extend class的field导进来 960 | extends_class_fields_map = self._get_extends_class_fields_map(class_id) 961 | extends_class_fields_map.update(field_map) 962 | 963 | if class_type == 'Enum': 964 | self._parse_enum(class_declaration.body, lines, class_id, import_map, field_map, package_name, filepath) 965 | 966 | # 处理 methods 信息 967 | self._parse_method(class_declaration.methods, lines, class_id, import_map, extends_class_fields_map, package_name, filepath) 968 | 969 | self._parse_constructors(class_declaration.constructors, lines, class_id, import_map, extends_class_fields_map, package_name, filepath) 970 | 971 | def parse_java_file(self, filepath: str, commit_or_branch: str, parse_import_first=True): 972 | if filepath + '_' + commit_or_branch in self.parsed_filepath or not filepath.endswith('.java'): 973 | return 974 | self.parsed_filepath.append(filepath + '_' + commit_or_branch) 975 | try: 976 | with open(filepath, encoding='UTF-8') as fp: 977 | file_content = fp.read() 978 | except: 979 | return 980 | lines = file_content.split('\n') 981 | try: 982 | tree = javalang.parse.parse(file_content) 983 | if not tree.types: 984 | return 985 | except Exception as e: 986 | logging.error(f"Error parsing {filepath}: {e}") 987 | return 988 | 989 | # 处理包信息 990 | package_name = tree.package.name if tree.package else 'unknown' 991 | class_declaration = tree.types[0] 992 | class_name = class_declaration.name 993 | package_class = package_name + '.' + class_name 994 | if not self.sibling_dirs: 995 | package_path = package_class.replace('.', '/') + '.java' 996 | base_filepath = filepath.replace(package_path, '') 997 | self.sibling_dirs = self._get_sibling_dirs(base_filepath.replace('src/main/java/', '')) 998 | # 处理 import 信息 999 | if parse_import_first: 1000 | self._parse_import_file(tree.imports, commit_or_branch, parse_import_first) 1001 | logging.info(f'Parsing java file: {filepath}') 1002 | self._parse_tree_class(class_declaration, filepath, tree.imports, package_name, commit_or_branch, lines, parse_import_first) 1003 | 1004 | def parse_java_file_list(self, filepath_list: list, commit_or_branch: str): 1005 | with ThreadPoolExecutor(max_workers=4) as executor: 1006 | futures = [executor.submit(self.parse_java_file, file, commit_or_branch) for file in filepath_list] 1007 | for _ in as_completed(futures): 1008 | continue 1009 | 1010 | 1011 | if __name__ == '__main__': 1012 | print('jcci') 1013 | --------------------------------------------------------------------------------