├── .github └── workflows │ └── sync-to-gitee.yml ├── .gitignore ├── LICENSE ├── cli.py ├── doc ├── images │ ├── desc.png │ ├── github-token-contents.png │ ├── github-token-owner.png │ ├── github-token-pull-request.png │ ├── github-token-repo.png │ ├── init.png │ └── trident.png ├── pr.md └── publish.md ├── lib ├── api │ ├── abstract_client.py │ ├── gitea.py │ ├── gitee.py │ ├── github.py │ └── index.py ├── handler │ ├── helper.py │ ├── init.py │ ├── remote.py │ └── sync.py ├── http.py ├── logger.py ├── model │ ├── config.py │ ├── opts.py │ ├── repo.py │ └── sync.py ├── util.py ├── util_git.py └── version.py ├── pyproject.toml ├── readme.md └── test ├── .gitignore ├── copy_script.py ├── main_test.py ├── pr_test.py └── sync.yaml /.github/workflows/sync-to-gitee.yml: -------------------------------------------------------------------------------- 1 | name: sync-to-gitee 2 | on: 3 | push: 4 | branches: [ "main" ] 5 | pull_request: 6 | branches: [ "main" ] 7 | # schedule: 8 | # - # 国际时间 19:17 执行，北京时间3:17 ↙↙↙ 改成你想要每天自动执行的时间 9 | # - cron: '17 19 * * *' 10 | permissions: 11 | contents: read 12 | 13 | jobs: 14 | sync: 15 | runs-on: ubuntu-latest 16 | steps: 17 | - name: Checkout work repo # 1. 检出当前仓库(certd-sync-work) 18 | uses: actions/checkout@v3 19 | with: 20 | fetch-depth: 0 21 | - name: Set git user # 2. 给git命令设置用户名和邮箱,↙↙↙ 改成你的name和email 22 | run: | 23 | git config --global user.name "xiaojunnuo" 24 | git config --global user.email "xiaojunnuo@qq.com" 25 | 26 | - name: Set git token # 3. 给git命令设置token，用于push到目标仓库 27 | uses: de-vri-es/setup-git-credentials@v2 28 | with: 29 | credentials: https://${{secrets.PUSH_TOKEN_GITEE}}@gitee.com 30 | 31 | - name: push to gitee # 4. 执行同步 32 | run: | 33 | git remote add upstream https://gitee.com/handsfree-work/trident-sync 34 | git push --set-upstream upstream main 35 | 36 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .idea 2 | dist 3 | venv 4 | sync 5 | poetry.lock 6 | __pycache__ 7 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2023 Greper 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /cli.py: -------------------------------------------------------------------------------- 1 | """ 2 | 异构仓库同步升级工具 3 | 4 | Precondition: 5 | 1. install git 6 | Usage: 7 | trident init [-r ROOT] [-c CONFIG] 8 | trident sync [-r ROOT] [-c CONFIG] [-t TOKEN] [-i] 9 | trident remote [-r ROOT] [-u URL] [-f] 10 | trident version 11 | trident -h 12 | Options: 13 | -h,--help show help menu, 显示帮助菜单 14 | -c,--config=CONFIG config file path, 配置文件 [default: sync.yaml] 15 | -r,--root=ROOT root dir, 根目录 [default: .] 16 | -t,--token=TOKEN PR token 17 | -u,--url=URL remote git url, 远程地址 18 | -f,--force force push, 强制推送 19 | -i,--init init first, 同步前先初始化 20 | Example: 21 | trident init 22 | trident sync 23 | trident remote --url=https://github.com/handsfree-work/trident-test-sync 24 | """ 25 | import os 26 | 27 | import yaml 28 | from docopt import docopt 29 | 30 | from lib.handler.init import InitHandler 31 | from lib.handler.remote import RemoteHandler 32 | from lib.handler.sync import SyncHandler 33 | from lib.logger import logger 34 | from lib.model.config import Config 35 | from lib.util import get_arg 36 | from lib.version import get_version 37 | 38 | 39 | def cli(): 40 | """ 41 | 异构仓库同步升级工具入口 42 | """ 43 | args = docopt(__doc__) 44 | 45 | version = get_version() 46 | 47 | print(f''' 48 | 49 | ████████╗██████╗ ██╗██████╗ ███████╗███╗ ██╗████████╗ 50 | ╚══██╔══╝██╔══██╗██║██╔══██╗██╔════╝████╗ ██║╚══██╔══╝ 51 | ██║ ██████╔╝██║██║ ██║█████╗ ██╔██╗ ██║ ██║ 52 | ██║ ██╔══██╗██║██║ ██║██╔══╝ ██║╚██╗██║ ██║ 53 | ██║ ██║ ██║██║██████╔╝███████╗██║ ╚████║ ██║ 54 | ╚═╝ ╚═╝ ╚═╝╚═╝╚═════╝ ╚══════╝╚═╝ ╚═══╝ ╚═╝ 55 | https://github.com/handsfree-work/trident-sync 56 | Don't be stingy with your star ( 请不要吝啬你的star ) 57 | Copyright © 2023 greper@handsfree.work v{version} 58 | 59 | ''') 60 | root = get_root(args) 61 | if not os.path.exists(root): 62 | os.makedirs(root) 63 | 64 | arg_config = get_arg(args, '--config') 65 | config_dict = read_config(arg_config) 66 | 67 | os.chdir(root) 68 | if args['remote']: 69 | remote_url = args['--url'] 70 | force = args['--force'] 71 | RemoteHandler(root, remote_url=remote_url, force=force).handle() 72 | return 73 | 74 | token_from_args = get_arg(args, '--token') 75 | config = Config(config_dict) 76 | config.set_default_token(token_from_args) 77 | 78 | if args['init']: 79 | InitHandler(root, config).handle() 80 | elif args['sync']: 81 | init_first = get_arg(args, '--init') 82 | if init_first: 83 | logger.info("init first") 84 | InitHandler(root, config).handle() 85 | SyncHandler(root, config).handle() 86 | else: 87 | logger.info(__doc__) 88 | 89 | 90 | def read_config(arg_config='./sync.yaml'): 91 | config_file = os.path.abspath(f"{arg_config}") 92 | f = open(config_file, 'r', encoding='utf-8') 93 | return yaml.load(f, Loader=yaml.FullLoader) 94 | 95 | 96 | def get_root(args): 97 | root = get_arg(args, '--root') 98 | return os.path.abspath(root) 99 | 100 | 101 | if __name__ == '__main__': 102 | cli() 103 | -------------------------------------------------------------------------------- /doc/images/desc.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/handsfree-work/trident-sync/74d23ab5f16b1f27ae92b1e890a619fea0a350d5/doc/images/desc.png -------------------------------------------------------------------------------- /doc/images/github-token-contents.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/handsfree-work/trident-sync/74d23ab5f16b1f27ae92b1e890a619fea0a350d5/doc/images/github-token-contents.png -------------------------------------------------------------------------------- /doc/images/github-token-owner.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/handsfree-work/trident-sync/74d23ab5f16b1f27ae92b1e890a619fea0a350d5/doc/images/github-token-owner.png -------------------------------------------------------------------------------- /doc/images/github-token-pull-request.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/handsfree-work/trident-sync/74d23ab5f16b1f27ae92b1e890a619fea0a350d5/doc/images/github-token-pull-request.png -------------------------------------------------------------------------------- /doc/images/github-token-repo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/handsfree-work/trident-sync/74d23ab5f16b1f27ae92b1e890a619fea0a350d5/doc/images/github-token-repo.png -------------------------------------------------------------------------------- /doc/images/init.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/handsfree-work/trident-sync/74d23ab5f16b1f27ae92b1e890a619fea0a350d5/doc/images/init.png -------------------------------------------------------------------------------- /doc/images/trident.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/handsfree-work/trident-sync/74d23ab5f16b1f27ae92b1e890a619fea0a350d5/doc/images/trident.png -------------------------------------------------------------------------------- /doc/pr.md: -------------------------------------------------------------------------------- 1 | # 启用PR 2 | 3 | 4 | ## PR配置 5 | 6 | 同步成功后，支持自动创建PR 7 | 8 | 要开启此功能需要给对应的 `target_repo` 配置 `token` 和 `type` 9 | 10 | ```yaml 11 | repo: 12 | target: 13 | url: 'xxxx' 14 | path: 'xxxx' 15 | branch: 'xxxx' 16 | type: 'github' # 远程仓库类型。可选值：[github / gitee / gitea ] 17 | token: "xxxxxxx" # 授权token，请根据下方说明创建 18 | auto_merge: true # PR没冲突时，是否自动合并 19 | ``` 20 | 21 | ## token配置方式 22 | token有两种配置方式 23 | 1. 如上所示，在sync.yaml文件中配置 24 | 2. 也可以通过环境变量设置，变量名格式为：trident_token_{type} 25 | ```shell 26 | # 例如： 27 | export trident_token_github=ghp_xxxxxxxxxxxxxxxxxxxx 28 | ``` 29 | 30 | 31 | ## token创建 32 | 33 | ### 1、 github token 34 | 35 | 从此处创建token: https://github.com/settings/tokens 36 | 37 | 如果是 `Fine-grained tokens (beta) ` 38 | 要注意如下几点： 39 | 40 | 1. 选择正确的 `Resource owner` 41 | ![](./images/github-token-owner.png) 42 | 2. `Repository access`,要选择 `All repositories` 或者 `Only select repositories(选择特定仓库)` 43 | 3. `Repository permissions` 中的 `Contents ` `Pull requests` 都要选择`read and write` 44 | ![](./images/github-token-contents.png) 45 | ![](./images/github-token-pull-request.png) 46 | 47 | 48 | 49 | 50 | 如果是 `Personal access tokens (classic)` 51 | 52 | 1. 只需要勾选 `repo` 即可 53 | ![](./images/github-token-repo.png) 54 | 55 | 56 | ## 2、 gitee token 57 | 58 | 从此处创建token 59 | https://gitee.com/profile/personal_access_tokens 60 | 61 | token所属账号要求（如果账号本身就是`仓库拥有者`，可以无视） 62 | 63 | 1. 需要有对仓库具备审查和测试权限，前往仓库设置页面设置：（owner/repo替换成你的项目地址） 64 | https://gitee.com///settings#set_pr_assigner 65 | 66 | 2. 需要对仓库具有管理员权限，前往成员管理页面设置： 67 | https://gitee.com/handsfree-test/pr-test/team?type=masters 68 | 69 | ## 3、 gitea token 70 | 71 | 从管理AccessToken栏目中创建token （将your.gitea.host 替换成你的实际gitea地址） 72 | https://your.gitea.host/user/settings/applications 73 | 74 | 75 | 76 | 77 | 78 | -------------------------------------------------------------------------------- /doc/publish.md: -------------------------------------------------------------------------------- 1 | 2 | ```shell 3 | poetry build 4 | poetry publish --build --username=greper --password=xxxx 5 | 6 | ``` -------------------------------------------------------------------------------- /lib/api/abstract_client.py: -------------------------------------------------------------------------------- 1 | import time 2 | from abc import abstractmethod 3 | 4 | from lib.logger import logger 5 | from lib.util import re_pick 6 | 7 | 8 | def pick_from_url(git_url): 9 | re_str = "(http[s]?)://([^/]+)/([^/]+)/([^/\\.]+)" 10 | res = re_pick(re_str, git_url) 11 | if len(res) < 4: 12 | raise Exception(f"异常的git_url: {git_url}") 13 | return { 14 | 'protocol': res[0], 15 | 'host': res[1], 16 | 'owner': res[2], 17 | 'repo': res[3], 18 | } 19 | 20 | 21 | class AbstractClient: 22 | '''repo api''' 23 | token = 'token' 24 | http = None 25 | url = None 26 | repo_path = None 27 | owner = None 28 | repo = None 29 | api_root = None 30 | 31 | def __init__(self, http, token, url): 32 | self.token = token 33 | self.http = http 34 | self.url = url 35 | res = pick_from_url(url) 36 | self.owner = res['owner'] 37 | self.repo = res['repo'] 38 | self.url_prefix = f"{res['protocol']}://{res['host']}" 39 | self.repo_path = f"{self.owner}/{self.repo}" 40 | 41 | def create_pull_request(self, title, body, head_branch, base_branch, auto_merge=True): 42 | ''' 43 | https://try.gitea.io/api/swagger#/repository/repoCreatePullRequest 44 | 45 | curl -X 'POST' \ 46 | 'http://gitea.baode.docmirror.cn/api/v1/repos/{owner}/{repo}/pulls?access_token=1a5461e976c423068cc7677e608681b3d992eefe' \ 47 | -H 'accept: application/json' 48 | ''' 49 | 50 | pull_res = self.query_pull_request(head_branch, base_branch) 51 | if pull_res is None: 52 | pull_res = self.post_pull_request(title, body, head_branch, base_branch) 53 | pull_id = pull_res['number'] 54 | merged = False 55 | if auto_merge: 56 | merged = self.auto_merge(pull_id) 57 | return pull_id, merged 58 | 59 | def can_auto_merge(self, res): 60 | if 'mergeable' not in res: 61 | logger.warning("Status unknown, please manually merge PR") 62 | return None 63 | mergeable = res['mergeable'] 64 | if mergeable is True: 65 | return True 66 | if mergeable is False: 67 | logger.warning("There may be conflicts, please merge PR manually") 68 | return False 69 | logger.warning("Status unknown, please manually merge PR") 70 | return None 71 | 72 | def auto_merge(self, pull_id): 73 | logger.info("Check for can auto merge after 5 seconds") 74 | time.sleep(5) 75 | res = self.get_pull_request_detail(pull_id) 76 | can_merge = self.can_auto_merge(res) 77 | if can_merge is True: 78 | logger.info("PR will be auto merged") 79 | # 准备自动合并 80 | self.post_merge(pull_id, res) 81 | logger.info("Auto merge success") 82 | return True 83 | return False 84 | 85 | @abstractmethod 86 | def post_pull_request(self, title, body, head_branch, base_branch): 87 | pass 88 | 89 | @abstractmethod 90 | def query_pull_request(self, head_branch, base_branch): 91 | pass 92 | 93 | @abstractmethod 94 | def get_pull_request_detail(self, pull_id): 95 | pass 96 | 97 | @abstractmethod 98 | def post_merge(self, pull_id, detail): 99 | pass 100 | -------------------------------------------------------------------------------- /lib/api/gitea.py: -------------------------------------------------------------------------------- 1 | from lib.api.abstract_client import AbstractClient 2 | 3 | 4 | class GiteaClient(AbstractClient): 5 | '''gitea api''' 6 | ''' 7 | http://gitea.baode.docmirror.cn/{repos}/{owner} 8 | 9 | https://try.gitea.io/api/swagger#/repository/repoMergePullRequest 10 | 11 | ''' 12 | 13 | def __init__(self, http, token, url): 14 | super().__init__(http, token, url) 15 | self.api_root = f"{self.url_prefix}/api/v1" 16 | 17 | def post_pull_request(self, title, body, head_branch, base_branch): 18 | api = f"{self.api_root}/repos/{self.repo_path}/pulls?access_token={self.token}" 19 | res = self.http.post(api, data={ 20 | "title": title, 21 | "body": body, 22 | # "head": f"{self.owner}:{src_branch}", 23 | "head": f"{head_branch}", 24 | "base": base_branch 25 | }, res_is_standard=False, res_is_json=True) 26 | return res 27 | 28 | def query_pull_request(self, head_branch, base_branch, state='open'): 29 | api = f"{self.api_root}/repos/{self.repo_path}/pulls?access_token={self.token}&head={head_branch}&base={base_branch}&state={state}" 30 | # api = f"{self.api_root}/repos/{self.repo_path}/pulls?access_token={self.token}&head={src_branch}&base={target_branch}&state=open" 31 | res = self.http.get(api, res_is_standard=False, res_is_json=True) 32 | if len(res) > 0: 33 | return res[0] 34 | return None 35 | 36 | def get_pull_request_detail(self, pull_id): 37 | api = f"{self.api_root}/repos/{self.repo_path}/pulls/{pull_id}?access_token={self.token}" 38 | res = self.http.get(api, res_is_standard=False, res_is_json=True) 39 | return res 40 | 41 | def post_merge(self, pull_id, detail): 42 | api = f"{self.api_root}/repos/{self.repo_path}/pulls/{pull_id}/merge?access_token={self.token}" 43 | self.http.post(api, data={ 44 | "Do": 'merge' 45 | }, res_is_standard=False, res_is_json=False) 46 | -------------------------------------------------------------------------------- /lib/api/gitee.py: -------------------------------------------------------------------------------- 1 | import time 2 | 3 | from lib.api.abstract_client import AbstractClient 4 | from lib.logger import logger 5 | from lib.util import get_dict_value, re_pick 6 | 7 | 8 | class GiteeClient(AbstractClient): 9 | '''gitee api''' 10 | ''' 11 | https://gitee.com/api/v5/swagger#/postV5ReposOwnerRepoPulls 12 | ''' 13 | 14 | def __init__(self, http, token, url): 15 | super().__init__(http, token, url) 16 | self.api_root = "https://gitee.com/api/v5" 17 | 18 | def post_pull_request(self, title, body, head_branch, base_branch): 19 | api = f"{self.api_root}/repos/{self.repo_path}/pulls" 20 | res = self.http.post(api, data={ 21 | "access_token": self.token, 22 | "title": title, 23 | "body": body, 24 | "head": f"{self.owner}:{head_branch}", 25 | "base": base_branch 26 | }, res_is_standard=False, res_is_json=True) 27 | return res 28 | 29 | def query_pull_request(self, head_branch, base_branch, state='open'): 30 | api = f"{self.api_root}/repos/{self.repo_path}/pulls?access_token={self.token}&head={self.owner}:{head_branch}&base={base_branch}&state={state}" 31 | res = self.http.get(api, res_is_standard=False, res_is_json=True) 32 | if len(res) > 0: 33 | return res[0] 34 | return None 35 | 36 | def get_pull_request_detail(self, pull_id): 37 | api = f"{self.api_root}/repos/{self.repo_path}/pulls/{pull_id}?access_token={self.token}" 38 | res = self.http.get(api, res_is_standard=False, res_is_json=True) 39 | return res 40 | 41 | def post_merge(self, pull_id, detail): 42 | if detail['assignees_number'] > 0: 43 | self.post_review(pull_id) 44 | if detail['testers_number'] > 0: 45 | self.post_test(pull_id) 46 | 47 | api = f"{self.api_root}/repos/{self.repo_path}/pulls/{pull_id}/merge" 48 | self.http.put(api, data={ 49 | "access_token": self.token 50 | }, res_is_standard=False, res_is_json=True) 51 | 52 | def post_review(self, pull_id): 53 | logger.info("force approve review") 54 | api = f"{self.api_root}/repos/{self.repo_path}/pulls/{pull_id}/review" 55 | self.http.post(api, data={ 56 | "access_token": self.token, 57 | "force": True 58 | }, res_is_standard=False, res_is_json=False) 59 | 60 | def post_test(self, pull_id): 61 | logger.info("force approve test") 62 | api = f"{self.api_root}/repos/{self.repo_path}/pulls/{pull_id}/test" 63 | self.http.post(api, data={ 64 | "access_token": self.token, 65 | "force": True 66 | }, res_is_standard=False, res_is_json=False) 67 | -------------------------------------------------------------------------------- /lib/api/github.py: -------------------------------------------------------------------------------- 1 | from lib.api.abstract_client import AbstractClient 2 | 3 | 4 | class GithubClient(AbstractClient): 5 | ''' 6 | https://docs.github.com/en/rest/pulls/pulls?apiVersion=2022-11-28 7 | 8 | curl \ 9 | -X POST \ 10 | -H "Accept: application/vnd.github+json" \ 11 | -H "Authorization: Bearer "\ 12 | -H "X-GitHub-Api-Version: 2022-11-28" \ 13 | https://api.github.com/repos/OWNER/REPO/pulls \ 14 | -d '{"title":"Amazing new feature","body":"Please pull these awesome changes in!","head":"octocat:new-feature","base":"master"}' 15 | ''' 16 | 17 | '''github api''' 18 | # 下面定义了一个类属性 19 | token = 'token' 20 | http = None 21 | url = None 22 | repo_path = None 23 | owner = None 24 | repo = None 25 | headers = None; 26 | 27 | def __init__(self, http, token, url): 28 | super().__init__(http, token, url) 29 | 30 | self.headers = { 31 | "Accept": "application/vnd.github+json", 32 | "Authorization": f"Bearer {self.token}", 33 | "X-GitHub-Api-Version": "2022-11-28" 34 | } 35 | 36 | def post_pull_request(self, title, body, head_branch, base_branch): 37 | """ 38 | head_branch 来源分支 39 | base_branch 被合并分支 40 | """ 41 | api = f"https://api.github.com/repos/{self.repo_path}/pulls" 42 | res = self.http.post(api, data={ 43 | "title": title, 44 | "body": body, 45 | "head": f"{self.owner}:{head_branch}", 46 | "base": base_branch, 47 | "maintainer_can_modify": True 48 | }, headers=self.headers, res_is_standard=False, res_is_json=True) 49 | return res 50 | 51 | def query_pull_request(self, head_branch, base_branch, state='open'): 52 | api = f"https://api.github.com/repos/{self.repo_path}/pulls?head={self.owner}:{head_branch}&base={base_branch}&state={state}" 53 | res = self.http.get(api, headers=self.headers, res_is_standard=False, res_is_json=True) 54 | if len(res) > 0: 55 | return res[0] 56 | return None 57 | 58 | def get_pull_request_detail(self, pull_id): 59 | api = f"https://api.github.com/repos/{self.repo_path}/pulls/{pull_id}" 60 | res = self.http.get(api, headers=self.headers, res_is_standard=False, res_is_json=True) 61 | return res 62 | 63 | def post_merge(self, pull_id, detail): 64 | api = f"https://api.github.com/repos/{self.repo_path}/pulls/{pull_id}/merge" 65 | self.http.put(api, data={}, headers=self.headers, res_is_standard=False, res_is_json=True) 66 | -------------------------------------------------------------------------------- /lib/api/index.py: -------------------------------------------------------------------------------- 1 | from lib.api.gitea import GiteaClient 2 | from lib.api.gitee import GiteeClient 3 | from lib.api.github import GithubClient 4 | 5 | api_clients = { 6 | "github": GithubClient, 7 | "gitee": GiteeClient, 8 | "gitea": GiteaClient 9 | } 10 | -------------------------------------------------------------------------------- /lib/handler/helper.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | import os 3 | import time 4 | 5 | from git import Repo 6 | 7 | from lib.logger import logger 8 | from lib.model.config import RunStatus 9 | from lib.util import shell, check_need_push 10 | from lib.util_git import get_git_modify_file_count 11 | 12 | 13 | def save_work_repo(repo: Repo, message, push=True, status: RunStatus = None): 14 | shell("git add .") 15 | if status is None: 16 | status = RunStatus() 17 | count = get_git_modify_file_count() 18 | 19 | if 'origin' in repo.remotes: 20 | try: 21 | shell("git pull") 22 | except Exception as e: 23 | logger.warning('git pull failed') 24 | 25 | if count <= 0: 26 | logger.info("No modification, no need to submit") 27 | else: 28 | status.change = True 29 | time.sleep(1) 30 | shell(f'git commit -m "{message}"') 31 | status.commit = True 32 | 33 | if push: 34 | need_push = check_need_push(repo, repo.active_branch) 35 | if need_push is None: 36 | logger.warning( 37 | "Skip push，The remote address is not set for the current repository") 38 | logger.warning( 39 | "Use the [trident remote --url=] command to set the remote address of the repository and save the synchronization progress") 40 | 41 | elif need_push is True: 42 | shell(f"git push") 43 | status.push = True 44 | else: 45 | logger.info("No need to push") 46 | -------------------------------------------------------------------------------- /lib/handler/init.py: -------------------------------------------------------------------------------- 1 | import os 2 | import time 3 | 4 | import git 5 | 6 | from lib.handler.helper import save_work_repo 7 | from lib.logger import logger 8 | from lib.model.config import Config 9 | from lib.model.repo import RepoConf 10 | from lib.util import shell, save_file 11 | 12 | 13 | class InitHandler: 14 | 15 | def __init__(self, work_root, config): 16 | self.work_root = work_root 17 | self.config = config 18 | 19 | def handle(self): 20 | """ 21 | 处理 init 命令 22 | """ 23 | work_root = self.work_root 24 | config: Config = self.config 25 | logger.info(f"git init : {work_root}") 26 | os.chdir(work_root) 27 | shell('git init') 28 | repo = git.Repo(path=work_root) 29 | if len(repo.heads) == 0: 30 | self.save_ignore_file() 31 | shell("git add .") 32 | time.sleep(1) 33 | shell('git commit -m "🔱: sync init start [trident-sync]"') 34 | logger.info("get submodules") 35 | sms = repo.iter_submodules() 36 | conf_repos = config.repo 37 | conf_options = config.options 38 | conf_repo_root = conf_options.repo_root 39 | for key in conf_repos: 40 | added = False 41 | for module in sms: 42 | if key == module.name: 43 | logger.info(f"{key} has been added to the submodule") 44 | added = True 45 | break 46 | if added: 47 | continue 48 | item: RepoConf = conf_repos[key] 49 | logger.info(f"add submodule:{item.url}") 50 | path = f"{conf_repo_root}/{item.path}" 51 | # repo.create_submodule(key, path, url=item['url'], branch=item['branch']) 52 | shell(f"git submodule add -b {item.branch} --name {key} {item.url} {path}") 53 | 54 | logger.info("Update all submodule") 55 | 56 | shell(f"git submodule update --init --recursive --progress") 57 | repo.iter_submodules() 58 | save_work_repo(repo, '🔱: sync init [trident-sync]', push=config.options.push) 59 | os.chdir(os.getcwd()) 60 | logger.info("init success") 61 | repo.close() 62 | 63 | def save_ignore_file(self): 64 | ignore_file = f"{self.work_root}/.gitignore" 65 | ignore = ''' 66 | .idea 67 | .vscode 68 | .git 69 | __pycache__ 70 | ''' 71 | save_file(ignore_file, ignore) 72 | -------------------------------------------------------------------------------- /lib/handler/remote.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | import git 4 | 5 | from lib.logger import logger 6 | from lib.util import shell 7 | 8 | 9 | class RemoteHandler: 10 | 11 | def __init__(self, work_root, remote_url=None, force=False): 12 | self.work_root = work_root 13 | self.remote_url = remote_url 14 | self.force = force 15 | 16 | def handle(self): 17 | os.chdir(self.work_root) 18 | repo = git.Repo(path=self.work_root) 19 | cur_branch_name = repo.head.reference 20 | url = self.remote_url 21 | if 'origin' in repo.remotes: 22 | logger.info("The remote origin already exists and no url parameter is needed") 23 | shell("git pull") 24 | else: 25 | if not url: 26 | logger.info( 27 | "Please use the [trident remote --url=] command to set the remote address first") 28 | return 29 | else: 30 | shell(f"git remote add origin {url}") 31 | # origin = repo.create_remote("origin", url) 32 | logger.info('add remote origin success:' + url) 33 | force = "" 34 | if self.force: 35 | force = " -f " 36 | shell(f"git push -u {force} origin {cur_branch_name}") 37 | logger.info('push success') 38 | -------------------------------------------------------------------------------- /lib/handler/sync.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | import json 3 | import os 4 | import shutil 5 | import time 6 | 7 | import git 8 | 9 | from lib.api.index import api_clients 10 | from lib.handler.helper import save_work_repo 11 | from lib.http import Http 12 | from lib.logger import logger 13 | from lib.model.config import Config 14 | from lib.model.sync import SyncTask 15 | from lib.util import shell, get_dict_value, check_need_push, set_dict_value, is_blank_dir 16 | from lib.util_git import force_checkout_main_branch, checkout_branch, collection_commit_message, \ 17 | get_git_modify_file_count 18 | 19 | 20 | def read_status(root): 21 | file_path = f'{root}/status.json' 22 | if not os.path.exists(file_path): 23 | return {} 24 | fo = open(file_path, "r") 25 | config_str = fo.read() 26 | fo.close() 27 | if config_str is None: 28 | return {} 29 | try: 30 | return json.loads(config_str) 31 | except Exception as e: 32 | print(e) 33 | return {} 34 | 35 | 36 | def save_status(root, status): 37 | # 创建配置文件 38 | file_path = f'{root}/status.json' 39 | # 写入配置文件 40 | config_str = json.dumps(status) 41 | fo = open(file_path, "w") 42 | fo.write(config_str) 43 | fo.close() 44 | return status 45 | 46 | 47 | def text_center(text: str, length=40): 48 | return text.center(length, '-') 49 | 50 | 51 | def sync_func(task: SyncTask, src_dir, target_dir): 52 | if task.copy_script is None or task.copy_script.strip() == '': 53 | shutil.copytree(src_dir, target_dir) 54 | else: 55 | if len(task.copy_script.splitlines()) > 1: 56 | # 多行 则表示是脚本，直接执行 57 | exec(task.copy_script) 58 | else: 59 | # 单行，表示是文件，加载文件模块，并执行copy方法 60 | filepath = os.path.abspath(task.copy_script) 61 | do_copy_from_file_module(filepath, task, src_dir, target_dir) 62 | 63 | 64 | def do_copy_from_file_module(filepath, task: SyncTask, src_dir, target_dir): 65 | # 加载文件模块，并执行copy方法 66 | import importlib.util 67 | import sys 68 | # For illustrative purposes. 69 | import tokenize 70 | file_path = tokenize.__file__ 71 | module_name = tokenize.__name__ 72 | 73 | spec = importlib.util.spec_from_file_location('copy_script', filepath) 74 | module = importlib.util.module_from_spec(spec) 75 | sys.modules[module_name] = module 76 | spec.loader.exec_module(module) 77 | module.copy(task, src_dir, target_dir) 78 | 79 | 80 | class SyncHandler: 81 | 82 | def __init__(self, work_root, config): 83 | self.work_root = work_root 84 | self.config: Config = config 85 | self.status = read_status(work_root) 86 | self.conf_repo = config.repo 87 | self.conf_options = config.options 88 | self.conf_repo_root = self.conf_options.repo_root 89 | 90 | proxy_fix = self.conf_options.proxy_fix 91 | use_system_proxy = self.conf_options.use_system_proxy 92 | self.http = Http(use_system_proxy=use_system_proxy, proxy_fix=proxy_fix) 93 | 94 | self.repo: git.Repo = git.Repo.init(path=work_root) 95 | 96 | def handle(self): 97 | """ 98 | 处理 sync 命令 99 | """ 100 | logger.info(text_center("sync start")) 101 | config = self.config 102 | os.chdir(self.work_root) 103 | # save_work_repo(self.repo, "save work repo before sync", self.config.options.push) 104 | is_init = False 105 | ref_count = sum(1 for ref in self.repo.refs) 106 | if ref_count > 0: 107 | # 初始化一下子项目 108 | shell(f"git submodule update --init --recursive --progress") 109 | self.repo.iter_submodules() 110 | sms = self.repo.submodules 111 | if sms and len(sms) > 0: 112 | is_init = True 113 | 114 | if not is_init: 115 | logger.error("Not initialized yet, please execute the [trident init] command first") 116 | raise Exception("Not initialized yet, please execute the [trident init] command first") 117 | 118 | sync_task_map = config.sync 119 | sms = self.repo.submodules 120 | try: 121 | for key in sync_task_map: 122 | sync_task: SyncTask = sync_task_map[key] 123 | # 执行同步任务 124 | task_executor = TaskExecutor(self.work_root, self.config, self.status, sms, self.http, sync_task) 125 | task_executor.do_task() 126 | 127 | self.config.status.success = True 128 | 129 | # 所有任务已完成 130 | # 提交同步仓库的变更 131 | self.commit_work_repo() 132 | self.repo.close() 133 | finally: 134 | self.render_result(sync_task_map) 135 | logger.info(text_center("sync end")) 136 | 137 | def render_result(self, conf_sync_map): 138 | def right(target: str, res: bool, label_length=8): 139 | return f"{target.rjust(label_length, ' ')}:{'✅' if res else '🚫'}" 140 | 141 | def fill(string: str): 142 | return string.ljust(15, " ") 143 | 144 | cs = self.config.status 145 | result = text_center(right('result', cs.success, 0)) 146 | for key in conf_sync_map: 147 | t: SyncTask = conf_sync_map[key] 148 | s = t.status 149 | task_result = f"\n 🏹 {fill(t.key)} --> {right('success', s.success)} {right('copy', s.copy)} {right('change', s.change)} {right('commit', s.commit)} {right('push', s.push)} {right('pr', s.pr)} {right('merge', s.merge)}" 150 | result += task_result 151 | result += f"\n 🔱 {fill('sync_work_repo')} --> {right('change', cs.change)} {right('commit', cs.commit)} {right('push', cs.push)} " 152 | # 输出结果 153 | logger.info(result) 154 | 155 | def commit_work_repo(self): 156 | now = datetime.datetime.now() 157 | message = f"🔱: sync all task at {now} [trident-sync]" 158 | os.chdir(self.work_root) 159 | save_work_repo(self.repo, message, self.config.options.push, status=self.config.status) 160 | 161 | 162 | class TaskExecutor: 163 | def __init__(self, work_root, config: Config, status: dict, sms, http, sync_task: SyncTask): 164 | self.key = sync_task.key 165 | self.work_root = work_root 166 | self.sync_task = sync_task 167 | self.sms = sms 168 | self.task_src = sync_task.src 169 | self.task_target = sync_task.target 170 | 171 | self.conf_options = config.options 172 | 173 | self.status = status 174 | self.http = http 175 | 176 | self.conf_src_repo = self.task_src.repo_ref 177 | self.conf_target_repo = self.task_target.repo_ref 178 | self.repo_src = sms[self.task_src.repo].module() 179 | self.repo_target = sms[self.task_target.repo].module() 180 | 181 | def do_task(self): 182 | 183 | logger.info(text_center(f"【{self.key}】 task start")) 184 | time.sleep(0.2) 185 | 186 | # 同步任务开始 187 | # 更新源仓库代码 188 | self.pull_src_repo() 189 | # 当前目录切换到目标项目 190 | os.chdir(self.repo_target.working_dir) 191 | # 先强制切换回主分支 192 | force_checkout_main_branch(self.task_target.repo_ref) 193 | # 创建同步分支，并checkout 194 | is_first = checkout_branch(self.repo_target, self.task_target.branch) 195 | # 开始复制文件 196 | 197 | self.do_sync(is_first) 198 | 199 | # 提交代码 200 | self.do_commit() 201 | # push更新 202 | has_push = self.do_push() 203 | # 创建PR 204 | self.do_pull_request(has_push) 205 | 206 | logger.info(text_center(f"【{self.key}】 task complete")) 207 | self.sync_task.status.success = True 208 | self.repo_src.close() 209 | self.repo_target.close() 210 | 211 | def pull_src_repo(self): 212 | os.chdir(self.repo_src.working_dir) 213 | logger.info(f"update src repo :{self.task_src.repo_ref.url}") 214 | shell(f"git clean -f && git checkout . && git checkout {self.task_src.repo_ref.branch} -f && git pull") 215 | logger.info(f"update submodule of src repo") 216 | shell(f"git submodule update --init --recursive --progress ") 217 | logger.info(f"update src repo success") 218 | 219 | def do_sync(self, is_first): 220 | dir_src_sync = f"{self.repo_src.working_dir}/{self.task_src.dir}" 221 | dir_target_sync = f"{self.repo_target.working_dir}/{self.task_target.dir}" 222 | logger.info(f"sync dir：{dir_src_sync}->{dir_target_sync}") 223 | # 检查源仓库目录是否有文件，如果没有文件，可能初始化仓库不正常 224 | src_is_blank = is_blank_dir(dir_src_sync) 225 | if src_is_blank: 226 | raise Exception( 227 | f"The src repo dir <{dir_src_sync}> is empty. It may not be fully initialized. Try to enter this directory and execute the [git pull] command") 228 | 229 | if is_first: 230 | # 第一次同步，目标目录必须为空 231 | target_is_blank = is_blank_dir(dir_target_sync) 232 | if not target_is_blank: 233 | logger.warning( 234 | f"For the first time, the target repo dir <{dir_target_sync}> is not empty") 235 | logger.warning( 236 | f"Please make sure that the dir is a copy of a version of the src repo, otherwise please change the directory!!") 237 | logger.warning( 238 | f"If you are sure that the directory is a copy of the source repository, you can try configuring \ 239 | and reruning [trident sync] command ,This will \ 240 | reset the sync_branch to first commit to see if an earlier version had the \ 241 | directory.") 242 | if not self.task_target.allow_reset_to_root: 243 | raise Exception( 244 | f"the target repo dir <{dir_target_sync}> is not empty, and allow_reset_to_root is False") 245 | else: 246 | logger.info(f"The allow_reset_to_root is True, Trying to reset the sync_branch to root commit") 247 | root_hash = shell("git rev-list --max-parents=0 HEAD", get_out=True) 248 | shell(f"git reset {root_hash.strip()}") 249 | shell("git clean -df ") 250 | logger.info(f"Reset the sync_branch to root commit success") 251 | # 再次检测目录是否为空 252 | target_is_blank = is_blank_dir(dir_target_sync) 253 | if not target_is_blank: 254 | logger.warning( 255 | f"The target repository directory <{dir_target_sync}> is still not empty, Some changes maybe lost !!!") 256 | logger.info("after 5 seconds will be continue") 257 | time.sleep(5) 258 | 259 | if self.task_target.remove_dir_before_copy and os.path.exists(dir_target_sync): 260 | logger.info(f"remove <{dir_target_sync}> ...") 261 | shutil.rmtree(dir_target_sync) 262 | time.sleep(0.2) 263 | logger.info(f"copy files") 264 | os.chdir(self.work_root) 265 | sync_func(self.sync_task, dir_src_sync, dir_target_sync) 266 | os.chdir(self.repo_target.working_dir) 267 | git_file = f"{dir_target_sync}/.git" 268 | if os.path.exists(git_file): 269 | os.unlink(git_file) 270 | logger.info(f"【{self.key}】 Copy completed, ready to submit : {self.task_target.dir}") 271 | time.sleep(1) 272 | self.sync_task.status.copy = True 273 | 274 | def do_commit(self): 275 | shell(f"git add .") 276 | time.sleep(1) 277 | count = get_git_modify_file_count() 278 | time.sleep(1) 279 | logger.info(f"modify count : {count}") 280 | key = self.key 281 | if count <= 0: 282 | logger.info(f"【{key}】 No change, no need to submit") 283 | return False 284 | else: 285 | self.sync_task.status.change = True 286 | last_commit = get_dict_value(self.status, f"sync.{key}.last_commit_src") 287 | messsges = collection_commit_message(self.repo_src, self.task_src.repo_ref.branch, last_commit) 288 | body = "" 289 | for msg in messsges: 290 | body += msg + "\n" 291 | now = datetime.datetime.now() 292 | message = f"🔱: [{key}] sync upgrade with {len(messsges)} commits [trident-sync] " 293 | # 提交更新 294 | shell(f'git commit -m "{message}" -m "{body}"') 295 | # repo_target.index.commit(f"sync {key} success [{now}]") 296 | logger.info(f"【{key}】 submit success") 297 | time.sleep(0.2) 298 | # 记录最后提交hash 299 | src_last_hash = self.repo_src.head.commit.hexsha 300 | target_last_hash = self.repo_target.head.commit.hexsha 301 | 302 | set_dict_value(self.status, f"sync.{key}.last_commit_src", src_last_hash) 303 | set_dict_value(self.status, f"sync.{key}.last_commit_target", target_last_hash) 304 | save_status(self.work_root, self.status) 305 | self.sync_task.status.commit = True 306 | return True 307 | 308 | def do_push(self): 309 | if not self.conf_options.push: 310 | return False 311 | logger.info("Check if push is needed") 312 | # 检测是否需要push 313 | key = self.key 314 | need_push = check_need_push(self.repo_target, self.task_target.branch) 315 | if need_push is False: 316 | logger.info("No commit to push") 317 | return False 318 | else: 319 | logger.info("need push") 320 | logger.info(f"【{key}】 pushing") 321 | shell(f'git push --set-upstream origin {self.task_target.branch}') 322 | logger.info(f"【{key}】 push success") 323 | time.sleep(0.2) 324 | self.sync_task.status.push = True 325 | return True 326 | 327 | def do_pull_request(self, has_push): 328 | key = self.key 329 | if not self.conf_options.pull_request: 330 | return False 331 | if not has_push: 332 | return False 333 | token = self.task_target.repo_ref.token 334 | repo_type = self.task_target.repo_ref.type 335 | auto_merge = self.conf_target_repo.auto_merge 336 | if not repo_type: 337 | logger.warning(f"[repo:{self.task_target.repo}] type is not configured, Unable to create PR") 338 | return False 339 | if not token: 340 | logger.warning(f"[repo:{self.task_target.repo}] token is not configured, Unable to create PR") 341 | return False 342 | else: 343 | client = api_clients[repo_type](self.http, token, self.task_target.repo_ref.url) 344 | title = f"[{key}] sync upgrade [trident-sync]" 345 | body = f"{self.task_src.repo}:{self.conf_src_repo.branch}:{self.task_src.dir} -> {self.task_target.repo}:\ 346 | {self.conf_target_repo.branch}:{self.task_target.dir} " 347 | logger.info( 348 | f"Ready to create PR, {self.task_target.branch} -> {self.conf_target_repo.branch} , url:{self.conf_target_repo.url}") 349 | try: 350 | pull_id, merged = client.create_pull_request(title, body, self.task_target.branch, 351 | self.conf_target_repo.branch, 352 | auto_merge=auto_merge) 353 | self.sync_task.status.pr = True 354 | if merged: 355 | self.sync_task.status.merge = True 356 | except Exception as e: 357 | # logger.opt(exception=e).error("提交PR出错") 358 | logger.error(f"Error creating PR：{e}") 359 | time.sleep(0.2) 360 | return True 361 | -------------------------------------------------------------------------------- /lib/http.py: -------------------------------------------------------------------------------- 1 | import json 2 | 3 | import requests 4 | import urllib 5 | 6 | from lib.logger import logger 7 | 8 | 9 | class HttpException(Exception): 10 | # 自定义异常类型的初始化 11 | def __init__(self, msg, code, data): 12 | self.msg = msg 13 | self.code = code 14 | self.data = data 15 | 16 | # 返回异常类对象的说明信息 17 | def __str__(self): 18 | return self.msg 19 | 20 | 21 | def standard_res_handle(res): 22 | if res['code'] == 0: 23 | return res['data'] 24 | raise Exception(res['msg'], res['code']) 25 | 26 | 27 | class Http: 28 | def __init__(self, use_system_proxy=True, proxy_fix=True): 29 | self.proxies = {'http': None, 'https': None} 30 | if use_system_proxy: 31 | proxies = urllib.request.getproxies() 32 | if proxy_fix and "https" in proxies: 33 | https_proxy = proxies['https'] 34 | if https_proxy.startswith('https://'): 35 | proxies['https'] = https_proxy.replace("https://", "http://") 36 | self.proxies = proxies 37 | self.verify = True 38 | self.logger = logger 39 | 40 | def set_proxies(self, proxies): 41 | self.proxies = proxies 42 | 43 | def options(self, url, cookies=None, headers=None, **kwargs): 44 | self.logger.debug(f"http request[options] url:{url}") 45 | if headers is None: 46 | headers = {} 47 | if 'Content-Type' not in headers: 48 | headers['Content-Type'] = 'application/json' 49 | session = requests.Session() 50 | session.trust_env = False 51 | response = session.options(url, proxies=self.proxies, headers=headers, cookies=cookies, **kwargs) 52 | self.logger.debug("http response: " + response.text) 53 | return 54 | 55 | def post(self, url, data, res_is_json=True, res_is_standard=True, cookies=None, headers=None, **kwargs): 56 | self.logger.debug(f"http request[post] url:{url}") 57 | if headers is None: 58 | headers = {} 59 | if 'Content-Type' not in headers: 60 | headers['Content-Type'] = 'application/json' 61 | session = requests.Session() 62 | session.headers.clear() 63 | session.headers.update(headers) 64 | session.trust_env = False 65 | response = session.post(url, json=data, proxies=self.proxies, headers=headers, cookies=cookies, 66 | verify=self.verify, **kwargs) 67 | self.logger.debug("http response: " + response.text) 68 | return self.res_handle(response, res_is_json, res_is_standard) 69 | 70 | def put(self, url, data, res_is_json=True, res_is_standard=True, cookies=None, headers=None, **kwargs): 71 | self.logger.debug(f"http request[put] url:{url}") 72 | if headers is None: 73 | headers = {} 74 | if 'Content-Type' not in headers: 75 | headers['Content-Type'] = 'application/json' 76 | session = requests.Session() 77 | session.headers.clear() 78 | session.headers.update(headers) 79 | session.trust_env = False 80 | response = session.put(url, json=data, proxies=self.proxies, headers=headers, cookies=cookies, 81 | verify=self.verify, **kwargs) 82 | self.logger.debug("http response: " + response.text) 83 | return self.res_handle(response, res_is_json, res_is_standard) 84 | 85 | def get(self, url, headers=None, res_is_json=True, res_is_standard=True): 86 | self.logger.debug(f"http request[get] url:{url}") 87 | if headers is None: 88 | headers = {} 89 | if 'Content-Type' not in headers: 90 | headers['Content-Type'] = 'application/json' 91 | session = requests.Session() 92 | session.trust_env = False 93 | response = session.get(url, proxies=self.proxies, verify=self.verify, headers=headers) 94 | self.logger.debug("http response: " + response.text) 95 | return self.res_handle(response, res_is_json, res_is_standard) 96 | 97 | def download(self, url, save_path, on_progress=None): 98 | session = requests.Session() 99 | session.trust_env = False 100 | down_res = session.get(url=url, proxies=self.proxies, stream=True) 101 | chunk_size = 1024 * 1024 102 | downloaded = 0 103 | content_size = int(down_res.headers['content-length']) # 内容体总大小 104 | with open(save_path, "wb") as file: 105 | for data in down_res.iter_content(chunk_size=chunk_size): 106 | file.write(data) 107 | downloaded += len(data) 108 | if on_progress is not None: 109 | on_progress({ 110 | 'total': content_size, 111 | 'downloaded': downloaded, 112 | 'status': 'downloading' 113 | }) 114 | if on_progress: 115 | on_progress({ 116 | 'total': content_size, 117 | 'downloaded': downloaded, 118 | 'status': 'finished' 119 | }) 120 | 121 | def res_handle(self, response, res_is_json, res_is_standard): 122 | 123 | if response.status_code < 200 or response.status_code > 299: 124 | return self.error_handle(response) 125 | 126 | if res_is_json: 127 | res = response.json() 128 | if res_is_standard: 129 | return standard_res_handle(res) 130 | return res 131 | else: 132 | return response.text 133 | 134 | def error_handle(self, response): 135 | data = {} 136 | try: 137 | data = json.loads(response.text) 138 | except Exception as e: 139 | pass 140 | raise HttpException(f'请求错误：({response.status_code}):{response.text}', 1, data) 141 | -------------------------------------------------------------------------------- /lib/logger.py: -------------------------------------------------------------------------------- 1 | from sys import stdout 2 | 3 | import loguru 4 | 5 | logger = loguru.logger 6 | logger.remove(0) 7 | logger.add(stdout, format="{time:YYYY-MM-DD HH:mm:ss} | {level:7} | - {message}", colorize=True) 8 | -------------------------------------------------------------------------------- /lib/model/config.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | from lib.model.opts import Options 4 | from lib.model.repo import RepoConf 5 | from lib.model.sync import SyncTask 6 | from lib.util import get_dict_value 7 | 8 | 9 | class RunStatus: 10 | success: bool = False 11 | change: bool = False 12 | commit: bool = False 13 | push: bool = False 14 | 15 | 16 | class Config: 17 | repo = {} 18 | sync = {} 19 | options: Options 20 | 21 | status: RunStatus 22 | 23 | def __init__(self, conf_dict): 24 | conf_repo = get_dict_value(conf_dict, 'repo') 25 | for key in conf_repo: 26 | self.repo[key] = RepoConf(key, conf_repo[key]) 27 | conf_sync = get_dict_value(conf_dict, 'sync') 28 | for key in conf_sync: 29 | self.sync[key] = SyncTask(key, conf_sync[key], self.repo) 30 | conf_options = get_dict_value(conf_dict, 'options') 31 | self.status = RunStatus() 32 | self.options = Options(conf_options) 33 | 34 | def set_default_token(self, token=None): 35 | for key in self.repo: 36 | repo: RepoConf = self.repo[key] 37 | if repo.token: 38 | continue 39 | if not token: 40 | token = os.getenv(f'{repo.type}_token') 41 | if not token: 42 | token = os.getenv(f'trident_token_{repo.type}') 43 | if token: 44 | repo.token = token 45 | -------------------------------------------------------------------------------- /lib/model/opts.py: -------------------------------------------------------------------------------- 1 | ''' 2 | options: #选项 3 | repo_root: repo # submodule保存根目录 4 | push: true # 同步后是否push 5 | pr: true # 是否创建pr，需要目标仓库配置token和type 6 | proxy_fix: true # 是否将https代理改成http://开头，解决开着代理情况下，无法发起https请求的问题 7 | use_system_proxy: false # 是否使用系统代理 8 | ''' 9 | from lib.util import merge_from_dict 10 | 11 | 12 | class Options: 13 | repo_root: str = 'repo' 14 | push: bool = True 15 | pull_request: bool = True 16 | proxy_fix: bool = True 17 | use_system_proxy: bool = True 18 | 19 | def __init__(self, conf_dict=None): 20 | if conf_dict is None: 21 | conf_dict = {} 22 | merge_from_dict(self, conf_dict) 23 | -------------------------------------------------------------------------------- /lib/model/repo.py: -------------------------------------------------------------------------------- 1 | ''' 2 | repo: 3 | test: # 你的项目（接受同步项目），可以任意命名 4 | url: "https://github.com/handsfree-work/trident-test" # 仓库地址 5 | path: "target" # submodule子路径 6 | branch: "main" # 你的代码开发主分支（接受合并的分支）例如dev、main等 7 | token: "" # 仓库token，用于提交PR, 前往 https://github.com/settings/tokens 创建token 8 | type: github # 仓库类型，用于提交PR，可选项：[github/gitee/gitea/gitlab] 9 | ''' 10 | from lib.util import merge_from_dict 11 | 12 | 13 | class RepoConf: 14 | key: str = None 15 | 16 | # submodule相关 17 | url: str = None 18 | branch: str = None 19 | path: str = None 20 | 21 | # pr 相关 22 | token: str = None 23 | type: str = None 24 | auto_merge: bool = True 25 | 26 | def __init__(self, key, conf_dict: dict): 27 | self.key = key 28 | merge_from_dict(self, conf_dict) 29 | if not self.url or not self.branch or not self.path: 30 | raise Exception(f"repo<{key}> 中 url/branch/path 必须配置") 31 | 32 | if self.url.startswith("https://github.com"): 33 | self.type = 'github' 34 | elif self.url.startswith("https://gitee.com"): 35 | self.type = 'gitee' 36 | -------------------------------------------------------------------------------- /lib/model/sync.py: -------------------------------------------------------------------------------- 1 | ''' 2 | sync: 3 | task1: 4 | src: # 源仓库 5 | repo: fs-admin # 源仓库名称，上面repo配置的仓库名称引用 6 | dir: '.' #要同步给target的目录 7 | target: #目标仓库 8 | repo: test # 目标仓库名称，上面repo配置的仓库名称引用 9 | dir: 'package/ui/certd-client' # 接收src同步过来的目录 10 | branch: 'client_sync' # 同步分支名称（需要配置一个未被占用的分支名称） 11 | ''' 12 | from lib.model.repo import RepoConf 13 | from lib.util import merge_from_dict 14 | 15 | 16 | class SyncTaskSrc: 17 | repo: str = None 18 | dir: str = None 19 | repo_ref: RepoConf 20 | 21 | def __init__(self, conf_dict): 22 | merge_from_dict(self, conf_dict) 23 | if not self.repo or not self.dir: 24 | raise Exception("sync.[key].src 中 < repo/dir > 必须配置") 25 | 26 | 27 | class SyncTaskTarget: 28 | repo: str = None 29 | dir: str = None 30 | branch: str = None 31 | allow_reset_to_root: bool = True 32 | remove_dir_before_copy: bool = True 33 | repo_ref: RepoConf 34 | 35 | def __init__(self, conf_dict): 36 | merge_from_dict(self, conf_dict) 37 | if not self.repo or not self.dir or not self.branch: 38 | raise Exception("sync.[key].target 中 < repo/dir/branch > 必须配置") 39 | 40 | 41 | class SyncStatus: 42 | success: bool = False 43 | change: bool = False 44 | copy: bool = False 45 | commit: bool = False 46 | push: bool = False 47 | pr: bool = False 48 | merge: bool = False 49 | 50 | 51 | class SyncTask: 52 | key: str = None 53 | src: SyncTaskSrc = None 54 | target: SyncTaskTarget = None 55 | copy_script: str = None 56 | 57 | status: SyncStatus 58 | 59 | def __init__(self, key, conf_sync: dict, repo_map): 60 | self.key = key 61 | 62 | if 'src' not in conf_sync: 63 | raise Exception(f"sync.{key}.src 必须配置") 64 | if 'target' not in conf_sync: 65 | raise Exception(f"sync.{key}.target 必须配置") 66 | 67 | merge_from_dict(self, conf_sync) 68 | self.status = SyncStatus() 69 | 70 | self.src = SyncTaskSrc(conf_sync['src']) 71 | self.target = SyncTaskTarget(conf_sync['target']) 72 | 73 | self.set_repo_ref(self.src, repo_map) 74 | self.set_repo_ref(self.target, repo_map) 75 | 76 | def set_repo_ref(self, task, repo_list): 77 | if task.repo in repo_list: 78 | task.repo_ref = repo_list[task.repo] 79 | else: 80 | raise Exception(f"任务[{self.key}]的{self.src.repo} 仓库配置不存在，请检查repo配置") 81 | -------------------------------------------------------------------------------- /lib/util.py: -------------------------------------------------------------------------------- 1 | import os 2 | import shutil 3 | import stat 4 | import subprocess as sp 5 | import re 6 | from lib.logger import logger 7 | 8 | 9 | def re_pick(re_str, input_str, flags=0): 10 | # inputStr = "['{0: 203, 11: 1627438682 [2021-07-28 10:18:02], 12:36 [通过蓝牙更改错误密码锁定计数], 13: 770449129, 19: 3, 20: 0, 100: 2141634486}']" 11 | 12 | reg = re.compile(re_str, flags) # 增加匹配效率的 S 多行匹配 13 | lists = re.findall(reg, str(input_str)) 14 | if len(lists) > 0: 15 | return lists[0] 16 | return [] 17 | 18 | 19 | def get_dict_value(dict_target, key, def_value=None): 20 | arr = str.split(key, '.') 21 | value = None 22 | parent = dict_target 23 | for key in arr: 24 | if parent and key and key in parent: 25 | value = parent[key] 26 | parent = value 27 | else: 28 | value = None 29 | if value is None: 30 | value = def_value 31 | return value 32 | 33 | 34 | def set_dict_value(dict_target: object, key: str, value: object) -> object: 35 | arr = str.split(key, '.') 36 | parent = dict_target 37 | last_key = arr[len(arr) - 1] 38 | for key in arr: 39 | if last_key == key: 40 | parent[key] = value 41 | return 42 | if key not in parent: 43 | parent[key] = {} 44 | parent = parent[key] 45 | 46 | 47 | def shell(cmd, ignore_errors=False, get_out=False): 48 | logger.info(cmd) 49 | out = None 50 | if get_out: 51 | out = sp.PIPE 52 | 53 | p = sp.run(cmd, shell=True, encoding='utf-8', stdout=out) 54 | if p.returncode != 0 and not ignore_errors: 55 | logger.debug(p.stdout) 56 | logger.debug(p.stderr) 57 | raise Exception(p.stderr) 58 | if get_out and p.stdout: 59 | logger.debug(p.stdout) 60 | return p.stdout 61 | 62 | 63 | def read_file(file_path): 64 | if not os.path.exists(file_path): 65 | return None 66 | fo = open(file_path, "r", encoding='utf-8') 67 | content = fo.read() 68 | fo.close() 69 | return content 70 | 71 | 72 | def save_file(file_path, content): 73 | dir_path = os.path.dirname(file_path) 74 | if not os.path.exists(dir_path): 75 | os.makedirs(dir_path) 76 | fo = open(file_path, "w", encoding='utf-8') 77 | fo.write(content) 78 | fo.close() 79 | 80 | 81 | def check_need_push(repo, branch): 82 | """ 83 | 检查是否需要push，hash相等返回false，hash不相等返回true，没有远程分支返回None 84 | :param repo: 85 | :param branch: 86 | :return: 87 | """ 88 | local_hash = repo.head.commit.hexsha 89 | remote_hash = None 90 | refs = repo.refs 91 | logger.info(f"refs:{refs}") 92 | origin_key = f"origin/{branch}" 93 | if origin_key in refs: 94 | remote_hash = refs[origin_key].commit.hexsha 95 | else: 96 | return None 97 | 98 | logger.info(f"local_hash:{local_hash} -> remote_hash:{remote_hash} ") 99 | if local_hash == remote_hash: 100 | return False 101 | return True 102 | 103 | 104 | def merge_from_dict(obj, dic): 105 | for key in dic: 106 | setattr(obj, key, dic[key]) 107 | 108 | 109 | def rm_dir(root): 110 | def readonly_handler(func, path, exe_info): 111 | os.chmod(path, stat.S_IWRITE) 112 | func(path) 113 | 114 | shutil.rmtree(root, onerror=readonly_handler) 115 | 116 | 117 | def get_arg(args, key): 118 | value = args[key] 119 | if isinstance(value, list): 120 | if len(value) > 0: 121 | value = value[0] 122 | else: 123 | return None 124 | return value 125 | 126 | 127 | def is_blank_dir(root): 128 | """ 129 | 检查目录是否为空，如果目录不存在也返回True 130 | """ 131 | if not os.path.exists(root): 132 | return True 133 | is_blank = True 134 | for file in os.listdir(root): 135 | if file == '.git': 136 | continue 137 | is_blank = False 138 | break 139 | return is_blank 140 | -------------------------------------------------------------------------------- /lib/util_git.py: -------------------------------------------------------------------------------- 1 | import os 2 | import time 3 | 4 | from git import Repo 5 | 6 | from lib.logger import logger 7 | from lib.model.config import RunStatus 8 | from lib.model.repo import RepoConf 9 | from lib.util import shell, check_need_push 10 | 11 | 12 | def add_and_commit(message): 13 | shell("git add .") 14 | count = get_git_modify_file_count() 15 | if count > 0: 16 | time.sleep(1) 17 | shell(f'git commit -m "{message}"') 18 | 19 | 20 | def get_git_modify_file_count(): 21 | ret = shell(f"git status", get_out=True) 22 | lines = ret.split("\n") 23 | file_list = [] 24 | # 忽略的package列表 25 | 26 | count = 0 27 | for line in lines: 28 | start = line.find(': ') 29 | if start < 0: 30 | start = line.find('： ') 31 | # TODO 兼容一下mac版 git中文国际化之后变成了这个冒号 32 | if start < 0: 33 | continue 34 | start += 1 35 | file = line[start:].strip() 36 | count += 1 37 | return count 38 | 39 | 40 | def force_checkout_main_branch(conf_repo_ref: RepoConf): 41 | # 切换回主分支 42 | shell(f"git clean -f && git checkout . && git checkout -f {conf_repo_ref.branch}") 43 | time.sleep(1) 44 | 45 | 46 | def checkout_branch(repo: Repo, branch: str): 47 | ''' 48 | 创建或更新分支，如果远程有，则从远程拉取 49 | ''' 50 | # 看看远程是否有对应分支 51 | logger.info(f"checkout branch：{branch}") 52 | 53 | origin_key = f"origin/{branch}" 54 | origin_exists = False 55 | local_exists = False 56 | if origin_key in repo.refs: 57 | origin_exists = True 58 | 59 | if branch in repo.heads: 60 | local_exists = True 61 | 62 | is_first = False 63 | if origin_exists and not local_exists: 64 | # 远程有，本地没有，从远程拉取 65 | shell(f"git branch {branch} --track origin/{branch}") 66 | elif not origin_exists and not local_exists: 67 | # 两边都没有，本地创建 68 | shell(f"git branch {branch}") 69 | is_first = True 70 | elif origin_exists and local_exists: 71 | # 两边都有 72 | shell(f"git checkout {branch}") 73 | shell(f"git pull") 74 | time.sleep(1) 75 | shell(f"git checkout {branch}") 76 | time.sleep(1) 77 | return is_first 78 | 79 | 80 | def collection_commit_message(repo, branch, last_commit=None, max_count=20): 81 | # 准备commit文本 82 | commits = repo.iter_commits(branch, max_count=max_count) 83 | messages = [] 84 | more = "..." 85 | for item in commits: 86 | if item.hexsha == last_commit: 87 | more = "" 88 | break 89 | messages.append(item.message.strip()) 90 | messages.append(more) 91 | return messages 92 | -------------------------------------------------------------------------------- /lib/version.py: -------------------------------------------------------------------------------- 1 | from pathlib import Path 2 | 3 | from lib.util import read_file 4 | 5 | 6 | def project_root() -> Path: 7 | """Find the project root directory by locating pyproject.toml.""" 8 | current_file = Path(__file__) 9 | for parent_directory in current_file.parents: 10 | if (parent_directory / "pyproject.toml").is_file(): 11 | return parent_directory 12 | raise FileNotFoundError("Could not find project root containing pyproject.toml") 13 | 14 | 15 | def get_version(): 16 | try: 17 | # Probably this is the pyproject.toml of a development install 18 | path_to_pyproject_toml = project_root() / "pyproject.toml" 19 | except FileNotFoundError: 20 | # Probably not a development install 21 | path_to_pyproject_toml = None 22 | 23 | if path_to_pyproject_toml is not None: 24 | content = read_file(path_to_pyproject_toml) 25 | lines = content.split('\n') 26 | for line in lines: 27 | line = line.strip() 28 | if line.startswith("version = "): 29 | version = line.replace("version = ", "").replace("\"", "") 30 | return version 31 | return "0.0.0" 32 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [tool.poetry] 2 | name = "trident-sync" 3 | version = "1.2.3" 4 | description = "三叉戟，二次开发项目同步升级工具，Secondary development repo sync and upgrade CLI" 5 | authors = ["xiaojunnuo "] 6 | license = "MIT" 7 | readme = "README.md" 8 | packages = [{ include = "lib" }, { include = "cli.py" }, { include = "pyproject.toml" }] 9 | 10 | [tool.poetry.dependencies] 11 | python = ">=3.8,<4" 12 | loguru = "^0.6.0" 13 | docopt = "^0.6.2" 14 | gitpython = "^3.1.30" 15 | pyyaml = "^6.0" 16 | requests = "^2.28.2" 17 | 18 | [tool.poetry.group.dev.dependencies] 19 | pytest = "^7.2.1" 20 | pytest-html = "^3.2.0" 21 | 22 | [build-system] 23 | requires = ["poetry-core"] 24 | build-backend = "poetry.core.masonry.api" 25 | 26 | [[tool.poetry.source]] 27 | name = "aliyun" 28 | url = "https://mirrors.aliyun.com/pypi/simple/" 29 | 30 | [tool.poetry.scripts] 31 | trident = "cli:cli" -------------------------------------------------------------------------------- /readme.md: -------------------------------------------------------------------------------- 1 | # 🔱 trident-sync 三叉戟同步 2 | 3 | 三叉戟同步，是一款异构项目同步升级工具，二次开发同步神器。 4 | 5 | [中文](./readme.md) 6 | 7 |
8 | star 11 | GitHub stars 14 |
15 | 16 | ## 1. 简介 17 | 18 | 当我们的项目内部使用了其他项目进行二次开发，那么这个模块就只能停留在当时的版本，无法方便的更新。 19 | 20 | 本工具可以自动获取变更并合并到你的项目仓库，让二次开发项目可以持续升级。 21 | 22 | 本工具适用于所有二次开发需要同步更新的场景： 23 | 24 | * 直接copy源项目进行二次开发。 25 | * 源项目与你项目目录结构不一致（异构）。 26 | * 源项目本身的submodule过多，一个个fork、merge太麻烦。 27 | * 源项目只是你项目的一个子模块，但你不想用submodule 28 | 29 | ## 2. 缘起 30 | 31 | 我有一个 [certd](https://github.com/certd/certd) 项目，这是一个自动更新ssl证书的工具，但这不是重点。 32 | 重点是它一开始只是一个独立的命令行工具。 33 | 目录结构如下： 34 | 35 | ``` 36 | src 37 | | --packages 38 | | --core 39 | | --plugins 40 | 41 | ``` 42 | 43 | 某一天我想开发v2版本，把它升级成一个带后台和界面的web项目。 44 | 恰好我找到了两个模版项目(其实也是我写的🤭),可以帮我快速实现以上需求。 45 | 46 | * [fs-admin-antdv](https://github.com/fast-crud/fs-admin-antdv) （前端admin模版） 47 | * [fs-server-js](https://github.com/fast-crud/fs-server-js) （服务端） 48 | 49 | 我把这两个项目复制到了`certd`项目中，进行二次开发。 50 | 此时`certd`项目目录结构变成如下： 51 | 52 | ``` 53 | src 54 | | --packages 55 | | --core 56 | | --plugins 57 | | --ui 58 | | --certd-client //这是fs-admin-antdv的副本 59 | | --certd-server //这是fs-server-js的副本 60 | ``` 61 | 62 | 为了使`certd-client`和`certd-server`能够随时同步`模版项目`的更新 63 | 我将使用本项目`trident-sync`来自动帮我升级。 64 | 65 |

66 | 67 |

像不像个三叉戟🔱？

68 |

69 | 70 | ## 3. 原理过程 71 | 72 | 初始化（init）： 73 | 74 | 1. 初始化`同步工作仓库`（sync_work_repo） 75 | 2. `clone` `源仓库`（src_repo）和`目标仓库`（target_repo），添加到`同步工作仓库`的`submodule` 76 | 3. 给`目标仓库`创建并切换到`同步分支`（sync_branch） 77 | 4. 将`源仓库内的文件`复制到`目标仓库对应的目录`，然后`commit、push` 78 | 5. 此时`目标仓库`内的`sync_branch`分支拥有`源仓库`的副本 79 | 80 | 同步（sync）： 81 | 82 | 1. 当`源仓库`有变更时、拉取`源仓库`更新 83 | 2. 删除`目标仓库对应的目录`，复制`源仓库所有文件`到`目标仓库对应的目录` 84 | 3. 此时`git add . && git commit` 提交的就是`源仓库变更部分` 85 | 4. 至此我们成功将`源仓库的更新`转化成了`目标仓库的commit`，后续就是常规的合并操作了。 86 | 5. 创建`target.sync_branch` -> `target.main`的`PR` 87 | 6. 处理`PR`，合并到开发主分支，升级完成 88 | 89 |

90 | 91 |

同步流程图

92 |

93 | 94 | > 没有冲突的话，同步过程可以全部自动化。 95 | > 解决冲突是唯一需要手动的部分。 96 | 97 | ## 4. 快速开始 98 | 99 | ### 4.1 准备工作 100 | 101 | * 安装 [python (3.8+)](https://www.python.org/downloads/) 102 | * 安装 `git` 103 | * 准备你的项目和要同步的源项目 104 | 105 | ### 4.2 安装本工具 106 | 107 | ```shell 108 | # 安装本工具，安装成功后就可以使用 trident 命令了 109 | pip install trident-sync --upgrade 110 | ``` 111 | 112 | ### 4.3 编写配置文件 113 | 114 | * 创建一个同步工作目录，你可以任意命名，接下来都在这个目录下进行操作 115 | 116 | ``` 117 | mkdir sync_work_repo 118 | cd sync_work_repo 119 | ``` 120 | 121 | * 编写`./sync_work_repo/sync.yaml`， 下面是示例，请根据其中注释说明改成你自己的内容 122 | 123 | ```yaml 124 | # ./sync_work_repo/sync.yaml 125 | repo: # 仓库列表，可以配置多个仓库 126 | fs-admin: # 上游项目1，可以任意命名 127 | url: "https://github.com/fast-crud/fs-admin-antdv" # 源仓库地址 128 | path: "fs-admin-antdv" # submodule保存路径，一般配置仓库名称即可 129 | branch: "main" # 要同步过来的分支 130 | certd: # 你的项目（接受同步项目），可以任意命名 131 | url: "https://github.com/certd/certd" # 目标仓库地址 132 | path: "certd" # submodule保存路径，一般配置仓库名称即可 133 | branch: "dev" # 你的代码开发主分支（接受合并的分支）例如dev、main、v1、v2等 134 | # 以下配置与PR相关，更多关于PR的文档请前往 https://github.com/handsfree-work/trident-sync/tree/main/doc/pr.md 135 | # 第一次使用，你可以暂时不配置，同步完之后需要手动操作合并 136 | token: "" # 仓库的token，用于提交PR 137 | type: github # 仓库类型，用于提交PR，可选项：[github/gitee/gitea] 138 | auto_merge: true # 是否自动合并,如果有冲突则需要手动处理 139 | # 注意： 初始化之后，不要修改url和path，以免出现意外。但是可以添加新的repo. 140 | 141 | sync: # 同步配置，可以配置多个同步任务 142 | client: # 同步任务1，可以任意命名 143 | src: # 源仓库 144 | repo: fs-admin # 源仓库名称，上面repo配置的仓库引用 145 | dir: '.' # 要同步给target的目录（不能为空目录） 146 | target: #接受合并的仓库，就是你的项目 147 | repo: certd # 目标仓库名称，上面repo配置的仓库引用 148 | dir: 'package/ui/certd-client' # 接收src同步过来的目录 149 | # ↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑ 如果你之前已经使用源仓库副本做了一部分特性开发，那么这里配置源仓库副本的目录） 150 | branch: 'client_sync' # 同步分支名称（需要配置一个未被占用的分支名称） 151 | 152 | options: #其他选项 153 | repo_root: repo # submodule保存根目录 154 | push: false # 同步后是否push 155 | # ↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑ 第一次使用，先本地测试，不push，没问题之后再改成true 156 | pull_request: true # 是否创建pull request，需要目标仓库配置token和type 157 | proxy_fix: true # 是否将https代理改成http://开头，解决python开启代理时无法发出https请求的问题 158 | use_system_proxy: true # 是否使用系统代理 159 | 160 | ``` 161 | 162 | ### 4.4 初始化 163 | 164 | 此命令会将`sync_work_repo`目录初始化成一个git仓库，然后将`sync.yaml`中配置的`repo` 添加为`submodule` 165 | 166 | ```shell 167 | cd sync_work_repo 168 | # 执行初始化操作 169 | trident init 170 | ``` 171 | 172 |

173 | 174 |

初始化执行效果

175 |

176 | 177 | > 只需运行一次即可，除非你添加了新的`repo` 178 | 179 | > 初始化过程会将多个仓库添加为submodule，此步骤在网络不好时容易出问题 180 | > 你可以删除目录下除`sync.yaml`之外的所有文件，重新运行`trident init`命令 181 | 182 | ### 4.5 进行同步 183 | 184 | 执行同步命令，将根据`sync.yaml`中`sync`配置的任务进行同步 185 | 你将得到一个与源项目文件完全一致的同步分支（`client_sync`）。 186 | 之后你需要将`client_sync`分支合并到你的开发主分支。 187 | 如果配置了`push=true`、`token`、`type`，会自动提交PR来合并，[你需要视情况处理PR](#5-合并同步分支) 188 | 189 | ```shell 190 | # 以后你只需要定时运行这个命令，即可保持同步升级 191 | trident sync 192 | ``` 193 | 194 | 运行效果 195 | 196 | ``` 197 | root:~/sync_work_repo$ trident sync 198 | . 199 | . 200 | . 201 | 2023-01-28 14:13:41 | INFO | - refs:[] 202 | 2023-01-28 14:13:41 | WARNING | - Skip push，The remote address is not set for the current repository. Use the [trident remote ] command to set the remote address of the repository and save the synchronization progress 203 | 2023-01-28 14:13:41 | INFO | - ----------------result:✅---------------- 204 | 🏹 task --> success:✅ copy:✅ change:✅ commit:✅ push:✅ pr:✅ merge:✅ 205 | 🔱 sync_work_repo --> change:✅ commit:✅ push:🚫 206 | 2023-01-28 14:13:41 | INFO | - ----------------sync end---------------- 207 | ``` 208 | 209 | > 注意：不要在同步分支内写你自己的任何代码（示例配置中为`client_sync`分支） 210 | 211 | 212 | 213 | ### 4.6 [可选] 保存 sync_work_repo 214 | 215 | 将`sync_work_repo`push到远程服务器，防止更换电脑丢失同步进度。 216 | 后续你只需要`clone` `sync_work_repo` ，然后直接运行`trident sync`即可继续同步 217 | 218 | ```shell 219 | # 给同步仓库设置远程地址，并push 220 | trident remote --url= 221 | 222 | # 或者运行如下命令，一样的 223 | git remote add origin 224 | git push 225 | ``` 226 | 227 | > 注意： `sync_work_repo_git_url` 应该是一个新的空的远程仓库 228 | > 如果不是空的，可以加 `-f` 选项强制push（sync_work_repo原有的内容会被覆盖）。 229 | 230 | ### 4.7 [可选] 定时运行 231 | 232 | 你可以将 `` 这个远程仓库和 `trident sync` 命令配置到任何`CI/DI`工具（例如jenkins、github 233 | action、drone等）自动定时同步 234 | 235 | ## 5 合并同步分支 236 | 237 | 源仓库如果有更新，那么同步完之后，将会有三种情况： 238 | 239 | * 启用了PR： [如何启用PR？](#启用PR) 240 | * 无冲突：自动创建PR，然后自动合并，你无需任何操作 241 | * 有冲突：自动创建PR，然后需要 [手动处理PR](#处理PR) 242 | * 未启用PR： 243 | * 你需要 [手动合并](#手动合并) 244 | 245 | #### 启用PR 246 | 247 | 要启用PR，你需要如下配置 248 | 249 | ```yaml 250 | repo: 251 | target: 252 | token: xxxx # 创建PR的token 253 | type: github # upstream类型，支持[ github | gitee | gitea ] 254 | auto_merge: true # 是否自动合并 255 | 256 | ``` 257 | 258 | [token如何获取？](./doc/pr.md) 259 | 260 | #### 处理PR 261 | 262 | 当PR有冲突时，就需要手动处理冲突，才能合并进入主分支 263 | 264 | * 其中 `github` `gitee`支持在web页面直接手动解决冲突 265 | * `gitea`需要线下解决，此时你仍然需要 [手动合并](#手动合并) 266 | 267 | #### 手动合并 268 | 269 | 一般出现冲突了，都建议在IDE上手动进行合并 270 | 271 | 1. 关闭PR（没有PR的话，请无视） 272 | 2. 本地更新所有分支 273 | 3. 通过IDE进行分支merge操作（rebase也行，用你平常熟悉的合并分支操作） 274 | 275 | ``` 276 | target: --------> target: 277 | 同步分支 merge 开发主分支 278 | ``` 279 | 280 | #### 避免冲突建议 281 | 282 | 我们应该尽量避免冲突，请实际开发中遵循以下原则： 283 | 284 | 1. 尽量不删除、不移动源项目的目录和文件（否则容易造成意想不到的难以解决的冲突） 285 | 2. 尽量少在源项目的文件上进行修改（可以改，但尽量少） 286 | 3. 新功能和新特性应该写在自己建立的新目录和新文件中 287 | 288 | 总结就是六个字： 不删、少改、多加。 289 | 290 | ## 6. 自动化 291 | 292 | ### 6.1 GitHub Actions 293 | GitHub Actions 可以免费的帮你进行自动化同步操作 294 | 295 | 请参考 [certd 自动化同步示例](https://github.com/certd/certd-sync-work) 296 | -------------------------------------------------------------------------------- /test/.gitignore: -------------------------------------------------------------------------------- 1 | .pytest_cache 2 | */pr_test_git 3 | tmp -------------------------------------------------------------------------------- /test/copy_script.py: -------------------------------------------------------------------------------- 1 | import shutil 2 | 3 | 4 | # custom copy script 5 | def copy(task, src_dir, target_dir): 6 | shutil.copytree(src_dir, target_dir) 7 | -------------------------------------------------------------------------------- /test/main_test.py: -------------------------------------------------------------------------------- 1 | import os.path 2 | import shutil 3 | 4 | from cli import read_config 5 | from lib.handler.init import InitHandler 6 | from lib.handler.remote import RemoteHandler 7 | from lib.handler.sync import SyncHandler 8 | from lib.logger import logger 9 | from lib.model.config import Config 10 | from lib.model.repo import RepoConf 11 | from lib.util import rm_dir, shell, save_file, read_file 12 | 13 | cur_root = os.path.abspath(os.path.curdir) 14 | print(cur_root) 15 | tmp = os.path.abspath("./tmp/sync") 16 | work_root = f"{tmp}/sync-work-repo" 17 | dir_repos = f"{tmp}/original" 18 | key_sub_repo = 'sync-src-submodule' 19 | sub_git_url = f"https://gitee.com/handsfree-test/{key_sub_repo}" 20 | save_git_url = f"https://gitee.com/handsfree-test/sync-work-repo" 21 | 22 | sub_main_branch = "main" 23 | dir_sub_repo = f"{dir_repos}/{key_sub_repo}" 24 | readme_file = './readme.md' 25 | key_src_repo = "sync-src" 26 | key_target_repo = "target" 27 | dir_src_repo = f"{dir_repos}/{key_src_repo}" 28 | target_sync_branch = "test_sync" 29 | sync_config_file_origin = os.path.abspath("sync.yaml") 30 | sync_config_file_save = f"{work_root}/sync.yaml" 31 | 32 | 33 | def get_config(): 34 | config_dict = read_config(sync_config_file_origin) 35 | config = Config(config_dict) 36 | config.set_default_token() 37 | return config 38 | 39 | 40 | def mkdirs(): 41 | if os.path.exists(tmp): 42 | rm_dir(tmp) 43 | os.makedirs(tmp) 44 | os.makedirs(work_root) 45 | 46 | os.makedirs(dir_repos) 47 | # 初始化 sub repo 48 | os.makedirs(dir_sub_repo) 49 | os.chdir(work_root) 50 | 51 | 52 | def prepare(): 53 | '''测试准备''' 54 | config = get_config() 55 | repo_src: RepoConf = config.repo[key_src_repo] 56 | mkdirs() 57 | os.chdir(dir_sub_repo) 58 | shell("git init") 59 | readme_content = 'submodule' 60 | save_file(readme_file, readme_content) 61 | shell("git add .") 62 | shell("git commit -m \"submodule init\"") 63 | shell(f"git remote add origin {sub_git_url}") 64 | shell(f"git push -f -u origin {sub_main_branch}") 65 | 66 | # 初始化 src repo 67 | os.makedirs(dir_src_repo) 68 | 69 | os.chdir(dir_src_repo) 70 | shell("git init") 71 | readme_content = 'src' 72 | save_file(readme_file, readme_content) 73 | shell("git add .") 74 | shell("git commit -m \"src init\"") 75 | shell(f"git remote add origin {repo_src.url}") 76 | 77 | # 添加submodule 78 | shell(f"git submodule add -b {sub_main_branch} --name {key_sub_repo} {sub_git_url} ./sub/{key_sub_repo}") 79 | shell("git add .") 80 | shell("git commit -m \"add sub\"") 81 | shell(f"git push -f -u origin {sub_main_branch}") 82 | 83 | # 初始化 target repo 84 | repo_target: RepoConf = config.repo[key_target_repo] 85 | dir_target_repo = f"{dir_repos}/{key_target_repo}" 86 | os.makedirs(dir_target_repo) 87 | 88 | os.chdir(dir_target_repo) 89 | shell("git init") 90 | readme_content = 'target' 91 | save_file(readme_file, readme_content) 92 | shell("git add .") 93 | shell("git commit -m \"target init\"") 94 | 95 | readme_content = 'src-copy' 96 | copy_readme_file = './package/sync-src/readme.md' 97 | save_file(copy_readme_file, readme_content) 98 | 99 | shell("git add .") 100 | shell("git commit -m \"sync-src-copy\"") 101 | 102 | shell(f"git remote add origin {repo_target.url}") 103 | shell(f"git push -f -u origin {repo_target.branch}") 104 | 105 | # 删除远程分支 106 | shell(f"git branch {target_sync_branch}") 107 | shell(f"git push -f -u origin {target_sync_branch}") 108 | shell(f"git push origin --delete {target_sync_branch}") 109 | 110 | shutil.copyfile(sync_config_file_origin, sync_config_file_save) 111 | logger.info('test prepare success') 112 | return config 113 | 114 | 115 | def submodule_update(): 116 | config = get_config() 117 | repo_src: RepoConf = config.repo[key_src_repo] 118 | submodule_dir = f"{dir_src_repo}/sub/{key_sub_repo}" 119 | os.chdir(submodule_dir) 120 | readme_content = "submodule v2" 121 | save_file(readme_file, readme_content) 122 | shell("git add .") 123 | shell("git commit -m \"sub update v2\"") 124 | shell(f"git push -f -u origin {sub_main_branch}") 125 | 126 | os.chdir(dir_src_repo) 127 | shell("git add .") 128 | shell("git commit -m \"src update v2\"") 129 | shell(f"git push -f -u origin {repo_src.branch}") 130 | 131 | 132 | def test_prepare(): 133 | prepare() 134 | 135 | 136 | def test_init(): 137 | config = get_config() 138 | InitHandler(work_root, config).handle() 139 | target_readme = f"{work_root}/repo/target/readme.md" 140 | target_readme_content = read_file(target_readme) 141 | assert target_readme_content == 'target' 142 | 143 | src_readme = f"{work_root}/repo/sync-src/readme.md" 144 | src_readme_content = read_file(src_readme) 145 | assert src_readme_content == 'src' 146 | 147 | sub_readme = f"{work_root}/repo/sync-src/sub/sync-src-submodule/readme.md" 148 | sub_readme_content = read_file(sub_readme) 149 | assert sub_readme_content == 'submodule' 150 | 151 | 152 | def test_sync_first(): 153 | config = get_config() 154 | # 测试第一次同步 155 | SyncHandler(work_root, config).handle() 156 | 157 | target_repo_dir = f"{work_root}/repo/target/" 158 | os.chdir(target_repo_dir) 159 | shell(f"git checkout {target_sync_branch}") 160 | 161 | readme = f"{target_repo_dir}/package/sync-src/readme.md" 162 | readme_content = read_file(readme) 163 | assert readme_content == 'src' 164 | 165 | readme = f"{target_repo_dir}/package/sync-src/sub/{key_sub_repo}/readme.md" 166 | readme_content = read_file(readme) 167 | assert readme_content == 'submodule' 168 | 169 | assert config.sync['task1'].status.merge is False 170 | assert config.status.push is False 171 | 172 | 173 | def test_set_remote(): 174 | os.chdir(work_root) 175 | RemoteHandler(work_root, remote_url=save_git_url, force=True).handle() 176 | 177 | 178 | def test_submodule_update(): 179 | # src的子模块有更新 180 | submodule_update() 181 | 182 | 183 | # 测试重新init 184 | def test_re_init(): 185 | # 重复init测试 186 | config = get_config() 187 | InitHandler(work_root, config).handle() 188 | 189 | 190 | def test_clone_save_repo(): 191 | # 删除save仓库 192 | os.chdir(tmp) 193 | rm_dir(work_root) 194 | shell(f"git clone {save_git_url}") 195 | 196 | 197 | def test_sync_second(): 198 | # 测试第二次同步 199 | config = get_config() 200 | SyncHandler(work_root, config).handle() 201 | target_repo_dir = f"{work_root}/repo/target/" 202 | os.chdir(target_repo_dir) 203 | shell(f"git checkout {target_sync_branch}") 204 | 205 | readme = f"{target_repo_dir}/package/sync-src/readme.md" 206 | readme_content = read_file(readme) 207 | assert readme_content == 'src' 208 | 209 | readme = f"{target_repo_dir}/package/sync-src/sub/{key_sub_repo}/readme.md" 210 | readme_content = read_file(readme) 211 | assert readme_content == 'submodule v2' 212 | 213 | assert config.sync['task1'].status.merge is False 214 | assert config.status.push is True 215 | -------------------------------------------------------------------------------- /test/pr_test.py: -------------------------------------------------------------------------------- 1 | import os.path 2 | 3 | import git 4 | 5 | from lib.api.abstract_client import pick_from_url 6 | from lib.api.gitea import GiteaClient 7 | from lib.api.gitee import GiteeClient 8 | from lib.api.github import GithubClient 9 | from lib.http import Http 10 | from lib.logger import logger 11 | from lib.util import shell, save_file, rm_dir 12 | 13 | main_branch = 'main' 14 | no_conflict_branch = 'no_conflict_branch' 15 | conflict_branch = 'conflict_branch' 16 | 17 | 18 | def prepare(origin_url): 19 | '''pr测试准备''' 20 | # 初始化git 21 | pr_test_git_dir = os.path.abspath("./tmp/pr_test_git") 22 | if os.path.exists(pr_test_git_dir): 23 | rm_dir(pr_test_git_dir) 24 | os.makedirs(pr_test_git_dir) 25 | os.chdir(pr_test_git_dir) 26 | 27 | shell("git init") 28 | readme = 'readme.md' 29 | readme_content = '1' 30 | save_file(readme, readme_content) 31 | shell("git add .") 32 | shell("git commit -m \"1\"") 33 | # origin = repo.create_remote("origin", url) 34 | shell(f"git remote add origin {origin_url}") 35 | 36 | # 创建基础分支 37 | repo = git.Repo.init(pr_test_git_dir) 38 | if repo.active_branch.name != main_branch: 39 | shell(f"git branch {main_branch}") 40 | shell(f"git checkout {main_branch}") 41 | shell(f"git push -f -u origin {main_branch}") 42 | 43 | # 创建特性分支1,追加一行,无冲突 44 | shell(f"git checkout {main_branch}") 45 | shell(f"git branch {no_conflict_branch}") 46 | shell(f"git checkout {no_conflict_branch}") 47 | readme = 'readme.md' 48 | readme_content = '1\n2' 49 | save_file(readme, readme_content) 50 | shell("git add .") 51 | shell("git commit -m \"no_conflict_branch\"") 52 | # origin = repo.create_remote("origin", url) 53 | shell(f"git push -f -u origin {no_conflict_branch}") 54 | 55 | # 创建特性分支2,冲突一行 56 | 57 | shell(f"git switch {main_branch}") 58 | shell(f"git branch {conflict_branch}") 59 | shell(f"git checkout {conflict_branch}") 60 | readme = 'readme.md' 61 | readme_content = '2' 62 | save_file(readme, readme_content) 63 | shell("git add .") 64 | shell("git commit -m \"conflict_branch\"") 65 | # origin = repo.create_remote("origin", url) 66 | shell(f"git push -f -u origin {conflict_branch}") 67 | 68 | logger.info('pr test prepare success') 69 | 70 | 71 | def create_github_client(origin_url): 72 | http = Http() 73 | token = os.getenv('GITHUB_TOKEN') 74 | client = GithubClient(http, token, origin_url) 75 | return client 76 | 77 | 78 | def create_gitee_client(origin_url): 79 | http = Http() 80 | token = os.getenv('GITEE_TOKEN') 81 | client = GiteeClient(http, token, origin_url) 82 | return client 83 | 84 | 85 | def create_gitea_client(origin_url): 86 | http = Http() 87 | token = os.getenv('GITEA_TOKEN') 88 | client = GiteaClient(http, token, origin_url) 89 | return client 90 | 91 | 92 | def test_pr_pick_git_url(): 93 | origin_url = "https://github.com/handsfree-test/pr-test" 94 | res = pick_from_url(origin_url) 95 | assert res['owner'] == 'handsfree-test' 96 | assert res['repo'] == 'pr-test' 97 | 98 | res = pick_from_url(origin_url + "/") 99 | assert res['owner'] == 'handsfree-test' 100 | assert res['repo'] == 'pr-test' 101 | 102 | res = pick_from_url(origin_url + ".git") 103 | assert res['owner'] == 'handsfree-test' 104 | assert res['repo'] == 'pr-test' 105 | 106 | res = pick_from_url("http://docmirror.cn:6789/handsfree-test/pr-test.git") 107 | assert res['owner'] == 'handsfree-test' 108 | assert res['repo'] == 'pr-test' 109 | 110 | 111 | def test_pr_github(): 112 | origin_url = "https://github.com/handsfree-test/pr-test" 113 | prepare(origin_url) 114 | client = create_github_client(origin_url) 115 | title = "no_conflict branch pr" 116 | body = "no_conflict branch pr" 117 | src_branch = no_conflict_branch 118 | target_branch = main_branch 119 | client.create_pull_request(title, body, src_branch, target_branch, auto_merge=False) 120 | 121 | client.create_pull_request(title, body, src_branch, target_branch, auto_merge=True) 122 | 123 | # 测试conflict pr 124 | title = "conflict branch pr" 125 | body = "conflict branch pr" 126 | src_branch = conflict_branch 127 | client.create_pull_request(title, body, src_branch, target_branch, auto_merge=True) 128 | 129 | 130 | def test_pr_gitee(): 131 | origin_url = "https://gitee.com/handsfree-test/pr-test" 132 | prepare(origin_url) 133 | client = create_gitee_client(origin_url) 134 | title = "no_conflict branch pr" 135 | body = "no_conflict branch pr" 136 | src_branch = no_conflict_branch 137 | target_branch = main_branch 138 | # 先不自动提交 139 | client.create_pull_request(title, body, src_branch, target_branch, auto_merge=False) 140 | 141 | # 自动提交 142 | client.create_pull_request(title, body, src_branch, target_branch, auto_merge=True) 143 | 144 | # 测试conflict pr 145 | title = "conflict branch pr" 146 | body = "conflict branch pr" 147 | src_branch = conflict_branch 148 | client.create_pull_request(title, body, src_branch, target_branch, auto_merge=True) 149 | 150 | 151 | def test_pr_gitea(): 152 | origin_url = "http://docmirror.cn:6789/handsfree-test/pr-test" 153 | prepare(origin_url) 154 | client = create_gitea_client(origin_url) 155 | title = "no_conflict branch pr" 156 | body = "no_conflict branch pr" 157 | src_branch = no_conflict_branch 158 | target_branch = main_branch 159 | # 先不自动提交 160 | client.create_pull_request(title, body, src_branch, target_branch, auto_merge=False) 161 | 162 | # 自动提交 163 | client.create_pull_request(title, body, src_branch, target_branch, auto_merge=True) 164 | 165 | # 测试conflict pr 166 | title = "conflict branch pr" 167 | body = "conflict branch pr" 168 | src_branch = conflict_branch 169 | client.create_pull_request(title, body, src_branch, target_branch, auto_merge=True) 170 | -------------------------------------------------------------------------------- /test/sync.yaml: -------------------------------------------------------------------------------- 1 | repo: 2 | sync-src: # 上游项目1，可以任意命名 3 | url: "https://gitee.com/handsfree-test/sync-src" # 仓库地址 4 | path: "sync-src" # submodule保存路径 5 | branch: "main" # 要同步过来的分支 6 | target: # 你的项目（接受同步项目），可以任意命名 7 | url: "https://gitee.com/handsfree-test/sync-target" # 仓库地址 8 | path: "target" # submodule子路径 9 | branch: "main" # 你的代码开发主分支（接受合并的分支）例如dev、main等 10 | token: "" # 仓库token，用于提交PR, 前往 https://github.com/settings/tokens 创建token 11 | type: gitee # 仓库类型，用于提交PR，可选项：[github/gitee/gitea/gitlab] 12 | auto_merge: true 13 | 14 | sync: # 同步配置，可以配置多个同步任务 15 | task1: # 同步任务1，可以任意命名 16 | src: # 源仓库 17 | repo: sync-src # 源仓库名称，上面repo配置的仓库名称引用 18 | dir: './' # 要同步给target的目录 19 | target: #目标仓库 20 | repo: target # 目标仓库名称，上面repo配置的仓库名称引用 21 | dir: 'package/sync-src' # 接收src同步过来的目录 22 | allow_reset_to_root: true # 是否允许重置同步分支到root commit记录，第一次同步时需要找到target.dir是一个空目录的commit，确保后续合并时不会丢失变更，root commit大概率应该是空目录 23 | remove_dir_before_copy: true # 复制文件前是否删除target.dir，如果target.dir与src.dir是相同的，保持默认true即可 24 | branch: 'test_sync' # 同步分支名称（需要配置一个未被占用的分支名称） 25 | copy_script: ../../../copy_script.py # 自定义同步脚本，不配置则整个目录复制，脚本中应定义方法:copy(task: {key,src:{上方配置},target:{上方配置}}, src_dir:str, target_dir:str) 26 | 27 | options: #选项 28 | repo_root: repo # submodule保存根目录 29 | push: true # 同步后是否push 30 | pull_request: true # 是否创建pr，需要目标仓库配置token和type 31 | proxy_fix: true # 是否将https代理改成http://开头 32 | use_system_proxy: true # 是否使用系统代理 --------------------------------------------------------------------------------