├── .gitignore ├── LICENSE ├── README.md ├── pyproject.toml └── src └── BetterADBSync ├── FileSystems ├── Android.py ├── Base.py ├── Local.py └── __init__.py ├── SAOLogging.py ├── __init__.py └── argparsing.py /.gitignore: -------------------------------------------------------------------------------- 1 | __pycache__/ 2 | /dist 3 | /src/*.egg-info 4 | /venv/ 5 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | 179 | APPENDIX: How to apply the Apache License to your work. 180 | 181 | To apply the Apache License to your work, attach the following 182 | boilerplate notice, with the fields enclosed by brackets "[]" 183 | replaced with your own identifying information. (Don't include 184 | the brackets!) The text should be enclosed in the appropriate 185 | comment syntax for the file format. We also recommend that a 186 | file or class name and description of purpose be included on the 187 | same "printed page" as the copyright notice for easier 188 | identification within third-party archives. 189 | 190 | Copyright [yyyy] [name of copyright owner] 191 | 192 | Licensed under the Apache License, Version 2.0 (the "License"); 193 | you may not use this file except in compliance with the License. 194 | You may obtain a copy of the License at 195 | 196 | http://www.apache.org/licenses/LICENSE-2.0 197 | 198 | Unless required by applicable law or agreed to in writing, software 199 | distributed under the License is distributed on an "AS IS" BASIS, 200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 201 | See the License for the specific language governing permissions and 202 | limitations under the License. 203 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Better ADB Sync 2 | 3 | An [rsync](https://wiki.archlinux.org/title/rsync)-like program to sync files between a computer and an Android device 4 | 5 | ## Installation 6 | 7 | Available on [PyPI](https://pypi.org/project/BetterADBSync/) 8 | 9 | ``` 10 | $ pip install BetterADBSync 11 | ``` 12 | 13 | ## QRD 14 | 15 | To push from your computer to your phone use 16 | ``` 17 | $ adbsync push LOCAL ANDROID 18 | ``` 19 | 20 | To pull from your phone to your computer use 21 | ``` 22 | $ adbsync pull ANDROID LOCAL 23 | ``` 24 | 25 | Full help is available with `$ adbsync --help` 26 | 27 | ## Intro 28 | 29 | This is a (pretty much from scratch) rewrite of Google's [adbsync](https://github.com/google/adb-sync) repo. 30 | 31 | The reason for the rewrite is to 32 | 33 | 1. Update the repo to Python 3 codestyle (strings are by default UTF-8, no more b"" and u"", classes don't need to inherit from object, 4 space indentation etc) 34 | 2. Add in support for `--exclude`, `--exclude-from`, `--del`, `--delete-excluded` like `rsync` has (this required a complete rewrite of the diffing algorithm) 35 | 36 | ## Additions 37 | 38 | - `--del` will delete files and folders on the destination end that are not present on the source end. This does not include exluded files. 39 | - `--delete-excluded` will delete excluded files and folders on the destination end. 40 | - `--exclude` can be used many times. Each should be a `fnmatch` pattern relative to the source. These patterns will be ignored unless `--delete-excluded` is specified. 41 | - `--exclude-from` can be used many times. Each should be a filename of a file containing `fnmatch` patterns relative to the source. 42 | 43 | ## Possible future TODOs 44 | 45 | I am satisfied with my code so far, however a few things could be added if they are ever needed 46 | 47 | - `--backup` and `--backup-dir-local` or `--backup-dir-android` to move outdated / to-delete files to another folder instead of deleting 48 | 49 | --- 50 | 51 | ---BEGIN ORIGINAL README.md--- 52 | 53 | adb-sync 54 | ======== 55 | 56 | adb-sync is a tool to synchronize files between a PC and an Android device 57 | using the ADB (Android Debug Bridge). 58 | 59 | Related Projects 60 | ================ 61 | 62 | Before getting used to this, please review this list of projects that are 63 | somehow related to adb-sync and may fulfill your needs better: 64 | 65 | * [rsync](http://rsync.samba.org/) is a file synchronization tool for local 66 | (including FUSE) file systems or SSH connections. This can be used even with 67 | Android devices if rooted or using an app like 68 | [SSHelper](https://play.google.com/store/apps/details?id=com.arachnoid.sshelper). 69 | * [adbfs](http://collectskin.com/adbfs/) is a FUSE file system that uses adb to 70 | communicate to the device. Requires a rooted device, though. 71 | * [adbfs-rootless](https://github.com/spion/adbfs-rootless) is a fork of adbfs 72 | that requires no root on the device. Does not play very well with rsync. 73 | * [go-mtpfs](https://github.com/hanwen/go-mtpfs) is a FUSE file system to 74 | connect to Android devices via MTP. Due to MTP's restrictions, only a certain 75 | set of file extensions is supported. To store unsupported files, just add 76 | .txt! Requires no USB debugging mode. 77 | 78 | Setup 79 | ===== 80 | 81 | Android Side 82 | ------------ 83 | 84 | First you need to enable USB debugging mode. This allows authorized computers 85 | (on Android before 4.4.3 all computers) to perform possibly dangerous 86 | operations on your device. If you do not accept this risk, do not proceed and 87 | try using [go-mtpfs](https://github.com/hanwen/go-mtpfs) instead! 88 | 89 | On your Android device: 90 | 91 | * Go to the Settings app. 92 | * If there is no "Developer Options" menu: 93 | * Select "About". 94 | * Tap "Build Number" seven times. 95 | * Go back. 96 | * Go to "Developer Options". 97 | * Enable "USB Debugging". 98 | 99 | PC Side 100 | ------- 101 | 102 | * Install the [Android SDK](http://developer.android.com/sdk/index.html) (the 103 | stand-alone Android SDK "for an existing IDE" is sufficient). Alternatively, 104 | some Linux distributions come with a package named like "android-tools-adb" 105 | that contains the required tool. 106 | * Make sure "adb" is in your PATH. If you use a package from your Linux 107 | distribution, this should already be the case; if you used the SDK, you 108 | probably will have to add an entry to PATH in your ~/.profile file, log out 109 | and log back in. 110 | * `git clone https://github.com/google/adb-sync` 111 | * `cd adb-sync` 112 | * Copy or symlink the adb-sync script somewhere in your PATH. For example: 113 | `cp adb-sync /usr/local/bin/` 114 | 115 | Usage 116 | ===== 117 | 118 | To get a full help, type: 119 | 120 | ``` 121 | adb-sync --help 122 | ``` 123 | 124 | To synchronize your music files from ~/Music to your device, type one of: 125 | 126 | ``` 127 | adb-sync ~/Music /sdcard 128 | adb-sync ~/Music/ /sdcard/Music 129 | ``` 130 | 131 | To synchronize your music files from ~/Music to your device, deleting files you 132 | removed from your PC, type one of: 133 | 134 | ``` 135 | adb-sync --delete ~/Music /sdcard 136 | adb-sync --delete ~/Music/ /sdcard/Music 137 | ``` 138 | 139 | To copy all downloads from your device to your PC, type: 140 | 141 | ``` 142 | adb-sync --reverse /sdcard/Download/ ~/Downloads 143 | ``` 144 | 145 | ADB Channel 146 | =========== 147 | 148 | This package also contains a separate tool called adb-channel, which is a 149 | convenience wrapper to connect a networking socket on the Android device to 150 | file descriptors on the PC side. It can even launch and shut down the given 151 | application automatically! 152 | 153 | It is best used as a `ProxyCommand` for SSH (install 154 | [SSHelper](https://play.google.com/store/apps/details?id=com.arachnoid.sshelper) 155 | first) using a configuration like: 156 | 157 | ``` 158 | Host sshelper 159 | Port 2222 160 | ProxyCommand adb-channel tcp:%p com.arachnoid.sshelper/.SSHelperActivity 1 161 | ``` 162 | 163 | After adding this to `~/.ssh/config`, run `ssh-copy-id sshelper`. 164 | 165 | Congratulations! You can now use `rsync`, `sshfs` etc. to the host name 166 | `sshelper`. 167 | 168 | Contributing 169 | ============ 170 | 171 | Patches to this project are very welcome. 172 | 173 | Before sending a patch or pull request, we ask you to fill out one of the 174 | Contributor License Agreements: 175 | 176 | * [Google Individual Contributor License Agreement, v1.1](https://developers.google.com/open-source/cla/individual) 177 | * [Google Software Grant and Corporate Contributor License Agreement, v1.1](https://developers.google.com/open-source/cla/corporate) 178 | 179 | Disclaimer 180 | ========== 181 | 182 | This is not an official Google product. 183 | 184 | 185 | ---END ORIGINAL README.md--- 186 | 187 | --- 188 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [project] 2 | name = "BetterADBSync" 3 | dynamic = ["version"] 4 | description = "Better version of adb-sync for Python3" 5 | readme = "README.md" 6 | license = {file = "LICENSE"} 7 | 8 | [project.urls] 9 | Homepage = "https://github.com/jb2170/better-adb-sync/" 10 | 11 | [project.scripts] 12 | adbsync = "BetterADBSync:main" 13 | 14 | [tool.setuptools.dynamic] 15 | version = {attr = "BetterADBSync.__version__"} 16 | 17 | [build-system] 18 | requires = ["setuptools>=61.0"] 19 | build-backend = "setuptools.build_meta" 20 | -------------------------------------------------------------------------------- /src/BetterADBSync/FileSystems/Android.py: -------------------------------------------------------------------------------- 1 | from typing import Iterable, Iterator, List, NoReturn, Tuple 2 | import logging 3 | import os 4 | import re 5 | import stat 6 | import shlex 7 | import datetime 8 | import subprocess 9 | 10 | from ..SAOLogging import logging_fatal 11 | 12 | from .Base import FileSystem 13 | 14 | class AndroidFileSystem(FileSystem): 15 | RE_TESTCONNECTION_NO_DEVICE = re.compile("^adb\\: no devices/emulators found$") 16 | RE_TESTCONNECTION_DAEMON_NOT_RUNNING = re.compile("^\\* daemon not running; starting now at tcp:\\d+$") 17 | RE_TESTCONNECTION_DAEMON_STARTED = re.compile("^\\* daemon started successfully$") 18 | 19 | RE_LS_TO_STAT = re.compile( 20 | r"""^ 21 | (?: 22 | (?P -) | 23 | (?P b) | 24 | (?P c) | 25 | (?P d) | 26 | (?P l) | 27 | (?P p) | 28 | (?P s)) 29 | [-r][-w][-xsS] 30 | [-r][-w][-xsS] 31 | [-r][-w][-xtT] # Mode string 32 | [ ]+ 33 | (?: 34 | [0-9]+ # Number of hard links 35 | [ ]+ 36 | )? 37 | [^ ]+ # User name/ID 38 | [ ]+ 39 | [^ ]+ # Group name/ID 40 | [ ]+ 41 | (?(S_IFBLK) [^ ]+[ ]+[^ ]+[ ]+) # Device numbers 42 | (?(S_IFCHR) [^ ]+[ ]+[^ ]+[ ]+) # Device numbers 43 | (?(S_IFDIR) (?P[0-9]+ [ ]+))? # Directory size 44 | (?(S_IFREG) (?P [0-9]+) [ ]+) # Size 45 | (?(S_IFLNK) ([0-9]+) [ ]+) # Link length 46 | (?P 47 | [0-9]{4}-[0-9]{2}-[0-9]{2} # Date 48 | [ ] 49 | [0-9]{2}:[0-9]{2}) # Time 50 | [ ] 51 | # Don't capture filename for symlinks (ambiguous). 52 | (?(S_IFLNK) .* | (?P .*)) 53 | $""", re.DOTALL | re.VERBOSE) 54 | 55 | RE_NO_SUCH_FILE = re.compile("^.*: No such file or directory$") 56 | RE_LS_NOT_A_DIRECTORY = re.compile("ls: .*: Not a directory$") 57 | RE_TOTAL = re.compile("^total \\d+$") 58 | 59 | RE_REALPATH_NO_SUCH_FILE = re.compile("^realpath: .*: No such file or directory$") 60 | RE_REALPATH_NOT_A_DIRECTORY = re.compile("^realpath: .*: Not a directory$") 61 | 62 | ADBSYNC_END_OF_COMMAND = "ADBSYNC END OF COMMAND" 63 | 64 | def __init__(self, adb_arguments: List[str], adb_encoding: str) -> None: 65 | super().__init__(adb_arguments) 66 | self.adb_encoding = adb_encoding 67 | self.proc_adb_shell = subprocess.Popen( 68 | self.adb_arguments + ["shell"], 69 | stdin = subprocess.PIPE, 70 | stdout = subprocess.PIPE, 71 | stderr = subprocess.STDOUT 72 | ) 73 | 74 | def __del__(self): 75 | self.proc_adb_shell.stdin.close() 76 | self.proc_adb_shell.wait() 77 | 78 | def adb_shell(self, commands: List[str]) -> Iterator[str]: 79 | self.proc_adb_shell.stdin.write(shlex.join(commands).encode(self.adb_encoding)) 80 | self.proc_adb_shell.stdin.write(" NoReturn: 96 | logging.critical("ADB line not captured") 97 | logging_fatal(line) 98 | 99 | def test_connection(self): 100 | for line in self.adb_shell([":"]): 101 | print(line) 102 | 103 | if self.RE_TESTCONNECTION_DAEMON_NOT_RUNNING.fullmatch(line) or self.RE_TESTCONNECTION_DAEMON_STARTED.fullmatch(line): 104 | continue 105 | 106 | raise BrokenPipeError 107 | 108 | def ls_to_stat(self, line: str) -> Tuple[str, os.stat_result]: 109 | if self.RE_NO_SUCH_FILE.fullmatch(line): 110 | raise FileNotFoundError 111 | elif self.RE_LS_NOT_A_DIRECTORY.fullmatch(line): 112 | raise NotADirectoryError 113 | elif match := self.RE_LS_TO_STAT.fullmatch(line): 114 | match_groupdict = match.groupdict() 115 | st_mode = stat.S_IRWXU | stat.S_IRGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IXOTH # 755 116 | if match_groupdict['S_IFREG']: 117 | st_mode |= stat.S_IFREG 118 | if match_groupdict['S_IFBLK']: 119 | st_mode |= stat.S_IFBLK 120 | if match_groupdict['S_IFCHR']: 121 | st_mode |= stat.S_IFCHR 122 | if match_groupdict['S_IFDIR']: 123 | st_mode |= stat.S_IFDIR 124 | if match_groupdict['S_IFIFO']: 125 | st_mode |= stat.S_IFIFO 126 | if match_groupdict['S_IFLNK']: 127 | st_mode |= stat.S_IFLNK 128 | if match_groupdict['S_IFSOCK']: 129 | st_mode |= stat.S_IFSOCK 130 | st_size = None if match_groupdict["st_size"] is None else int(match_groupdict["st_size"]) 131 | st_mtime = int(datetime.datetime.strptime(match_groupdict["st_mtime"], "%Y-%m-%d %H:%M").timestamp()) 132 | 133 | # Fill the rest with dummy values. 134 | st_ino = 1 135 | st_rdev = 0 136 | st_nlink = 1 137 | st_uid = -2 # Nobody. 138 | st_gid = -2 # Nobody. 139 | st_atime = st_ctime = st_mtime 140 | 141 | return match_groupdict["filename"], os.stat_result((st_mode, st_ino, st_rdev, st_nlink, st_uid, st_gid, st_size, st_atime, st_mtime, st_ctime)) 142 | else: 143 | self.line_not_captured(line) 144 | 145 | @property 146 | def sep(self) -> str: 147 | return "/" 148 | 149 | def unlink(self, path: str) -> None: 150 | for line in self.adb_shell(["rm", path]): 151 | self.line_not_captured(line) 152 | 153 | def rmdir(self, path: str) -> None: 154 | for line in self.adb_shell(["rm", "-r", path]): 155 | self.line_not_captured(line) 156 | 157 | def makedirs(self, path: str) -> None: 158 | for line in self.adb_shell(["mkdir", "-p", path]): 159 | self.line_not_captured(line) 160 | 161 | def realpath(self, path: str) -> str: 162 | for line in self.adb_shell(["realpath", path]): 163 | if self.RE_REALPATH_NO_SUCH_FILE.fullmatch(line): 164 | raise FileNotFoundError 165 | elif self.RE_REALPATH_NOT_A_DIRECTORY.fullmatch(line): 166 | raise NotADirectoryError 167 | else: 168 | return line 169 | # permission error possible? 170 | 171 | def lstat(self, path: str) -> os.stat_result: 172 | for line in self.adb_shell(["ls", "-lad", path]): 173 | return self.ls_to_stat(line)[1] 174 | 175 | def lstat_in_dir(self, path: str) -> Iterable[Tuple[str, os.stat_result]]: 176 | for line in self.adb_shell(["ls", "-la", path]): 177 | if self.RE_TOTAL.fullmatch(line): 178 | continue 179 | else: 180 | yield self.ls_to_stat(line) 181 | 182 | def utime(self, path: str, times: Tuple[int, int]) -> None: 183 | atime = datetime.datetime.fromtimestamp(times[0]).strftime("%Y%m%d%H%M") 184 | mtime = datetime.datetime.fromtimestamp(times[1]).strftime("%Y%m%d%H%M") 185 | for line in self.adb_shell(["touch", "-at", atime, "-mt", mtime, path]): 186 | self.line_not_captured(line) 187 | 188 | def join(self, base: str, leaf: str) -> str: 189 | return os.path.join(base, leaf).replace("\\", "/") # for Windows 190 | 191 | def split(self, path: str) -> Tuple[str, str]: 192 | head, tail = os.path.split(path) 193 | return head.replace("\\", "/"), tail # for Windows 194 | 195 | def normpath(self, path: str) -> str: 196 | return os.path.normpath(path).replace("\\", "/") 197 | 198 | def push_file_here(self, source: str, destination: str, show_progress: bool = False) -> None: 199 | if show_progress: 200 | kwargs_call = {} 201 | else: 202 | kwargs_call = { 203 | "stdout": subprocess.DEVNULL, 204 | "stderr": subprocess.DEVNULL 205 | } 206 | if subprocess.call(self.adb_arguments + ["push", source, destination], **kwargs_call): 207 | logging_fatal("Non-zero exit code from adb push") 208 | -------------------------------------------------------------------------------- /src/BetterADBSync/FileSystems/Base.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | from typing import Iterable, List, Tuple, Union 3 | import logging 4 | import os 5 | import stat 6 | 7 | from ..SAOLogging import perror 8 | 9 | class FileSystem(): 10 | def __init__(self, adb_arguments: List[str]) -> None: 11 | self.adb_arguments = adb_arguments 12 | 13 | def _get_files_tree(self, tree_path: str, tree_path_stat: os.stat_result, follow_links: bool = False): 14 | # the reason to have two functions instead of one purely recursive one is to use self.lstat_in_dir ie ls 15 | # which is much faster than individually stat-ing each file. Hence we have get_files_tree's special first lstat 16 | if stat.S_ISLNK(tree_path_stat.st_mode): 17 | if not follow_links: 18 | logging.warning(f"Ignoring symlink {tree_path}") 19 | return None 20 | logging.debug(f"Following symlink {tree_path}") 21 | try: 22 | tree_path_realpath = self.realpath(tree_path) 23 | tree_path_stat_realpath = self.lstat(tree_path_realpath) 24 | except (FileNotFoundError, NotADirectoryError, PermissionError) as e: 25 | perror(f"Skipping symlink {tree_path}", e) 26 | return None 27 | return self._get_files_tree(tree_path_realpath, tree_path_stat_realpath, follow_links = follow_links) 28 | elif stat.S_ISDIR(tree_path_stat.st_mode): 29 | tree = {".": (60 * (int(tree_path_stat.st_atime) // 60), 60 * (int(tree_path_stat.st_mtime) // 60))} 30 | for filename, stat_object_child, in self.lstat_in_dir(tree_path): 31 | if filename in [".", ".."]: 32 | continue 33 | tree[filename] = self._get_files_tree( 34 | self.join(tree_path, filename), 35 | stat_object_child, 36 | follow_links = follow_links) 37 | return tree 38 | elif stat.S_ISREG(tree_path_stat.st_mode): 39 | return (60 * (int(tree_path_stat.st_atime) // 60), 60 * (int(tree_path_stat.st_mtime) // 60)) # minute resolution 40 | else: 41 | raise NotImplementedError 42 | 43 | def get_files_tree(self, tree_path: str, follow_links: bool = False): 44 | statObject = self.lstat(tree_path) 45 | return self._get_files_tree(tree_path, statObject, follow_links = follow_links) 46 | 47 | def remove_tree(self, tree_path: str, tree: Union[Tuple[int, int], dict], dry_run: bool = True) -> None: 48 | if isinstance(tree, tuple): 49 | logging.info(f"Removing {tree_path}") 50 | if not dry_run: 51 | self.unlink(tree_path) 52 | elif isinstance(tree, dict): 53 | remove_folder = tree.pop(".", False) 54 | for key, value in tree.items(): 55 | self.remove_tree(self.normpath(self.join(tree_path, key)), value, dry_run = dry_run) 56 | if remove_folder: 57 | logging.info(f"Removing folder {tree_path}") 58 | if not dry_run: 59 | self.rmdir(tree_path) 60 | else: 61 | raise NotImplementedError 62 | 63 | def push_tree_here(self, 64 | tree_path: str, 65 | relative_tree_path: str, # for logging paths of files / folders copied relative to the source root / destination root 66 | # nicely instead of repeating the root every time; rsync does this nice logging 67 | tree: Union[Tuple[int, int], dict], 68 | destination_root: str, 69 | fs_source: FileSystem, 70 | dry_run: bool = True, 71 | show_progress: bool = False 72 | ) -> None: 73 | if isinstance(tree, tuple): 74 | if dry_run: 75 | logging.info(f"{relative_tree_path}") 76 | else: 77 | if not show_progress: 78 | # log this instead of letting adb display output 79 | logging.info(f"{relative_tree_path}") 80 | self.push_file_here(tree_path, destination_root, show_progress = show_progress) 81 | self.utime(destination_root, tree) 82 | elif isinstance(tree, dict): 83 | try: 84 | tree.pop(".") # directory needs making 85 | logging.info(f"{relative_tree_path}{self.sep}") 86 | if not dry_run: 87 | self.makedirs(destination_root) 88 | except KeyError: 89 | pass 90 | for key, value in tree.items(): 91 | self.push_tree_here( 92 | fs_source.normpath(fs_source.join(tree_path, key)), 93 | fs_source.join(relative_tree_path, key), 94 | value, 95 | self.normpath(self.join(destination_root, key)), 96 | fs_source, 97 | dry_run = dry_run, 98 | show_progress = show_progress 99 | ) 100 | else: 101 | raise NotImplementedError 102 | 103 | # Abstract methods below implemented in Local.py and Android.py 104 | 105 | @property 106 | def sep(self) -> str: 107 | raise NotImplementedError 108 | 109 | def unlink(self, path: str) -> None: 110 | raise NotImplementedError 111 | 112 | def rmdir(self, path: str) -> None: 113 | raise NotImplementedError 114 | 115 | def makedirs(self, path: str) -> None: 116 | raise NotImplementedError 117 | 118 | def realpath(self, path: str) -> str: 119 | raise NotImplementedError 120 | 121 | def lstat(self, path: str) -> os.stat_result: 122 | raise NotImplementedError 123 | 124 | def lstat_in_dir(self, path: str) -> Iterable[Tuple[str, os.stat_result]]: 125 | raise NotImplementedError 126 | 127 | def utime(self, path: str, times: Tuple[int, int]) -> None: 128 | raise NotImplementedError 129 | 130 | def join(self, base: str, leaf: str) -> str: 131 | raise NotImplementedError 132 | 133 | def split(self, path: str) -> Tuple[str, str]: 134 | raise NotImplementedError 135 | 136 | def normpath(self, path: str) -> str: 137 | raise NotImplementedError 138 | 139 | def push_file_here(self, source: str, destination: str, show_progress: bool = False) -> None: 140 | raise NotImplementedError 141 | -------------------------------------------------------------------------------- /src/BetterADBSync/FileSystems/Local.py: -------------------------------------------------------------------------------- 1 | from typing import Iterable, Tuple 2 | import os 3 | import subprocess 4 | 5 | from ..SAOLogging import logging_fatal 6 | 7 | from .Base import FileSystem 8 | 9 | class LocalFileSystem(FileSystem): 10 | @property 11 | def sep(self) -> str: 12 | return os.path.sep 13 | 14 | def unlink(self, path: str) -> None: 15 | os.unlink(path) 16 | 17 | def rmdir(self, path: str) -> None: 18 | os.rmdir(path) 19 | 20 | def makedirs(self, path: str) -> None: 21 | os.makedirs(path, exist_ok = True) 22 | 23 | def realpath(self, path: str) -> str: 24 | return os.path.realpath(path) 25 | 26 | def lstat(self, path: str) -> os.stat_result: 27 | return os.lstat(path) 28 | 29 | def lstat_in_dir(self, path: str) -> Iterable[Tuple[str, os.stat_result]]: 30 | for filename in os.listdir(path): 31 | yield filename, self.lstat(self.join(path, filename)) 32 | 33 | def utime(self, path: str, times: Tuple[int, int]) -> None: 34 | os.utime(path, times) 35 | 36 | def join(self, base: str, leaf: str) -> str: 37 | return os.path.join(base, leaf) 38 | 39 | def split(self, path: str) -> Tuple[str, str]: 40 | return os.path.split(path) 41 | 42 | def normpath(self, path: str) -> str: 43 | return os.path.normpath(path) 44 | 45 | def push_file_here(self, source: str, destination: str, show_progress: bool = False) -> None: 46 | if show_progress: 47 | kwargs_call = {} 48 | else: 49 | kwargs_call = { 50 | "stdout": subprocess.DEVNULL, 51 | "stderr": subprocess.DEVNULL 52 | } 53 | if subprocess.call(self.adb_arguments + ["pull", source, destination], **kwargs_call): 54 | logging_fatal("Non-zero exit code from adb pull") 55 | -------------------------------------------------------------------------------- /src/BetterADBSync/FileSystems/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jb2170/better-adb-sync/2d19f23b2ae8a74e4d4e038df049112408335a6b/src/BetterADBSync/FileSystems/__init__.py -------------------------------------------------------------------------------- /src/BetterADBSync/SAOLogging.py: -------------------------------------------------------------------------------- 1 | """Nice logging, with colors on Linux.""" 2 | 3 | from typing import Any, Union 4 | import logging 5 | import sys 6 | 7 | class ColoredFormatter(logging.Formatter): 8 | """Logging Formatter to add colors""" 9 | 10 | fg_bright_blue = "\x1b[94m" 11 | fg_yellow = "\x1b[33m" 12 | fg_red = "\x1b[31m" 13 | fg_bright_red_bold = "\x1b[91;1m" 14 | reset = "\x1b[0m" 15 | 16 | def __init__(self, fmt, datefmt): 17 | super().__init__() 18 | self.messagefmt = fmt 19 | self.datefmt = datefmt 20 | 21 | self.formats = { 22 | logging.DEBUG: "{}{}{}".format(self.fg_bright_blue, self.messagefmt, self.reset), 23 | logging.INFO: "{}".format(self.messagefmt), 24 | logging.WARNING: "{}{}{}".format(self.fg_yellow, self.messagefmt, self.reset), 25 | logging.ERROR: "{}{}{}".format(self.fg_red, self.messagefmt, self.reset), 26 | logging.CRITICAL: "{}{}{}".format(self.fg_bright_red_bold, self.messagefmt, self.reset) 27 | } 28 | 29 | self.formatters = { 30 | logging.DEBUG: logging.Formatter(self.formats[logging.DEBUG], datefmt = self.datefmt), 31 | logging.INFO: logging.Formatter(self.formats[logging.INFO], datefmt = self.datefmt), 32 | logging.WARNING: logging.Formatter(self.formats[logging.WARNING], datefmt = self.datefmt), 33 | logging.ERROR: logging.Formatter(self.formats[logging.ERROR], datefmt = self.datefmt), 34 | logging.CRITICAL: logging.Formatter(self.formats[logging.CRITICAL], datefmt = self.datefmt) 35 | } 36 | 37 | def format(self, record): 38 | formatter = self.formatters[record.levelno] 39 | return formatter.format(record) 40 | 41 | def setup_root_logger( 42 | no_color: bool = False, 43 | verbosity_level: int = 0, 44 | quietness_level: int = 0, 45 | messagefmt: str = "[%(asctime)s][%(levelname)s] %(message)s (%(filename)s:%(lineno)d)", 46 | messagefmt_verbose: str = "[%(asctime)s][%(levelname)s] %(message)s (%(filename)s:%(lineno)d)", 47 | datefmt: str = "%Y-%m-%d %H:%M:%S" 48 | ): 49 | messagefmt_to_use = messagefmt_verbose if verbosity_level else messagefmt 50 | logging_level = 10 * (2 + quietness_level - verbosity_level) 51 | if not no_color and sys.platform == "linux": 52 | formatter_class = ColoredFormatter 53 | else: 54 | formatter_class = logging.Formatter 55 | 56 | root_logger = logging.getLogger() 57 | root_logger.setLevel(logging_level) 58 | console_handler = logging.StreamHandler() 59 | console_handler.setFormatter(formatter_class(fmt = messagefmt_to_use, datefmt = datefmt)) 60 | root_logger.addHandler(console_handler) 61 | 62 | def logging_fatal(message, log_stack_info: bool = True, exit_code: int = 1): 63 | logging.critical(message) 64 | logging.debug("Stack Trace", stack_info = log_stack_info) 65 | logging.critical("Exiting") 66 | raise SystemExit(exit_code) 67 | 68 | def log_tree(title, tree, finals = None, log_leaves_types = True, logging_level = logging.INFO): 69 | """Log tree nicely if it is a dictionary. 70 | log_leaves_types can be False to log no leaves, True to log all leaves, or a tuple of types for which to log.""" 71 | if finals is None: 72 | finals = [] 73 | if not isinstance(tree, dict): 74 | logging.log(msg = "{}{}{}".format( 75 | "".join([" " if final else "│" for final in finals[:-1]] + ["└" if final else "├" for final in finals[-1:]]), 76 | title, 77 | ": {}".format(tree) if log_leaves_types is not False and (log_leaves_types is True or isinstance(tree, log_leaves_types)) else "" 78 | ), level = logging_level) 79 | else: 80 | logging.log(msg = "{}{}".format( 81 | "".join([" " if final else "│" for final in finals[:-1]] + ["└" if final else "├" for final in finals[-1:]]), 82 | title 83 | ), level = logging_level) 84 | tree_items = list(tree.items()) 85 | for key, value in tree_items[:-1]: 86 | log_tree(key, value, finals = finals + [False], log_leaves_types = log_leaves_types, logging_level = logging_level) 87 | for key, value in tree_items[-1:]: 88 | log_tree(key, value, finals = finals + [True], log_leaves_types = log_leaves_types, logging_level = logging_level) 89 | 90 | # like logging.CRITICAl, logging.DEBUG etc 91 | FATAL = 60 92 | 93 | def perror(s: Union[str, Any], e: Exception, logging_level: int = logging.ERROR): 94 | strerror = e.strerror if (isinstance(e, OSError) and e.strerror is not None) else e.__class__.__name__ 95 | msg = f"{s}{': ' if s else ''}{strerror}" 96 | if logging_level == FATAL: 97 | logging_fatal(msg) 98 | else: 99 | logging.log(logging_level, msg) 100 | -------------------------------------------------------------------------------- /src/BetterADBSync/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | """Sync files between a computer and an Android device""" 4 | 5 | __version__ = "1.4.0" 6 | 7 | from typing import List, Tuple, Union 8 | import logging 9 | import os 10 | import stat 11 | import fnmatch 12 | 13 | from .argparsing import get_cli_args 14 | from .SAOLogging import logging_fatal, log_tree, setup_root_logger, perror, FATAL 15 | 16 | from .FileSystems.Base import FileSystem 17 | from .FileSystems.Local import LocalFileSystem 18 | from .FileSystems.Android import AndroidFileSystem 19 | 20 | class FileSyncer(): 21 | @classmethod 22 | def diff_trees(cls, 23 | source: Union[dict, Tuple[int, int], None], 24 | destination: Union[dict, Tuple[int, int], None], 25 | path_source: str, 26 | path_destination: str, 27 | destination_exclude_patterns: List[str], 28 | path_join_function_source, 29 | path_join_function_destination, 30 | folder_file_overwrite_error: bool = True, 31 | ) -> Tuple[ 32 | Union[dict, Tuple[int, int], None], # delete 33 | Union[dict, Tuple[int, int], None], # copy 34 | Union[dict, Tuple[int, int], None], # excluded_source 35 | Union[dict, Tuple[int, int], None], # unaccounted_destination 36 | Union[dict, Tuple[int, int], None] # excluded_destination 37 | ]: 38 | 39 | exclude = False 40 | for destination_exclude_pattern in destination_exclude_patterns: 41 | if fnmatch.fnmatch(path_destination, destination_exclude_pattern): 42 | exclude = True 43 | break 44 | 45 | if source is None: 46 | if destination is None: 47 | delete = None 48 | copy = None 49 | excluded_source = None 50 | unaccounted_destination = None 51 | excluded_destination = None 52 | elif isinstance(destination, tuple): 53 | if exclude: 54 | delete = None 55 | copy = None 56 | excluded_source = None 57 | unaccounted_destination = None 58 | excluded_destination = destination 59 | else: 60 | delete = None 61 | copy = None 62 | excluded_source = None 63 | unaccounted_destination = destination 64 | excluded_destination = None 65 | elif isinstance(destination, dict): 66 | if exclude: 67 | delete = {".": None} 68 | copy = None 69 | excluded_source = None 70 | unaccounted_destination = {".": None} 71 | excluded_destination = destination 72 | else: 73 | delete = {".": None} 74 | copy = None 75 | excluded_source = None 76 | unaccounted_destination = {".": destination["."]} 77 | excluded_destination = {".": None} 78 | destination.pop(".") 79 | for key, value in destination.items(): 80 | delete[key], _, _, unaccounted_destination[key], excluded_destination[key] = cls.diff_trees( 81 | None, 82 | value, 83 | path_join_function_source(path_source, key), 84 | path_join_function_destination(path_destination, key), 85 | destination_exclude_patterns, 86 | path_join_function_source, 87 | path_join_function_destination, 88 | folder_file_overwrite_error = folder_file_overwrite_error 89 | ) 90 | else: 91 | raise NotImplementedError 92 | 93 | elif isinstance(source, tuple): 94 | if destination is None: 95 | if exclude: 96 | delete = None 97 | copy = None 98 | excluded_source = source 99 | unaccounted_destination = None 100 | excluded_destination = None 101 | else: 102 | delete = None 103 | copy = source 104 | excluded_source = None 105 | unaccounted_destination = None 106 | excluded_destination = None 107 | elif isinstance(destination, tuple): 108 | if exclude: 109 | delete = None 110 | copy = None 111 | excluded_source = source 112 | unaccounted_destination = None 113 | excluded_destination = destination 114 | else: 115 | if source[1] > destination[1]: 116 | delete = destination 117 | copy = source 118 | excluded_source = None 119 | unaccounted_destination = None 120 | excluded_destination = None 121 | else: 122 | delete = None 123 | copy = None 124 | excluded_source = None 125 | unaccounted_destination = None 126 | excluded_destination = None 127 | elif isinstance(destination, dict): 128 | if exclude: 129 | delete = {".": None} 130 | copy = None 131 | excluded_source = source 132 | unaccounted_destination = {".": None} 133 | excluded_destination = destination 134 | else: 135 | delete = destination 136 | copy = source 137 | excluded_source = None 138 | unaccounted_destination = {".": None} 139 | excluded_destination = {".": None} 140 | if folder_file_overwrite_error: 141 | logging.critical(f"Refusing to overwrite directory {path_destination} with file {path_source}") 142 | logging_fatal("Use --force if you are sure!") 143 | else: 144 | logging.warning(f"Overwriting directory {path_destination} with file {path_source}") 145 | else: 146 | raise NotImplementedError 147 | 148 | elif isinstance(source, dict): 149 | if destination is None: 150 | if exclude: 151 | delete = None 152 | copy = {".": None} 153 | excluded_source = source 154 | unaccounted_destination = None 155 | excluded_destination = None 156 | else: 157 | delete = None 158 | copy = {".": source["."]} 159 | excluded_source = {".": None} 160 | unaccounted_destination = None 161 | excluded_destination = None 162 | source.pop(".") 163 | for key, value in source.items(): 164 | _, copy[key], excluded_source[key], _, _ = cls.diff_trees( 165 | value, 166 | None, 167 | path_join_function_source(path_source, key), 168 | path_join_function_destination(path_destination, key), 169 | destination_exclude_patterns, 170 | path_join_function_source, 171 | path_join_function_destination, 172 | folder_file_overwrite_error = folder_file_overwrite_error 173 | ) 174 | elif isinstance(destination, tuple): 175 | if exclude: 176 | delete = None 177 | copy = {".": None} 178 | excluded_source = source 179 | unaccounted_destination = None 180 | excluded_destination = destination 181 | else: 182 | delete = destination 183 | copy = {".": source["."]} 184 | excluded_source = {".": None} 185 | unaccounted_destination = None 186 | excluded_destination = None 187 | source.pop(".") 188 | for key, value in source.items(): 189 | _, copy[key], excluded_source[key], _, _ = cls.diff_trees( 190 | value, 191 | None, 192 | path_join_function_source(path_source, key), 193 | path_join_function_destination(path_destination, key), 194 | destination_exclude_patterns, 195 | path_join_function_source, 196 | path_join_function_destination, 197 | folder_file_overwrite_error = folder_file_overwrite_error 198 | ) 199 | if folder_file_overwrite_error: 200 | logging.critical(f"Refusing to overwrite file {path_destination} with directory {path_source}") 201 | logging_fatal("Use --force if you are sure!") 202 | else: 203 | logging.warning(f"Overwriting file {path_destination} with directory {path_source}") 204 | excluded_destination = None 205 | elif isinstance(destination, dict): 206 | if exclude: 207 | delete = {".": None} 208 | copy = {".": None} 209 | excluded_source = source 210 | unaccounted_destination = {".": None} 211 | excluded_destination = destination 212 | else: 213 | delete = {".": None} 214 | copy = {".": None} 215 | excluded_source = {".": None} 216 | unaccounted_destination = {".": None} 217 | excluded_destination = {".": None} 218 | source.pop(".") 219 | for key, value in source.items(): 220 | delete[key], copy[key], excluded_source[key], unaccounted_destination[key], excluded_destination[key] = cls.diff_trees( 221 | value, 222 | destination.pop(key, None), 223 | path_join_function_source(path_source, key), 224 | path_join_function_destination(path_destination, key), 225 | destination_exclude_patterns, 226 | path_join_function_source, 227 | path_join_function_destination, 228 | folder_file_overwrite_error = folder_file_overwrite_error 229 | ) 230 | destination.pop(".") 231 | for key, value in destination.items(): 232 | delete[key], _, _, unaccounted_destination[key], excluded_destination[key] = cls.diff_trees( 233 | None, 234 | value, 235 | path_join_function_source(path_source, key), 236 | path_join_function_destination(path_destination, key), 237 | destination_exclude_patterns, 238 | path_join_function_source, 239 | path_join_function_destination, 240 | folder_file_overwrite_error = folder_file_overwrite_error 241 | ) 242 | else: 243 | raise NotImplementedError 244 | 245 | else: 246 | raise NotImplementedError 247 | 248 | return delete, copy, excluded_source, unaccounted_destination, excluded_destination 249 | 250 | @classmethod 251 | def remove_excluded_folders_from_unaccounted_tree(cls, unaccounted: Union[dict, Tuple[int, int]], excluded: Union[dict, None]) -> dict: 252 | # For when we have --del but not --delete-excluded selected; we do not want to delete unaccounted folders that are the 253 | # parent of excluded items. At the point in the program that this function is called at either 254 | # 1) unaccounted is a tuple (file) and excluded is None 255 | # 2) unaccounted is a dict and excluded is a dict or None 256 | # trees passed to this function are already pruned; empty dictionary (sub)trees don't exist 257 | if excluded is None: 258 | return unaccounted 259 | else: 260 | unaccounted_non_excluded = {} 261 | for unaccounted_key, unaccounted_value in unaccounted.items(): 262 | if unaccounted_key == ".": 263 | continue 264 | unaccounted_non_excluded[unaccounted_key] = cls.remove_excluded_folders_from_unaccounted_tree( 265 | unaccounted_value, 266 | excluded.get(unaccounted_key, None) 267 | ) 268 | return unaccounted_non_excluded 269 | 270 | @classmethod 271 | def prune_tree(cls, tree): 272 | """Remove all Nones from a tree. May return None if tree is None however.""" 273 | if not isinstance(tree, dict): 274 | return tree 275 | else: 276 | return_dict = {} 277 | for key, value in tree.items(): 278 | value_pruned = cls.prune_tree(value) 279 | if value_pruned is not None: 280 | return_dict[key] = value_pruned 281 | return return_dict or None 282 | 283 | @classmethod 284 | def sort_tree(cls, tree): 285 | if not isinstance(tree, dict): 286 | return tree 287 | return { 288 | k: cls.sort_tree(v) 289 | for k, v in sorted(tree.items()) 290 | } 291 | 292 | @classmethod 293 | def paths_to_fixed_destination_paths(cls, 294 | path_source: str, 295 | fs_source: FileSystem, 296 | path_destination: str, 297 | fs_destination: FileSystem 298 | ) -> Tuple[str, str]: 299 | """Modify sync paths according to how a trailing slash on the source path should be treated""" 300 | # TODO I'm not exactly sure if this covers source and destination being symlinks (lstat vs stat etc) 301 | # we only need to consider when the destination is a directory 302 | try: 303 | lstat_destination = fs_destination.lstat(path_destination) 304 | except FileNotFoundError: 305 | return path_source, path_destination 306 | except (NotADirectoryError, PermissionError) as e: 307 | perror(path_source, e, FATAL) 308 | 309 | if stat.S_ISLNK(lstat_destination.st_mode): 310 | logging_fatal("Destination is a symlink. Not sure what to do. See GitHub issue #8") 311 | 312 | if not stat.S_ISDIR(lstat_destination.st_mode): 313 | return path_source, path_destination 314 | 315 | # we know the destination is a directory at this point 316 | try: 317 | lstat_source = fs_source.lstat(path_source) 318 | except FileNotFoundError: 319 | return path_source, path_destination 320 | except (NotADirectoryError, PermissionError) as e: 321 | perror(path_source, e, FATAL) 322 | 323 | if stat.S_ISREG(lstat_source.st_mode) or (stat.S_ISDIR(lstat_source.st_mode) and path_source[-1] not in ["/", "\\"]): 324 | path_destination = fs_destination.join( 325 | path_destination, 326 | fs_destination.split(path_source)[1] 327 | ) 328 | return path_source, path_destination 329 | 330 | def main(): 331 | args = get_cli_args(__doc__, __version__) 332 | 333 | setup_root_logger( 334 | no_color = args.logging_no_color, 335 | verbosity_level = args.logging_verbosity_verbose, 336 | quietness_level = args.logging_verbosity_quiet, 337 | messagefmt = "[%(levelname)s] %(message)s" if os.name == "nt" else "%(message)s" 338 | ) 339 | 340 | for exclude_from_pathname in args.exclude_from: 341 | with exclude_from_pathname.open("r") as f: 342 | args.exclude.extend(line for line in f.read().splitlines() if line) 343 | 344 | adb_arguments = [args.adb_bin] + [f"-{arg}" for arg in args.adb_flags] 345 | for option, value in args.adb_options: 346 | adb_arguments.append(f"-{option}") 347 | adb_arguments.append(value) 348 | 349 | fs_android = AndroidFileSystem(adb_arguments, args.adb_encoding) 350 | fs_local = LocalFileSystem(adb_arguments) 351 | 352 | try: 353 | fs_android.test_connection() 354 | except BrokenPipeError: 355 | logging_fatal("Connection test failed") 356 | 357 | if args.direction == "push": 358 | path_source = args.direction_push_local 359 | fs_source = fs_local 360 | path_destination = args.direction_push_android 361 | fs_destination = fs_android 362 | else: 363 | path_source = args.direction_pull_android 364 | fs_source = fs_android 365 | path_destination = args.direction_pull_local 366 | fs_destination = fs_local 367 | 368 | path_source, path_destination = FileSyncer.paths_to_fixed_destination_paths(path_source, fs_source, path_destination, fs_destination) 369 | 370 | path_source = fs_source.normpath(path_source) 371 | path_destination = fs_destination.normpath(path_destination) 372 | 373 | try: 374 | files_tree_source = fs_source.get_files_tree(path_source, follow_links = args.copy_links) 375 | except (FileNotFoundError, NotADirectoryError, PermissionError) as e: 376 | perror(path_source, e, FATAL) 377 | 378 | try: 379 | files_tree_destination = fs_destination.get_files_tree(path_destination, follow_links = args.copy_links) 380 | except FileNotFoundError: 381 | files_tree_destination = None 382 | except (NotADirectoryError, PermissionError) as e: 383 | perror(path_destination, e, FATAL) 384 | 385 | logging.info("Source tree:") 386 | if files_tree_source is not None: 387 | log_tree(path_source, files_tree_source) 388 | logging.info("") 389 | 390 | logging.info("Destination tree:") 391 | if files_tree_destination is not None: 392 | log_tree(path_destination, files_tree_destination) 393 | logging.info("") 394 | 395 | if isinstance(files_tree_source, dict): 396 | excludePatterns = [fs_destination.normpath( 397 | fs_destination.join(path_destination, exclude) 398 | ) for exclude in args.exclude] 399 | else: 400 | excludePatterns = [fs_destination.normpath( 401 | path_destination + exclude 402 | ) for exclude in args.exclude] 403 | logging.debug("Exclude patterns:") 404 | logging.debug(excludePatterns) 405 | logging.debug("") 406 | 407 | tree_delete, tree_copy, tree_excluded_source, tree_unaccounted_destination, tree_excluded_destination = FileSyncer.diff_trees( 408 | files_tree_source, 409 | files_tree_destination, 410 | path_source, 411 | path_destination, 412 | excludePatterns, 413 | fs_source.join, 414 | fs_destination.join, 415 | folder_file_overwrite_error = not args.dry_run and not args.force 416 | ) 417 | 418 | tree_delete = FileSyncer.prune_tree(tree_delete) 419 | tree_copy = FileSyncer.prune_tree(tree_copy) 420 | tree_excluded_source = FileSyncer.prune_tree(tree_excluded_source) 421 | tree_unaccounted_destination = FileSyncer.prune_tree(tree_unaccounted_destination) 422 | tree_excluded_destination = FileSyncer.prune_tree(tree_excluded_destination) 423 | 424 | tree_delete = FileSyncer.sort_tree(tree_delete) 425 | tree_copy = FileSyncer.sort_tree(tree_copy) 426 | tree_excluded_source = FileSyncer.sort_tree(tree_excluded_source) 427 | tree_unaccounted_destination = FileSyncer.sort_tree(tree_unaccounted_destination) 428 | tree_excluded_destination = FileSyncer.sort_tree(tree_excluded_destination) 429 | 430 | logging.info("Delete tree:") 431 | if tree_delete is not None: 432 | log_tree(path_destination, tree_delete, log_leaves_types = False) 433 | logging.info("") 434 | 435 | logging.info("Copy tree:") 436 | if tree_copy is not None: 437 | log_tree(f"{path_source} --> {path_destination}", tree_copy, log_leaves_types = False) 438 | logging.info("") 439 | 440 | logging.info("Source excluded tree:") 441 | if tree_excluded_source is not None: 442 | log_tree(path_source, tree_excluded_source, log_leaves_types = False) 443 | logging.info("") 444 | 445 | logging.info("Destination unaccounted tree:") 446 | if tree_unaccounted_destination is not None: 447 | log_tree(path_destination, tree_unaccounted_destination, log_leaves_types = False) 448 | logging.info("") 449 | 450 | logging.info("Destination excluded tree:") 451 | if tree_excluded_destination is not None: 452 | log_tree(path_destination, tree_excluded_destination, log_leaves_types = False) 453 | logging.info("") 454 | 455 | 456 | tree_unaccounted_destination_non_excluded = None 457 | if tree_unaccounted_destination is not None: 458 | tree_unaccounted_destination_non_excluded = FileSyncer.prune_tree( 459 | FileSyncer.remove_excluded_folders_from_unaccounted_tree( 460 | tree_unaccounted_destination, 461 | tree_excluded_destination 462 | ) 463 | ) 464 | 465 | logging.info("Non-excluded-supporting destination unaccounted tree:") 466 | if tree_unaccounted_destination_non_excluded is not None: 467 | log_tree(path_destination, tree_unaccounted_destination_non_excluded, log_leaves_types = False) 468 | logging.info("") 469 | 470 | logging.info("SYNCING") 471 | logging.info("") 472 | 473 | if tree_delete is not None: 474 | logging.info("Deleting delete tree") 475 | fs_destination.remove_tree(path_destination, tree_delete, dry_run = args.dry_run) 476 | else: 477 | logging.info("Empty delete tree") 478 | logging.info("") 479 | 480 | if args.delete_excluded and args.delete: 481 | if tree_excluded_destination is not None: 482 | logging.info("Deleting destination excluded tree") 483 | fs_destination.remove_tree(path_destination, tree_excluded_destination, dry_run = args.dry_run) 484 | else: 485 | logging.info("Empty destination excluded tree") 486 | logging.info("") 487 | if tree_unaccounted_destination is not None: 488 | logging.info("Deleting destination unaccounted tree") 489 | fs_destination.remove_tree(path_destination, tree_unaccounted_destination, dry_run = args.dry_run) 490 | else: 491 | logging.info("Empty destination unaccounted tree") 492 | logging.info("") 493 | elif args.delete_excluded: 494 | if tree_excluded_destination is not None: 495 | logging.info("Deleting destination excluded tree") 496 | fs_destination.remove_tree(path_destination, tree_excluded_destination, dry_run = args.dry_run) 497 | else: 498 | logging.info("Empty destination excluded tree") 499 | logging.info("") 500 | elif args.delete: 501 | if tree_unaccounted_destination_non_excluded is not None: 502 | logging.info("Deleting non-excluded-supporting destination unaccounted tree") 503 | fs_destination.remove_tree(path_destination, tree_unaccounted_destination_non_excluded, dry_run = args.dry_run) 504 | else: 505 | logging.info("Empty non-excluded-supporting destination unaccounted tree") 506 | logging.info("") 507 | 508 | if tree_copy is not None: 509 | logging.info("Copying copy tree") 510 | fs_destination.push_tree_here( 511 | path_source, 512 | fs_destination.split(path_source)[1] if isinstance(tree_copy, tuple) else ".", 513 | tree_copy, 514 | path_destination, 515 | fs_source, 516 | dry_run = args.dry_run, 517 | show_progress = args.show_progress 518 | ) 519 | else: 520 | logging.info("Empty copy tree") 521 | logging.info("") 522 | 523 | if __name__ == "__main__": 524 | main() 525 | -------------------------------------------------------------------------------- /src/BetterADBSync/argparsing.py: -------------------------------------------------------------------------------- 1 | from typing import List, Optional 2 | from dataclasses import dataclass 3 | import argparse 4 | from pathlib import Path 5 | 6 | @dataclass 7 | class Args(): 8 | logging_no_color: bool 9 | logging_verbosity_verbose: int 10 | logging_verbosity_quiet: int 11 | 12 | dry_run: bool 13 | copy_links: bool 14 | exclude: List[str] 15 | exclude_from: List[Path] 16 | delete: bool 17 | delete_excluded: bool 18 | force: bool 19 | show_progress: bool 20 | adb_encoding: str 21 | 22 | adb_bin: str 23 | adb_flags: List[str] 24 | adb_options: List[List[str]] 25 | 26 | direction: str 27 | 28 | direction_push_local: Optional[str] 29 | direction_push_android: Optional[str] 30 | 31 | direction_pull_android: Optional[str] 32 | direction_pull_local: Optional[str] 33 | 34 | def get_cli_args(docstring: str, version: str) -> Args: 35 | parser = argparse.ArgumentParser(description = docstring) 36 | parser.add_argument("--version", 37 | action = "version", 38 | version = version 39 | ) 40 | 41 | parser_logging = parser.add_argument_group(title = "logging") 42 | parser_logging.add_argument("--no-color", 43 | help = "Disable colored logging (Linux only)", 44 | action = "store_true", 45 | dest = "logging_no_color" 46 | ) 47 | parser_logging_verbosity = parser_logging.add_mutually_exclusive_group(required = False) 48 | parser_logging_verbosity.add_argument("-v", "--verbose", 49 | help = "Increase logging verbosity: -v for debug", 50 | action = "count", 51 | dest = "logging_verbosity_verbose", 52 | default = 0 53 | ) 54 | parser_logging_verbosity.add_argument("-q", "--quiet", 55 | help = "Decrease logging verbosity: -q for warning, -qq for error, -qqq for critical, -qqqq for no logging messages", 56 | action = "count", 57 | dest = "logging_verbosity_quiet", 58 | default = 0 59 | ) 60 | 61 | parser.add_argument("-n", "--dry-run", 62 | help = "Perform a dry run; do not actually copy and delete etc", 63 | action = "store_true", 64 | dest = "dry_run" 65 | ) 66 | parser.add_argument("-L", "--copy-links", 67 | help = "Follow symlinks and copy their referent file / directory", 68 | action = "store_true", 69 | dest = "copy_links" 70 | ) 71 | parser.add_argument("--exclude", 72 | help = "fnmatch pattern to ignore relative to source (reusable)", 73 | action = "append", 74 | dest = "exclude", 75 | default = [] 76 | ) 77 | parser.add_argument("--exclude-from", 78 | help = "Filename of file containing fnmatch patterns to ignore relative to source (reusable)", 79 | metavar = "EXCLUDE_FROM", 80 | type = Path, 81 | action = "append", 82 | dest = "exclude_from", 83 | default = [] 84 | ) 85 | parser.add_argument("--del", 86 | help = "Delete files at the destination that are not in the source", 87 | action = "store_true", 88 | dest = "delete" 89 | ) 90 | parser.add_argument("--delete-excluded", 91 | help = "Delete files at the destination that are excluded", 92 | action = "store_true", 93 | dest = "delete_excluded" 94 | ) 95 | parser.add_argument("--force", 96 | help = "Allows files to overwrite folders and folders to overwrite files. This is false by default to prevent large scale accidents", 97 | action = "store_true", 98 | dest = "force" 99 | ) 100 | parser.add_argument("--show-progress", 101 | help = "Show progress from 'adb push' and 'adb pull' commands", 102 | action = "store_true", 103 | dest = "show_progress" 104 | ) 105 | parser.add_argument("--adb-encoding", 106 | help = "Which encoding to use when talking to adb. Defaults to UTF-8. Relevant to GitHub issue #22", 107 | dest = "adb_encoding", 108 | default = "UTF-8" 109 | ) 110 | 111 | parser_adb = parser.add_argument_group(title = "ADB arguments", 112 | description = "By default ADB works for me without touching any of these, but if you have any specific demands then go ahead. See 'adb --help' for a full list of adb flags and options" 113 | ) 114 | parser_adb.add_argument("--adb-bin", 115 | help = "Use the given adb binary. Defaults to 'adb' ie whatever is on path", 116 | dest = "adb_bin", 117 | default = "adb") 118 | parser_adb.add_argument("--adb-flag", 119 | help = "Add a flag to call adb with, eg '--adb-flag d' for adb -d, that is return an error if more than one device is connected", 120 | metavar = "ADB_FLAG", 121 | action = "append", 122 | dest = "adb_flags", 123 | default = [] 124 | ) 125 | parser_adb.add_argument("--adb-option", 126 | help = "Add an option to call adb with, eg '--adb-option P 5037' for adb -P 5037, that is use port 5037 for the adb server", 127 | metavar = ("OPTION", "VALUE"), 128 | nargs = 2, 129 | action = "append", 130 | dest = "adb_options", 131 | default = [] 132 | ) 133 | 134 | parser_direction = parser.add_subparsers(title = "direction", 135 | dest = "direction", 136 | required = True 137 | ) 138 | 139 | parser_direction_push = parser_direction.add_parser("push", 140 | help = "Push from computer to phone" 141 | ) 142 | parser_direction_push.add_argument("direction_push_local", 143 | metavar = "LOCAL", 144 | help = "Local path" 145 | ) 146 | parser_direction_push.add_argument("direction_push_android", 147 | metavar = "ANDROID", 148 | help = "Android path" 149 | ) 150 | 151 | parser_direction_pull = parser_direction.add_parser("pull", 152 | help = "Pull from phone to computer" 153 | ) 154 | parser_direction_pull.add_argument("direction_pull_android", 155 | metavar = "ANDROID", 156 | help = "Android path" 157 | ) 158 | parser_direction_pull.add_argument("direction_pull_local", 159 | metavar = "LOCAL", 160 | help = "Local path" 161 | ) 162 | 163 | args = parser.parse_args() 164 | 165 | if args.direction == "push": 166 | args_direction_ = ( 167 | args.direction_push_local, 168 | args.direction_push_android, 169 | None, 170 | None 171 | ) 172 | else: 173 | args_direction_ = ( 174 | None, 175 | None, 176 | args.direction_pull_android, 177 | args.direction_pull_local 178 | ) 179 | 180 | args = Args( 181 | args.logging_no_color, 182 | args.logging_verbosity_verbose, 183 | args.logging_verbosity_quiet, 184 | 185 | args.dry_run, 186 | args.copy_links, 187 | args.exclude, 188 | args.exclude_from, 189 | args.delete, 190 | args.delete_excluded, 191 | args.force, 192 | args.show_progress, 193 | args.adb_encoding, 194 | 195 | args.adb_bin, 196 | args.adb_flags, 197 | args.adb_options, 198 | 199 | args.direction, 200 | *args_direction_ 201 | ) 202 | 203 | return args 204 | --------------------------------------------------------------------------------