├── .gitignore ├── .travis.yml ├── LICENSE ├── README.CN.md ├── README.rst ├── scripts └── deploy.sh ├── setup.py ├── syncrypto ├── __init__.py ├── __main__.py ├── cli.py ├── core.py ├── crypto.py ├── filetree.py ├── package_info.py └── util.py └── tests ├── requirements.txt ├── test_cli.py ├── test_crypto.py ├── test_filetree.py ├── test_rule.py ├── test_sync.py └── util.py /.gitignore: -------------------------------------------------------------------------------- 1 | .idea 2 | *.swp 3 | *.pyc 4 | dist 5 | build 6 | *.egg-info -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: python 2 | python: 3 | - '2.6' 4 | - '2.7' 5 | - '3.3' 6 | - '3.4' 7 | - '3.5' 8 | install: 9 | - if [[ $TRAVIS_PYTHON_VERSION != 2.6 ]]; then pip install pexpect; fi 10 | - pip install -r tests/requirements.txt 11 | - pip install . 12 | script: nosetests --rednose --with-cov 13 | after_success: 14 | - codecov 15 | deploy: 16 | provider: pypi 17 | user: liangqing 18 | password: 19 | secure: eabEthT57DZikaIj8HhPRHA7LX3GcwU+klZch6xATPtrX4U8KLSgmsTL6pgCXfMqTwlFUxuStPexRLr8ZDJP6clGY9sUU4CkWGVdMbnuVQYqj0reY1tM/UE/M5RloApZ+S3P5sH0zBvBiVAl1WZN0sYqf1LHYH4k/ErFIGKGaExPu0Zc5y9JEb+pD8mYmwiM19SjeQknuTREmQUmAyUe/mqbUj7eTh5uhv9zQmfJglRn85mj2wCbcRfiD0YE7BfXVjAICwdjQmdGGxQW13kkDHXfFiPgU9MNu4qsWV/CXihVfTwJ5rz/YdX/07ZIKRyIJaq1onSmblunH6/I7QvMc6RShNtnSBtpYMv51BszOJlytfBQHKpRgL+ekXiij31bS3dOOio/icYpxAuVlkMwsWE3mrk2e3pyi5KUn0D8oW7rcFgLxv2I2NYz7+gmUHi0n7KiwdFa4gwX8KX8Qe4kbeQ7ofcP1DTyZJ6NdK6oNztDeUQ3+rDBiRo7luwB6q5YlkPZfMrr3zN+BsWRpgT23EqErQloOgsb2zMP5nqKDvsJ3h1D5UW2zJ0VcjV3fgnUgnxIZsSrXBvlOcFQO9WTJNK5GSo7qo2X9DHQOjumV9OZJuynqZ9UDYuYHRyms3cciB1gVjFF/A1hZ5EPtUjd+4kGwZvQzhUgOPFAMZsWqxQ= 20 | on: 21 | tags: true -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright 2015 Qing Liang (https://github.com/liangqing) 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. -------------------------------------------------------------------------------- /README.CN.md: -------------------------------------------------------------------------------- 1 | syncrypto - 文件夹加密双向同步 2 | ============================== 3 | 4 | [![最新版本](https://img.shields.io/pypi/v/syncrypto.svg)](https://pypi.python.org/pypi/syncrypto/)[![集成测试](https://travis-ci.org/liangqing/syncrypto.svg?branch=master)](https://travis-ci.org/liangqing/syncrypto)[![代码覆盖率](https://codecov.io/github/liangqing/syncrypto/coverage.svg?branch=master)](https://codecov.io/github/liangqing/syncrypto?branch=master)[![代码健康度](https://landscape.io/github/liangqing/syncrypto/master/landscape.svg?style=flat)](https://landscape.io/github/liangqing/syncrypto/master) 5 | 6 | 7 | ## 介绍 8 | 9 | `syncrypto`可以用来将一个文件夹里面所有文件同步到另外一个加密的文件夹中， 10 | 一般来说，可以这样使用： 11 | 12 | ``` 13 | syncrypto syncrypto 14 | 文件夹A <---------------------> 加密文件夹B <-------------------> 文件夹C 15 | 机器X (可以放在不安全的地方，例如云存储) 机器Y 16 | 17 | ``` 18 | 19 | 加密文件夹B中的文件都是加密过的，所以可以把它放在不怎么安全的地方（例如云盘，公共硬盘等） 20 | 21 | 明文文件和加密文件是一一对应的关系，所以每次同步只会添加（删除/修改）那些需要的文件，这样 22 | 很适合那些基于文件系统的同步工具，例如云盘、rsync等。 23 | 24 | **同步过程是双向的**，所以，文件不仅仅是从明文文件夹同步到加密文件夹，同样也会从加密文件夹同步到 25 | 明文文件夹，`syncrypto`会根据算法选择最新的。 26 | 27 | 如果有冲突的话，`syncrypto`会将明文文件重命名（加上单词conflict），然后将密文文件同步过来 28 | 29 | `syncrypto`是不会删除文件的，如果同步过程中需要删除或者覆盖文件，`syncrypto`会将文件移动 30 | 到废纸篓里面。密文文件夹的废纸篓在_syncrypto/trash下，明文文件夹的废纸篓在 31 | .syncrypto/trash下。密文文件夹废纸篓里面的文件一样是加密过的。 32 | 33 | ## 安装 34 | 35 | ### 支持的平台和系统 36 | 37 | `syncrypto`支持python2, python3，并且在下面的平台下[测试](https://travis-ci.org/liangqing/syncrypto)通过： 38 | 39 | * python2.6 40 | * python2.7 41 | * python3.3 42 | * python3.4 43 | * python3.5 44 | 45 | 支持Windows，Linux，OS X 46 | 47 | ### 安装依赖 48 | 49 | **如果是Windows的话，可以直接跳过** 50 | 51 | 由于依赖[cryptography](https://github.com/pyca/cryptography)，在*Linux*上需要先安装一些依赖: 52 | 53 | 在Debian/Ubuntu系列中运行 54 | ```shell 55 | sudo apt-get install build-essential libssl-dev libffi-dev python-dev 56 | ``` 57 | 或者，在Fedora/RHEL系列中运行 58 | ```shell 59 | sudo yum install gcc libffi-devel python-devel openssl-devel 60 | ``` 61 | 62 | 如果是OS X系统，需要运行 63 | ```shell 64 | xcode-select --install 65 | ``` 66 | 67 | ### 安装与更新 68 | 69 | 安装完所有依赖后，即可通过[pip](https://pip.pypa.io/en/latest/installing.html) 70 | 安装``syncrypto``: 71 | 72 | ```shell 73 | pip install syncrypto 74 | ``` 75 | 76 | 或者通过下面的命令更新 77 | ```shell 78 | pip install -U syncrypto 79 | ``` 80 | 81 | ## 使用 82 | 83 | ### 同步 84 | 85 | ```shell 86 | syncrypto [加密文件夹] [明文文件夹] # 注意，加密文件夹放在前面 87 | ``` 88 | 可以使用这个命令来同步，运行后会提示输入密码，第一次在该加密目录下运行的话是设置密码，之后 89 | 运行的话会进行密码匹配，如果不匹配则不能进行同步（放心，`syncrypto`不会保存明文的密码） 90 | 91 | 如果不想通过命令行交互的方式输入密码，可以通过--password-file选项来通过文件给出密码： 92 | 93 | ```shell 94 | syncrypto [加密文件夹] [明文文件夹] --password-file [密码文件] 95 | ``` 96 | 密码文件里面保存的是明文密码 97 | 98 | ### 为同步添加rule 99 | 100 | 有时候，有些文件（例如一些临时文件）没有必要进行加密同步，这个时候你可以通过rule来排除 101 | 这些文件: 102 | 103 | ```shell 104 | syncrypto [加密文件夹] [明文文件夹] --rule 'ignore: name match *.swp' 105 | ``` 106 | 上面这条命令会在同步过程中忽略那些文件名匹配"*.swp"的文件 107 | 108 | 可以添加多条rule： 109 | 110 | ```shell 111 | syncrypto [加密文件夹] [明文文件夹] --rule 'include: name eq README.md' --rule 'ignore: name match *.md' 112 | ``` 113 | 114 | 上面这条命令会在同步过程中忽略那些文件名匹配"*.md"的文件，但是保留文件名为"README.md"的文件。 115 | 116 | 如果有多条rule的话，会按照顺序优先选择第一条匹配的rule 117 | 118 | 也可以通过文件，而不是命令行的方式配置rule，--rule-file选项可以做到： 119 | 120 | ```shell 121 | syncrypto [加密文件夹] [明文文件夹] --rule-file [rule文件] 122 | ``` 123 | 124 | rule文件可以这样写： 125 | 126 | ``` 127 | include: name eq README.md 128 | 129 | # ignore all markdown files, this is a comment 130 | ignore: name match *.md 131 | ``` 132 | 133 | 默认的--rule-file指向`[明文文件夹]/.syncrypto/rules` 134 | 135 | 如果同时给定了--rule, --rule-file选项，那会--rule指定的规则优先级更高。 136 | 137 | 138 | rule的格式： 139 | ``` 140 | [action]: [file attribute] [operand] [value] 141 | ``` 142 | 143 | `[action]`是指匹配规则后的动作，可以为'include', 'ignore', 'exclude'。 144 | 145 | 'include'表示包含，'ignore'表示忽略，'exclude'和ignore是同样的含义 146 | 147 | `[file attribute]`是指参与匹配的文件的属性，支持： 148 | 149 | * `name`, 文件名，包括扩展名 150 | * `path`, 文件路径，从明文文件夹的根目录算起，例如 "a/b/c.txt" 151 | * `size`, 文件大小 152 | * `ctime`, 文件的change time（windows下指的是创建时间） 153 | * `mtime`, 文件的修改时间 154 | 155 | `[operand]`: 156 | * `eq`, `==` 157 | * `gt`, `>` 158 | * `lt`, `<` 159 | * `gte`, `>=` 160 | * `lte`, `<=` 161 | * `ne`, `!=`, `<>` 162 | * `match`, 通配符匹配 163 | * `regexp`, 正则表达式匹配 164 | 165 | `[value]` 就是参与比较或者匹配的值，如果operand是size的话，默认单位是字节，可以带单位， 166 | 例如K,M,G; 2K表示2048字节。 167 | 168 | 如果是`ctime`, `mtime`的话，时间的格式是："%Y-%m-%d %H:%M:%S" 169 | 170 | ### 加密一个文件 171 | 172 | 如果只想加密一个文件，可以使用： 173 | 174 | ```shell 175 | syncrypto --encrypt-file [文件路径] 176 | ``` 177 | 178 | 这条命令默认会将加密后的文件放在明文文件相同目录下，如果想放到别的地方，可以加上--out-file 179 | 参数: 180 | 181 | ```shell 182 | syncrypto --encrypt-file [明文文件路径] --out-file [加密后文件路径] 183 | ``` 184 | 185 | ### 解密一个文件 186 | 187 | 如果想解密任何一个通过``syncrypto``加密过的文件，可以使用： 188 | 189 | ```shell 190 | syncrypto --decrypt-file [文件路径] 191 | ``` 192 | 193 | 这条命令默认会将解密后的文件放在**当前目录下**，如果想放到别的地方，同样可以加上--out-file 194 | 参数: 195 | 196 | ```shell 197 | syncrypto --decrypt-file [密文文件] --out-file [解密后文件] 198 | ``` 199 | 200 | ### 修改密码 201 | 202 | 修改一个已经加密同步过后的密文目录中的密码 203 | 204 | ```shell 205 | syncrypto --change-password [密文目录] 206 | ``` 207 | 这条命令首先会提示输入当前密码，之后会提示设置新密码，设置成功后会将密文目录下的所有文件 208 | 重新加密一遍。 209 | 210 | 211 | ### 帮助 212 | 213 | ```shell 214 | syncrypto -h 215 | ``` 216 | 217 | 218 | ## License 219 | 220 | Apache License, Version 2.0 221 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | Two-way synchronization between a folder and its ciphertext 2 | =========================================================== 3 | 4 | .. image:: https://img.shields.io/pypi/v/syncrypto.svg 5 | :target: https://pypi.python.org/pypi/syncrypto/ 6 | :alt: Latest Version 7 | 8 | .. image:: https://travis-ci.org/liangqing/syncrypto.svg?branch=master 9 | :target: https://travis-ci.org/liangqing/syncrypto 10 | :alt: Build And Test Status 11 | 12 | .. image:: https://codecov.io/github/liangqing/syncrypto/coverage.svg?branch=master 13 | :target: https://codecov.io/github/liangqing/syncrypto?branch=master 14 | :alt: Code Coverage 15 | 16 | .. image:: https://landscape.io/github/liangqing/syncrypto/master/landscape.svg?style=flat 17 | :target: https://landscape.io/github/liangqing/syncrypto/master 18 | :alt: Code Health 19 | 20 | Introduction 21 | ============ 22 | You can use ``syncrypto`` to encrypt a folder to another folder which contains the 23 | corresponding encrypted content. 24 | 25 | The most common scenario is\: 26 | 27 | .. code-block:: text 28 | 29 | syncrypto syncrypto 30 | plaintext folder A <-------------> encrypted folder B <-----------> plaintext folder C 31 | in machine X in cloud storage in machine Y 32 | 33 | The files in encrypted folder B are encrypted, so you can store it in any unsafe 34 | environment, such as cloud service(Dropbox/OneDrive), USB storage or any other 35 | storage that you can not control. 36 | 37 | Each plaintext file has a corresponding encrypted file in the encrypted folder, 38 | so if you modify one file in plaintext folder, there will be only one file 39 | modified in the encrypted folder after synchronization. This make sure the 40 | synchronization only changes the necessary content in encrypted folder, and is 41 | very useful for file based cloud storage service to synchronizing minimal contents. 42 | 43 | **The synchronization is two-way**, files not only syncing from plain text folder to 44 | encrypted folder, but also syncing from encrypted folder to plain text folder. 45 | ``syncrypto`` will choose the newest file. 46 | 47 | If conflict happens, ``syncrypto`` will rename the plaintext file(add 'conflict' 48 | word in it), and sync the encrypted file. 49 | 50 | ``syncrypto`` never delete files, if files or folders should be deleted or over 51 | written by the syncing algorithm, ``syncrypto`` just move the files or folders 52 | to the trash, the trash in encrypted folder located at _syncrypto/trash, 53 | at .syncrypto/trash in plaintext folder. Files in encrypted folder's trash are 54 | also encrypted. You can delete any files in trash in any time if you make sure 55 | the files in it are useless. 56 | 57 | 58 | Installation 59 | ============ 60 | 61 | Support Platform 62 | ---------------- 63 | 64 | ``syncrypto`` supports both python 2 and python 3, and is tested_ in\: 65 | 66 | .. _tested: https://travis-ci.org/liangqing/syncrypto 67 | 68 | * python 2.6 69 | * python 2.7 70 | * python 3.3 71 | * python 3.4 72 | * python 3.5 73 | 74 | And support Linux, OS X, Windows operating systems 75 | 76 | Install Dependencies 77 | -------------------- 78 | 79 | **If you are using windows, just jump to next** 80 | 81 | Because ``syncrypto`` rely on cryptography_ , so need to install some 82 | dependencies before install ``syncrypto``\: 83 | 84 | .. _cryptography: https://github.com/pyca/cryptography 85 | 86 | For Debian and Ubuntu, the following command will ensure that the required 87 | dependencies are installed\: 88 | 89 | .. code-block:: 90 | 91 | sudo apt-get install build-essential libssl-dev libffi-dev python-dev 92 | 93 | 94 | For Fedora and RHEL-derivatives, the following command will ensure that the 95 | required dependencies are installed\: 96 | 97 | .. code-block:: 98 | 99 | sudo yum install gcc libffi-devel python-devel openssl-devel 100 | 101 | For OS X, run\: 102 | 103 | .. code-block:: 104 | 105 | xcode-select --install 106 | 107 | 108 | Install And Update By pip 109 | ------------------------- 110 | 111 | After installing all dependencies, you can install ``syncrypto`` by pip_ \: 112 | 113 | .. _pip: https://pip.pypa.io/en/latest/installing.html 114 | 115 | .. code-block:: 116 | 117 | pip install syncrypto 118 | 119 | or update by\: 120 | 121 | .. code-block:: 122 | 123 | pip install -U syncrypto 124 | 125 | Usage 126 | ===== 127 | 128 | Synchronization 129 | --------------- 130 | 131 | .. code-block:: 132 | 133 | syncrypto [encrypted folder] [plaintext folder] 134 | 135 | It will prompt you to input a password, if the encrypted folder is empty, 136 | the input password will be set to the encrypted folder, or it will be used 137 | to verify the password you set before (take it easy, ``syncrypto`` never store 138 | plaintext password) 139 | 140 | If you don't want input password in interactive mode, you can use --password-file 141 | option\: 142 | 143 | .. code-block:: 144 | 145 | syncrypto [encrypted folder] [plaintext folder] --password-file [password file path] 146 | 147 | The password file contains the password in it. 148 | 149 | Notice that the first argument is encrypted folder, and the second one is 150 | plaintext folder. 151 | 152 | 153 | Add rule for Synchronization 154 | ---------------------------- 155 | 156 | Sometimes, it is unnecessary to encrypt and sync some files 157 | (for example, some temporary files), 158 | if you want ignore these files, you can add rule\: 159 | 160 | .. code-block:: 161 | 162 | syncrypto [encrypted folder] [plaintext folder] --rule 'ignore: name match *.swp' 163 | 164 | the command above ignores files which name matches \*.swp 165 | 166 | You can add rules multiple times\: 167 | 168 | .. code-block:: 169 | 170 | syncrypto [encrypted folder] [plaintext folder] --rule 'include: name eq README.md' --rule 'ignore: name match *.md' 171 | 172 | the command above ignores files matching "\*.md" but includes files named "README.md". 173 | 174 | The rules are ordered, it means that the rules in front have higher priority than 175 | later, if a rule matches, the matching process will returned immediately. 176 | 177 | You can add rules in a file looks like\: 178 | 179 | .. code-block:: 180 | 181 | include: name eq README.md 182 | 183 | # ignore all markdown files, this is a comment 184 | ignore: name match *.md 185 | 186 | and use the rules by "--rule-file" option\: 187 | 188 | .. code-block:: 189 | 190 | syncrypto [encrypted folder] [plaintext folder] --rule-file [rule file path] 191 | 192 | the default rule file path is "[plaintext folder]/.syncrypto/rules", so you can 193 | add rules in "[plaintext folder]/.syncrypto/rules", and don't need specify the 194 | "--rule-file" option explicitly. 195 | 196 | If you give some rules in command line, and write some rules in rule file at 197 | the same time, the rules in command line will have higher priority than rules 198 | in file. 199 | 200 | The format of a rule\: 201 | 202 | .. code-block:: 203 | 204 | [action]: [file attribute] [operand] [value] 205 | 206 | ``action`` can be ``include``, ``exclude``, ``ignore`` 207 | 208 | ``include`` means the file matching the rule will syncing, ``exclude`` means the 209 | file matching the rule will not syncing. 210 | 211 | ``ignore`` equals ``exclude``. 212 | 213 | ``syncrypto`` supports a lot of file attributes while matching rules, the complete 214 | list is\: 215 | 216 | * ``name``, the name of the file, include file extension. 217 | * ``path``, the relative path from the root of the plaintext folder. 218 | * ``size``, the size of the file 219 | * ``ctime``, the change time of the file, (in windows, it is creation time) 220 | * ``mtime``, the modification time of the file 221 | 222 | operands\: 223 | 224 | * ``eq``, ``==`` 225 | * ``gt``, ``>`` 226 | * ``lt``, ``<`` 227 | * ``gte``, ``>=`` 228 | * ``lte``, ``<=`` 229 | * ``ne``, ``!=``, ``<>`` 230 | * ``match``, match by glob, for example, "\*.md" matches all files end with "md" 231 | * ``regexp``, perform a regular expression match 232 | 233 | The unit of value in ``size`` rules are "byte" by default, you can also use 234 | "K", "M" "G", for example specify the value "2K" means 2048 bytes 235 | 236 | The format of value in ``ctime``, ``mtime`` is "%Y-%m-%d %H:%M:%S" 237 | 238 | Encrypt a file 239 | -------------- 240 | 241 | .. code-block:: 242 | 243 | syncrypto --encrypt-file [plaintext file path] 244 | 245 | This command will encrypt the plaintext file to its parent folder with the filename 246 | add a "encrypted" word 247 | 248 | You can also specify the target encrypted file by --out-file option, such as\: 249 | 250 | .. code-block:: 251 | 252 | syncrypto --encrypt-file [plaintext file path] --out-file [encrypted file path] 253 | 254 | Decrypt a file 255 | -------------- 256 | 257 | .. code-block:: 258 | 259 | syncrypto --decrypt-file [encrypted file path] 260 | 261 | This command will decrypt the encrypted file to **current working directory** 262 | 263 | You can also specify the target plaintext file by --out-file option, such as\: 264 | 265 | .. code-block:: 266 | 267 | syncrypto --decrypt-file [encrypted file path] --out-file [plaintext file path] 268 | 269 | 270 | Change the password 271 | ------------------- 272 | 273 | .. code-block:: 274 | 275 | syncrypto --change-password [encrypted folder] 276 | 277 | Change the password of the encrypted folder, this will re-encrypt all files within 278 | the encrypted folder 279 | 280 | 281 | Show the help 282 | ------------- 283 | 284 | .. code-block:: 285 | 286 | syncrypto -h 287 | 288 | 289 | License 290 | ======= 291 | 292 | Apache License, Version 2.0 293 | -------------------------------------------------------------------------------- /scripts/deploy.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | cd `dirname $0`/.. 4 | 5 | rm -f dist/* 6 | 7 | python setup.py bdist_wheel --universal && twine upload dist/* 8 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup, find_packages 2 | from codecs import open 3 | from os import path 4 | import syncrypto 5 | 6 | here = path.abspath(path.dirname(__file__)) 7 | 8 | with open(path.join(here, 'README.rst'), encoding='utf-8') as f: 9 | long_description = f.read() 10 | 11 | setup( 12 | name='syncrypto', 13 | version=syncrypto.__version__, 14 | description=syncrypto.__doc__, 15 | long_description=long_description, 16 | url='https://github.com/liangqing/syncrypto', 17 | author=syncrypto.__author__, 18 | author_email='liangqing226@gmail.com', 19 | license='http://www.apache.org/licenses/LICENSE-2.0', 20 | classifiers=[ 21 | 'License :: OSI Approved :: Apache Software License', 22 | 'Programming Language :: Python :: 2', 23 | 'Programming Language :: Python :: 2.6', 24 | 'Programming Language :: Python :: 2.7', 25 | 'Programming Language :: Python :: 3', 26 | 'Programming Language :: Python :: 3.3', 27 | 'Programming Language :: Python :: 3.4', 28 | 'Topic :: Communications :: File Sharing', 29 | ], 30 | packages=find_packages(), 31 | install_requires=['cryptography', 'lockfile'], 32 | package_data={ 33 | 'syncrypto': ['README.rst', 'LICENSE'], 34 | }, 35 | entry_points={ 36 | 'console_scripts': [ 37 | 'syncrypto = syncrypto.__main__:main', 38 | ], 39 | }, 40 | ) 41 | -------------------------------------------------------------------------------- /syncrypto/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright 2015 Qing Liang (https://github.com/liangqing) 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | 18 | from .filetree import FileEntry, FileTree, FileRule, FileRuleSet, \ 19 | InvalidRegularExpression 20 | from .crypto import Crypto 21 | from .core import Syncrypto, InvalidFolder 22 | from .core import main as cli 23 | from .package_info import __version__, __author__, __doc__ 24 | -------------------------------------------------------------------------------- /syncrypto/__main__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # 3 | # Copyright 2015 Qing Liang (https://github.com/liangqing) 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | """The main entry point. Invoke as `syncrypto' or `python -m syncrypto'. 18 | 19 | """ 20 | import sys 21 | from .core import main 22 | 23 | 24 | if __name__ == '__main__': 25 | sys.exit(main()) 26 | -------------------------------------------------------------------------------- /syncrypto/cli.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright 2015 Qing Liang (https://github.com/liangqing) 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | 18 | from __future__ import unicode_literals 19 | import argparse 20 | from .package_info import __doc__ as description 21 | from .util import command_text 22 | 23 | 24 | parser = argparse.ArgumentParser( 25 | description=description 26 | ) 27 | 28 | parser.add_argument( 29 | 'encrypted_folder', 30 | help='The encrypted folder', 31 | type=command_text, 32 | nargs='?' 33 | ) 34 | 35 | parser.add_argument( 36 | 'plaintext_folder', 37 | help='The plaintext folder', 38 | type=command_text, 39 | nargs='?' 40 | ) 41 | 42 | parser.add_argument( 43 | '--password-file', 44 | type=command_text, 45 | help=("Use the password in the file instead of " 46 | "getting it from interactive input") 47 | ) 48 | 49 | parser.add_argument( 50 | '--change-password', 51 | action='store_true', 52 | help='Change the password of an encrypted folder' 53 | ) 54 | 55 | parser.add_argument( 56 | '--print-encrypted-tree', 57 | action='store_true', 58 | help='Print the file tree in encrypted folder' 59 | ) 60 | 61 | parser.add_argument( 62 | '--decrypt-file', 63 | type=command_text, 64 | help=('Decrypt a file, it will store the result plaintext file in current ' 65 | 'directory unless you specify --out-file option') 66 | ) 67 | 68 | parser.add_argument( 69 | '--encrypt-file', 70 | type=command_text, 71 | help=('Encrypt a file, it will store the result encrypted file in the same ' 72 | 'directory unless you specify --out-file option') 73 | ) 74 | 75 | parser.add_argument( 76 | '--out-file', 77 | type=command_text, 78 | help=('When encrypting/decrypting a file, ' 79 | 'specify the output file path') 80 | ) 81 | 82 | parser.add_argument( 83 | '--interval', 84 | type=int, 85 | help='Sync directory every interval seconds' 86 | ) 87 | 88 | parser.add_argument( 89 | '--rule-file', 90 | type=command_text, 91 | help='Specify the rule file, default is [plaintext folder]/.syncrypto/rules' 92 | ) 93 | 94 | parser.add_argument( 95 | '--rule', 96 | type=command_text, 97 | action="append", 98 | help='Add include or exclude rules' 99 | ) 100 | 101 | parser.add_argument( 102 | '--debug', 103 | action="store_true", 104 | help='Debug mode' 105 | ) 106 | 107 | parser.add_argument( 108 | '--version', 109 | action="store_true", 110 | help='Display the version' 111 | ) 112 | -------------------------------------------------------------------------------- /syncrypto/core.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright 2015 Qing Liang (https://github.com/liangqing) 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | 18 | from __future__ import print_function 19 | from __future__ import absolute_import 20 | from __future__ import unicode_literals 21 | from io import open 22 | import os 23 | import sys 24 | import os.path 25 | import shutil 26 | import json 27 | from datetime import datetime 28 | from time import sleep, time 29 | from lockfile.mkdirlockfile import MkdirLockFile as LockFile 30 | from random import randint 31 | from stat import S_IWUSR, S_IRUSR 32 | from .crypto import Crypto, DecryptError 33 | from .filetree import FileTree, FileRuleSet, FileEntry 34 | from .util import printable_text, string_digest, getpass 35 | 36 | try: 37 | from cStringIO import StringIO as BytesIO 38 | except ImportError: 39 | from io import BytesIO 40 | 41 | 42 | class GenerateEncryptedFilePathError(Exception): 43 | pass 44 | 45 | 46 | class ChangeTheSamePassword(Exception): 47 | pass 48 | 49 | 50 | class InvalidFolder(Exception): 51 | pass 52 | 53 | 54 | DEFAULT_RULES = b"""ignore: name eq .Trashes 55 | ignore: name eq .fseventsd 56 | ignore: name eq Thumb.db 57 | ignore: name eq node_modules 58 | ignore: name eq .sass-cache 59 | ignore: name eq .idea 60 | ignore: name eq .git 61 | ignore: name eq .svn 62 | ignore: name eq .hg 63 | ignore: name eq .cvs 64 | ignore: name match *.pyc 65 | ignore: name match *.class 66 | ignore: name match .*TemporaryItems 67 | ignore: name match .*DS_Store 68 | ignore: name match *.swp 69 | ignore: name match *.swo""" 70 | 71 | 72 | class Syncrypto(object): 73 | 74 | def __init__(self, crypto, encrypted_folder, plain_folder=None, 75 | encrypted_tree=None, plain_tree=None, snapshot_tree=None, 76 | rule_set=None, rule_file=None, debug=False): 77 | 78 | self.crypto = crypto 79 | self.encrypted_folder = encrypted_folder 80 | self.plain_folder = plain_folder 81 | self.encrypted_tree = encrypted_tree 82 | self.plain_tree = plain_tree 83 | self.snapshot_tree = snapshot_tree 84 | self.rule_set = rule_set 85 | self._debug = debug 86 | self._encrypted_folder_is_new = False 87 | self._trash_name = self._generate_trash_name() 88 | self._snapshot_trash_name = None 89 | self._snapshot_tree_name = string_digest(self.encrypted_folder) 90 | self._encrypted_filetree_entry = None 91 | 92 | if not os.path.isdir(self.encrypted_folder): 93 | if os.path.exists(self.encrypted_folder): 94 | raise InvalidFolder("Encrypted folder path is not correct: " + 95 | self.encrypted_folder) 96 | else: 97 | os.makedirs(self.encrypted_folder) 98 | 99 | if os.path.exists(os.path.join(self.encrypted_folder, ".syncrypto")): 100 | raise InvalidFolder("Encrypted folder can not has .syncrypto folder" 101 | " within it, do you pass the wrong arguments?") 102 | 103 | if plain_folder is not None: 104 | if not os.path.isdir(self.plain_folder): 105 | if os.path.exists(self.plain_folder): 106 | raise InvalidFolder( 107 | "Plaintext folder path is not correct: " + 108 | self.plain_folder) 109 | else: 110 | os.makedirs(self.plain_folder) 111 | 112 | if os.path.exists( 113 | os.path.join(self.plain_folder, "_syncrypto")): 114 | raise InvalidFolder( 115 | "Plaintext folder can not has _syncrypto folder within it" 116 | ", do you pass the wrong arguments?") 117 | 118 | if self.rule_set is None: 119 | self.rule_set = FileRuleSet() 120 | 121 | if rule_file is None: 122 | rule_file = self._plain_rule_path() 123 | if not os.path.exists(rule_file): 124 | with open(rule_file, "wb") as f: 125 | f.write(DEFAULT_RULES) 126 | 127 | if os.path.exists(rule_file): 128 | with open(rule_file, 'rb') as f: 129 | for line in f: 130 | line = line.strip() 131 | if line == b"" or line[0] == b'#': 132 | continue 133 | self.rule_set.add_rule_by_string(line.decode("ascii")) 134 | 135 | def debug(self, message): 136 | if self._debug: 137 | print("[DEBUG]", printable_text(message)) 138 | 139 | @staticmethod 140 | def info(message): 141 | print(printable_text(message)) 142 | 143 | @staticmethod 144 | def error(message): 145 | print(printable_text(message), file=sys.stderr) 146 | 147 | @staticmethod 148 | def _generate_trash_name(): 149 | return datetime.now().isoformat().replace(':', '_') 150 | 151 | def _generate_encrypted_path(self, encrypted_file): 152 | dirname, name = encrypted_file.split() 153 | digest = string_digest(name) 154 | i = 2 155 | while True: 156 | if dirname == '': 157 | fs_pathname = digest[:i] 158 | else: 159 | parent = self.encrypted_tree.get(dirname) 160 | if parent is None: 161 | self.error("Can not find file entry for %s" % 162 | dirname) 163 | raise GenerateEncryptedFilePathError() 164 | fs_pathname = parent.fs_pathname + '/' + digest[:i] 165 | if not self.encrypted_tree.has_fs_pathname(fs_pathname): 166 | encrypted_file.fs_pathname = fs_pathname 167 | return 168 | i += 1 169 | raise GenerateEncryptedFilePathError() 170 | 171 | def _encrypt_file(self, pathname): 172 | plain_file = self.plain_tree.get(pathname) 173 | plain_path = plain_file.fs_path(self.plain_folder) 174 | encrypted_file = self.encrypted_tree.get(pathname) 175 | if not os.path.exists(plain_path): 176 | self.error("%s not exists!" % plain_path) 177 | return encrypted_file 178 | if encrypted_file is None: 179 | encrypted_file = plain_file.clone() 180 | if pathname.startswith(".syncrypto/"): 181 | encrypted_file.fs_pathname = '_'+plain_file.fs_pathname[1:] 182 | else: 183 | try: 184 | self._generate_encrypted_path(encrypted_file) 185 | except GenerateEncryptedFilePathError: 186 | return None 187 | encrypted_path = encrypted_file.fs_path(self.encrypted_folder) 188 | mtime = plain_file.mtime 189 | if plain_file.isdir: 190 | if not os.path.exists(encrypted_path): 191 | os.makedirs(encrypted_path) 192 | encrypted_file.copy_attr_from(plain_file) 193 | return encrypted_file 194 | if os.path.exists(encrypted_path): 195 | self._move_to_encrypted_trash(encrypted_file) 196 | directory = os.path.dirname(encrypted_path) 197 | if not os.path.isdir(directory): 198 | os.makedirs(directory) 199 | plain_fd = open(plain_path, 'rb') 200 | encrypted_fd = open(encrypted_path, 'wb') 201 | self.crypto.encrypt_fd(plain_fd, encrypted_fd, plain_file) 202 | encrypted_file.copy_attr_from(plain_file) 203 | if plain_file.mode is not None: 204 | os.chmod(encrypted_path, plain_file.mode) 205 | os.utime(encrypted_path, (mtime, mtime)) 206 | plain_fd.close() 207 | encrypted_fd.close() 208 | return encrypted_file 209 | 210 | def _decrypt_file(self, pathname): 211 | encrypted_file = self.encrypted_tree.get(pathname) 212 | encrypted_path = encrypted_file.fs_path(self.encrypted_folder) 213 | plain_file = self.plain_tree.get(pathname) 214 | if not os.path.exists(encrypted_path): 215 | self.error("%s not exists!" % encrypted_path) 216 | return plain_file 217 | if plain_file is None: 218 | plain_file = encrypted_file.clone() 219 | plain_file.fs_pathname = plain_file.pathname 220 | plain_path = plain_file.fs_path(self.plain_folder) 221 | mtime = encrypted_file.mtime 222 | if encrypted_file.isdir: 223 | if not os.path.exists(plain_path): 224 | os.makedirs(plain_path) 225 | if encrypted_file.mode is not None: 226 | os.chmod(plain_path, encrypted_file.mode | S_IWUSR | S_IRUSR) 227 | os.utime(plain_path, (mtime, mtime)) 228 | plain_file.copy_attr_from(encrypted_file) 229 | return plain_file 230 | if os.path.exists(plain_path): 231 | self._move_to_plain_trash(plain_file) 232 | directory = os.path.dirname(plain_path) 233 | if not os.path.isdir(directory): 234 | os.makedirs(directory) 235 | plain_fd = open(plain_path, 'wb') 236 | encrypted_fd = open(encrypted_path, 'rb') 237 | self.crypto.decrypt_fd(encrypted_fd, plain_fd) 238 | plain_file.copy_attr_from(encrypted_file) 239 | plain_fd.close() 240 | encrypted_fd.close() 241 | if encrypted_file.mode is not None: 242 | os.chmod(plain_path, encrypted_file.mode) 243 | os.utime(plain_path, (mtime, mtime)) 244 | return plain_file 245 | 246 | @staticmethod 247 | def _conflict_path(path): 248 | dirname = os.path.dirname(path) 249 | filename = os.path.basename(path) 250 | dot_pos = filename.rfind(".") 251 | if dot_pos > 0: 252 | name = filename[:dot_pos] 253 | ext = filename[dot_pos:] 254 | else: 255 | name = filename 256 | ext = "" 257 | name += ".conflict" 258 | conflict_path = os.path.join(dirname, name+ext) 259 | i = 1 260 | if os.path.exists(conflict_path): 261 | conflict_path = \ 262 | os.path.join(dirname, name+"."+str(i)+ext) 263 | i += 1 264 | return conflict_path 265 | 266 | def _is_ignore(self, plain_file, encrypted_file): 267 | return (self.rule_set.test(plain_file) != 'include' or 268 | self.rule_set.test(encrypted_file) != 'include') 269 | 270 | @staticmethod 271 | def _is_equal(file_entry, file_entry_compare): 272 | if file_entry is None or file_entry_compare is None: 273 | return False 274 | if file_entry.isdir and file_entry_compare.isdir: 275 | return True 276 | if file_entry.digest is not None \ 277 | and file_entry_compare.digest is not None: 278 | return file_entry.digest == file_entry_compare.digest 279 | return \ 280 | file_entry.size == file_entry_compare.size and \ 281 | int(file_entry.mtime) == int(file_entry_compare.mtime) 282 | 283 | def _compare_file(self, encrypted_file, plain_file, snapshot_file): 284 | if self._is_ignore(plain_file, encrypted_file): 285 | return "ignore" 286 | if self._encrypted_folder_is_new: 287 | return "encrypt" 288 | if self._is_equal(plain_file, encrypted_file): 289 | return 'same' 290 | plain_file_changed = not self._is_equal(plain_file, snapshot_file) 291 | encrypted_file_changed = not self._is_equal(encrypted_file, 292 | snapshot_file) 293 | if plain_file is not None and encrypted_file is not None: 294 | if plain_file_changed and not encrypted_file_changed: 295 | return "encrypt" 296 | elif encrypted_file_changed and not plain_file_changed: 297 | return "decrypt" 298 | elif not encrypted_file_changed and not plain_file_changed: 299 | return "same" 300 | else: 301 | return 'conflict' 302 | elif plain_file is not None: 303 | if plain_file_changed: 304 | return "encrypt" 305 | else: 306 | return "remove plain" 307 | elif encrypted_file is not None: 308 | if encrypted_file_changed: 309 | return "decrypt" 310 | else: 311 | return "remove encrypted" 312 | return None 313 | 314 | def _move_to_encrypted_trash(self, file_entry): 315 | trash_path = self._trash_path_in_encrypted_folder(file_entry) 316 | if os.path.exists(trash_path): 317 | if os.path.isdir(trash_path): 318 | shutil.rmtree(trash_path) 319 | else: 320 | os.remove(trash_path) 321 | shutil.move(file_entry.fs_path(self.encrypted_folder), trash_path) 322 | 323 | def _move_to_plain_trash(self, file_entry): 324 | trash_path = self._trash_path_in_plain_folder(file_entry) 325 | if os.path.exists(trash_path): 326 | if os.path.isdir(trash_path): 327 | shutil.rmtree(trash_path) 328 | else: 329 | os.remove(trash_path) 330 | shutil.move(file_entry.fs_path(self.plain_folder), trash_path) 331 | 332 | def _trash_path_in_encrypted_folder(self, file_entry): 333 | path = file_entry.fs_path( 334 | os.path.join(self.encrypted_folder, '_syncrypto', 'trash', 335 | self._trash_name)) 336 | self._ensure_dir(path) 337 | return path 338 | 339 | def _trash_path_in_plain_folder(self, file_entry): 340 | path = file_entry.fs_path( 341 | os.path.join(self.plain_folder, '.syncrypto', 'trash', 342 | self._trash_name)) 343 | self._ensure_dir(path) 344 | return path 345 | 346 | def _plain_rule_path(self): 347 | return self._plain_folder_path("rules") 348 | 349 | def _encrypted_rule_path(self): 350 | return self._plain_folder_path("rules") 351 | 352 | def _encrypted_tree_path(self): 353 | return self._encrypted_folder_path("filetree") 354 | 355 | def _snapshot_tree_path(self): 356 | return self._plain_folder_path(self._snapshot_tree_name+'.filetree') 357 | 358 | def _plain_folder_path(self, sub_file): 359 | filename = ".syncrypto" 360 | path = os.path.join(self.plain_folder, filename, sub_file) 361 | self._ensure_dir(path) 362 | return path 363 | 364 | def _encrypted_folder_path(self, sub_file): 365 | filename = "_syncrypto" 366 | path = os.path.join(self.encrypted_folder, filename, sub_file) 367 | self._ensure_dir(path) 368 | return path 369 | 370 | def _save_trees(self): 371 | self._save_encrypted_tree() 372 | self._save_snapshot_tree() 373 | 374 | def _save_encrypted_tree(self): 375 | fp = open(self._encrypted_tree_path(), "wb") 376 | tree_dict = self.encrypted_tree.to_dict() 377 | tree_dict["snapshot_tree_name"] = self._snapshot_tree_name 378 | self.crypto.encrypt_fd(BytesIO(json.dumps(tree_dict).encode("utf-8")), 379 | fp, self._encrypted_filetree_entry, 380 | Crypto.COMPRESS) 381 | fp.close() 382 | 383 | def _load_encrypted_tree(self): 384 | encrypted_tree_path = self._encrypted_tree_path() 385 | if not os.path.exists(encrypted_tree_path): 386 | self.encrypted_tree = FileTree() 387 | self._encrypted_folder_is_new = True 388 | snapshot_tree_path = \ 389 | os.path.join(self.plain_folder, ".syncrypto", 390 | self._snapshot_tree_name+".filetree") 391 | if not os.path.exists(snapshot_tree_path): 392 | self._snapshot_tree_name = string_digest( 393 | os.path.abspath(self.encrypted_folder)+str(time())) 394 | else: 395 | fp = open(encrypted_tree_path, "rb") 396 | try: 397 | tree_fd = BytesIO() 398 | self._encrypted_filetree_entry = \ 399 | self.crypto.decrypt_fd(fp, tree_fd) 400 | tree_fd.seek(0) 401 | tree_dict = json.loads(tree_fd.getvalue().decode("utf-8")) 402 | if "snapshot_tree_name" in tree_dict: 403 | self._snapshot_tree_name = tree_dict["snapshot_tree_name"] 404 | self.encrypted_tree = FileTree.from_dict(tree_dict) 405 | finally: 406 | fp.close() 407 | 408 | def _save_snapshot_tree(self): 409 | fp = open(self._snapshot_tree_path(), 'wb') 410 | snapshot_tree_dict = self.snapshot_tree.to_dict() 411 | snapshot_tree_dict["trash_name"] = self._trash_name 412 | self.crypto.compress_fd( 413 | BytesIO(json.dumps(snapshot_tree_dict).encode("utf-8")), fp) 414 | fp.close() 415 | 416 | def _load_plain_tree(self): 417 | self.plain_tree = FileTree.from_fs(self.plain_folder, 418 | rule_set=self.rule_set) 419 | 420 | def _load_snapshot_tree(self): 421 | snapshot_tree_path = self._snapshot_tree_path() 422 | if not os.path.exists(snapshot_tree_path): 423 | self.snapshot_tree = FileTree() 424 | else: 425 | fp = open(snapshot_tree_path, "rb") 426 | try: 427 | tree_fd = BytesIO() 428 | self.crypto.decompress_fd(fp, tree_fd) 429 | tree_fd.seek(0) 430 | snapshot_tree_dict = \ 431 | json.loads(tree_fd.getvalue().decode("utf-8")) 432 | if "trash_name" in snapshot_tree_dict: 433 | self._snapshot_trash_name = snapshot_tree_dict["trash_name"] 434 | self.snapshot_tree = FileTree.from_dict(snapshot_tree_dict) 435 | finally: 436 | fp.close() 437 | 438 | @staticmethod 439 | def _ensure_dir(path): 440 | target_dir = os.path.dirname(path) 441 | if not os.path.isdir(target_dir): 442 | os.makedirs(target_dir) 443 | 444 | def _delete_file(self, pathname, is_in_encrypted_folder): 445 | tree, root, target = None, None, None 446 | if is_in_encrypted_folder: 447 | tree = self.encrypted_tree 448 | root = self.encrypted_folder 449 | target = "encrypted folder" 450 | else: 451 | tree = self.plain_tree 452 | root = self.plain_folder 453 | target = "plaintext folder" 454 | file_entry = tree.get(pathname) 455 | fs_path = file_entry.fs_path(root) 456 | if os.path.isdir(fs_path): 457 | if is_in_encrypted_folder: 458 | self._move_to_encrypted_trash(file_entry) 459 | else: 460 | self._move_to_plain_trash(file_entry) 461 | self.info("Delete folder %s in %s" % (file_entry.pathname, 462 | target)) 463 | elif os.path.exists(fs_path): 464 | if is_in_encrypted_folder: 465 | self._move_to_encrypted_trash(file_entry) 466 | else: 467 | self._move_to_plain_trash(file_entry) 468 | self.info("Delete file %s in %s" % (file_entry.pathname, 469 | target)) 470 | tree.remove(pathname) 471 | 472 | @staticmethod 473 | def _revise_folder(tree, root): 474 | for entry in tree.folders(): 475 | fs_path = entry.fs_path(root) 476 | os.utime(fs_path, (entry.mtime, entry.mtime)) 477 | 478 | def _do_sync_folder(self): 479 | 480 | if self.plain_folder is None: 481 | raise Exception("please specify the plaintext folder to sync files") 482 | 483 | pathnames = list(set(self.plain_tree.pathnames() + 484 | self.encrypted_tree.pathnames())) 485 | pathnames.sort() 486 | encrypted_remove_list = [] 487 | plain_remove_list = [] 488 | self.info(("Start synchronizing between encrypted folder '%s' " 489 | "and plaintext folder '%s'") % ( 490 | self.encrypted_folder, self.plain_folder 491 | )) 492 | self.debug("encrypted_tree:") 493 | self.debug(self.encrypted_tree) 494 | self.debug("plain_tree:") 495 | self.debug(self.plain_tree) 496 | self.debug("snapshot_tree:") 497 | self.debug(self.snapshot_tree) 498 | if os.path.exists(self._plain_rule_path()) \ 499 | or os.path.exists(self._encrypted_rule_path()): 500 | pathnames.append(".syncrypto/rules") 501 | self.plain_tree.set(".syncrypto/rules", 502 | FileEntry.from_file(self._plain_rule_path(), 503 | ".syncrypto/rules")) 504 | ignore_prefix = None 505 | for pathname in pathnames: 506 | if ignore_prefix is not None \ 507 | and pathname.startswith(ignore_prefix): 508 | self.plain_tree.remove(pathname) 509 | self.encrypted_tree.remove(pathname) 510 | encrypted_file = self.encrypted_tree.get(pathname) 511 | plain_file = self.plain_tree.get(pathname) 512 | action = self._compare_file(encrypted_file, plain_file, 513 | self.snapshot_tree.get(pathname)) 514 | self.debug("%s: %s, %s" % (action, encrypted_file, plain_file)) 515 | if action == "remove encrypted": 516 | encrypted_remove_list.append(pathname) 517 | elif action == "remove plain": 518 | plain_remove_list.append(pathname) 519 | elif action == "encrypt": 520 | encrypted_file = self._encrypt_file(pathname) 521 | if encrypted_file is None: 522 | continue 523 | self.encrypted_tree.set(pathname, encrypted_file) 524 | if not encrypted_file.isdir: 525 | self.info("Encrypt %s to %s" % 526 | (plain_file.fs_pathname, 527 | encrypted_file.fs_pathname)) 528 | elif action == "decrypt": 529 | plain_file = self._decrypt_file(pathname) 530 | if plain_file is None: 531 | continue 532 | self.plain_tree.set(pathname, plain_file) 533 | if not plain_file.isdir: 534 | self.info("Decrypt %s to %s" % 535 | (encrypted_file.fs_pathname, 536 | plain_file.fs_pathname)) 537 | elif action == "same": 538 | if not encrypted_file.isdir: 539 | self.debug("%s is not changed " % plain_file.fs_pathname) 540 | elif action == 'conflict': 541 | if plain_file.isdir and encrypted_file.isdir: 542 | continue 543 | plain_path = plain_file.fs_path(self.plain_folder) 544 | shutil.move(plain_path, self._conflict_path(plain_path)) 545 | if plain_file.isdir: 546 | ignore_prefix = pathname 547 | plain_file = self._decrypt_file(pathname) 548 | self.plain_tree.set(pathname, plain_file) 549 | self.info("%s has conflict!" % plain_file.fs_pathname) 550 | elif action == 'ignore': 551 | if encrypted_file is not None: 552 | encrypted_remove_list.append(pathname) 553 | if (plain_file is not None and plain_file.isdir) \ 554 | or \ 555 | (encrypted_file is not None and encrypted_file.isdir): 556 | ignore_prefix = pathname+'/' 557 | 558 | for pathname in encrypted_remove_list: 559 | self._delete_file(pathname, True) 560 | for pathname in plain_remove_list: 561 | self._delete_file(pathname, False) 562 | 563 | self._revise_folder(self.plain_tree, self.plain_folder) 564 | 565 | self.debug("encrypted_tree:") 566 | self.debug(self.encrypted_tree) 567 | self.debug("plain_tree:") 568 | self.debug(self.plain_tree) 569 | self.snapshot_tree = self.encrypted_tree 570 | self._save_trees() 571 | self.info(("Finish synchronizing between encrypted folder '%s' " 572 | "and plaintext folder '%s'") % ( 573 | self.encrypted_folder, self.plain_folder 574 | )) 575 | self._trash_name = self._generate_trash_name() 576 | 577 | def sync_folder(self, reload_tree=True): 578 | encrypted_folder_lock = LockFile(self.encrypted_folder) 579 | if encrypted_folder_lock.is_locked(): 580 | self.info("Acquiring the lock of encrypted folder...") 581 | else: 582 | self.debug("Encrypted folder is not locked") 583 | with encrypted_folder_lock: 584 | self.debug("Acquired the encrypted folder's lock") 585 | plain_folder_lock = LockFile(self.plain_folder) 586 | if plain_folder_lock.is_locked(): 587 | self.info("Acquiring the lock of plaintext folder...") 588 | else: 589 | self.debug("Plaintext folder is not locked") 590 | with plain_folder_lock: 591 | self.debug("Acquired the plaintext folder's lock") 592 | if reload_tree: 593 | self._load_encrypted_tree() 594 | self._load_plain_tree() 595 | self._load_snapshot_tree() 596 | if self.snapshot_tree is None: 597 | self._load_snapshot_tree() 598 | self._do_sync_folder() 599 | 600 | def change_password(self, newpass): 601 | if self.encrypted_tree is None: 602 | self._load_encrypted_tree() 603 | newpass = newpass.encode('utf-8') 604 | oldpass = self.crypto.password 605 | if oldpass == newpass: 606 | raise ChangeTheSamePassword() 607 | for file_entry in self.encrypted_tree.files(): 608 | fs_path = file_entry.fs_path(self.encrypted_folder) 609 | 610 | self.crypto.password = oldpass 611 | fp = open(fs_path, 'rb') 612 | string = BytesIO() 613 | self.crypto.decrypt_fd(fp, string) 614 | fp.close() 615 | 616 | string.seek(0) 617 | self.crypto.password = newpass 618 | fp = open(fs_path, 'wb') 619 | self.crypto.encrypt_fd(string, fp, file_entry) 620 | fp.close() 621 | self.crypto.password = newpass 622 | self._save_encrypted_tree() 623 | 624 | 625 | def _generate_tmp_path(folder=None): 626 | if folder is None: 627 | folder = os.getcwd() 628 | while True: 629 | path = os.path.join(folder, 630 | "%d_%d" % (int(time()), randint(1000, 9999))) 631 | if not os.path.exists(path): 632 | return path 633 | 634 | 635 | def cli_decrypt_file(crypto, encrypted_path, plain_path=None): 636 | if not os.path.isfile(encrypted_path): 637 | print(printable_text(encrypted_path+" is not a file")) 638 | return 1 639 | if plain_path is not None: 640 | file_entry = crypto.decrypt_file(encrypted_path, plain_path) 641 | else: 642 | tmp_path = _generate_tmp_path() 643 | file_entry = crypto.decrypt_file(encrypted_path, tmp_path) 644 | plain_path = file_entry.name() 645 | os.rename(tmp_path, plain_path) 646 | if file_entry.mode is not None: 647 | os.chmod(plain_path, file_entry.mode) 648 | os.utime(plain_path, (file_entry.mtime, file_entry.mtime)) 649 | return 0 650 | 651 | 652 | def cli_encrypt_file(crypto, plain_path, encrypted_path=None): 653 | if not os.path.isfile(plain_path): 654 | print(printable_text(plain_path+" is not a file")) 655 | return 1 656 | filename = os.path.basename(plain_path) 657 | pos = filename.rfind('.') 658 | if pos > 0: 659 | name = filename[:pos] 660 | ext = filename[pos:] 661 | else: 662 | name = filename 663 | ext = '' 664 | file_entry = FileEntry.from_file(plain_path, filename) 665 | if encrypted_path is not None: 666 | crypto.encrypt_file(plain_path, encrypted_path, file_entry) 667 | else: 668 | encrypted_path = os.path.join(os.path.dirname(plain_path), 669 | name+'.encrypted'+ext) 670 | crypto.encrypt_file(plain_path, encrypted_path, file_entry) 671 | return 0 672 | 673 | 674 | def main(args=sys.argv[1:]): 675 | 676 | from .cli import parser 677 | 678 | args = parser.parse_args(args=args) 679 | 680 | if args.version: 681 | from .package_info import __version__ 682 | print(__version__) 683 | return 1 684 | 685 | password = None 686 | 687 | if args.password_file is not None and os.path.exists(args.password_file): 688 | with open(args.password_file) as f: 689 | password = f.read().strip("\n") 690 | 691 | rule_set = FileRuleSet() 692 | 693 | if args.rule is not None: 694 | for rule_string in args.rule: 695 | rule_set.add_rule_by_string(rule_string) 696 | 697 | if password is None: 698 | password = getpass('Please input the password:') 699 | 700 | crypto = Crypto(password) 701 | 702 | try: 703 | 704 | if args.decrypt_file is not None: 705 | return cli_decrypt_file(crypto, args.decrypt_file, args.out_file) 706 | 707 | if args.encrypt_file is not None: 708 | return cli_encrypt_file(crypto, args.encrypt_file, args.out_file) 709 | 710 | if args.encrypted_folder is None: 711 | parser.print_help() 712 | return 1 713 | 714 | syncrypto = Syncrypto(crypto, 715 | args.encrypted_folder, 716 | args.plaintext_folder, 717 | rule_set=rule_set, 718 | rule_file=args.rule_file, 719 | debug=args.debug) 720 | if args.change_password: 721 | newpass1 = None 722 | while True: 723 | newpass1 = getpass('Please input the new password:') 724 | newpass2 = getpass('Please re input the new password:') 725 | if len(newpass1) < 6: 726 | print("new password is too short") 727 | elif newpass1 != newpass2: 728 | print("two inputs are not match") 729 | else: 730 | break 731 | syncrypto.change_password(newpass1) 732 | elif args.print_encrypted_tree: 733 | print(printable_text(syncrypto.encrypted_tree)) 734 | elif args.plaintext_folder is not None: 735 | if args.interval: 736 | while True: 737 | syncrypto.sync_folder() 738 | sleep(args.interval) 739 | else: 740 | syncrypto.sync_folder() 741 | return 0 742 | except DecryptError: 743 | print("Your password is not correct") 744 | return 3 745 | except InvalidFolder as e: 746 | print(e.args[0]) 747 | return 4 748 | -------------------------------------------------------------------------------- /syncrypto/crypto.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright 2015 Qing Liang (https://github.com/liangqing) 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | 18 | from __future__ import print_function 19 | from __future__ import absolute_import 20 | from __future__ import unicode_literals 21 | from __future__ import division 22 | from io import open 23 | from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes 24 | from cryptography.hazmat.backends import default_backend 25 | import os 26 | import zlib 27 | import hashlib 28 | from struct import pack, unpack 29 | from time import time 30 | from io import BytesIO 31 | from .filetree import FileEntry 32 | 33 | 34 | class InvalidKey(Exception): 35 | pass 36 | 37 | 38 | class DecryptError(Exception): 39 | pass 40 | 41 | 42 | class VersionNotCompatible(Exception): 43 | pass 44 | 45 | 46 | class Crypto(object): 47 | 48 | VERSION = 0x1 49 | 50 | COMPRESS = 0x1 51 | 52 | BUFFER_SIZE = 1024 * 16 53 | 54 | def __init__(self, password, key_size=32): 55 | 56 | self.password = password.encode("utf-8") 57 | self.key_size = key_size 58 | self.block_size = 16 59 | 60 | def encrypt_file(self, plain_path, encrypted_path, plain_file_entry): 61 | with open(plain_path, 'rb') as plain_fd: 62 | with open(encrypted_path, 'wb') as encrypted_fd: 63 | self.encrypt_fd(plain_fd, encrypted_fd, plain_file_entry) 64 | 65 | def decrypt_file(self, encrypted_path, plain_path): 66 | with open(encrypted_path, 'rb') as encrypted_fd: 67 | with open(plain_path, 'wb') as plain_fd: 68 | return self.decrypt_fd(encrypted_fd, plain_fd) 69 | 70 | @staticmethod 71 | def compress_fd(in_fd, out_fd): 72 | compress_obj = zlib.compressobj() 73 | while True: 74 | data = in_fd.read(Crypto.BUFFER_SIZE) 75 | if len(data) > 0: 76 | out_fd.write(compress_obj.compress(data)) 77 | else: 78 | break 79 | out_fd.write(compress_obj.flush()) 80 | 81 | @staticmethod 82 | def decompress_fd(in_fd, out_fd): 83 | decompress_obj = zlib.decompressobj() 84 | while True: 85 | data = in_fd.read(Crypto.BUFFER_SIZE) 86 | if len(data) > 0: 87 | out_fd.write(decompress_obj.decompress(data)) 88 | else: 89 | break 90 | out_fd.write(decompress_obj.flush()) 91 | 92 | def gen_key_and_iv(self, salt): 93 | d = d_i = b'' 94 | while len(d) < self.key_size + self.block_size: 95 | d_i = hashlib.md5(d_i + self.password + salt).digest() 96 | d += d_i 97 | return d[:self.key_size], d[self.key_size:self.key_size+self.block_size] 98 | 99 | @staticmethod 100 | def _build_footer(file_entry): 101 | return file_entry.digest + \ 102 | pack(b'!Q', file_entry.size) + \ 103 | pack(b'!I', int(file_entry.mtime)) + \ 104 | pack(b'!i', file_entry.mode or 0) 105 | 106 | @staticmethod 107 | def _unpack_footer(pathname, footer): 108 | (size, mtime, mode) = unpack(b'!QIi', footer[16:32]) 109 | if mode == 0: 110 | mode = None 111 | return FileEntry(pathname, size, None, mtime, mode, 112 | footer[:16], False) 113 | 114 | def encrypt_fd(self, in_fd, out_fd, file_entry, flags=0): 115 | """ 116 | +-----------------------------------------------------+ 117 | | Version(1) | Flags(1) | Pathname size(2) | Salt(12) | 118 | +-----------------------------------------------------+ 119 | | Encrypted Pathname | 120 | +-----------------------------------------------------+ 121 | | Encrypted Content | 122 | | ... | 123 | +-----------------------------------------------------+ 124 | | Encrypted Content Digest(16) | 125 | +-----------------------------------------------------+ 126 | | size(8)* | mtime(4) | mode(4) | 127 | +-----------------------------------------------------+ 128 | | Encrypted Entire Digest(16) | 129 | +-----------------------------------------------------+ 130 | 131 | * size, mtime, mode are also encrypted 132 | """ 133 | bs = self.block_size 134 | if file_entry is None: 135 | file_entry = FileEntry('file_entry.tmp', 0, time(), time(), 0) 136 | if file_entry.salt is None: 137 | file_entry.salt = os.urandom(bs - 4) 138 | key, iv = self.gen_key_and_iv(file_entry.salt) 139 | cipher = Cipher(algorithms.AES(key), modes.CBC(iv), 140 | backend=default_backend()) 141 | encryptor = cipher.encryptor() 142 | pathname = file_entry.pathname.encode("utf-8")[:2**16] 143 | pathname_size = len(pathname) 144 | pathname_padding = b'' 145 | if pathname_size % bs != 0: 146 | padding_length = (bs - pathname_size % bs) 147 | pathname_padding = padding_length * b'\0' 148 | 149 | flags &= 0xFF 150 | out_fd.write(pack(b'BB', self.VERSION, flags)) 151 | out_fd.write(pack(b'!H', pathname_size)) 152 | out_fd.write(file_entry.salt) 153 | out_fd.write(encryptor.update(pathname+pathname_padding)) 154 | compress_obj = None 155 | if flags & Crypto.COMPRESS: 156 | compress_obj = zlib.compressobj() 157 | 158 | finished = False 159 | md5 = hashlib.md5() 160 | rest = b'' 161 | end = False 162 | while not finished: 163 | if compress_obj is not None: 164 | buf = BytesIO() 165 | buf.write(rest) 166 | compress_size = len(rest) 167 | while compress_size < self.BUFFER_SIZE: 168 | in_data = in_fd.read(self.BUFFER_SIZE) 169 | if len(in_data) == 0: 170 | end = True 171 | try: 172 | buf.write(compress_obj.flush()) 173 | except Exception: 174 | pass 175 | break 176 | md5.update(in_data) 177 | compress_data = compress_obj.compress(in_data) 178 | compress_size += len(compress_data) 179 | buf.write(compress_data) 180 | data = buf.getvalue() 181 | data_size = len(data) 182 | if end: 183 | chunk = data 184 | elif data_size < self.BUFFER_SIZE: 185 | rest_size = data_size % bs 186 | chunk = data[:-rest_size] 187 | rest = data[-rest_size:] 188 | else: 189 | chunk = data[:self.BUFFER_SIZE] 190 | rest = data[self.BUFFER_SIZE:] 191 | else: 192 | chunk = in_fd.read(self.BUFFER_SIZE) 193 | md5.update(chunk) 194 | if len(chunk) == 0 or len(chunk) % bs != 0: 195 | padding_length = (bs - len(chunk) % bs) or bs 196 | chunk += padding_length * pack(b'B', padding_length) 197 | finished = True 198 | out_fd.write(encryptor.update(chunk)) 199 | 200 | file_entry.digest = md5.digest() 201 | footer = self._build_footer(file_entry) 202 | md5.update(footer) 203 | entire_digest = md5.digest() 204 | out_fd.write(encryptor.update(footer)) 205 | out_fd.write(encryptor.update(entire_digest)) 206 | out_fd.write(encryptor.finalize()) 207 | return file_entry 208 | 209 | def decrypt_fd(self, in_fd, out_fd): 210 | (version, flags, salt, pathname, decryptor) = \ 211 | self.extract_header(in_fd) 212 | md5 = hashlib.md5() 213 | next_chunk = '' 214 | finished = False 215 | file_entry = None 216 | decompress_obj = None 217 | if flags & self.COMPRESS: 218 | decompress_obj = zlib.decompressobj() 219 | footer_size = 48 220 | entire_digest = None 221 | entire_digest_check = None 222 | content_digest_check = None 223 | footer = None 224 | while not finished: 225 | chunk, next_chunk = next_chunk, in_fd.read(self.BUFFER_SIZE) 226 | if not chunk: 227 | continue 228 | plaintext = decryptor.update(chunk) 229 | if len(next_chunk) < self.BUFFER_SIZE: 230 | plaintext += decryptor.update(next_chunk) 231 | plaintext += decryptor.finalize() 232 | entire_digest = plaintext[-16:] 233 | footer = plaintext[-footer_size:-16] 234 | file_entry = self._unpack_footer(pathname, footer) 235 | padding_length = 0 236 | if len(plaintext) > footer_size: 237 | padding_length = \ 238 | bytearray(plaintext[-footer_size-1:-footer_size])[0] 239 | plaintext = plaintext[:-padding_length-footer_size] 240 | finished = True 241 | if decompress_obj is not None: 242 | decompress_error = False 243 | try: 244 | plaintext = decompress_obj.decompress(plaintext) 245 | except zlib.error: 246 | decompress_error = True 247 | if decompress_error: 248 | raise DecryptError() 249 | md5.update(plaintext) 250 | out_fd.write(plaintext) 251 | if finished: 252 | if decompress_obj is not None: 253 | rest = decompress_obj.flush() 254 | md5.update(rest) 255 | out_fd.write(rest) 256 | content_digest_check = md5.digest() 257 | md5.update(footer) 258 | entire_digest_check = md5.digest() 259 | 260 | if file_entry is None or entire_digest is None \ 261 | or entire_digest_check is None or content_digest_check is None: 262 | raise DecryptError() 263 | 264 | file_entry.salt = salt 265 | if file_entry.digest != content_digest_check or entire_digest != \ 266 | entire_digest_check: 267 | raise DecryptError() 268 | 269 | return file_entry 270 | 271 | def extract_header(self, in_fd): 272 | bs = self.block_size 273 | line = in_fd.read(bs) 274 | if len(line) < bs: 275 | raise DecryptError( 276 | "header line size is not correct, expect %d, got %d" % 277 | (bs, len(line))) 278 | ints = bytearray(line[:2]) 279 | version = ints[0] 280 | if version > self.VERSION: 281 | raise VersionNotCompatible("Unrecognized version: (%d)" % version) 282 | flags = ints[1] 283 | (pathname_size,) = unpack(b'!H', line[2:4]) 284 | salt = line[4:] 285 | key, iv = self.gen_key_and_iv(salt) 286 | cipher = Cipher(algorithms.AES(key), modes.CBC(iv), 287 | backend=default_backend()) 288 | decryptor = cipher.decryptor() 289 | pathname_block_size = pathname_size 290 | if pathname_size % bs != 0: 291 | pathname_block_size = int((pathname_size/bs+1) * bs) 292 | pathname_data = in_fd.read(pathname_block_size) 293 | if len(pathname_data) < pathname_block_size: 294 | raise DecryptError( 295 | "pathname length is not correct, expect %d, got %d" % 296 | (pathname_block_size, len(pathname_data))) 297 | pathname_data = decryptor.update(pathname_data)[:pathname_size] 298 | try: 299 | pathname = pathname_data.decode("utf-8") 300 | except UnicodeDecodeError: 301 | raise DecryptError() 302 | 303 | return version, flags, salt, pathname, decryptor 304 | 305 | def extract_entry(self, in_fd): 306 | (version, flags, salt, pathname, decryptor) = \ 307 | self.extract_header(in_fd) 308 | -------------------------------------------------------------------------------- /syncrypto/filetree.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright 2015 Qing Liang (https://github.com/liangqing) 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | 18 | from __future__ import print_function 19 | from __future__ import absolute_import 20 | from __future__ import unicode_literals 21 | import binascii 22 | import os 23 | import os.path 24 | import re 25 | from datetime import datetime 26 | import time 27 | from fnmatch import fnmatch 28 | from .util import unicode_text, file_digest 29 | 30 | 31 | class InvalidRuleString(Exception): 32 | pass 33 | 34 | 35 | class InvalidRegularExpression(Exception): 36 | pass 37 | 38 | 39 | class FileEntry(object): 40 | 41 | def __init__(self, pathname, size, ctime, mtime, mode, digest=None, 42 | isdir=False, fs_pathname=None, salt=None): 43 | self.pathname = pathname 44 | self.isdir = isdir 45 | self.size = size 46 | self.ctime = ctime 47 | self.mtime = mtime 48 | self.mode = mode 49 | self.digest = digest 50 | self.fs_pathname = fs_pathname 51 | self.salt = salt 52 | 53 | def __str__(self): 54 | t = datetime.fromtimestamp(self.mtime) 55 | if self.isdir: 56 | return " ".join(['directory', self.pathname, ':', 57 | unicode_text(t), self.fs_pathname]) 58 | else: 59 | return " ".join(['file', self.pathname, ':', 60 | unicode_text(t), self.fs_pathname]) 61 | 62 | def name(self): 63 | return (self.split())[1] 64 | 65 | def split(self): 66 | pos = self.pathname.rfind('/') 67 | if pos < 0: 68 | return '', self.pathname 69 | return self.pathname[:pos], self.pathname[pos+1:] 70 | 71 | def fs_path(self, root): 72 | if os.path.sep != '/': 73 | return root + os.path.sep + self.fs_pathname.replace('/', 74 | os.path.sep) 75 | return root + os.path.sep + self.fs_pathname 76 | 77 | def to_dict(self): 78 | d = {} 79 | for k in FileEntry.properties(): 80 | v = getattr(self, k) 81 | if v is not None and (k == 'digest' or k == 'salt'): 82 | d[k] = binascii.hexlify(v).decode('utf-8') 83 | else: 84 | d[k] = v 85 | return d 86 | 87 | def clone(self): 88 | return FileEntry(self.pathname, self.size, self.ctime, self.mtime, 89 | self.mode, self.digest, self.isdir, self.fs_pathname) 90 | 91 | def copy_attr_from(self, target): 92 | self.isdir = target.isdir 93 | self.size = target.size 94 | self.ctime = target.ctime 95 | self.mtime = target.mtime 96 | if target.mode is not None: 97 | self.mode = target.mode 98 | self.salt = target.salt 99 | self.digest = target.digest 100 | 101 | @classmethod 102 | def from_dict(cls, d): 103 | if 'digest' in d and d['digest'] is not None: 104 | d['digest'] = binascii.unhexlify(d['digest']) 105 | if 'salt' in d and d['salt'] is not None: 106 | d['salt'] = binascii.unhexlify(d['salt']) 107 | return cls(**d) 108 | 109 | @classmethod 110 | def from_file(cls, path, pathname): 111 | stat = os.stat(path) 112 | mode = stat.st_mode 113 | if os.name == 'nt': 114 | mode = None 115 | size = stat.st_size 116 | isdir = os.path.isdir(path) 117 | digest = None 118 | if not isdir and size <= 10240: 119 | digest = file_digest(path) 120 | return cls(pathname, size, stat.st_ctime, stat.st_mtime, 121 | mode, isdir=isdir, 122 | fs_pathname=pathname, digest=digest) 123 | 124 | @staticmethod 125 | def properties(): 126 | return ["pathname", "isdir", "size", "ctime", 127 | "mtime", "mode", "digest", "fs_pathname", "salt"] 128 | 129 | 130 | class FileRule(object): 131 | 132 | _OP_MAP = { 133 | ">": "gt", 134 | "<": "lt", 135 | ">=": "gte", 136 | "<=": "lte", 137 | "=": "eq", 138 | "==": "eq", 139 | "!=": "ne", 140 | "<>": "ne" 141 | } 142 | 143 | _SUPPORTED_ATTRIBUTES = [ 144 | "path", "name", "size", "ctime", "mtime" 145 | ] 146 | 147 | def __init__(self, attr, op, value, action): 148 | if op in FileRule._OP_MAP: 149 | op = FileRule._OP_MAP[op] 150 | if op not in ['eq', 'ne', 'lt', 'lte', 'gt', 'gte', 'match', 'regexp']: 151 | raise ValueError("Unsupported file filter op: "+op) 152 | if attr != 'name' and attr not in self._SUPPORTED_ATTRIBUTES: 153 | raise ValueError("Unsupported file filter attribute: "+attr) 154 | self.attr = attr 155 | if attr == 'size': 156 | value = unicode_text(value).lower() 157 | unit = value[-1] 158 | if unit == 'g': 159 | self.value = int(value[:-1]) << 30 160 | elif unit == 'm': 161 | self.value = int(value[:-1]) << 20 162 | elif unit == 'k': 163 | self.value = int(value[:-1]) << 10 164 | else: 165 | self.value = int(value) 166 | elif attr == 'ctime' or attr == 'mtime': 167 | self.value = time.mktime(datetime.strptime( 168 | value, "%Y-%m-%d %H:%M:%S").timetuple()) 169 | elif op == 'regexp': 170 | if value[0] != '^': 171 | value = '^'+value 172 | if value[-1] != '$': 173 | value += '$' 174 | try: 175 | self.value = re.compile(value) 176 | except re.error: 177 | self.value = None 178 | if self.value is None: 179 | raise InvalidRegularExpression( 180 | "Regular expression '"+value+"' not correct") 181 | else: 182 | self.value = value 183 | 184 | self.op = op 185 | self.action = action 186 | 187 | def test(self, file_entry): 188 | if file_entry is None: 189 | return None 190 | attr = self.attr 191 | if attr == 'name' or attr == 'path': 192 | attr = 'pathname' 193 | value = getattr(file_entry, attr) 194 | if self.attr == 'name': 195 | value = os.path.basename(value) 196 | method = getattr(self, self.op) 197 | if method(value, self.value): 198 | return self.action 199 | return None 200 | 201 | @staticmethod 202 | def eq(a, b): 203 | return a == b 204 | 205 | @staticmethod 206 | def ne(a, b): 207 | return a != b 208 | 209 | @staticmethod 210 | def lt(a, b): 211 | return a < b 212 | 213 | @staticmethod 214 | def gt(a, b): 215 | return a > b 216 | 217 | @staticmethod 218 | def lte(a, b): 219 | return a <= b 220 | 221 | @staticmethod 222 | def gte(a, b): 223 | return a >= b 224 | 225 | @staticmethod 226 | def match(value, pattern): 227 | return fnmatch(value, pattern) 228 | 229 | @staticmethod 230 | def regexp(value, regexp): 231 | return regexp.match(value) is not None 232 | 233 | def to_dict(self): 234 | value = self.value 235 | return { 236 | 'attr': self.attr, 237 | 'value': value, 238 | 'op': self.op 239 | } 240 | 241 | @classmethod 242 | def from_dict(cls, d): 243 | return cls(**d) 244 | 245 | 246 | class FileRuleSet(object): 247 | 248 | _RULE_STRING_REGEXP = re.compile( 249 | r"\s*(\w+)\s+(\S+)\s+(\".+\"|'.+'|.+)\s*") 250 | 251 | _RULE_STRING_REGEXP_WITH_ACTION = re.compile( 252 | r"\s*(include|exclude|ignore)\s*:\s*(\w+)\s+(\S+)\s+(\".+\"|'.+'|.+)\s*" 253 | ) 254 | 255 | def __init__(self, default_action="include"): 256 | self._rules = [] 257 | self.default_action = default_action 258 | 259 | def add(self, attr, op, value, action): 260 | self._rules.append(FileRule(attr, op, value, action)) 261 | 262 | def add_rule(self, rule): 263 | self._rules.append(rule) 264 | 265 | def add_rule_by_string(self, rule_string, action=None): 266 | self._rules.append(self.parse(rule_string, action)) 267 | 268 | def test(self, file_entry): 269 | for rule in self._rules: 270 | action = rule.test(file_entry) 271 | if action is not None: 272 | return action 273 | return self.default_action 274 | 275 | @classmethod 276 | def parse(cls, rule_string, action=None): 277 | if action is None: 278 | match = cls._RULE_STRING_REGEXP_WITH_ACTION.match(rule_string) 279 | else: 280 | match = cls._RULE_STRING_REGEXP.match(rule_string) 281 | if match is None: 282 | raise InvalidRuleString() 283 | if action is None: 284 | return FileRule(match.group(2).strip(), 285 | match.group(3).strip(), match.group(4).strip('"\''), 286 | match.group(1).strip()) 287 | return FileRule(match.group(1).strip(), 288 | match.group(2).strip(), match.group(3).strip('"\''), 289 | action) 290 | 291 | 292 | class FileTree(object): 293 | 294 | def __init__(self, table=None): 295 | self._table = table 296 | if self._table is None: 297 | self._table = {} 298 | 299 | def pathnames(self): 300 | return list(self._table) 301 | 302 | def files(self): 303 | files = [] 304 | for pathname in self._table: 305 | f = self._table[pathname] 306 | if not f.isdir: 307 | files.append(f) 308 | return files 309 | 310 | def folders(self): 311 | folders = [] 312 | for pathname in self._table: 313 | f = self._table[pathname] 314 | if f.isdir: 315 | folders.append(f) 316 | return folders 317 | 318 | def get(self, pathname): 319 | if pathname in self._table: 320 | return self._table[pathname] 321 | return None 322 | 323 | def set(self, pathname, file_entry): 324 | self._table[pathname] = file_entry 325 | 326 | def remove(self, pathname): 327 | if pathname in self._table: 328 | del self._table[pathname] 329 | 330 | def has(self, pathname): 331 | return pathname in self._table 332 | 333 | def has_fs_pathname(self, fs_pathname): 334 | for f in self._table.values(): 335 | if f.fs_pathname == fs_pathname: 336 | return True 337 | return False 338 | 339 | def walk_tree(self, path, rule_set, pathname=''): 340 | isfile = os.path.isfile(path) 341 | isdir = os.path.isdir(path) 342 | if (isfile or isdir) and pathname != '': 343 | file_entry = FileEntry.from_file(path, pathname) 344 | if rule_set is None: 345 | self._table[pathname] = file_entry 346 | else: 347 | action = rule_set.test(file_entry) 348 | if action == "include": 349 | self._table[pathname] = file_entry 350 | else: 351 | return 352 | if not isdir: 353 | return 354 | for name in os.listdir(path): 355 | if name == '.' or name == '..' \ 356 | or name == '.syncrypto' or name == '_syncrypto': 357 | continue 358 | sub_pathname = pathname+'/'+name 359 | if pathname == '': 360 | sub_pathname = name 361 | self.walk_tree(path+os.path.sep+name, rule_set, sub_pathname) 362 | 363 | def __str__(self): 364 | table = self._table 365 | s = "" 366 | for key in table: 367 | item = table[key] 368 | s += unicode_text(item)+"\n" 369 | return s 370 | 371 | def to_dict(self): 372 | table = {} 373 | for pathname in self._table: 374 | f = self._table[pathname] 375 | if f is not None: 376 | table[pathname] = f.to_dict() 377 | return { 378 | 'table': table, 379 | } 380 | 381 | @classmethod 382 | def from_fs(cls, root, table=None, rule_set=None): 383 | filetree = cls(table) 384 | filetree.walk_tree(root, rule_set) 385 | return filetree 386 | 387 | @classmethod 388 | def from_dict(cls, d): 389 | table = {} 390 | if 'table' in d: 391 | t = d['table'] 392 | for pathname in t: 393 | f = t[pathname] 394 | table[pathname] = FileEntry.from_dict(f) 395 | return cls(table) 396 | -------------------------------------------------------------------------------- /syncrypto/package_info.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright 2015 Qing Liang (https://github.com/liangqing) 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | """ 18 | Two-way synchronization between a folder and its ciphertext 19 | """ 20 | __title__ = "syncrypto" 21 | __version__ = "0.4.2" 22 | __author__ = 'Qing Liang' 23 | __license__ = 'Apache 2.0' 24 | __copyright__ = 'Copyright 2015 Qing Liang (https://github.com/liangqing)' 25 | -------------------------------------------------------------------------------- /syncrypto/util.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright 2015 Qing Liang (https://github.com/liangqing) 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | 18 | from __future__ import unicode_literals 19 | import sys 20 | import os 21 | import hashlib 22 | import binascii 23 | from getpass import getpass as builtin_getpass 24 | 25 | py3 = sys.version_info[0] == 3 26 | py2 = sys.version_info[0] == 2 27 | py2_6 = (py2 and sys.version_info[1] == 6) 28 | is_windows = os.name == "nt" 29 | fs_encoding = sys.getfilesystemencoding() 30 | 31 | if py3: 32 | 33 | def unicode_text(s, encoding="utf-8"): 34 | if isinstance(s, str): 35 | return s 36 | elif isinstance(s, bytes): 37 | return str(s, encoding) 38 | else: 39 | return str(s) 40 | 41 | 42 | def printable_text(s, encoding="utf-8"): 43 | return unicode_text(s, encoding) 44 | 45 | 46 | def command_text(s): 47 | return unicode_text(s, fs_encoding) 48 | 49 | def command_encoded(s): 50 | return s 51 | 52 | else: 53 | 54 | def unicode_text(s, encoding="utf-8"): 55 | if isinstance(s, unicode): 56 | return s 57 | if not isinstance(s, str) or not isinstance(s, bytes): 58 | s = s.__str__() 59 | if isinstance(s, unicode): 60 | return s 61 | return unicode(s, encoding=encoding) 62 | 63 | def printable_text(s, encoding="utf-8"): 64 | if isinstance(s, str): 65 | return s 66 | s = unicode_text(s) 67 | return s.encode(encoding) 68 | 69 | 70 | def command_text(s): 71 | return unicode_text(s, fs_encoding) 72 | 73 | def command_encoded(s): 74 | return s.encode(fs_encoding) 75 | 76 | 77 | def file_digest(path, buffer_size=10240): 78 | md5_obj = hashlib.md5() 79 | with open(path, 'rb') as f: 80 | while True: 81 | data = f.read(buffer_size) 82 | if len(data) <= 0: 83 | break 84 | md5_obj.update(data) 85 | return md5_obj.digest() 86 | 87 | 88 | def string_digest(string, encoding="utf-8"): 89 | md5_obj = hashlib.md5() 90 | md5_obj.update(string.encode(encoding)) 91 | return hexlify(md5_obj.digest()) 92 | 93 | 94 | def hexlify(data): 95 | return unicode_text(binascii.hexlify(data)) 96 | 97 | 98 | def file_hexlify_digest(path): 99 | return hexlify(file_digest(path)) 100 | 101 | 102 | def getpass(text="password:"): 103 | if is_windows and py2: 104 | text = text.encode("utf8") 105 | return builtin_getpass(text) 106 | -------------------------------------------------------------------------------- /tests/requirements.txt: -------------------------------------------------------------------------------- 1 | coverage 2 | nose 3 | nose-cov 4 | rednose 5 | unittest2 6 | cryptography 7 | lockfile 8 | codecov -------------------------------------------------------------------------------- /tests/test_cli.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright 2015 Qing Liang (https://github.com/liangqing) 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | 18 | from __future__ import print_function 19 | from __future__ import unicode_literals 20 | from io import open 21 | import unittest 22 | import os 23 | import os.path 24 | import shutil 25 | from tempfile import mkdtemp, mkstemp 26 | from filecmp import cmp as file_cmp 27 | from syncrypto import cli as syncrypto_cli_raw 28 | from time import time 29 | from subprocess import Popen, PIPE 30 | from util import clear_folder, prepare_filetree, tree_cmp 31 | from syncrypto.util import is_windows, command_encoded, py2_6 32 | 33 | try: 34 | from cStringIO import StringIO as BytesIO 35 | except ImportError: 36 | from io import BytesIO 37 | 38 | 39 | if not is_windows and not py2_6: 40 | import pexpect 41 | 42 | 43 | def syncrypto_cli(args): 44 | args = [command_encoded(arg) for arg in args] 45 | return syncrypto_cli_raw(args) 46 | 47 | 48 | class CliTestCase(unittest.TestCase): 49 | 50 | def setUp(self): 51 | self.plain_folder = mkdtemp() 52 | self.plain_folder_check = mkdtemp() 53 | self.encrypted_folder = mkdtemp() 54 | fd, self.password_file = mkstemp() 55 | os.write(fd, b"password test") 56 | os.close(fd) 57 | 58 | def tearDown(self): 59 | shutil.rmtree(self.plain_folder) 60 | shutil.rmtree(self.plain_folder_check) 61 | shutil.rmtree(self.encrypted_folder) 62 | os.remove(self.password_file) 63 | 64 | def tree_cmp(self, folder1, folder2): 65 | return tree_cmp(folder1, folder2, ignores=[".syncrypto"]) 66 | 67 | def check_result(self): 68 | cmp_result = self.tree_cmp(self.plain_folder, self.plain_folder_check) 69 | self.assertEqual(cmp_result.left_only, []) 70 | self.assertEqual(cmp_result.right_only, []) 71 | self.assertEqual(cmp_result.diff_files, []) 72 | 73 | def cli(self, args): 74 | self.assertEqual(syncrypto_cli(args), 0) 75 | 76 | def pipe(self, args): 77 | os.chdir(os.path.dirname(os.path.dirname(__file__))) 78 | if py2_6: 79 | args = ["python", "-m", "syncrypto.__main__"] + args 80 | else: 81 | args = ["python", "-m", "syncrypto"] + args 82 | return Popen(args, stdout=PIPE, stdin=PIPE, stderr=PIPE) 83 | 84 | def pexpect(self, args): 85 | os.chdir(os.path.dirname(os.path.dirname(__file__))) 86 | if py2_6: 87 | args = ["python", "-m", "syncrypto.__main__"] + args 88 | else: 89 | args = ["python", "-m", "syncrypto"] + args 90 | return pexpect.spawn(" ".join(args)) 91 | 92 | def check_result_after_sync(self): 93 | self.cli(["--password-file", self.password_file, self.encrypted_folder, 94 | self.plain_folder]) 95 | self.cli(["--password-file", self.password_file, self.encrypted_folder, 96 | self.plain_folder_check]) 97 | self.check_result() 98 | 99 | def check_result_after_sync_from_check_folder(self): 100 | self.cli(["--password-file", self.password_file, self.encrypted_folder, 101 | self.plain_folder_check]) 102 | self.cli(["--password-file", self.password_file, self.encrypted_folder, 103 | self.plain_folder]) 104 | self.check_result() 105 | 106 | def clear_folders(self): 107 | clear_folder(self.plain_folder) 108 | clear_folder(self.plain_folder_check) 109 | clear_folder(self.encrypted_folder) 110 | 111 | def norm_path(self, folder, pathname): 112 | return folder+os.path.sep+pathname.replace("/", os.path.sep) 113 | 114 | def modify_file(self, folder, pathname, content): 115 | path = self.norm_path(folder, pathname) 116 | fd = open(path, 'wb') 117 | fd.write(content.encode("utf-8")) 118 | fd.close() 119 | os.utime(path, (time(), time()+1)) 120 | 121 | def add_file(self, folder, pathname, content): 122 | path = self.norm_path(folder, pathname) 123 | fd = open(path, 'wb') 124 | fd.write(content.encode("utf-8")) 125 | fd.close() 126 | 127 | def add_folder(self, folder, pathname): 128 | os.makedirs(self.norm_path(folder, pathname)) 129 | 130 | def delete_file(self, folder, pathname): 131 | os.remove(self.norm_path(folder, pathname)) 132 | 133 | def delete_folder(self, folder, pathname): 134 | shutil.rmtree(self.norm_path(folder, pathname)) 135 | 136 | def rename(self, folder, pathname, pathname2): 137 | os.rename(self.norm_path(folder, pathname), 138 | self.norm_path(folder, pathname2)) 139 | 140 | def test_invalid_password(self): 141 | self.cli(["--password-file", self.password_file, self.encrypted_folder, 142 | self.plain_folder]) 143 | fd, path = mkstemp() 144 | try: 145 | os.write(fd, b"wrong password") 146 | os.close(fd) 147 | self.assertEqual(syncrypto_cli(["--password-file", path, 148 | self.encrypted_folder, 149 | self.plain_folder]), 3) 150 | finally: 151 | os.remove(path) 152 | 153 | def test_interactive_input_password(self): 154 | if is_windows or py2_6: 155 | return 156 | self.cli(["--password-file", self.password_file, self.encrypted_folder, 157 | self.plain_folder]) 158 | child = self.pexpect([self.encrypted_folder, self.plain_folder]) 159 | child.expect("password:") 160 | child.sendline("password test") 161 | child.expect(pexpect.EOF) 162 | child.close() 163 | self.assertEqual(child.exitstatus, 0) 164 | 165 | def test_interactive_change_password(self): 166 | if is_windows or py2_6: 167 | return 168 | self.cli(["--password-file", self.password_file, self.encrypted_folder, 169 | self.plain_folder]) 170 | child = self.pexpect(["--change-password", self.encrypted_folder]) 171 | child.expect("password:") 172 | child.sendline("password test") 173 | child.expect("password:") 174 | child.sendline("password new") 175 | child.expect("password:") 176 | child.sendline("password new") 177 | child.expect(pexpect.EOF) 178 | child.close() 179 | self.assertEqual(child.exitstatus, 0) 180 | child = self.pexpect([self.encrypted_folder, self.plain_folder]) 181 | child.expect("password:") 182 | child.sendline("password new") 183 | child.expect(pexpect.EOF) 184 | child.close() 185 | self.assertEqual(child.exitstatus, 0) 186 | child = self.pexpect([self.encrypted_folder, self.plain_folder]) 187 | child.expect("password:") 188 | child.sendline("password wrong") 189 | child.expect(pexpect.EOF) 190 | child.close() 191 | self.assertEqual(child.exitstatus, 3) 192 | 193 | def test_interactive_invalid_password(self): 194 | if is_windows or py2_6: 195 | return 196 | self.cli(["--password-file", self.password_file, self.encrypted_folder, 197 | self.plain_folder]) 198 | child = self.pexpect([self.encrypted_folder, self.plain_folder]) 199 | child.expect("password:") 200 | child.sendline("wrong password") 201 | child.expect(pexpect.EOF) 202 | child.close() 203 | self.assertEqual(child.exitstatus, 3) 204 | 205 | def test_basic_sync(self): 206 | self.clear_folders() 207 | prepare_filetree(self.plain_folder, ''' 208 | simple_file: hello world 209 | file/in/sub/folder: hello world 210 | empty_dir/ 211 | ''') 212 | self.check_result_after_sync() 213 | 214 | def test_basic_sync_multiple_times(self): 215 | self.clear_folders() 216 | prepare_filetree(self.plain_folder, ''' 217 | simple_file: hello world 218 | file/in/sub/folder: hello world 219 | empty_dir/ 220 | ''') 221 | self.check_result_after_sync() 222 | self.check_result_after_sync() 223 | self.check_result_after_sync_from_check_folder() 224 | self.check_result_after_sync() 225 | self.check_result_after_sync_from_check_folder() 226 | self.check_result_after_sync_from_check_folder() 227 | self.check_result_after_sync() 228 | 229 | def test_modify_file(self): 230 | self.clear_folders() 231 | prepare_filetree(self.plain_folder, ''' 232 | keep_same: same file 233 | will_modify: modify the file please 234 | file/in/sub/folder/will_modify: modify the file please! 235 | empty_dir/ 236 | ''') 237 | self.check_result_after_sync() 238 | self.modify_file(self.plain_folder, "will_modify", "it is modified") 239 | self.modify_file(self.plain_folder, "file/in/sub/folder/will_modify", 240 | "it is modified") 241 | self.check_result_after_sync() 242 | 243 | def test_modify_file_in_check_folder(self): 244 | self.clear_folders() 245 | prepare_filetree(self.plain_folder, ''' 246 | keep_same: same file 247 | will_modify: modify the file please 248 | file/in/sub/folder/will_modify: modify the file please! 249 | empty_dir/ 250 | ''') 251 | self.check_result_after_sync() 252 | self.modify_file(self.plain_folder_check, 253 | "will_modify", "it is modified") 254 | self.modify_file(self.plain_folder_check, 255 | "file/in/sub/folder/will_modify", "it is modified") 256 | self.check_result_after_sync_from_check_folder() 257 | 258 | def test_rename_file(self): 259 | self.clear_folders() 260 | prepare_filetree(self.plain_folder, ''' 261 | keep_same: same file 262 | will_rename: rename 263 | will_rename2: rename 264 | file/in/sub/folder/will_rename: rename 265 | empty_dir/ 266 | ''') 267 | self.check_result_after_sync() 268 | self.rename(self.plain_folder, "will_rename", "renamed") 269 | self.rename(self.plain_folder, "file/in/sub/folder/will_rename", 270 | "renamed2") 271 | self.rename(self.plain_folder, "will_rename2", 272 | "file/in/sub/folder/renamed2") 273 | self.check_result_after_sync() 274 | 275 | def test_rename_file_in_check_folder(self): 276 | self.clear_folders() 277 | prepare_filetree(self.plain_folder, ''' 278 | keep_same: same file 279 | will_rename: rename 280 | will_rename2: rename 281 | file/in/sub/folder/will_rename: rename 282 | empty_dir/ 283 | ''') 284 | self.check_result_after_sync() 285 | self.rename(self.plain_folder_check, "will_rename", "renamed") 286 | self.rename(self.plain_folder_check, "file/in/sub/folder/will_rename", 287 | "renamed2") 288 | self.rename(self.plain_folder_check, "will_rename2", 289 | "file/in/sub/folder/renamed2") 290 | self.check_result_after_sync_from_check_folder() 291 | 292 | def test_add_file(self): 293 | self.clear_folders() 294 | prepare_filetree(self.plain_folder, ''' 295 | simple_file: simple file 296 | file/in/sub/folder/simple_file: file in the folder! 297 | empty_dir/ 298 | ''') 299 | self.check_result_after_sync() 300 | self.add_file(self.plain_folder, "add_file", "add file") 301 | self.add_file(self.plain_folder, "file/in/sub/folder/add_file", 302 | "add file!") 303 | self.check_result_after_sync() 304 | 305 | def test_add_file_in_check_folder(self): 306 | self.clear_folders() 307 | prepare_filetree(self.plain_folder, ''' 308 | simple_file: simple file 309 | file/in/sub/folder/simple_file: file in the folder! 310 | empty_dir/ 311 | ''') 312 | self.check_result_after_sync() 313 | self.add_file(self.plain_folder_check, "add_file", "add file") 314 | self.add_file(self.plain_folder_check, "file/in/sub/folder/add_file", 315 | "add file!") 316 | self.check_result_after_sync_from_check_folder() 317 | 318 | def test_add_folder(self): 319 | self.clear_folders() 320 | prepare_filetree(self.plain_folder, ''' 321 | simple_file: simple file 322 | file/in/sub/folder/simple_file: file in the folder! 323 | empty_dir/ 324 | ''') 325 | self.check_result_after_sync() 326 | self.add_folder(self.plain_folder, "empty_dir/add_folder") 327 | self.add_folder(self.plain_folder, "folder/with/file") 328 | self.add_file(self.plain_folder, "folder/with/file/test", "test\ntest!") 329 | self.check_result_after_sync() 330 | 331 | def test_add_folder_in_check_folder(self): 332 | self.clear_folders() 333 | prepare_filetree(self.plain_folder, ''' 334 | simple_file: simple file 335 | file/in/sub/folder/simple_file: file in the folder! 336 | empty_dir/ 337 | ''') 338 | self.check_result_after_sync() 339 | self.add_folder(self.plain_folder_check, "empty_dir/add_folder") 340 | self.add_folder(self.plain_folder_check, "folder/with/file") 341 | self.add_file(self.plain_folder_check, "folder/with/file/test", 342 | "test\ntest!") 343 | self.check_result_after_sync_from_check_folder() 344 | 345 | def test_delete_file(self): 346 | self.clear_folders() 347 | prepare_filetree(self.plain_folder, ''' 348 | delete_me: delete me! 349 | file/in/sub/folder/delete_me: oh, please delete me! 350 | ''') 351 | self.check_result_after_sync() 352 | self.delete_file(self.plain_folder, "delete_me") 353 | self.delete_file(self.plain_folder, "file/in/sub/folder/delete_me") 354 | self.check_result_after_sync() 355 | 356 | def test_delete_file_in_check_folder(self): 357 | self.clear_folders() 358 | prepare_filetree(self.plain_folder, ''' 359 | delete_me: delete me! 360 | file/in/sub/folder/delete_me: oh, please delete me! 361 | ''') 362 | self.check_result_after_sync() 363 | self.delete_file(self.plain_folder_check, "delete_me") 364 | self.delete_file(self.plain_folder_check, 365 | "file/in/sub/folder/delete_me") 366 | self.check_result_after_sync_from_check_folder() 367 | 368 | def test_delete_folder(self): 369 | self.clear_folders() 370 | prepare_filetree(self.plain_folder, ''' 371 | file_reserve: test 372 | folder/reserve: lol 373 | empty_folder1/ 374 | empty_folder2/in/sub/folder/ 375 | non_empty_folder1/file: test 1 376 | non_empty_folder2/in/sub/folder/file: test 2 377 | ''') 378 | self.check_result_after_sync() 379 | self.delete_folder(self.plain_folder, "empty_folder1") 380 | self.delete_folder(self.plain_folder, "empty_folder2/in/sub/folder/") 381 | self.delete_folder(self.plain_folder, "non_empty_folder1") 382 | self.delete_folder(self.plain_folder, "non_empty_folder2/in/sub/folder") 383 | self.check_result_after_sync() 384 | 385 | def test_delete_folder_in_check_folder(self): 386 | self.clear_folders() 387 | prepare_filetree(self.plain_folder, ''' 388 | file_reserve: test 389 | folder/reserve: LOL 390 | empty_folder1/ 391 | empty_folder2/in/sub/folder/ 392 | non_empty_folder1/file: test 1 393 | non_empty_folder2/in/sub/folder/file: test 2 394 | ''') 395 | self.check_result_after_sync() 396 | self.delete_folder(self.plain_folder_check, "empty_folder1") 397 | self.delete_folder(self.plain_folder_check, 398 | "empty_folder2/in/sub/folder/") 399 | self.delete_folder(self.plain_folder_check, "non_empty_folder1") 400 | self.delete_folder(self.plain_folder_check, 401 | "non_empty_folder2/in/sub/folder") 402 | self.check_result_after_sync_from_check_folder() 403 | 404 | def test_rule_set(self): 405 | self.clear_folders() 406 | prepare_filetree(self.plain_folder, ''' 407 | filename_sync: 1 408 | filename_not_sync: 2 409 | filename_not_sync_encrypted: 3 410 | ''') 411 | self.cli(["--password-file", self.password_file, 412 | "--rule", 413 | "exclude: name match *_not_sync", 414 | self.encrypted_folder, self.plain_folder]) 415 | self.cli(["--password-file", self.password_file, 416 | "--rule", 417 | "exclude: name match *_encrypted", 418 | self.encrypted_folder, self.plain_folder_check]) 419 | cmp_result = self.tree_cmp(self.plain_folder, self.plain_folder_check) 420 | self.assertEqual(sorted(cmp_result.left_only), 421 | ["filename_not_sync", "filename_not_sync_encrypted"]) 422 | self.assertEqual(len(cmp_result.right_only), 0) 423 | self.assertEqual(len(cmp_result.diff_files), 0) 424 | 425 | def test_rule_file(self): 426 | self.clear_folders() 427 | prepare_filetree(self.plain_folder, ''' 428 | filename_sync: 1 429 | filename_not_sync: 2 430 | filename_not_sync_encrypted: 3 431 | ''') 432 | dot_folder = os.path.join(self.plain_folder, ".syncrypto") 433 | dot_folder_check = os.path.join(self.plain_folder_check, ".syncrypto") 434 | if not os.path.exists(dot_folder): 435 | os.mkdir(dot_folder) 436 | if not os.path.exists(dot_folder_check): 437 | os.mkdir(dot_folder_check) 438 | with open(os.path.join(dot_folder, "rules"), 'wb') as f: 439 | f.write(b'exclude: name match *_not_sync') 440 | with open(os.path.join(dot_folder_check, "rules"), 'wb') as f: 441 | f.write(b'exclude: name match *_encrypted') 442 | self.cli(["--password-file", self.password_file, 443 | self.encrypted_folder, self.plain_folder]) 444 | self.assertTrue(os.path.exists( 445 | os.path.join(self.encrypted_folder, "_syncrypto", "rules"))) 446 | self.cli(["--password-file", self.password_file, 447 | self.encrypted_folder, self.plain_folder_check]) 448 | self.assertTrue(file_cmp(os.path.join(dot_folder, "rules"), 449 | os.path.join(dot_folder_check, "rules"))) 450 | cmp_result = self.tree_cmp(self.plain_folder, self.plain_folder_check) 451 | self.assertEqual(sorted(cmp_result.left_only), 452 | ["filename_not_sync", "filename_not_sync_encrypted"]) 453 | self.assertEqual(len(cmp_result.right_only), 0) 454 | self.assertEqual(len(cmp_result.diff_files), 0) 455 | 456 | def test_encrypted_file_name(self): 457 | self.clear_folders() 458 | prepare_filetree(self.plain_folder, ''' 459 | normal: hello 460 | normal_folder/file: hello 461 | 211: 1 462 | 117/hello: 2 463 | ''') 464 | self.check_result_after_sync() 465 | 466 | def test_conflict_starting(self): 467 | self.clear_folders() 468 | prepare_filetree(self.plain_folder, ''' 469 | files.txt: text file 470 | folder/no_extension: no extension file 471 | folder/no_conflict1:1 472 | ''') 473 | prepare_filetree(self.plain_folder_check, ''' 474 | files.txt: different text file! 475 | folder/no_extension: no extension file! 476 | folder/no_conflict2: 2 477 | ''') 478 | self.cli(["--password-file", self.password_file, self.encrypted_folder, 479 | self.plain_folder]) 480 | self.cli(["--password-file", self.password_file, self.encrypted_folder, 481 | self.plain_folder_check]) 482 | cmp_result = self.tree_cmp(self.plain_folder, self.plain_folder_check) 483 | self.assertEqual(cmp_result.left_only, []) 484 | self.assertEqual(sorted(cmp_result.right_only), 485 | ["files.conflict.txt", 486 | "folder/no_conflict2", 487 | "folder/no_extension.conflict"]) 488 | self.assertEqual(cmp_result.diff_files, []) 489 | self.assertEqual(open(os.path.join(self.plain_folder_check, 490 | "files.conflict.txt")).read(), 491 | "different text file!") 492 | self.assertEqual(open( 493 | os.path.join(self.plain_folder_check, 494 | "folder/no_extension.conflict")).read(), 495 | "no extension file!") 496 | 497 | def test_conflict_after_syncing(self): 498 | self.clear_folders() 499 | prepare_filetree(self.plain_folder, ''' 500 | files.txt: text file 501 | folder/no_extension: no extension file 502 | folder/no_conflict1:1 503 | ''') 504 | self.check_result_after_sync() 505 | self.modify_file(self.plain_folder, "files.txt", "modified") 506 | self.modify_file(self.plain_folder_check, "files.txt", "modified 2") 507 | self.cli(["--password-file", self.password_file, self.encrypted_folder, 508 | self.plain_folder]) 509 | self.cli(["--password-file", self.password_file, self.encrypted_folder, 510 | self.plain_folder_check]) 511 | cmp_result = self.tree_cmp(self.plain_folder, self.plain_folder_check) 512 | self.assertEqual(cmp_result.left_only, []) 513 | self.assertEqual(sorted(cmp_result.right_only), 514 | ["files.conflict.txt"]) 515 | 516 | def test_encrypt_file_no_out_file(self): 517 | self.clear_folders() 518 | prepare_filetree(self.plain_folder, ''' 519 | test_simple_file: hello 520 | ext.txt: with extension 521 | ''') 522 | self.cli(["--password-file", self.password_file, "--encrypt-file", 523 | os.path.join(self.plain_folder, "test_simple_file")]) 524 | self.assertTrue(os.path.exists( 525 | os.path.join(self.plain_folder, "test_simple_file.encrypted"))) 526 | self.cli([ 527 | "--password-file", self.password_file, 528 | "--decrypt-file", 529 | os.path.join(self.plain_folder, "test_simple_file.encrypted"), 530 | "--out-file", 531 | os.path.join(self.plain_folder, "test_simple_file_decrypted") 532 | ]) 533 | self.assertTrue( 534 | file_cmp( 535 | os.path.join(self.plain_folder, "test_simple_file_decrypted"), 536 | os.path.join(self.plain_folder, "test_simple_file"), False)) 537 | 538 | self.cli(["--password-file", self.password_file, "--encrypt-file", 539 | os.path.join(self.plain_folder, "ext.txt")]) 540 | self.assertTrue(os.path.exists( 541 | os.path.join(self.plain_folder, "ext.encrypted.txt"))) 542 | self.cli([ 543 | "--password-file", self.password_file, 544 | "--decrypt-file", 545 | os.path.join(self.plain_folder, "ext.encrypted.txt"), 546 | "--out-file", 547 | os.path.join(self.plain_folder, "ext.decrypted.txt") 548 | ]) 549 | self.assertTrue( 550 | file_cmp( 551 | os.path.join(self.plain_folder, "ext.decrypted.txt"), 552 | os.path.join(self.plain_folder, "ext.txt"), False)) 553 | 554 | def test_encrypt_file_given_out_file(self): 555 | self.clear_folders() 556 | prepare_filetree(self.plain_folder, ''' 557 | test_simple_file: hello 558 | ''') 559 | self.cli(["--password-file", self.password_file, "--encrypt-file", 560 | os.path.join(self.plain_folder, "test_simple_file")]) 561 | self.assertTrue(os.path.exists( 562 | os.path.join(self.plain_folder, "test_simple_file.encrypted"))) 563 | self.cli([ 564 | "--password-file", self.password_file, 565 | "--decrypt-file", 566 | os.path.join(self.plain_folder, "test_simple_file.encrypted"), 567 | "--out-file", 568 | os.path.join(self.plain_folder, "test_simple_file_decrypted") 569 | ]) 570 | self.assertTrue( 571 | file_cmp( 572 | os.path.join(self.plain_folder, "test_simple_file_decrypted"), 573 | os.path.join(self.plain_folder, "test_simple_file"), False)) 574 | 575 | def test_decrypt_file_no_out_file(self): 576 | self.clear_folders() 577 | prepare_filetree(self.plain_folder, ''' 578 | test_simple_file: hello 579 | ''') 580 | self.cli(["--password-file", self.password_file, self.encrypted_folder, 581 | self.plain_folder]) 582 | encrypted_path = None 583 | for name in os.listdir(self.encrypted_folder): 584 | if name.startswith(".") or name.startswith('_'): 585 | continue 586 | encrypted_path = name 587 | self.assertFalse(encrypted_path is None) 588 | origin = os.getcwd() 589 | os.chdir(self.plain_folder_check) 590 | self.cli(["--password-file", self.password_file, "--decrypt-file", 591 | os.path.join(self.encrypted_folder, encrypted_path)]) 592 | self.assertTrue(os.path.exists("test_simple_file")) 593 | with open("test_simple_file") as f: 594 | self.assertEqual(f.read(), "hello") 595 | os.chdir(origin) 596 | 597 | def test_decrypt_file_given_out_file(self): 598 | self.clear_folders() 599 | prepare_filetree(self.plain_folder, ''' 600 | test_simple_file: hello 601 | ''') 602 | self.cli(["--password-file", self.password_file, self.encrypted_folder, 603 | self.plain_folder]) 604 | encrypted_path = None 605 | for name in os.listdir(self.encrypted_folder): 606 | if name.startswith(".") or name.startswith('_'): 607 | continue 608 | encrypted_path = name 609 | self.assertFalse(encrypted_path is None) 610 | plain_path = os.path.join(self.plain_folder_check, "decrypted_file") 611 | self.cli(["--password-file", self.password_file, "--decrypt-file", 612 | os.path.join(self.encrypted_folder, encrypted_path), 613 | '--out-file', plain_path]) 614 | # origin = os.getcwd() 615 | # os.chdir(self.plain_folder_check) 616 | # self.assertTrue(os.path.exists(plain_path)) 617 | # with open(plain_path) as f: 618 | # self.assertEqual(f.read(), "hello") 619 | os.remove(plain_path) 620 | # os.chdir(origin) 621 | 622 | def test_pass_wrong_arguments(self): 623 | self.clear_folders() 624 | prepare_filetree(self.plain_folder, ''' 625 | test_simple_file: hello 626 | ''') 627 | self.check_result_after_sync() 628 | self.assertNotEqual(syncrypto_cli(["--password-file", 629 | self.password_file, 630 | self.plain_folder, 631 | self.encrypted_folder]), 0) 632 | 633 | def test_not_ascii_arguments(self): 634 | plain_folder = mkdtemp("中文") 635 | plain_folder_check = mkdtemp("中文") 636 | encrypted_folder = mkdtemp("中文") 637 | try: 638 | prepare_filetree(plain_folder, ''' 639 | 文件: 你好 640 | ''') 641 | self.cli(["--password-file", self.password_file, 642 | encrypted_folder, 643 | plain_folder]) 644 | self.cli(["--password-file", self.password_file, 645 | encrypted_folder, 646 | plain_folder_check]) 647 | cmp_result = self.tree_cmp(plain_folder, plain_folder_check) 648 | self.assertEqual(cmp_result.left_only, []) 649 | self.assertEqual(cmp_result.right_only, []) 650 | self.assertEqual(cmp_result.diff_files, []) 651 | finally: 652 | shutil.rmtree(plain_folder) 653 | shutil.rmtree(plain_folder_check) 654 | shutil.rmtree(encrypted_folder) 655 | 656 | 657 | if __name__ == '__main__': 658 | unittest.main() 659 | -------------------------------------------------------------------------------- /tests/test_crypto.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright 2015 Qing Liang (https://github.com/liangqing) 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | 18 | from __future__ import print_function 19 | from __future__ import unicode_literals 20 | from io import open 21 | import unittest 22 | import os 23 | import os.path 24 | from tempfile import mkstemp 25 | from syncrypto import FileEntry, Crypto 26 | import hashlib 27 | import binascii 28 | 29 | 30 | def _hex(data): 31 | return binascii.hexlify(data) 32 | 33 | 34 | try: 35 | from cStringIO import StringIO as BytesIO 36 | except ImportError: 37 | from io import BytesIO 38 | 39 | 40 | class CryptoTestCase(unittest.TestCase): 41 | 42 | def setUp(self): 43 | file_fp, file_path = mkstemp() 44 | self.password = 'password' 45 | self.crypto = Crypto(self.password) 46 | self.file_path = file_path 47 | self.file_entry = FileEntry.from_file(file_path, 48 | os.path.basename(file_path)) 49 | os.close(file_fp) 50 | 51 | def tearDown(self): 52 | os.remove(self.file_path) 53 | 54 | def test_basic_encrypt(self): 55 | in_fd = BytesIO() 56 | middle_fd = BytesIO() 57 | out_fd = BytesIO() 58 | in_fd.write(b"hello") 59 | in_fd.seek(0) 60 | self.crypto.encrypt_fd(in_fd, middle_fd, self.file_entry) 61 | middle_fd.seek(0) 62 | self.crypto.decrypt_fd(middle_fd, out_fd) 63 | self.assertEqual(in_fd.getvalue(), out_fd.getvalue()) 64 | 65 | def test_file_api(self): 66 | fd1, file_path1 = mkstemp() 67 | fd2, file_path2 = mkstemp() 68 | fd3, file_path3 = mkstemp() 69 | os.write(fd1, b'hello world') 70 | os.close(fd1) 71 | os.close(fd2) 72 | os.close(fd3) 73 | self.crypto.encrypt_file(file_path1, file_path2, self.file_entry) 74 | self.crypto.decrypt_file(file_path2, file_path3) 75 | self.assertEqual(open(file_path3, 'rb').read(), b'hello world') 76 | os.remove(file_path1) 77 | os.remove(file_path2) 78 | os.remove(file_path3) 79 | 80 | def test_large_encrypt(self): 81 | in_fd = BytesIO() 82 | middle_fd = BytesIO() 83 | out_fd = BytesIO() 84 | in_fd.write(os.urandom(1024 * 1024)) 85 | in_fd.seek(0) 86 | self.crypto.encrypt_fd(in_fd, middle_fd, self.file_entry) 87 | middle_fd.seek(0) 88 | self.crypto.decrypt_fd(middle_fd, out_fd) 89 | self.assertEqual(in_fd.getvalue(), out_fd.getvalue()) 90 | 91 | def test_encrypt_twice(self): 92 | in_fd = BytesIO() 93 | out_fd1 = BytesIO() 94 | out_fd2 = BytesIO() 95 | in_fd.write(b"hello") 96 | in_fd.seek(0) 97 | self.crypto.encrypt_fd(in_fd, out_fd1, self.file_entry) 98 | in_fd.seek(0) 99 | self.crypto.encrypt_fd(in_fd, out_fd2, self.file_entry) 100 | self.assertEqual(out_fd1.getvalue(), out_fd2.getvalue()) 101 | 102 | def test_repeat_encrypt(self): 103 | in_fd = BytesIO() 104 | out_fd1 = BytesIO() 105 | out_fd2 = BytesIO() 106 | in_fd.write(os.urandom(1024 * 1024)) 107 | in_fd.seek(0) 108 | self.crypto.encrypt_fd(in_fd, out_fd1, self.file_entry) 109 | in_fd.seek(0) 110 | self.crypto.encrypt_fd(in_fd, out_fd2, self.file_entry) 111 | self.assertEqual(out_fd1.getvalue(), out_fd2.getvalue()) 112 | 113 | def test_compress(self): 114 | in_fd = BytesIO() 115 | middle_fd = BytesIO() 116 | out_fd = BytesIO() 117 | in_fd.write(os.urandom(1024 * 1024)) 118 | in_fd.seek(0) 119 | self.crypto.compress_fd(in_fd, middle_fd) 120 | middle_fd.seek(0) 121 | self.crypto.decompress_fd(middle_fd, out_fd) 122 | self.assertEqual(in_fd.getvalue(), out_fd.getvalue()) 123 | 124 | def test_encrypted_compress(self): 125 | in_fd = BytesIO() 126 | middle_fd = BytesIO() 127 | out_fd = BytesIO() 128 | in_fd.write(os.urandom(1024)) 129 | in_fd.seek(0) 130 | self.crypto.encrypt_fd(in_fd, middle_fd, self.file_entry, 131 | Crypto.COMPRESS) 132 | middle_fd.seek(0) 133 | self.crypto.decrypt_fd(middle_fd, out_fd) 134 | self.assertEqual(in_fd.getvalue(), out_fd.getvalue()) 135 | 136 | def test_encrypted_compress_large(self): 137 | in_fd = BytesIO() 138 | middle_fd = BytesIO() 139 | out_fd = BytesIO() 140 | data = "\n".join([str(x) for x in range(1000000, 1000000+8*1024+39)]) 141 | in_fd.write(data.encode("ascii")) 142 | in_fd.seek(0) 143 | self.crypto.encrypt_fd(in_fd, middle_fd, self.file_entry, 144 | Crypto.COMPRESS) 145 | middle_fd.seek(0) 146 | self.crypto.decrypt_fd(middle_fd, out_fd) 147 | self.assertEqual(in_fd.getvalue(), out_fd.getvalue()) 148 | 149 | if __name__ == '__main__': 150 | unittest.main() 151 | -------------------------------------------------------------------------------- /tests/test_filetree.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright 2015 Qing Liang (https://github.com/liangqing) 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | 18 | from __future__ import print_function 19 | from __future__ import unicode_literals 20 | import unittest 21 | import os 22 | import os.path 23 | import shutil 24 | from tempfile import mkstemp, mkdtemp 25 | from syncrypto import FileEntry, FileTree, FileRuleSet 26 | from time import time 27 | from util import prepare_filetree, clear_folder 28 | from syncrypto.util import file_hexlify_digest, file_digest, hexlify, is_windows 29 | 30 | 31 | try: 32 | from cStringIO import StringIO as BytesIO 33 | except ImportError: 34 | from io import BytesIO 35 | 36 | 37 | class FileEntryTestCase(unittest.TestCase): 38 | 39 | def setUp(self): 40 | file_fp, file_path = mkstemp() 41 | self.file_path = file_path 42 | stat = os.stat(self.file_path) 43 | self.file_attrs = { 44 | 'pathname': os.path.basename(self.file_path), 45 | 'size': stat.st_size, 46 | 'ctime': int(time()), 47 | 'mtime': stat.st_mtime, 48 | 'mode': stat.st_mode, 49 | } 50 | self.file_object = FileEntry(**self.file_attrs) 51 | os.close(file_fp) 52 | 53 | def tearDown(self): 54 | os.remove(self.file_path) 55 | 56 | def test_property(self): 57 | stat = os.stat(self.file_path) 58 | self.assertEqual(self.file_object.isdir, False) 59 | self.assertEqual(self.file_object.digest, None) 60 | self.assertEqual(self.file_object.size, stat.st_size) 61 | self.assertEqual(self.file_object.ctime, int(stat.st_ctime)) 62 | self.assertEqual(self.file_object.mtime, stat.st_mtime) 63 | self.assertEqual(self.file_object.mode, stat.st_mode) 64 | self.assertEqual(self.file_object.pathname, 65 | os.path.basename(self.file_path)) 66 | 67 | def test_to_dict(self): 68 | d = self.file_object.to_dict() 69 | for k in self.file_attrs: 70 | self.assertEqual(d[k], self.file_attrs[k]) 71 | 72 | def test_from_file(self): 73 | stat = os.stat(self.file_path) 74 | d = { 75 | 'pathname': os.path.basename(self.file_path), 76 | 'fs_pathname': os.path.basename(self.file_path), 77 | 'size': stat.st_size, 78 | 'ctime': stat.st_ctime, 79 | 'mtime': stat.st_mtime, 80 | 'mode': stat.st_mode, 81 | 'digest': file_hexlify_digest(self.file_path), 82 | 'isdir': False, 83 | 'salt': None 84 | } 85 | if is_windows: 86 | d['mode'] = None 87 | file_object = FileEntry.from_file(self.file_path, d['pathname']) 88 | self.assertEqual(d, file_object.to_dict()) 89 | 90 | def test_from_dict(self): 91 | stat = os.stat(self.file_path) 92 | d = { 93 | 'pathname': os.path.basename(self.file_path), 94 | 'fs_pathname': os.path.basename(self.file_path), 95 | 'size': stat.st_size, 96 | 'ctime': int(time()), 97 | 'mtime': stat.st_mtime, 98 | 'mode': stat.st_mode, 99 | 'digest': file_digest(self.file_path), 100 | 'isdir': False, 101 | 'salt': None 102 | } 103 | file_object = FileEntry(**d) 104 | d['digest'] = hexlify(d['digest']) 105 | self.assertEqual(d, file_object.to_dict()) 106 | 107 | 108 | class FileTreeTestCase(unittest.TestCase): 109 | 110 | def setUp(self): 111 | self.directory_path = mkdtemp() 112 | 113 | def tearDown(self): 114 | shutil.rmtree(self.directory_path) 115 | 116 | def test_walk_tree(self): 117 | prepare_filetree(self.directory_path, ''' 118 | a 119 | b/ 120 | c/d/e/f 121 | 1/2 122 | ''') 123 | filetree = FileTree.from_fs(self.directory_path) 124 | self.assertEqual(filetree.get('a').isdir, False) 125 | self.assertEqual(filetree.get('b').isdir, True) 126 | self.assertEqual(filetree.get('c').isdir, True) 127 | self.assertEqual(filetree.get('c/d').isdir, True) 128 | self.assertEqual(filetree.get('c/d/e').isdir, True) 129 | self.assertEqual(filetree.get('c/d/e/f').isdir, False) 130 | self.assertEqual(filetree.get('1').isdir, True) 131 | self.assertEqual(filetree.get('1/2').isdir, False) 132 | self.assertEqual(len(filetree.files()), 3) 133 | self.assertEqual(len(filetree.folders()), 5) 134 | 135 | def test_walk_tree_with_rule(self): 136 | clear_folder(self.directory_path) 137 | prepare_filetree(self.directory_path, ''' 138 | ignore/b/c:1 139 | ignore/bb:2 140 | include/d/e: 3 141 | include/dd: 4 142 | whatever_file: 5 143 | ''') 144 | rule_set = FileRuleSet() 145 | rule_set.add_rule_by_string("ignore: name eq ignore") 146 | filetree = FileTree.from_fs(self.directory_path, rule_set=rule_set) 147 | self.assertEqual(len(filetree.files()), 3) 148 | self.assertEqual(len(filetree.folders()), 2) 149 | 150 | 151 | if __name__ == '__main__': 152 | unittest.main() 153 | -------------------------------------------------------------------------------- /tests/test_rule.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright 2015 Qing Liang (https://github.com/liangqing) 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | 18 | from __future__ import print_function 19 | from __future__ import unicode_literals 20 | import unittest 21 | import os 22 | import os.path 23 | from tempfile import mkstemp 24 | from syncrypto import FileEntry, FileRule, FileRuleSet, InvalidRegularExpression 25 | from util import format_datetime 26 | 27 | from time import time 28 | 29 | try: 30 | from cStringIO import StringIO as BytesIO 31 | except ImportError: 32 | from io import BytesIO 33 | 34 | 35 | class FileRuleTestCase(unittest.TestCase): 36 | 37 | def setUp(self): 38 | file_fp, file_path = mkstemp() 39 | self.file_path = file_path 40 | self.file_entry = FileEntry.from_file(self.file_path, os.path.basename( 41 | self.file_path)) 42 | os.close(file_fp) 43 | 44 | def tearDown(self): 45 | os.remove(self.file_path) 46 | 47 | def test_eq(self): 48 | 49 | for op in ['eq', '=', '==']: 50 | f1 = FileRule('name', op, os.path.basename(self.file_path), 51 | 'include') 52 | f2 = FileRule('name', op, "...", 'include') 53 | self.assertEqual(f1.test(self.file_entry), "include") 54 | self.assertEqual(f2.test(self.file_entry), None) 55 | 56 | def test_ne(self): 57 | for op in ['ne', '!=', '<>']: 58 | f1 = FileRule('name', op, os.path.basename(self.file_path), 59 | 'exclude') 60 | f2 = FileRule('name', op, "...", 'exclude') 61 | self.assertEqual(f2.test(self.file_entry), "exclude") 62 | self.assertEqual(f1.test(self.file_entry), None) 63 | 64 | def test_lt(self): 65 | for op in ['lt', '<']: 66 | f1 = FileRule('size', op, 10, 'include') 67 | f2 = FileRule('size', op, 0, 'include') 68 | self.assertEqual(f1.test(self.file_entry), 'include') 69 | self.assertEqual(f2.test(self.file_entry), None) 70 | 71 | def test_gt(self): 72 | for op in ['gt', '>']: 73 | f1 = FileRule('mtime', op, 74 | format_datetime(time()-3600), 'exclude') 75 | f2 = FileRule('mtime', op, 76 | format_datetime(time()+3600), 'exclude') 77 | self.assertEqual(f1.test(self.file_entry), 'exclude') 78 | self.assertEqual(f2.test(self.file_entry), None) 79 | 80 | def test_gte(self): 81 | for op in ['gte', '>=']: 82 | f1 = FileRule('mtime', op, 83 | format_datetime(time()-3600), 'exclude') 84 | f2 = FileRule('mtime', op, 85 | format_datetime(time()+3600), 'exclude') 86 | self.assertEqual(f1.test(self.file_entry), 'exclude') 87 | self.assertEqual(f2.test(self.file_entry), None) 88 | 89 | def test_lte(self): 90 | self.file_entry.ctime = int(self.file_entry.ctime) 91 | f1 = FileRule('ctime', 'lte', 92 | format_datetime(self.file_entry.ctime), 'include') 93 | f2 = FileRule('ctime', 'lte', 94 | format_datetime(time()-3600), 'include') 95 | f3 = FileRule('ctime', 'eq', 96 | format_datetime(self.file_entry.ctime), 'include') 97 | self.assertEqual(f1.test(self.file_entry), 'include') 98 | self.assertEqual(f2.test(self.file_entry), None) 99 | self.assertEqual(f3.test(self.file_entry), 'include') 100 | 101 | def test_match(self): 102 | f1 = FileRule('name', 'match', "*", 'include') 103 | f2 = FileRule('name', 'match', "", 'include') 104 | f3 = FileRule('name', 'match', 105 | os.path.basename(self.file_entry.pathname), 'include') 106 | self.assertEqual(f1.test(self.file_entry), 'include') 107 | self.assertEqual(f2.test(self.file_entry), None) 108 | self.assertEqual(f3.test(self.file_entry), 'include') 109 | 110 | file_entry = self.file_entry.clone() 111 | file_entry.pathname = "t.test" 112 | f = FileRule("name", "match", "*.test", "include") 113 | self.assertEqual(f.test(file_entry), "include") 114 | file_entry.pathname = "test" 115 | self.assertEqual(f.test(file_entry), None) 116 | 117 | def test_size(self): 118 | self.file_entry.size = 2 119 | for unit in ["k", "M", "G"]: 120 | 121 | f = FileRule('size', '>', "1"+unit, 'include') 122 | self.assertEqual(f.test(self.file_entry), None) 123 | f = FileRule('size', '<', "1"+unit, 'include') 124 | self.assertEqual(f.test(self.file_entry), 'include') 125 | 126 | self.file_entry.size <<= 10 127 | 128 | f = FileRule('size', '>', "1"+unit, 'include') 129 | self.assertEqual(f.test(self.file_entry), 'include') 130 | f = FileRule('size', '<', "1"+unit, 'include') 131 | self.assertEqual(f.test(self.file_entry), None) 132 | f = FileRule('size', 'eq', "2"+unit, 'include') 133 | self.assertEqual(f.test(self.file_entry), 'include') 134 | 135 | # no unit 136 | self.file_entry.size = 99 137 | f = FileRule('size', '>', 100, 'include') 138 | self.assertEqual(f.test(self.file_entry), None) 139 | f = FileRule('size', '<', 100, 'include') 140 | self.assertEqual(f.test(self.file_entry), 'include') 141 | 142 | def regexp_invalid(self): 143 | FileRule('name', 'regexp', "*.txt", 'include') 144 | 145 | def test_regexp(self): 146 | self.file_entry.pathname = "test_file.txt" 147 | f = FileRule('name', 'regexp', "test.*", 'include') 148 | self.assertEqual(f.test(self.file_entry), 'include') 149 | f = FileRule('name', 'regexp', "test*", 'include') 150 | self.assertEqual(f.test(self.file_entry), None) 151 | self.assertRaises(InvalidRegularExpression, self.regexp_invalid) 152 | 153 | 154 | class FileRuleSetTestCase(unittest.TestCase): 155 | 156 | def setUp(self): 157 | file_fp, file_path = mkstemp() 158 | self.file_path = file_path 159 | self.file_entry = FileEntry.from_file(self.file_path, os.path.basename( 160 | self.file_path)) 161 | os.close(file_fp) 162 | 163 | def tearDown(self): 164 | os.remove(self.file_path) 165 | 166 | def test_basic(self): 167 | rule_set = FileRuleSet() 168 | rule_set.add("size", ">", 1024, "include") 169 | rule_set.add("path", "eq", self.file_entry.pathname, "exclude") 170 | self.assertEqual(rule_set.test(self.file_entry), "exclude") 171 | 172 | def test_basic_parse(self): 173 | rule_set = FileRuleSet() 174 | rule_set.add_rule_by_string("size > 1024", "include") 175 | rule_set.add_rule_by_string("path = "+self.file_entry.pathname, 176 | "exclude") 177 | self.assertEqual(rule_set.test(self.file_entry), "exclude") 178 | 179 | def test_basic_parse_with_quotes(self): 180 | rule_set = FileRuleSet() 181 | rule_set.add_rule_by_string("size > '1024'", "include") 182 | rule_set.add_rule_by_string("path = \""+self.file_entry.pathname+"\"", 183 | "exclude") 184 | self.assertEqual(rule_set.test(self.file_entry), "exclude") 185 | 186 | def test_parse_with_no_action(self): 187 | f = FileRuleSet.parse("include: size > 1024") 188 | self.assertEqual(f.action, "include") 189 | self.assertEqual(f.attr, "size") 190 | self.assertEqual(f.op, "gt") 191 | self.assertEqual(f.value, 1024) 192 | 193 | def test_parse_with_action(self): 194 | f = FileRuleSet.parse("size > 1024", "exclude") 195 | self.assertEqual(f.action, "exclude") 196 | self.assertEqual(f.attr, "size") 197 | self.assertEqual(f.op, "gt") 198 | self.assertEqual(f.value, 1024) 199 | 200 | def test_default_action(self): 201 | rule_set = FileRuleSet() 202 | rule_set.add_rule_by_string("exclude: size > 1024000") 203 | self.assertEqual(rule_set.test(self.file_entry), 204 | rule_set.default_action) 205 | rule_set = FileRuleSet(default_action="laf") 206 | rule_set.add_rule_by_string("exclude: size > 1024000") 207 | self.assertEqual(rule_set.test(self.file_entry), "laf") 208 | 209 | def test_match_pattern(self): 210 | f = FileRuleSet.parse("exclude: name match *_not_sync") 211 | self.assertEqual(f.action, "exclude") 212 | self.assertEqual(f.attr, "name") 213 | self.assertEqual(f.op, "match") 214 | self.assertEqual(f.value, "*_not_sync") 215 | 216 | def test_eq(self): 217 | tmp = self.file_entry.pathname 218 | f = FileRuleSet.parse("exclude: name eq .git") 219 | self.file_entry.pathname = ".gitignore" 220 | self.assertEqual(f.test(self.file_entry), None) 221 | self.file_entry.pathname = tmp 222 | 223 | if __name__ == '__main__': 224 | unittest.main() 225 | -------------------------------------------------------------------------------- /tests/test_sync.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright 2015 Qing Liang (https://github.com/liangqing) 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | 18 | from __future__ import print_function 19 | from __future__ import unicode_literals 20 | from io import open 21 | import unittest 22 | import os 23 | import os.path 24 | import shutil 25 | from tempfile import mkdtemp 26 | from syncrypto import FileTree, Crypto, Syncrypto, InvalidFolder 27 | from filecmp import dircmp 28 | from syncrypto.crypto import DecryptError 29 | from util import clear_folder, prepare_filetree 30 | 31 | try: 32 | from cStringIO import StringIO as BytesIO 33 | except ImportError: 34 | from io import BytesIO 35 | 36 | 37 | class SyncTestCase(unittest.TestCase): 38 | 39 | def setUp(self): 40 | self.crypto = Crypto('password') 41 | self.plain_folder = mkdtemp() 42 | self.plain_folder_check = mkdtemp() 43 | self.encrypted_folder = mkdtemp() 44 | prepare_filetree(self.plain_folder, ''' 45 | sync_file_modify:hello world 46 | sync_file_delete:delete 47 | sync/file/modify:hello world 48 | empty_dir_delete/ 49 | not_empty_dir/dir2/dir3/file 50 | dir2/file2 51 | ''') 52 | self.plain_tree = self.plain_tree = FileTree.from_fs(self.plain_folder) 53 | self.plain_tree_check = FileTree() 54 | self.encrypted_tree = FileTree() 55 | self.snapshot_tree = FileTree() 56 | 57 | def tearDown(self): 58 | shutil.rmtree(self.plain_folder) 59 | shutil.rmtree(self.plain_folder_check) 60 | shutil.rmtree(self.encrypted_folder) 61 | 62 | def isPass(self): 63 | sync = Syncrypto(self.crypto, self.encrypted_folder, self.plain_folder, 64 | self.encrypted_tree, self.plain_tree) 65 | sync2 = Syncrypto(self.crypto, self.encrypted_folder, 66 | self.plain_folder_check, 67 | self.encrypted_tree, self.plain_tree_check) 68 | sync.sync_folder(False) 69 | sync2.sync_folder(False) 70 | directory_cmp = dircmp(self.plain_folder, self.plain_folder_check) 71 | self.assertEqual(len(directory_cmp.left_only), 0) 72 | self.assertEqual(len(directory_cmp.right_only), 0) 73 | self.assertEqual(len(directory_cmp.diff_files), 0) 74 | 75 | def test_init(self): 76 | self.isPass() 77 | 78 | def pass_invalid_encrypted_folder(self): 79 | invalid_folder = self.encrypted_folder+os.path.sep+"invalid_folder" 80 | with open(invalid_folder, 'wb') as f: 81 | f.write(b'Test') 82 | Syncrypto(self.crypto, invalid_folder, self.plain_folder) 83 | os.remove(invalid_folder) 84 | 85 | def pass_invalid_plaintext_folder(self): 86 | invalid_folder = self.plain_folder+os.path.sep+"invalid_folder" 87 | with open(invalid_folder, 'wb') as f: 88 | f.write(b'Test') 89 | Syncrypto(self.crypto, self.encrypted_folder, invalid_folder) 90 | os.remove(invalid_folder) 91 | 92 | def test_false_directory(self): 93 | self.assertRaises(InvalidFolder, self.pass_invalid_encrypted_folder) 94 | self.assertRaises(InvalidFolder, self.pass_invalid_plaintext_folder) 95 | 96 | def test_add_file(self): 97 | path = self.plain_folder + os.path.sep + "add_file" 98 | fp = open(path, "wb") 99 | fp.write(b"hello world") 100 | fp.close() 101 | self.plain_tree = FileTree.from_fs(self.plain_folder) 102 | self.isPass() 103 | 104 | def test_add_file_and_modify(self): 105 | path = self.plain_folder + os.path.sep + "add_file_and_modify" 106 | fp = open(path, "wb") 107 | fp.write(b"hello world") 108 | fp.close() 109 | self.plain_tree = FileTree.from_fs(self.plain_folder) 110 | self.isPass() 111 | 112 | path = self.plain_folder + os.path.sep + "add_file_and_modify" 113 | fp = open(path, "wb") 114 | fp.write(b"hello world again") 115 | fp.close() 116 | self.plain_tree = FileTree.from_fs(self.plain_folder) 117 | self.plain_tree.get("add_file_and_modify").mtime += 1 118 | self.isPass() 119 | 120 | def test_modify_file(self): 121 | path = self.plain_tree.get("sync_file_modify").fs_path( 122 | self.plain_folder) 123 | fp = open(path, "wb") 124 | fp.write(b"hello world again") 125 | fp.close() 126 | self.plain_tree = FileTree.from_fs(self.plain_folder) 127 | self.plain_tree.get("sync_file_modify").mtime += 1 128 | self.isPass() 129 | 130 | def test_modify_file_in_folder(self): 131 | path = self.plain_tree.get("sync/file/modify").fs_path( 132 | self.plain_folder) 133 | fp = open(path, "wb") 134 | fp.write(b"hello world again") 135 | fp.close() 136 | self.plain_tree = FileTree.from_fs(self.plain_folder) 137 | self.plain_tree.get("sync/file/modify").mtime += 1 138 | self.isPass() 139 | 140 | def test_delete_file(self): 141 | path = self.plain_tree.get("sync_file_delete").fs_path( 142 | self.plain_folder) 143 | os.remove(path) 144 | self.plain_tree = FileTree.from_fs(self.plain_folder) 145 | self.isPass() 146 | 147 | def test_delete_empty_folder(self): 148 | path = self.plain_tree.get("empty_dir_delete").fs_path( 149 | self.plain_folder) 150 | shutil.rmtree(path) 151 | self.plain_tree = FileTree.from_fs(self.plain_folder) 152 | self.isPass() 153 | 154 | def test_delete_non_empty_folder(self): 155 | path = self.plain_tree.get("not_empty_dir").fs_path(self.plain_folder) 156 | shutil.rmtree(path) 157 | self.plain_tree = FileTree.from_fs(self.plain_folder) 158 | self.isPass() 159 | 160 | def test_change_password(self): 161 | sync = Syncrypto(self.crypto, self.encrypted_folder, self.plain_folder, 162 | self.encrypted_tree, self.plain_tree, 163 | self.snapshot_tree) 164 | sync.sync_folder(False) 165 | oldpass = self.crypto.password 166 | newpass = "new password" 167 | sync.change_password(newpass) 168 | self.crypto.password = oldpass 169 | self.assertRaises(DecryptError, sync._load_encrypted_tree) 170 | self.crypto.password = newpass.encode("ascii") 171 | sync.sync_folder(False) 172 | 173 | 174 | if __name__ == '__main__': 175 | unittest.main() 176 | -------------------------------------------------------------------------------- /tests/util.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright 2015 Qing Liang (https://github.com/liangqing) 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | 18 | from __future__ import print_function 19 | from __future__ import unicode_literals 20 | from io import open 21 | import os 22 | import os.path 23 | import shutil 24 | from filecmp import cmp as file_cmp 25 | from tempfile import mkdtemp 26 | from time import strftime, localtime 27 | from fnmatch import fnmatch 28 | 29 | try: 30 | from cStringIO import StringIO as BytesIO 31 | except ImportError: 32 | from io import BytesIO 33 | 34 | 35 | def format_datetime(t): 36 | return strftime("%Y-%m-%d %H:%M:%S", localtime(t)) 37 | 38 | 39 | def clear_folder(folder): 40 | for name in os.listdir(folder): 41 | if name == '.' or name == '..': 42 | continue 43 | path = folder+os.path.sep+name 44 | if os.path.isdir(path): 45 | shutil.rmtree(path) 46 | else: 47 | os.remove(path) 48 | 49 | 50 | def print_folder(folder): 51 | for root, dirs, files in os.walk(folder): 52 | for d in dirs: 53 | print(root + "/" + d) 54 | for f in files: 55 | print(root + "/" + f) 56 | 57 | 58 | def prepare_filetree(root, tree_string): 59 | lines = tree_string.split("\n") 60 | for line in lines: 61 | line = line.strip() 62 | if line == '' or line[0] == '#': 63 | continue 64 | pos = line.find(':') 65 | content = '' 66 | if pos >= 0: 67 | content = line[pos+1:].strip() 68 | line = line[:pos] 69 | pathname = line.strip().replace("/", os.path.sep) 70 | path = root + os.path.sep + pathname 71 | if pathname.endswith(os.path.sep) and not os.path.exists(path): 72 | os.makedirs(path) 73 | continue 74 | directory = os.path.dirname(path) 75 | if not os.path.exists(directory): 76 | os.makedirs(directory) 77 | fp = open(path, 'wb') 78 | fp.write(content.encode("utf-8")) 79 | fp.close() 80 | 81 | 82 | class TreeCmpResult: 83 | 84 | def __init__(self): 85 | self.left_only = [] 86 | self.right_only = [] 87 | self.diff_files = [] 88 | 89 | def __str__(self): 90 | return "left_only: %s\n right_only: %s\n diff_files: %s" % \ 91 | (self.left_only, self.right_only, self.diff_files) 92 | 93 | 94 | def tree_cmp(left, right, pathname="", ignores=None): 95 | is_dir_left = left is not None and os.path.isdir(left) 96 | is_dir_right = right is not None and os.path.isdir(right) 97 | exists_left = left is not None and os.path.exists(left) 98 | exists_right = right is not None and os.path.exists(right) 99 | cmp_result = TreeCmpResult() 100 | if is_dir_left and is_dir_right: 101 | sub_files = list(set(os.listdir(left)+os.listdir(right))) 102 | for sub_file in sub_files: 103 | if sub_file == "." or sub_file == "..": 104 | continue 105 | ignore = False 106 | if ignores is not None: 107 | for pattern in ignores: 108 | if fnmatch(sub_file, pattern): 109 | ignore = True 110 | break 111 | if ignore: 112 | continue 113 | if pathname == "": 114 | sub_pathname = sub_file 115 | else: 116 | sub_pathname = pathname+"/"+sub_file 117 | sub_cmp = \ 118 | tree_cmp(os.path.join(left, sub_file), 119 | os.path.join(right, sub_file), 120 | sub_pathname) 121 | cmp_result.left_only += sub_cmp.left_only 122 | cmp_result.right_only += sub_cmp.right_only 123 | cmp_result.diff_files += sub_cmp.diff_files 124 | elif is_dir_left: 125 | if not exists_right: 126 | cmp_result.left_only.append(pathname) 127 | else: 128 | cmp_result.diff_files.append(pathname) 129 | sub_files = os.listdir(left) 130 | for sub_file in sub_files: 131 | if sub_file == "." or sub_file == "..": 132 | continue 133 | ignore = False 134 | if ignores is not None: 135 | for pattern in ignores: 136 | if fnmatch(sub_file, pattern): 137 | ignore = True 138 | break 139 | if ignore: 140 | continue 141 | if pathname == "": 142 | sub_pathname = sub_file 143 | else: 144 | sub_pathname = pathname+"/"+sub_file 145 | sub_cmp = \ 146 | tree_cmp(os.path.join(left, sub_file), 147 | None, 148 | sub_pathname) 149 | cmp_result.left_only += sub_cmp.left_only 150 | elif is_dir_right: 151 | if not exists_left: 152 | cmp_result.right_only.append(pathname) 153 | else: 154 | cmp_result.diff_files.append(pathname) 155 | sub_files = os.listdir(right) 156 | for sub_file in sub_files: 157 | if sub_file == "." or sub_file == "..": 158 | continue 159 | if pathname == "": 160 | sub_pathname = sub_file 161 | else: 162 | sub_pathname = pathname+"/"+sub_file 163 | sub_cmp = \ 164 | tree_cmp(None, os.path.join(right, sub_file), 165 | sub_pathname) 166 | cmp_result.right_only += sub_cmp.right_only 167 | elif exists_left and exists_right: 168 | if not file_cmp(left, right, False): 169 | cmp_result.diff_files.append(pathname) 170 | elif exists_left: 171 | cmp_result.left_only.append(pathname) 172 | elif exists_right: 173 | cmp_result.right_only.append(pathname) 174 | return cmp_result 175 | 176 | if __name__ == "__main__": 177 | folder1 = mkdtemp() 178 | folder2 = mkdtemp() 179 | prepare_filetree(folder1, """ 180 | a/b/c:1 181 | x/y:2 182 | x/z:11 183 | w:3 184 | .dot_file:dot file 185 | """) 186 | prepare_filetree(folder2, """ 187 | x/z:22 188 | a/b/c/d:2 189 | """) 190 | # cmp2 = dircmp(os.path.join(folder1, "a", "b"), 191 | # os.path.join(folder2, "a", "b")) 192 | result = tree_cmp(folder1, folder2, ignores=[".*"]) 193 | print(result) 194 | shutil.rmtree(folder1) 195 | shutil.rmtree(folder2) 196 | --------------------------------------------------------------------------------