├── .codeclimate.yml ├── .coveralls.yml ├── .gitignore ├── .travis.yml ├── LICENSE ├── README.md ├── onedrive_client ├── __init__.py ├── data │ ├── config_schema.json │ ├── ignore_v2.txt │ ├── items_db.sql │ ├── ngrok_default_conf.yaml │ └── security_config.yml ├── lang │ └── od_pref.en_US.json ├── od_api_helper.py ├── od_api_session.py ├── od_auth.py ├── od_context.py ├── od_dateutils.py ├── od_hashutils.py ├── od_i18n.py ├── od_main.py ├── od_models │ ├── __init__.py │ ├── account_profile.py │ ├── bidict.py │ ├── dict_guard │ │ ├── __init__.py │ │ └── exceptions.py │ ├── drive_config.py │ ├── path_filter.py │ ├── pretty_api.py │ └── webhook_notification.py ├── od_pref.py ├── od_repo.py ├── od_stringutils.py ├── od_task.py ├── od_tasks │ ├── __init__.py │ ├── base.py │ ├── delete_item.py │ ├── download_file.py │ ├── merge_dir.py │ ├── move_item.py │ ├── start_repo.py │ ├── update_item_base.py │ ├── update_mtime.py │ ├── update_subscriptions.py │ └── upload_file.py ├── od_threads.py ├── od_watcher.py ├── od_webhook.py └── od_webhooks │ ├── __init__.py │ ├── http_server.py │ └── ngrok_server.py ├── report.txt ├── requirements.txt ├── setup.cfg ├── setup.py ├── sideci.yml └── tests ├── __init__.py ├── data ├── drive_config_item.json ├── drive_response.json ├── folder_child_item.json ├── folder_item.json ├── ignore_list.txt ├── image_item.json ├── list_drives.json ├── me_profile_business_response.json ├── me_profile_response.json ├── quota_response.json ├── sample_config_schema.json ├── session_response.json ├── subfolder_item.json ├── subscription_response.json └── webhook_notification.json ├── test_api_helper.py ├── test_api_session.py ├── test_auth.py ├── test_context.py ├── test_dateutils.py ├── test_dict_guard.py ├── test_hashutils.py ├── test_main_cli.py ├── test_models.py ├── test_pref_cli.py ├── test_repo.py ├── test_stringutils.py ├── test_task_pool.py ├── test_tasks.py ├── test_threads.py ├── test_watcher.py └── test_webhook_worker.py /.codeclimate.yml: -------------------------------------------------------------------------------- 1 | --- 2 | engines: 3 | duplication: 4 | enabled: true 5 | config: 6 | languages: 7 | - python 8 | fixme: 9 | enabled: true 10 | radon: 11 | enabled: true 12 | markdownlint: 13 | enabled: true 14 | pep8: 15 | enabled: true 16 | ratings: 17 | paths: 18 | - "**.inc" 19 | - "**.module" 20 | - "**.py" 21 | - "**.md" 22 | exclude_paths: 23 | - tests/ 24 | -------------------------------------------------------------------------------- /.coveralls.yml: -------------------------------------------------------------------------------- 1 | service_name: travis-pro 2 | repo_token: os2XVGo4REre6aZRjao6NrPpOUw0NHcsL 3 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | 2 | # Created by https://www.gitignore.io/api/linux,windows,macos,python,pycharm,vim,sublimetext 3 | 4 | ### Linux ### 5 | *~ 6 | 7 | # temporary files which can be created if a process still has a handle open of a deleted file 8 | .fuse_hidden* 9 | 10 | # KDE directory preferences 11 | .directory 12 | 13 | # Linux trash folder which might appear on any partition or disk 14 | .Trash-* 15 | 16 | # .nfs files are created when an open file is removed but is still being accessed 17 | .nfs* 18 | 19 | 20 | ### Windows ### 21 | # Windows thumbnail cache files 22 | Thumbs.db 23 | ehthumbs.db 24 | ehthumbs_vista.db 25 | 26 | # Folder config file 27 | Desktop.ini 28 | 29 | # Recycle Bin used on file shares 30 | $RECYCLE.BIN/ 31 | 32 | # Windows Installer files 33 | *.cab 34 | *.msi 35 | *.msm 36 | *.msp 37 | 38 | # Windows shortcuts 39 | *.lnk 40 | 41 | 42 | ### macOS ### 43 | *.DS_Store 44 | .AppleDouble 45 | .LSOverride 46 | 47 | # Icon must end with two \r 48 | Icon 49 | # Thumbnails 50 | ._* 51 | # Files that might appear in the root of a volume 52 | .DocumentRevisions-V100 53 | .fseventsd 54 | .Spotlight-V100 55 | .TemporaryItems 56 | .Trashes 57 | .VolumeIcon.icns 58 | .com.apple.timemachine.donotpresent 59 | # Directories potentially created on remote AFP share 60 | .AppleDB 61 | .AppleDesktop 62 | Network Trash Folder 63 | Temporary Items 64 | .apdisk 65 | 66 | 67 | ### Python ### 68 | # Byte-compiled / optimized / DLL files 69 | __pycache__/ 70 | *.py[cod] 71 | *$py.class 72 | 73 | # C extensions 74 | *.so 75 | 76 | # Distribution / packaging 77 | .Python 78 | env/ 79 | build/ 80 | develop-eggs/ 81 | dist/ 82 | downloads/ 83 | eggs/ 84 | .eggs/ 85 | lib/ 86 | lib64/ 87 | parts/ 88 | sdist/ 89 | var/ 90 | wheels/ 91 | *.egg-info/ 92 | .installed.cfg 93 | *.egg 94 | 95 | # PyInstaller 96 | # Usually these files are written by a python script from a template 97 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 98 | *.manifest 99 | *.spec 100 | 101 | # Installer logs 102 | pip-log.txt 103 | pip-delete-this-directory.txt 104 | 105 | # Unit test / coverage reports 106 | htmlcov/ 107 | .tox/ 108 | .coverage 109 | .coverage.* 110 | .cache 111 | nosetests.xml 112 | coverage.xml 113 | *,cover 114 | .hypothesis/ 115 | 116 | # Translations 117 | *.mo 118 | *.pot 119 | 120 | # Django stuff: 121 | *.log 122 | local_settings.py 123 | 124 | # Flask stuff: 125 | instance/ 126 | .webassets-cache 127 | 128 | # Scrapy stuff: 129 | .scrapy 130 | 131 | # Sphinx documentation 132 | docs/_build/ 133 | 134 | # PyBuilder 135 | target/ 136 | 137 | # Jupyter Notebook 138 | .ipynb_checkpoints 139 | 140 | # pyenv 141 | .python-version 142 | 143 | # celery beat schedule file 144 | celerybeat-schedule 145 | 146 | # dotenv 147 | .env 148 | 149 | # virtualenv 150 | .venv/ 151 | venv/ 152 | ENV/ 153 | 154 | # Spyder project settings 155 | .spyderproject 156 | 157 | # Rope project settings 158 | .ropeproject 159 | 160 | 161 | ### PyCharm ### 162 | # Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio and Webstorm 163 | # Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 164 | 165 | # User-specific stuff: 166 | .idea/workspace.xml 167 | .idea/tasks.xml 168 | 169 | # Sensitive or high-churn files: 170 | .idea/dataSources/ 171 | .idea/dataSources.ids 172 | .idea/dataSources.xml 173 | .idea/dataSources.local.xml 174 | .idea/sqlDataSources.xml 175 | .idea/dynamic.xml 176 | .idea/uiDesigner.xml 177 | 178 | # Gradle: 179 | .idea/gradle.xml 180 | .idea/libraries 181 | 182 | # Mongo Explorer plugin: 183 | .idea/mongoSettings.xml 184 | 185 | ## File-based project format: 186 | *.iws 187 | 188 | ## Plugin-specific files: 189 | 190 | # IntelliJ 191 | /out/ 192 | 193 | # mpeltonen/sbt-idea plugin 194 | .idea_modules/ 195 | 196 | # JIRA plugin 197 | atlassian-ide-plugin.xml 198 | 199 | # Crashlytics plugin (for Android Studio and IntelliJ) 200 | com_crashlytics_export_strings.xml 201 | crashlytics.properties 202 | crashlytics-build.properties 203 | fabric.properties 204 | 205 | ### PyCharm Patch ### 206 | # Comment Reason: https://github.com/joeblau/gitignore.io/issues/186#issuecomment-215987721 207 | 208 | # *.iml 209 | # modules.xml 210 | # .idea/misc.xml 211 | # *.ipr 212 | 213 | 214 | ### Vim ### 215 | # swap 216 | [._]*.s[a-v][a-z] 217 | [._]*.sw[a-p] 218 | [._]s[a-v][a-z] 219 | [._]sw[a-p] 220 | # session 221 | Session.vim 222 | # temporary 223 | .netrwhist 224 | # auto-generated tag files 225 | tags 226 | 227 | 228 | ### SublimeText ### 229 | # cache files for sublime text 230 | *.tmlanguage.cache 231 | *.tmPreferences.cache 232 | *.stTheme.cache 233 | 234 | # workspace files are user-specific 235 | *.sublime-workspace 236 | 237 | # project files should be checked into the repository, unless a significant 238 | # proportion of contributors will probably not be using SublimeText 239 | # *.sublime-project 240 | 241 | # sftp configuration file 242 | sftp-config.json 243 | 244 | # Package control specific files 245 | Package Control.last-run 246 | Package Control.ca-list 247 | Package Control.ca-bundle 248 | Package Control.system-ca-bundle 249 | Package Control.cache/ 250 | Package Control.ca-certs/ 251 | bh_unicode_properties.cache 252 | 253 | # Sublime-github package stores a github token in this file 254 | # https://packagecontrol.io/packages/sublime-github 255 | GitHub.sublime-settings 256 | 257 | # End of https://www.gitignore.io/api/linux,windows,macos,python,pycharm,vim,sublimetext 258 | 259 | \.vscode/ 260 | 261 | session\.pickle 262 | 263 | onedrive_config\.yml 264 | 265 | \.pytest_cache 266 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: python 2 | 3 | os: 4 | - linux 5 | 6 | python: 7 | - 3.4 8 | - 3.5 9 | - 3.5-dev 10 | - 3.6 11 | - 3.6-dev 12 | - 3.7-dev 13 | 14 | cache: pip 15 | sudo: false 16 | 17 | addons: 18 | apt: 19 | packages: 20 | - build-essential 21 | - inotify-tools 22 | - python3-dev 23 | - libssl-dev 24 | - libglib2.0-dev 25 | - libdbus-1-dev 26 | - libdbus-glib-1-dev 27 | - python3-dbus 28 | 29 | install: 30 | - pip install -U pip 31 | - pip install -U setuptools 32 | - pip install requests_mock 33 | - pip install coverage coveralls pytest codeclimate-test-reporter 34 | - python setup.py install 35 | 36 | before_script: 37 | - python --version 38 | 39 | script: 40 | - MOCK_KEYRING=1 python -m pytest 41 | - MOCK_KEYRING=1 coverage3 run --branch --source=onedrive_client setup.py test 42 | 43 | after_success: 44 | - coverage3 report -m 45 | - coveralls 46 | - CODECLIMATE_REPO_TOKEN=299d832e7f28d822ec417baf31e79973d7d0a0e9225c8fecabf359867e70e67e codeclimate-test-reporter 47 | - bash <(curl -s https://codecov.io/bash) 48 | 49 | notifications: 50 | email: false 51 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2016 Xiangyu Bu 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # onedriveClient 2 | 3 | [![GitHub version](https://badge.fury.io/gh/derrix060%2FonedriveClient.svg)](https://badge.fury.io/gh/derrix060%2FonedriveClient) 4 | [![Build Status](https://travis-ci.com/derrix060/onedriveClient.svg?branch=master)](https://travis-ci.com/derrix060/onedriveClient) 5 | [![License](https://img.shields.io/github/license/derrix060/onedriveClient.svg "MIT License")](LICENSE) 6 | [![codecov](https://codecov.io/gh/derrix060/onedriveClient/branch/master/graph/badge.svg)](https://codecov.io/gh/derrix060/onedriveClient) 7 | [![Coverage Status](https://coveralls.io/repos/github/derrix060/onedriveClient/badge.svg)](https://coveralls.io/github/derrix060/onedriveClient) 8 | [![Code Climate](https://codeclimate.com/github/derrix060/onedriveClient/badges/gpa.svg)](https://codeclimate.com/github/derrix060/onedriveClient) 9 | [![Codacy Badge](https://api.codacy.com/project/badge/Grade/9c0ff0f7b1e64120bcd66dc2e72f932e)](https://www.codacy.com/app/derrix060/onedriveClient?utm_source=github.com&utm_medium=referral&utm_content=derrix060/onedriveClient&utm_campaign=Badge_Grade) 10 | 11 | 12 | 13 | # Not being supported anymore. Check out [this project](https://github.com/abraunegg/onedrive) instead 14 | 15 | 16 | 17 | ## Introduction 18 | 19 | `onedrive_client` is a client program for [Microsoft OneDrive](https://onedrive.com) 20 | for Linux. It enables you to sync local directories with remote OneDrive 21 | repositories (a.k.a., _"Drive"_) of one or more OneDrive Personal account 22 | (OneDrive for Business accounts are in beta tests, use with caution!). 23 | 24 | The program is written in Python3, and uses 25 | [official OneDrive Python SDK](https://github.com/OneDrive/onedrive-sdk-python) 26 | to communicate with OneDrive server, 27 | [Keyring](https://pypi.python.org/pypi/keyring) to securely store account 28 | credentials, and [Linux inotify API](https://linux.die.net/man/7/inotify) to 29 | monitor file system changes. 30 | 31 | **IN DEVELOPMENT. USE WITH CAUTION.** 32 | 33 | ## Installation 34 | 35 | To install `onedrive_client`, install all pre-requisite packages, make sure old 36 | versions of `onedrive_client` are uninstalled, and lastly install `onedrive_client`. 37 | Each of those steps will be addressed in following subsections. 38 | 39 | The guide that follows will assume an environment with Python3 interpreter 40 | installed. To check the version of your Python3 interpreter, run command 41 | 42 | ```bash 43 | $ python3 --version 44 | Python 3.5.2 45 | ``` 46 | 47 | If `python3` command is not found, or its version is below `3.4`, please 48 | install the latest `python3` package. For example, on Ubuntu 49 | 50 | ```bash 51 | $ sudo apt-get install python3 52 | ``` 53 | 54 | It's strongly suggested that you 55 | [use the latest PIP](https://pip.pypa.io/en/stable/installing/#installing-with-get-pip-py) 56 | to manage Python package dependencies. To get the latest `pip` from source, 57 | run command 58 | 59 | ```bash 60 | # Download pip installation script from official site using wget. 61 | $ wget -O- https://bootstrap.pypa.io/get-pip.py | sudo python3 62 | # Upgrade the components (e.g., setuptools) to latest version. 63 | $ sudo pip3 install -U pip setuptools 64 | ``` 65 | 66 | To run on Raspbery PI 3, is necessary add more packages. 67 | ```bash 68 | $ sudo apt-get install python3 build-essential python3-dev libssl-dev inotify-tools python3-dbus libffi-dev dbus-devel libdbus-glib-1-dev -y 69 | $ sudo pip3 install pydbus 70 | ``` 71 | 72 | ### Pre-requisites 73 | 74 | The use of low-level tools and APIs like `inotify` and `keyring` introduces 75 | low-level dependencies that need to be installed manually. On Ubuntu the 76 | following packages are needed: 77 | 78 | * `gcc` 79 | * `python3-dev` 80 | * `libssl-dev` 81 | * `inotify-tools` 82 | * `python3-dbus` (or probably `libdbus-glib-1-dev`) 83 | 84 | On other distros like Fedora, names of those packages may vary. 85 | 86 | Note that `keyring`, which provides secure local storage for OneDrive 87 | credentials (the leak of which may result in total compromise of your OneDrive 88 | data), may require additional packages (for example, D-Bus or FreeDesktop 89 | Secret Service) depending on your Linux distro and desktop manager. Please 90 | refer to its 91 | [installation instructions](https://pypi.python.org/pypi/keyring#installation-instructions) 92 | for more details. If your environment requires `keyring.alt` package, make 93 | sure to use the latest version (`sudo pip3 install -U keyrings.alt`). 94 | 95 | To install those dependencies on Ubuntu, use `apt-get` command: 96 | 97 | ```bash 98 | # Install gcc and other C-level pre-requisites. 99 | $ sudo apt install build-essential python3-dev libssl-dev inotify-tools python3-dbus libdbus-1-dev libdbus-glib-1-dev 100 | 101 | # Install keyring to store the passwords 102 | $ sudo apt install gnome-keyring 103 | $ eval `gnome-keyring-daemon` 104 | $ eval `dbus-launch` 105 | 106 | # install ngrok from own website (ngrok.com) and install in /usr/local/bin 107 | # Don't install ngrok with sudo apt-get install ngrok-client, it does not install the 'good' ngrok! 108 | ``` 109 | 110 | Python-level pre-requisites are listed in `requirements.txt` and will be 111 | installed automatically when installing `onedrive_client`. 112 | 113 | ### Uninstall older `onedrive_client` 114 | 115 | If you have old versions of `onedrive_client` (also named `onedrived` in the 116 | past) in system, please uninstall them before proceeding. The packages 117 | can be easily removed with `pip`. 118 | 119 | ```bash 120 | # Remove Python packages of older onedrive-d. 121 | $ sudo pip3 uninstall onedrive_d onedrive_client 122 | 123 | # Remove useless config files. 124 | $ rm -rf ~/.onedrived ~/.onedrive_client 125 | ``` 126 | 127 | ### Install `onedrive_client` 128 | 129 | First pull the code from GitHub repository: 130 | 131 | ```bash 132 | $ git clone https://github.com/derrix060/onedriveClient.git 133 | $ cd onedrive_client 134 | ``` 135 | Then install `onedrive_client`: 136 | 137 | ```bash 138 | $ python3 setup.py install 139 | ``` 140 | 141 | ## Usage 142 | 143 | `onedrive_client` exposes two commands -- `onedrive-client` and `onedrive-client-pref`. The 144 | former is the "synchronizer" and the latter is the "configurator". If you 145 | want to run it directly in code repository without installing the package, in 146 | the following example commands replace `onedrive-client` with 147 | `python3 -m onedrive_client.od_main` and replace `onedrive-client-pref` with 148 | `python3 -m onedrive_client.od_pref`. 149 | 150 | ### Configure `onedrive_client` 151 | 152 | Before running `onedrive_client` for the first time, or whenever you need to change 153 | the configurations, you will need to use `onedrive-client-pref` command. The 154 | subsections that follow introduces the basic usage scenarios. For more usage 155 | scenarios, refer to "More Usages" section. 156 | 157 | To read the complete usage of `onedrive-client-pref`, use argument `--help`: 158 | 159 | ```bash 160 | $ onedrive-client-pref --help 161 | Usage: od_pref.py [OPTIONS] COMMAND [ARGS]... 162 | 163 | Options: 164 | --version Show the version and exit. 165 | -h, --help Show this message and exit. 166 | 167 | Commands: 168 | account Add new OneDrive account to onedrive_client, list all existing ones, or 169 | remove some. 170 | config Modify config (e.g., proxies, intervals) for current user. 171 | drive List all remote OneDrive repositories (Drives) of linked accounts, 172 | add new Drives to sync, edit configurations of existing Drives, or 173 | remove a Drive from local list. 174 | ``` 175 | 176 | #### Authorizing accounts 177 | 178 | Operations related to configuring accounts can be listed by command 179 | `onedrive-client-pref account` 180 | 181 | ```bash 182 | $ onedrive-client-pref account --help 183 | Usage: od_pref.py account [OPTIONS] COMMAND [ARGS]... 184 | 185 | Options: 186 | -h, --help Show this message and exit. 187 | 188 | Commands: 189 | add Add a new OneDrive account to onedrive_client. 190 | del De-authorize and delete an existing account from onedrive_client. 191 | list List all linked accounts. 192 | ``` 193 | 194 | To add an OneDrive account to `onedrive_client`, you will need command 195 | `onedrive-client-pref account add`. Help message for this command is as follows: 196 | 197 | ```bash 198 | $ onedrive-client-pref account add --help 199 | Usage: od_pref.py account add [OPTIONS] 200 | 201 | Options: 202 | -u, --get-auth-url If set, print the authentication URL and exit. 203 | -c, --code TEXT Skip interactions and try authenticating with the code 204 | directly. 205 | -b, --for-business If set, add an OneDrive for Business account. 206 | -h, --help Show this message and exit. 207 | ``` 208 | 209 | More specifically, the CLI offers two modes to add an account -- _interactive 210 | mode_, in which the CLI guides you step by step, and _command mode_, in which 211 | you provide the information from command line arguments. 212 | 213 | ##### Interactive mode 214 | 215 | In interactive mode, the program will provide you with an URL to visit. Open 216 | this URL with a web browser (e.g., Chrome, Firefox), sign in with your 217 | Microsoft Account and authorize `onedrive_client` to access your OneDrive data. The 218 | web page will eventually land to a blank page whose URL starts with 219 | "https://login.live.com/oauth20_desktop.srf". Paste this URL 220 | (a.k.a., _callback URL_) back to the program. 221 | 222 | Note that `onedrive_client` needs your basic account information (e.g., email 223 | address) to distinguish different accounts (otherwise OneDrive returns 224 | "tokens" from which you cannot tell which account they stand for). 225 | 226 | ```bash 227 | $ onedrive-client-pref account add 228 | 229 | NOTE: To better manage your OneDrive accounts, onedrive_client needs permission to access your account info (e.g., email 230 | address to distinguish different accounts) and read/write your OneDrive files. 231 | 232 | Paste this URL into your browser to sign in and authorize onedrive_client: 233 | 234 | https://login.live.com/oauth20_authorize.srf?response_type=code&scope=wl.signin+wl.emails+wl.offline_access+ 235 | onedrive.readwrite&client_id=000000004010C916&redirect_uri=https%3A%2F%2Flogin.live.com%2Foauth20_desktop.srf 236 | 237 | The authentication web page will finish with a blank page whose URL starts with 238 | "https://login.live.com/oauth20_desktop.srf". Paste this URL here. 239 | Paste URL here: https://login.live.com/oauth20_desktop.srf?code=&lc=1033 240 | 241 | Successfully authorized onedrive_client. 242 | Successfully added account for Xiangyu Bu (xybu92@live.com, )! 243 | 244 | All OneDrive accounts associated with user "xb": 245 | 246 | # Account ID Owner Name Email Address 247 | --- ------------------ ------------ --------------- 248 | 0 Xiangyu Bu xybu92@live.com 249 | ``` 250 | 251 | ###### One Drive for Business Support 252 | Be careful, it's still on beta test. As explained before, the onedrive needs some informations. In case of Business, it needs to use 2 different services, one of them to do the background tasks (add, remove, rename some item) and another one just to get your name and email. That's why you need to authenticate in two different services. 253 | 254 | To add a business account, just insert the tag '-b' at end: 255 | 256 | ```bash 257 | $ onedrive-client-pref account add -b 258 | ``` 259 | 260 | As happen in normal account, you need to click on link showed and copy the entire link (not only the code!!), but as explained before, you need to do this twice (one for each service). 261 | 262 | ##### Command mode 263 | 264 | Instead of giving the sign-in URL and then prompting for the callback URL, use 265 | the following command to get the sign-in URL: 266 | 267 | ```bash 268 | $ onedrive-client-pref account add --get-auth-url 269 | NOTE: To better manage your OneDrive accounts, onedrive_client needs permission to access your account info (e.g., email 270 | address to distinguish different accounts) and read/write your OneDrive files. 271 | 272 | Paste this URL into your browser to sign in and authorize onedrive_client: 273 | 274 | https://login.live.com/oauth20_authorize.srf?response_type=code&client_id=000000004010C916&redirect_uri=https%3A%2F%2F 275 | login.live.com%2Foauth20_desktop.srf&scope=wl.signin+wl.emails+wl.offline_access+onedrive.readwrite 276 | ``` 277 | 278 | Visit the URL and do the same steps as interactive mode until you get the 279 | blank page. Copy the URL and copy the `code` parameter from the URL. For 280 | example, in URL 281 | `https://login.live.com/oauth20_desktop.srf?code=&lc=1033`, 282 | find the part `?code=&` and the code is the part 283 | ``. 284 | 285 | Use command `onedrive-client-pref account add --code `, where 286 | `` is the code, to add your account. 287 | 288 | #### Adding Drives to `onedrive_client` 289 | 290 | After you authorize `onedrive_client` to access your OneDrive data, you are now able 291 | to add Drives. Each OneDrive account has one or more Drive associated, and 292 | `onedrive_client` allows you to choose which Drive to sync. Similar to the step of 293 | authorizing `onedrive_client`, the CLI provides both interactive mode and command 294 | mode. 295 | 296 | #### Interactive mode 297 | 298 | ```bash 299 | $ onedrive-client-pref drive set 300 | Reading drives information from OneDrive server... 301 | 302 | All available Drives of authorized accounts: 303 | 304 | # Account Email Drive ID Type Quota Status 305 | --- --------------- ---------------- -------- --------------------------- -------- 306 | 0 personal 5.3 GB Used / 33.0 GB Total active 307 | 308 | Please enter row number of the Drive to add or modify (CTRL+C to abort): 0 309 | 310 | Going to add/edit Drive "" of account ""... 311 | Enter the directory path to sync with this Drive [/home/xb/OneDrive]: 312 | Syncing with directory "/home/xb/OneDrive"? [y/N]: y 313 | Enter the path to ignore file for this Drive [/home/xb/.config/onedrive_client/ignore_v2.txt]: 314 | 315 | Successfully configured Drive of account (): 316 | Local directory: /home/xb/OneDrive 317 | Ignore file path: /home/xb/.config/onedrive_client/ignore_v2.txt 318 | ``` 319 | 320 | If you have more than one account authorized, all drives of all authorized 321 | accounts will appear in the table. 322 | 323 | #### Command mode 324 | 325 | Please find the available command-line arguments from help message using 326 | command `onedrive-client-pref drive set --help`. 327 | 328 | ### Set up webhook 329 | 330 | #### Webhook explained 331 | 332 | For now, refer to issue #19. More details TBA. 333 | 334 | #### Using `ngrok`-based webhook 335 | 336 | Download and install [ngrok](https://ngrok.com). 337 | 338 | By default, `onedrive_client` will look for `ngrok` binary from `PATH`. To specify 339 | path to the binary manually, set up environment variable `NGROK` when running 340 | `onedrive-client`. For example, `NGROK=~/utils/ngrok onedrive-client start --debug`. 341 | 342 | To use a custom config file for `ngrok`, set environment variable 343 | `NGROK_CONFIG_FILE` to path of your desired config file. Note that `onedrive_client` 344 | will create a HTTPS tunnel automatically and there is no need to specify 345 | tunnels. The purpose of using a custom `ngrok` config file should be to adjust 346 | resource usage, or link `ngrok` process with your paid `ngrok` account. The 347 | default `ngrok` config file shipped with `onedrive_client` turns off terminal output 348 | of `ngrok` and disables inspection database. 349 | 350 | #### Using direct connection 351 | 352 | TBA. Not applicable to most end-user machines. 353 | 354 | ### Run `onedrive_client` in debug mode 355 | 356 | Use argument `--debug` so that `onedrive_client` runs in debug mode, using 357 | debug-level log verbosity and printing log to `stderr`. 358 | 359 | ```bash 360 | onedrive-client start --debug 361 | ``` 362 | 363 | To stop `onedrive_client` process which is running in debug mode, send `SIGINT` to 364 | the process or hitting CTRL+C if it runs in a terminal. 365 | 366 | ### Run `onedrive_client` as daemon 367 | 368 | It's suggested that you set up a log file before running in daemon mode: 369 | 370 | ``` 371 | $ onedrive-client-pref config set logfile_path PATH_TO_SOME_WRITABLE_FILE 372 | ``` 373 | 374 | To start the program as daemon, 375 | 376 | ```bash 377 | onedrive-client start 378 | ``` 379 | 380 | To stop the daemon, 381 | 382 | ```bash 383 | onedrive-client stop 384 | ``` 385 | 386 | or send `SIGTERM` to the process (Ctrl + C). 387 | 388 | ### More Usages 389 | 390 | #### Run `onedrive_client` with proxies 391 | 392 | `onedrive_client` follows behavior of standard Python library function 393 | [`getproxies()`](https://docs.python.org/3/library/urllib.request.html#urllib.request.getproxies) 394 | to read proxies information from the OS. That is, run the command with 395 | environment variable `HTTP_PROXY` (or `http_proxy`) to set up a HTTP proxy, and 396 | variable `HTTPS_PROXY` (or `https_proxy`) to set up a HTTPS proxy. For example, 397 | 398 | ```bash 399 | $ HTTPS_PROXY=https://user:pass@host:port/some_path onedrive-client start --debug 400 | ``` 401 | 402 | A HTTPS proxy must have a verifiable SSL certificate. 403 | 404 | #### List all authorized OneDrive accounts 405 | 406 | #### Remove an authorized account 407 | 408 | #### List all remote Drives 409 | 410 | #### Edit configuration of an existing Drive 411 | 412 | #### Edit ignore list (selective sync) 413 | 414 | #### Remove a Drive from `onedrive_client` 415 | 416 | ##### Interactive mode 417 | 418 | ```bash 419 | $ onedrive-client-pref drive del 420 | Drives that have been set up: 421 | 422 | #0 - Drive "": 423 | Account: () 424 | Local root: /home/xb/OneDrive 425 | Ignore file: /home/xb/.config/onedrive_client/ignore_v2.txt 426 | 427 | Please enter the # number of the Drive to delete (CTRL+C to abort): 0 428 | Continue to delete Drive "" (its local directory will NOT be deleted)? [y/N]: y 429 | Successfully deleted Drive "" from onedrive_client. 430 | ``` 431 | 432 | ##### Command mode 433 | 434 | The command-mode equivalent is: 435 | 436 | ```bash 437 | onedrive-client-pref drive del --drive-id [--yes] 438 | ``` 439 | 440 | If argument `--yes` is used, the specified Drive, if already added, will be 441 | deleted without confirmation. 442 | 443 | #### Adjusting parameters of `onedrive_client` 444 | 445 | #### Check latest version of `onedrive_client` 446 | 447 | ## Uninstallation 448 | 449 | Use `pip3` to uninstall `onedrive_client` from system: 450 | 451 | ```bash 452 | $ pip3 uninstall onedrive_client 453 | ``` 454 | 455 | If `--user` argument was not used when installing (that is, `onedrive_client` was 456 | installed as a system-level package), you will need root permission to run 457 | the command above. 458 | 459 | ## License 460 | 461 | MIT License. 462 | -------------------------------------------------------------------------------- /onedrive_client/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | onedrive-d 3 | A Microsoft OneDrive client for Linux. 4 | :copyright: (c) Xiangyu Bu 5 | :license: MIT 6 | """ 7 | 8 | import os 9 | import pkgutil 10 | 11 | __project__ = 'onedriveClient' 12 | __author__ = 'Mario Apra' 13 | __email__ = 'mariotapra@gmail.com' 14 | __version__ = '2.0.1' 15 | __homepage__ = 'https://github.com/derrix060/onedriveClient' 16 | 17 | 18 | def mkdir(path, uid, mode=0o700, exist_ok=True): 19 | """Create a path and set up owner uid.""" 20 | os.makedirs(path, mode, exist_ok=exist_ok) 21 | os.chown(path, uid, -1) 22 | 23 | 24 | def fix_owner_and_timestamp(path, uid, t): 25 | """ 26 | :param str path: 27 | :param int uid: 28 | :param int | float t: 29 | :return: 30 | """ 31 | os.chown(path, uid, -1) 32 | os.utime(path, (t, t)) 33 | 34 | 35 | def get_resource(rel_path, pkg_name='onedrive_client', is_text=True): 36 | """ 37 | Read a resource file in data/. 38 | :param str rel_path: 39 | :param str pkg_name: 40 | :param True | False is_text: True to indicate the text is UTF-8 encoded. 41 | :return str | bytes: Content of the file. 42 | """ 43 | content = pkgutil.get_data(pkg_name, rel_path) 44 | if is_text: 45 | content = content.decode('utf-8') 46 | return content 47 | -------------------------------------------------------------------------------- /onedrive_client/data/config_schema.json: -------------------------------------------------------------------------------- 1 | { 2 | "scan_interval_sec": { 3 | "type": "integer", 4 | "minimum": 60, 5 | "description": "@lang['config.scan_interval_sec.desc']" 6 | }, 7 | "num_workers": { 8 | "type": "integer", 9 | "minimum": 1, 10 | "description": "@lang['config.num_workers.desc']" 11 | }, 12 | "webhook_renew_interval_sec": { 13 | "type": "integer", 14 | "minimum": 30, 15 | "description": "@lang['config.webhook_renew_interval_sec.desc']" 16 | }, 17 | "webhook_port": { 18 | "type": "integer", 19 | "minimum": 0, 20 | "maximum": 65535, 21 | "description": "@lang['config.webhook_port.desc']" 22 | }, 23 | "start_delay_sec": { 24 | "type": "integer", 25 | "minimum": 0, 26 | "description": "@lang['config.start_delay_sec.desc']" 27 | }, 28 | "logfile_path": { 29 | "type": "string", 30 | "subtype": "file", 31 | "to_abspath": true, 32 | "create_if_missing": true, 33 | "allow_empty": true, 34 | "permission": "a", 35 | "description": "@lang['config.logfile_path.desc']" 36 | }, 37 | "webhook_type": { 38 | "type": "string", 39 | "choices": ["direct", "ngrok"], 40 | "description": "@lang['config.webhook_type.desc']" 41 | }, 42 | "webhook_host": { 43 | "type": "string", 44 | "allow_empty": true, 45 | "description": "@lang['config.webhook_host.desc']" 46 | }, 47 | "webhook_action_delay_sec": { 48 | "type": "integer", 49 | "minimum": 10, 50 | "description": "@lang['config.webhook_action_delay_sec.desc']" 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /onedrive_client/data/ignore_v2.txt: -------------------------------------------------------------------------------- 1 | # Hard-coded rules: 2 | # .*.odtemp! -- onedrive_client temp files. 3 | # *[<>?*:"|]* -- NTFS namespace violation. 4 | # .* -- NTFS namespace violation. 5 | 6 | # Created by https://www.gitignore.io/api/linux,windows,macos 7 | 8 | ### Linux ### 9 | *~ 10 | 11 | # temporary files which can be created if a process still has a handle open of a deleted file 12 | # .fuse_hidden* 13 | 14 | # KDE directory preferences 15 | # .directory 16 | 17 | # Linux trash folder which might appear on any partition or disk 18 | # .Trash-* 19 | 20 | # .nfs files are created when an open file is removed but is still being accessed 21 | # .nfs* 22 | 23 | 24 | ### Windows ### 25 | # Windows thumbnail cache files 26 | Thumbs.db 27 | ehthumbs.db 28 | ehthumbs_vista.db 29 | 30 | # Folder config file 31 | Desktop.ini 32 | 33 | # Recycle Bin used on file shares 34 | $RECYCLE.BIN/ 35 | 36 | # Windows shortcuts 37 | *.lnk 38 | 39 | 40 | ### macOS ### 41 | *.DS_Store 42 | # .AppleDouble 43 | # .LSOverride 44 | 45 | # Icon must end with two \r 46 | Icon 47 | # Thumbnails 48 | # ._* 49 | # Files that might appear in the root of a volume 50 | # .DocumentRevisions-V100 51 | # .fseventsd 52 | # .Spotlight-V100 53 | # .TemporaryItems 54 | # .Trashes 55 | # .VolumeIcon.icns 56 | # .com.apple.timemachine.donotpresent 57 | # Directories potentially created on remote AFP share 58 | # .AppleDB 59 | # .AppleDesktop 60 | Network Trash Folder 61 | Temporary Items 62 | # .apdisk 63 | 64 | # End of https://www.gitignore.io/api/linux,windows,macos 65 | -------------------------------------------------------------------------------- /onedrive_client/data/items_db.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE IF NOT EXISTS items ( 2 | id TEXT UNIQUE ON CONFLICT REPLACE, 3 | type INT, 4 | name TEXT, 5 | parent_id TEXT, 6 | parent_path TEXT, 7 | etag TEXT, 8 | ctag TEXT, 9 | size UNSIGNED BIG INT, 10 | size_local UNSIGNED BIG INT, 11 | created_time TEXT, 12 | modified_time TEXT, 13 | status INT, 14 | sha1_hash TEXT, 15 | record_time TEXT, 16 | PRIMARY KEY (parent_path, name) ON CONFLICT REPLACE 17 | ); 18 | -------------------------------------------------------------------------------- /onedrive_client/data/ngrok_default_conf.yaml: -------------------------------------------------------------------------------- 1 | console_ui: false 2 | inspect_db_size: -1 3 | -------------------------------------------------------------------------------- /onedrive_client/data/security_config.yml: -------------------------------------------------------------------------------- 1 | BUSINESS_V1: 2 | CLIENT_ID: '6fdb55b4-c905-4612-bd23-306c3918217c' 3 | CLIENT_SECRET: 'HThkLCvKhqoxTDV9Y9uS+EvdQ72fbWr/Qrn2PFBZ/Ow=' 4 | REDIRECT: 'https://od.cnbeining.com' 5 | 6 | BUSINESS_V2: 7 | CLIENT_ID: '0e170d2c-0ac5-4a4f-9099-c6bb0fb52d0c' 8 | CLIENT_SECRET: 'xdGsBCTOiCHxBWJcKyK2WpA' 9 | REDIRECT: 'https://onedrivesite.mario-apra.tk/' 10 | 11 | PERSONAL: 12 | CLIENT_ID: '000000004010C916' 13 | CLIENT_SECRET: 'PimIrUibJfsKsMcd0SqwPBwMTV7NDgYi' 14 | REDIRECT: 'https://login.live.com/oauth20_desktop.srf' -------------------------------------------------------------------------------- /onedrive_client/lang/od_pref.en_US.json: -------------------------------------------------------------------------------- 1 | { 2 | "config.scan_interval_sec.desc": "Interval, in seconds, between two actions of scanning the entire repository.", 3 | "config.num_workers.desc": "Total number of worker threads.", 4 | "config.webhook_renew_interval_sec.desc": "Renew webhook after this amount of time, in seconds. Ideal value should be slightly larger than the lifespan of onedrive_client process.", 5 | "config.start_delay_sec.desc": "Amount of time, in seconds, to sleep before main starts working.", 6 | "config.logfile_path.desc": "Path to log file. Empty string means writing to stdout.", 7 | "config.webhook_type.desc": "Type of webhook. Use \"direct\" only if your machine can be reached from public network.", 8 | "config.webhook_host.desc": "Hostname in webhook URL. Used in \"direct\" webhook and must resolve to local host. Leave blank to use public IP of the machine.", 9 | "config.webhook_port.desc": "Port number for webhook. Default: 0 (let OS allocate a free port).", 10 | "config.webhook_action_delay_sec.desc": "Wait for this period of time before acting (usually a merge action) after a webhook notification is received. During this period duplicate notifications can be merged.", 11 | 12 | "configurator.error_invalid_key": "Config key \"{key}\" does not exist or is read-only.", 13 | "configurator.error_int_value_required": "Invalid value \"{value}\" for \"{key}\": must be an integer.", 14 | "configurator.error_int_below_minimum": "Invalid value for \"{key}\": no less than {minimum}; got {value}.", 15 | "configurator.error_int_above_maximum": "Invalid value for \"{key}\": no more than {maximum}; got {value}.", 16 | "configurator.error_str_invalid_choice": "Invalid value \"{value}\" for \"{key}\": must be a choice from {choices}.", 17 | "configurator.error_str_not_startswith": "Invalid value \"{value}\" for \"{key}\": must start with {starts_with}.", 18 | "configurator.error_path_not_exist": "Invalid value for \"{key}\": path \"{path}\" does not exist.", 19 | "configurator.error_path_not_file": "Invalid value for \"{key}\": path \"{path}\" is not a file.", 20 | "configurator.error_generic": "Invalid value for \"{key}\": {error_message}.", 21 | 22 | "od_pref.drive.submain.short_help": "List all remote OneDrive repositories (Drives) of linked accounts, add new Drives to sync, edit configurations of existing Drives, or remove a Drive from local list.", 23 | "od_pref.list_drive.short_help": "List all available Drives.", 24 | "od_pref.del_drive.short_help": "Stop syncing a Drive with local directory.", 25 | "od_pref.del_drive.specify_drive_to_delete": "Please specify the Drive ID to delete.", 26 | "od_pref.del_drive.choose_index": "Please enter the # number of the Drive to delete (CTRL+C to abort)", 27 | "od_pref.del_drive.error_del_db_file": "Error deleting drive database: {error}.", 28 | 29 | "od_pref.config.submain.short_help": "Modify config (e.g., intervals, number of workers, etc.) for current user.", 30 | "od_pref.set_config.short_help": "Update a config parameter.", 31 | "od_pref.print_config.short_help": "Print all config parameters along with their descriptions and values.", 32 | 33 | "od_pref.account.submain.short_help": "Add new OneDrive account to onedrive_client, list or remove existing ones.", 34 | "od_pref.authenticate_account.short_help": "Add a new OneDrive account to onedrive_client.", 35 | "od_pref.authenticate_account.get_auth_url.help": "If set, print the authentication URL and exit.", 36 | "od_pref.authenticate_account.code.help": "Skip interactions and try authenticating with the code directly.", 37 | "od_pref.authenticate_account.for_business.help": "If set, add an OneDrive for Business account.", 38 | "od_pref.save_account.success": "Successfully added account for {profile.account_name} ({profile.account_email})!", 39 | "od_pref.save_account.print_header": "All OneDrive accounts associated with user \"{context.user_name}\":", 40 | "od_pref.save_account.error": "Failed to save account: {error_message}.", 41 | "od_pref.authenticate_account.for_business_unsupported": "OneDrive for Business is not yet supported.", 42 | "od_pref.authenticate_account.permission_note": "NOTE: To better manage your OneDrive accounts, onedrive_client needs permission to access your account info (e.g., email address to distinguish different accounts) and read/write your OneDrive files.\n", 43 | "od_pref.authenticate_account.paste_url_note": "Paste this URL into your browser to sign in and authorize onedrive_client:", 44 | "od_pref.authenticate_account.paste_url_instruction": "The authentication web page will finish with a blank page whose URL starts with \"{redirect_url}\". Paste this URL after the prompt.", 45 | "od_pref.authenticate_account.second_authentication": "Need authenticate again to get personal information'", 46 | "od_pref.authenticate_account.paste_url_prompt": "Paste URL here", 47 | "od_pref.authenticate_account.error.code_not_found_in_url": "Error: did not find authorization code in URL.", 48 | "od_pref.authenticate_account.success.authorized": "Successfully authorized onedrive_client.", 49 | "od_pref.authenticate_account.error.authorization": "Failed to authorize onedrive_client: {error_message}.", 50 | 51 | "od_pref.print_all_drives.fetching_drives.note": "Reading drives information from OneDrive server...", 52 | "od_pref.print_all_drives.all_drives_table.note": "All available Drives of authorized accounts:", 53 | "od_pref.print_all_drives.all_drives_table.header.index": "#", 54 | "od_pref.print_all_drives.all_drives_table.header.account_email": "Account Email", 55 | "od_pref.print_all_drives.all_drives_table.header.drive_id": "Drive ID", 56 | "od_pref.print_all_drives.all_drives_table.header.type": "Type", 57 | "od_pref.print_all_drives.all_drives_table.header.quota": "Quota", 58 | "od_pref.print_all_drives.all_drives_table.header.status": "Status", 59 | 60 | "api.drive.quota.short_format": "{used} Used / {total} Total" 61 | } 62 | -------------------------------------------------------------------------------- /onedrive_client/od_api_helper.py: -------------------------------------------------------------------------------- 1 | import json 2 | import logging 3 | import time 4 | 5 | import onedrivesdk 6 | import onedrivesdk.error 7 | import requests 8 | 9 | from onedrive_client import od_dateutils 10 | 11 | THROTTLE_PAUSE_SEC = 60 12 | 13 | 14 | def get_drive_request_builder(repo): 15 | return onedrivesdk.DriveRequestBuilder( 16 | request_url=repo.authenticator.client.base_url + 'drives/' + repo.drive.id, 17 | client=repo.authenticator.client) 18 | 19 | 20 | def create_subscription(folder_item_request, repo, webhook_url, expiration_time): 21 | """ 22 | :param onedrivesdk.ItemRequestBuilder folder_item_request: 23 | :param onedrive_client.od_repo.OneDriveLocalRepository repo: 24 | :param str webhook_url: 25 | :param datetime.datetime.datetime expiration_time: 26 | :return onedrivesdk.Subscription: 27 | """ 28 | subscriptions_collection_req = folder_item_request.subscriptions 29 | subscription_req_builder = onedrivesdk.SubscriptionRequestBuilder(subscriptions_collection_req._request_url, 30 | subscriptions_collection_req._client) 31 | subscription_req = item_request_call(repo, subscription_req_builder.request) 32 | subscription_req.content_type = "application/json" 33 | subscription_req.method = "POST" 34 | subscription = onedrivesdk.Subscription() 35 | subscription.notification_url = webhook_url 36 | subscription.expiration_date_time = expiration_time 37 | return onedrivesdk.Subscription(json.loads(subscription_req.send(subscription).content)) 38 | 39 | 40 | def update_subscription(self, subscription): 41 | """ A temp patch for bug https://github.com/OneDrive/onedrive-sdk-python/issues/95. """ 42 | self.content_type = "application/json" 43 | self.method = "PATCH" 44 | entity = onedrivesdk.Subscription(json.loads(self.send(subscription).content)) 45 | return entity 46 | 47 | 48 | onedrivesdk.SubscriptionRequest.update = update_subscription 49 | 50 | 51 | def get_item_modified_datetime(item): 52 | """ 53 | :param onedrivesdk.Item item: 54 | :return [arrow.Arrow, True | False]: Return a 2-tuple (datetime, bool) in which the bool indicates modifiable. 55 | """ 56 | # SDK Bug: the API can return some non-standard datetime string that SDK can't handle. 57 | # https://github.com/OneDrive/onedrive-sdk-python/issues/89 58 | # Until the bug is fixed I'll avoid the SDK calls and use the value directly. 59 | try: 60 | return od_dateutils.str_to_datetime(item.file_system_info._prop_dict['lastModifiedDateTime']), True 61 | except AttributeError: 62 | # OneDrive for Business does not have FileSystemInfo facet. Fall back to read-only mtime attribute. 63 | return od_dateutils.str_to_datetime(item._prop_dict['lastModifiedDateTime']), False 64 | 65 | 66 | def get_item_created_datetime(item): 67 | return od_dateutils.str_to_datetime(item._prop_dict['createdDateTime']) 68 | 69 | 70 | def item_request_call(repo, request_func, *args, **kwargs): 71 | while True: 72 | try: 73 | return request_func(*args, **kwargs) 74 | except onedrivesdk.error.OneDriveError as e: 75 | logging.error('Encountered API Error: %s.', e) 76 | if e.code == onedrivesdk.error.ErrorCode.ActivityLimitReached: 77 | time.sleep(THROTTLE_PAUSE_SEC) 78 | elif e.code == onedrivesdk.error.ErrorCode.Unauthenticated: 79 | repo.authenticator.refresh_session(repo.account_id) 80 | else: 81 | raise e 82 | except requests.ConnectionError as e: 83 | logging.error('Encountered connection error: %s. Retry in %d sec.', e, THROTTLE_PAUSE_SEC) 84 | time.sleep(THROTTLE_PAUSE_SEC) 85 | -------------------------------------------------------------------------------- /onedrive_client/od_api_session.py: -------------------------------------------------------------------------------- 1 | import base64 2 | import pickle 3 | import zlib 4 | from time import time 5 | 6 | import keyring 7 | import onedrivesdk.session 8 | 9 | 10 | def get_keyring_key(account_id): 11 | return OneDriveAPISession.KEYRING_ACCOUNT_KEY_PREFIX + account_id 12 | 13 | 14 | class OneDriveAPISession(onedrivesdk.session.Session): 15 | 16 | SESSION_ARG_KEYNAME = 'key' 17 | KEYRING_SERVICE_NAME = 'onedrive_client_v2' 18 | KEYRING_ACCOUNT_KEY_PREFIX = 'user.' 19 | PICKLE_PROTOCOL = 3 20 | 21 | @property 22 | def expires_in_sec(self): 23 | return self._expires_at - time() 24 | 25 | def save_session(self, **save_session_kwargs): 26 | if self.SESSION_ARG_KEYNAME not in save_session_kwargs: 27 | raise ValueError('"%s" must be specified in save_session() argument.' % self.SESSION_ARG_KEYNAME) 28 | data = base64.b64encode(zlib.compress(pickle.dumps(self, self.PICKLE_PROTOCOL))).decode('utf-8') 29 | keyring.set_password(self.KEYRING_SERVICE_NAME, save_session_kwargs[self.SESSION_ARG_KEYNAME], data) 30 | 31 | @staticmethod 32 | def load_session(**load_session_kwargs): 33 | """ 34 | :param dict[str, str] load_session_kwargs: 35 | :return onedrive_client.od_api_session.OneDriveAPISession: 36 | """ 37 | keyarg = OneDriveAPISession.SESSION_ARG_KEYNAME 38 | if keyarg not in load_session_kwargs: 39 | raise ValueError('"%s" must be specified in load_session() argument.' % keyarg) 40 | saved_data = keyring.get_password(OneDriveAPISession.KEYRING_SERVICE_NAME, load_session_kwargs[keyarg]) 41 | 42 | if saved_data is None: 43 | raise ValueError("Don't find anything") 44 | 45 | data = zlib.decompress(base64.b64decode(saved_data.encode('utf-8'))) 46 | return pickle.loads(data) 47 | -------------------------------------------------------------------------------- /onedrive_client/od_auth.py: -------------------------------------------------------------------------------- 1 | """ 2 | od_auth.py 3 | Core component for user authentication and authorization. 4 | :copyright: (c) Xiangyu Bu 5 | :license: MIT 6 | """ 7 | 8 | import logging 9 | 10 | import requests 11 | from requests.utils import getproxies 12 | import onedrivesdk 13 | import onedrivesdk.error 14 | from onedrivesdk.helpers.resource_discovery import ResourceDiscoveryRequest 15 | import os 16 | import yaml 17 | 18 | from onedrive_client import od_api_session 19 | from onedrive_client.od_models import account_profile 20 | 21 | PATH = os.path.abspath(os.path.dirname(os.path.dirname(__file__))) 22 | with open(os.path.join(PATH, 'onedrive_client', 'data', 'security_config.yml')) as config: 23 | SECURITY_CONFIG = yaml.safe_load(config) 24 | 25 | 26 | def get_authenticator_and_drives(context, account_id): 27 | # TODO: Ideally we should recursively get all drives because the API pages 28 | # them. 29 | account_type = context.config['accounts'][account_id]['account_type'] 30 | if account_type == account_profile.AccountTypes.PERSONAL: 31 | authenticator = OneDriveAuthenticator() 32 | elif account_type == account_profile.AccountTypes.BUSINESS: 33 | endpoint = context.config['accounts'][account_id]['webUrl'] 34 | endpoint = endpoint[:endpoint.find( 35 | '-my.sharepoint.com/')] + '-my.sharepoint.com/' 36 | authenticator = OneDriveBusinessAuthenticator(endpoint) 37 | else: 38 | logging.error("Error loading session: account_type don't exists") 39 | return 40 | 41 | try: 42 | authenticator.load_session( 43 | key=od_api_session.get_keyring_key(account_id)) 44 | if account_type == account_profile.AccountTypes.BUSINESS: 45 | authenticator.refresh_session(account_id) 46 | drives = authenticator.client.drives.get() 47 | except (onedrivesdk.error.OneDriveError, RuntimeError) as e: 48 | logging.error('Error loading session: %s. Try refreshing token.', e) 49 | authenticator.refresh_session(account_id) 50 | drives = authenticator.client.drives.get() 51 | return authenticator, drives 52 | 53 | 54 | class OneDriveBusinessAuthenticator: 55 | 56 | # This is to use OAuth v1 57 | APP_CLIENT_ID_BUSINESS = SECURITY_CONFIG['BUSINESS_V1']['CLIENT_ID'] 58 | APP_CLIENT_SECRET_BUSINESS = SECURITY_CONFIG['BUSINESS_V1']['CLIENT_SECRET'] 59 | APP_REDIRECT_URL = SECURITY_CONFIG['BUSINESS_V1']['REDIRECT'] 60 | # This is to use OAuth v2 (Graph) 61 | APP_ID = SECURITY_CONFIG['BUSINESS_V2']['CLIENT_ID'] 62 | APP_SECRET = SECURITY_CONFIG['BUSINESS_V2']['CLIENT_SECRET'] 63 | REDIRECT_URL = SECURITY_CONFIG['BUSINESS_V2']['REDIRECT'] 64 | 65 | ACCOUNT_TYPE = account_profile.AccountTypes.BUSINESS 66 | APP_DISCOVERY_URL_BUSINESS = 'https://api.office.com/discovery/' 67 | APP_AUTH_SERVER_URL_BUSINESS = 'https://login.microsoftonline.com/common/oauth2/authorize' 68 | APP_TOKEN_URL_BUSINESS = 'https://login.microsoftonline.com/common/oauth2/token' 69 | 70 | BASE_URL = 'https://graph.microsoft.com/v1.0/' 71 | ACCESS_TOKEN_URL = 'https://login.microsoftonline.com/common/oauth2/v2.0/token' 72 | AUTORIZE_URL = 'https://login.microsoftonline.com/common/oauth2/v2.0/authorize' 73 | SCOPES = 'User.Read offline_access' 74 | 75 | def __init__(self, endpoint=None): 76 | self.code = None 77 | proxies = getproxies() 78 | if len(proxies) == 0: 79 | self.http_provider = onedrivesdk.HttpProvider() 80 | else: 81 | from onedrivesdk.helpers.http_provider_with_proxy import HttpProviderWithProxy 82 | self.http_provider = HttpProviderWithProxy( 83 | proxies, verify_ssl=True) 84 | 85 | self.auth_provider = onedrivesdk.AuthProvider( 86 | self.http_provider, 87 | self.APP_CLIENT_ID_BUSINESS, 88 | session_type=od_api_session.OneDriveAPISession, 89 | auth_server_url=self.APP_AUTH_SERVER_URL_BUSINESS, 90 | auth_token_url=self.APP_TOKEN_URL_BUSINESS, 91 | scopes=self.SCOPES, 92 | ) 93 | 94 | if endpoint is not None: 95 | self.client = onedrivesdk.OneDriveClient( 96 | endpoint + '_api/v2.0/', self.auth_provider, self.http_provider) 97 | 98 | def get_auth_url(self): 99 | return self.auth_provider.get_auth_url(self.APP_REDIRECT_URL) 100 | 101 | def authenticate(self, code): 102 | logging.info('Authenticating...') 103 | self.auth_provider.authenticate( 104 | code, 105 | self.APP_REDIRECT_URL, 106 | self.APP_CLIENT_SECRET_BUSINESS, 107 | resource=self.APP_DISCOVERY_URL_BUSINESS) 108 | 109 | # this step can be slow 110 | service_info = ResourceDiscoveryRequest().get_service_info( 111 | self.auth_provider.access_token) 112 | 113 | self.APP_ENDPOINT = str(service_info[0]).split()[1] 114 | 115 | logging.info('Refreshing token...') 116 | self.auth_provider.redeem_refresh_token(self.APP_ENDPOINT) 117 | logging.info('Updating client') 118 | self.client = onedrivesdk.OneDriveClient( 119 | self.APP_ENDPOINT + '_api/v2.0/', 120 | self.auth_provider, 121 | self.http_provider) 122 | logging.info('Authenticated!') 123 | 124 | def get_profile(self): 125 | """ 126 | Discover the OneDrive for Business resource URI 127 | reference: https://github.com/OneDrive/onedrive-api-docs/blob/master/auth/aad_oauth.md 128 | util link: https://github.com/OneDrive/onedrive-api-docs 129 | """ 130 | # more detailed: ?$expand=children 131 | url = self.APP_ENDPOINT + '_api/v1.0/me/files/root' 132 | headers = {'Authorization': 'Bearer ' + 133 | self.auth_provider.access_token} 134 | proxies = getproxies() 135 | if len(proxies) == 0: 136 | proxies = None 137 | response = requests.get( 138 | url, 139 | headers=headers, 140 | proxies=proxies, 141 | verify=True) 142 | data = response.json() 143 | if response.status_code != requests.codes.ok: 144 | raise ValueError( 145 | 'Failed to read user profile:' + 146 | data['error']['message']) 147 | data['account_type'] = self.ACCOUNT_TYPE 148 | url = self.ACCESS_TOKEN_URL 149 | 150 | content = { 151 | 'grant_type': 'authorization_code', 152 | 'code': self.code, 153 | 'redirect_uri': self.REDIRECT_URL, 154 | 'scope': self.SCOPES, 155 | 'client_id': self.APP_ID, 156 | 'client_secret': self.APP_SECRET 157 | } 158 | 159 | response = requests.post(url, content) 160 | resp = response.json() 161 | token = resp['access_token'] 162 | data['refresh_token'] = token 163 | 164 | url = self.BASE_URL + 'me/' 165 | headers = {'Authorization': 'Bearer ' + token} 166 | 167 | response = requests.get(url, headers=headers) 168 | resp = response.json() 169 | 170 | data['name'] = resp['displayName'] 171 | data['first_name'] = resp['givenName'] 172 | data['last_name'] = resp['surname'] 173 | data['emails'] = resp['mail'] 174 | 175 | # End user information 176 | return account_profile.OneDriveAccountBusiness(data) 177 | 178 | @property 179 | def authentication_url(self): 180 | return "{auth_url}?client_id={app_id}&response_type=code" \ 181 | "&redirect_uri={redirect_url}&scope={scope}".format( 182 | auth_url=self.AUTORIZE_URL, 183 | app_id=self.APP_ID, 184 | redirect_url=self.REDIRECT_URL, 185 | scope=self.SCOPES) 186 | 187 | @property 188 | def session_expires_in_sec(self): 189 | return self.client.auth_provider._session.expires_in_sec 190 | 191 | def refresh_session(self, account_id): 192 | self.client.auth_provider.refresh_token() 193 | self.save_session(key=od_api_session.get_keyring_key(account_id)) 194 | 195 | def save_session(self, key): 196 | args = {od_api_session.OneDriveAPISession.SESSION_ARG_KEYNAME: key} 197 | self.client.auth_provider.save_session(**args) 198 | 199 | def load_session(self, key): 200 | args = {od_api_session.OneDriveAPISession.SESSION_ARG_KEYNAME: key} 201 | self.client.auth_provider.load_session(**args) 202 | 203 | 204 | class OneDriveAuthenticator: 205 | 206 | ACCOUNT_TYPE = account_profile.AccountTypes.PERSONAL 207 | APP_CLIENT_ID = SECURITY_CONFIG['PERSONAL']['CLIENT_ID'] 208 | APP_CLIENT_SECRET = SECURITY_CONFIG['PERSONAL']['CLIENT_SECRET'] 209 | APP_REDIRECT_URL = SECURITY_CONFIG['PERSONAL']['REDIRECT'] 210 | 211 | APP_BASE_URL = 'https://api.onedrive.com/v1.0/' 212 | APP_SCOPES = [ 213 | 'wl.signin', 214 | 'wl.emails', 215 | 'wl.offline_access', 216 | 'onedrive.readwrite'] 217 | 218 | def __init__(self): 219 | proxies = getproxies() 220 | if len(proxies) == 0: 221 | http_provider = onedrivesdk.HttpProvider() 222 | else: 223 | from onedrivesdk.helpers.http_provider_with_proxy import HttpProviderWithProxy 224 | http_provider = HttpProviderWithProxy(proxies, verify_ssl=True) 225 | auth_provider = onedrivesdk.AuthProvider( 226 | http_provider=http_provider, 227 | client_id=self.APP_CLIENT_ID, 228 | session_type=od_api_session.OneDriveAPISession, 229 | scopes=self.APP_SCOPES) 230 | self.client = onedrivesdk.OneDriveClient( 231 | self.APP_BASE_URL, auth_provider, http_provider) 232 | 233 | def get_auth_url(self): 234 | return self.client.auth_provider.get_auth_url(self.APP_REDIRECT_URL) 235 | 236 | def authenticate(self, code): 237 | self.client.auth_provider.authenticate( 238 | code, self.APP_REDIRECT_URL, self.APP_CLIENT_SECRET) 239 | 240 | def get_profile(self, user_id='me'): 241 | """ 242 | Fetch basic profile of the specified user (Live ID). 243 | :param str user_id: (Optional) ID of the target user. 244 | :return od_models.account_profile.OneDriveUserProfile: 245 | An OneDriveUserProfile object that od_models the user info. 246 | """ 247 | url = 'https://apis.live.net/v5.0/' + user_id 248 | headers = {'Authorization': 'Bearer ' + 249 | self.client.auth_provider.access_token} 250 | proxies = getproxies() 251 | if len(proxies) == 0: 252 | proxies = None 253 | response = requests.get( 254 | url, 255 | headers=headers, 256 | proxies=proxies, 257 | verify=True) 258 | if response.status_code != requests.codes.ok: 259 | raise ValueError('Failed to read user profile.') 260 | data = response.json() 261 | data['account_type'] = self.ACCOUNT_TYPE 262 | return account_profile.OneDriveAccountPersonal(data) 263 | 264 | @property 265 | def session_expires_in_sec(self): 266 | return self.client.auth_provider._session.expires_in_sec 267 | 268 | def refresh_session(self, account_id): 269 | self.client.auth_provider.refresh_token() 270 | self.save_session(key=od_api_session.get_keyring_key(account_id)) 271 | 272 | def save_session(self, key): 273 | args = {od_api_session.OneDriveAPISession.SESSION_ARG_KEYNAME: key} 274 | self.client.auth_provider.save_session(**args) 275 | 276 | def load_session(self, key): 277 | args = {od_api_session.OneDriveAPISession.SESSION_ARG_KEYNAME: key} 278 | self.client.auth_provider.load_session(**args) 279 | -------------------------------------------------------------------------------- /onedrive_client/od_context.py: -------------------------------------------------------------------------------- 1 | import json 2 | import logging 3 | import logging.handlers 4 | import os 5 | from pwd import getpwnam 6 | 7 | import click 8 | 9 | 10 | from onedrive_client import mkdir, get_resource, od_webhooks 11 | from onedrive_client.od_models import account_profile 12 | from onedrive_client.od_models import drive_config as _drive_config 13 | 14 | 15 | def is_invalid_username(s): 16 | return s is None or not isinstance(s, str) or len(s.strip()) == 0 17 | 18 | 19 | def get_login_username(): 20 | # If allow for sudo, prepend SUDO_USER. 21 | for key in ('LOGNAME', 'USER', 'LNAME', 'USERNAME'): 22 | s = os.getenv(key) 23 | if not is_invalid_username(s): 24 | return s 25 | raise ValueError('Cannot find login name of current user.') 26 | 27 | 28 | def load_context(loop=None): 29 | ctx = UserContext(loop=loop) 30 | try: 31 | ctx.load_config(ctx.DEFAULT_CONFIG_FILENAME) 32 | except OSError as e: 33 | logging.warning('Failed to load config file: %s. Use default.', e) 34 | return ctx 35 | 36 | 37 | def save_context(ctx): 38 | try: 39 | ctx.save_config(ctx.DEFAULT_CONFIG_FILENAME) 40 | except OSError as e: 41 | logging.error('Failed to save config file: %s. Changes were discarded.', e) 42 | 43 | 44 | class UserContext: 45 | """Stores config params for a single local user.""" 46 | 47 | DEFAULT_CONFIG = { 48 | 'accounts': {}, 49 | 'drives': {}, 50 | 'scan_interval_sec': 21600, # Poll every 6 hours. 51 | 'webhook_type': od_webhooks.DEFAULT_WEBHOOK_TYPE, 52 | 'webhook_host': '', 53 | 'webhook_port': 0, 54 | 'webhook_renew_interval_sec': 7200, # Renew webhook every 2 hours. 55 | 'webhook_action_delay_sec': 120, 56 | 'num_workers': 16, 57 | 'start_delay_sec': 0, 58 | 'logfile_path': '' 59 | } 60 | 61 | DEFAULT_CONFIG_FILENAME = 'onedrive_client_config_v2.json' 62 | DEFAULT_IGNORE_FILENAME = 'ignore_v2.txt' 63 | DEFAULT_NGROK_CONF_FILENAME = 'ngrok_conf.yaml' 64 | 65 | def __init__(self, loop): 66 | """ 67 | :param asyncio.AbstractEventLoop | None loop: 68 | """ 69 | # Information about host and user. 70 | self.host_name = os.uname()[1] 71 | self.user_name = get_login_username() 72 | self.user_uid = getpwnam(self.user_name).pw_uid 73 | self.user_home = os.path.expanduser('~' + self.user_name) 74 | self.config_dir = click.get_app_dir('onedrive_client') 75 | self._create_config_dir_if_missing() 76 | self.config = self.DEFAULT_CONFIG 77 | self.loop = loop 78 | self._watcher = None 79 | 80 | def _create_config_dir_if_missing(self): 81 | if os.path.exists(self.config_dir) and not os.path.isdir(self.config_dir): 82 | logging.info('Config dir "' + self.config_dir + '" is not a directory. Delete it.') 83 | os.remove(self.config_dir) 84 | if not os.path.exists(self.config_dir): 85 | logging.info('Config dir "' + self.config_dir + '" does not exist. Create it.') 86 | mkdir(self.config_dir, self.user_uid, mode=0o700, exist_ok=True) 87 | self._copy_default_config_file('ignore_v2.txt', self.DEFAULT_IGNORE_FILENAME) 88 | self._copy_default_config_file('ngrok_default_conf.yaml', self.DEFAULT_NGROK_CONF_FILENAME) 89 | 90 | def _copy_default_config_file(self, resource_filename, target_filename): 91 | with open(self.config_dir + '/' + target_filename, 'w') as f: 92 | f.write(get_resource('data/' + resource_filename)) 93 | 94 | @property 95 | def loop(self): 96 | """ 97 | :return asyncio.SelectorEventLoop | None: 98 | """ 99 | return self._loop 100 | 101 | # noinspection PyAttributeOutsideInit 102 | @loop.setter 103 | def loop(self, v): 104 | self._loop = v 105 | 106 | @property 107 | def watcher(self): 108 | """ 109 | :return onedrive_client.od_watcher.LocalRepositoryWatcher: 110 | """ 111 | return self._watcher 112 | 113 | @watcher.setter 114 | def watcher(self, watcher): 115 | self._watcher = watcher 116 | 117 | @staticmethod 118 | def set_logger(min_level=logging.WARNING, path=None): 119 | logging_config = {'level': min_level, 'format': '[%(asctime)-15s] %(levelname)s: %(threadName)s: %(message)s'} 120 | if path: 121 | logging_config['filename'] = path 122 | logging.basicConfig(**logging_config) 123 | 124 | def _add_and_return(self, config_key, id_key, obj, data): 125 | self.config[config_key][getattr(obj, id_key)] = data 126 | return obj 127 | 128 | def add_account(self, account_profile): 129 | """ 130 | Add a new account to config file. 131 | :param od_models.account_profile.OneDriveAccountProfile account_profile: 132 | :return od_models.account_profile.OneDriveAccountProfile: The account profile argument. 133 | """ 134 | 135 | return self._add_and_return('accounts', 'account_id', account_profile, account_profile.data) 136 | 137 | def get_account(self, account_id): 138 | """ 139 | Return profile of a saved account. 140 | :param str account_id: ID of the account to query. 141 | :return od_models.account_profile.OneDriveAccount: 142 | An OneDriveAccount object of the account profile. 143 | """ 144 | account = account_profile.OneDriveAccount(self.config['accounts'][account_id]) 145 | return account.get_account() 146 | 147 | def delete_account(self, account_id): 148 | """ 149 | Delete a saved account from config. 150 | :param str account_id: ID of the account to delete. 151 | """ 152 | del self.config['accounts'][account_id] 153 | 154 | def all_accounts(self): 155 | """Return a list of all linked account IDs.""" 156 | return sorted(self.config['accounts'].keys()) 157 | 158 | def add_drive(self, drive_config): 159 | """ 160 | Add a new drive to local config. 161 | :param od_models.drive_config.LocalDriveConfig drive_config: 162 | :return od_models.drive_config.LocalDriveConfig drive_config: 163 | """ 164 | return self._add_and_return('drives', 'drive_id', drive_config, drive_config._asdict()) 165 | 166 | def get_drive(self, drive_id): 167 | """ 168 | :param str drive_id: 169 | :return od_models.drive_config.LocalDriveConfig: 170 | """ 171 | return _drive_config.LocalDriveConfig(**self.config['drives'][drive_id]) 172 | 173 | def delete_drive(self, drive_id): 174 | del self.config['drives'][drive_id] 175 | 176 | def all_drives(self): 177 | return sorted(self.config['drives'].keys()) 178 | 179 | def load_config(self, filename): 180 | with open(self.config_dir + '/' + filename, 'r') as f: 181 | config = json.load(f) 182 | for k, v in config.items(): 183 | self.config[k] = v 184 | 185 | def save_config(self, filename): 186 | with open(self.config_dir + '/' + filename, 'w') as f: 187 | json.dump(self.config, f, sort_keys=True, indent=4, separators=(',', ': ')) 188 | -------------------------------------------------------------------------------- /onedrive_client/od_dateutils.py: -------------------------------------------------------------------------------- 1 | import arrow 2 | 3 | 4 | def datetime_to_str(d): 5 | """ 6 | :param arrow.Arrow d: 7 | :return str: 8 | """ 9 | datetime_str = d.to('utc').isoformat() 10 | if datetime_str.endswith('+00:00'): 11 | datetime_str = datetime_str[:-6] + 'Z' 12 | return datetime_str 13 | 14 | 15 | def str_to_datetime(s): 16 | """ 17 | :param str s: 18 | :return arrow.Arrow: 19 | """ 20 | return arrow.get(s) 21 | 22 | 23 | def datetime_to_timestamp(d): 24 | """ 25 | :param arrow.Arrow d: A datetime object. 26 | :return float: An equivalent UNIX timestamp. 27 | """ 28 | return d.float_timestamp 29 | 30 | 31 | def diff_timestamps(t1, t2): 32 | t1 = t1 - t2 33 | return 1 if t1 > 0.01 else -1 if t1 < -0.01 else 0 34 | -------------------------------------------------------------------------------- /onedrive_client/od_hashutils.py: -------------------------------------------------------------------------------- 1 | import hashlib 2 | 3 | 4 | def hash_match(local_abspath, remote_item): 5 | """ 6 | :param str local_abspath: 7 | :param onedrivesdk.model.item.Item remote_item: 8 | :return True | False: 9 | """ 10 | file_facet = remote_item.file 11 | if file_facet: 12 | hash_facet = file_facet.hashes 13 | if hash_facet: 14 | return hash_facet.sha1_hash and hash_facet.sha1_hash == sha1_value(local_abspath) 15 | return False 16 | 17 | 18 | def sha1_value(file_path, block_size=2 << 22): 19 | """ 20 | Calculate the MD5 or SHA hash value of the data of the specified file. 21 | :param str file_path: 22 | :param int block_size: 23 | :return str: 24 | """ 25 | alg = hashlib.sha1() 26 | with open(file_path, 'rb') as f: 27 | data = f.read(block_size) 28 | while len(data): 29 | alg.update(data) 30 | data = f.read(block_size) 31 | return alg.hexdigest().upper() 32 | -------------------------------------------------------------------------------- /onedrive_client/od_i18n.py: -------------------------------------------------------------------------------- 1 | import json 2 | 3 | from onedrive_client import get_resource 4 | 5 | 6 | class Translator: 7 | 8 | DEFAULT_LOCALE = 'en_US' 9 | 10 | def __init__(self, lang_resources, locale_str='en_US'): 11 | """ 12 | :param [str] lang_resources: List of language resource files to load. 13 | Example: ['od_pref', 'configurator'] to load lang/od_pref 14 | :param str locale_str: Locale to load. 15 | """ 16 | self.string_resources = dict() 17 | for lang in lang_resources: 18 | try: 19 | t = get_resource('lang/%s.%s.json' % (lang, locale_str), pkg_name='onedrive_client') 20 | except FileNotFoundError: 21 | t = get_resource('lang/%s.%s.json' % (lang, self.DEFAULT_LOCALE), pkg_name='onedrive_client') 22 | data = json.loads(t) 23 | self.string_resources.update(data) 24 | 25 | def __getitem__(self, item): 26 | return self.string_resources[item] 27 | -------------------------------------------------------------------------------- /onedrive_client/od_main.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import asyncio 4 | import gc 5 | import itertools 6 | import logging 7 | import os 8 | import subprocess 9 | import sys 10 | import weakref 11 | 12 | import click 13 | import daemonocle.cli 14 | 15 | from onedrive_client import od_repo 16 | from onedrive_client import od_task 17 | from onedrive_client import od_threads 18 | from onedrive_client import od_webhook 19 | from onedrive_client.od_tasks import start_repo, merge_dir, update_subscriptions 20 | from onedrive_client.od_auth import get_authenticator_and_drives 21 | from onedrive_client.od_context import load_context 22 | from onedrive_client.od_watcher import LocalRepositoryWatcher 23 | 24 | 25 | context = load_context(asyncio.get_event_loop()) 26 | pidfile = context.config_dir + '/onedrive_client.pid' 27 | task_workers = weakref.WeakSet() 28 | task_pool = None 29 | webhook_server = None 30 | webhook_worker = None 31 | 32 | 33 | def init_task_pool_and_workers(): 34 | global task_pool 35 | task_pool = od_task.TaskPool() 36 | for _ in range(context.config['num_workers']): 37 | w = od_threads.TaskWorkerThread(name='Worker-%d' % len(task_workers), task_pool=task_pool) 38 | w.start() 39 | task_workers.add(w) 40 | 41 | 42 | def shutdown_workers(): 43 | for w in task_workers: 44 | if w: 45 | w.stop() 46 | if task_pool: 47 | task_pool.close(len(task_workers)) 48 | for w in task_workers: 49 | if w: 50 | w.join() 51 | 52 | 53 | def init_webhook(): 54 | global webhook_server, webhook_worker 55 | try: 56 | webhook_server = od_webhook.get_webhook_server(context) 57 | except RuntimeError as e: 58 | logging.critical('Error initializing webhook: %s', e) 59 | raise SystemExit() 60 | webhook_worker = od_webhook.WebhookWorkerThread(webhook_url=webhook_server.webhook_url, 61 | callback_func=repo_updated_callback, 62 | action_delay_sec=context.config['webhook_action_delay_sec']) 63 | webhook_server.set_worker(webhook_worker) 64 | webhook_worker.start() 65 | webhook_server.start() 66 | 67 | 68 | def shutdown_webhook(): 69 | global webhook_server 70 | if webhook_server: 71 | webhook_server.stop() 72 | webhook_server.join() 73 | webhook_server = None 74 | 75 | 76 | # noinspection PyUnusedLocal 77 | def shutdown_callback(code, _): 78 | logging.info('Shutting down. Code: %s.', str(code)) 79 | asyncio.gather(*asyncio.Task.all_tasks()).cancel() 80 | context.loop.stop() 81 | shutdown_webhook() 82 | shutdown_workers() 83 | if context and context.watcher: 84 | context.watcher.close() 85 | context.watcher = None 86 | logging.shutdown() 87 | logging.info('Shut down complete.') 88 | 89 | 90 | def get_repo_table(ctx): 91 | """ 92 | :param onedrive_client.od_context.UserContext ctx: 93 | :return dict[str, [onedrive_client.od_repo.OneDriveLocalRepository]]: 94 | """ 95 | all_accounts = {} 96 | all_account_ids = ctx.all_accounts() 97 | if len(all_account_ids) == 0: 98 | logging.critical('onedrive_client is not linked with any OneDrive account. Please configure onedrive_client first.') 99 | sys.exit(1) 100 | for account_id in all_account_ids: 101 | authenticator, drives = get_authenticator_and_drives(ctx, account_id) 102 | local_repos = [od_repo.OneDriveLocalRepository(ctx, authenticator, d, ctx.get_drive(d.id)) 103 | for d in drives if d.id in ctx.config['drives']] 104 | if len(local_repos) > 0: 105 | all_accounts[account_id] = local_repos 106 | else: 107 | profile = ctx.get_account(account_id) 108 | logging.info('No Drive associated with account "%s" (%s).', profile.account_email, account_id) 109 | return all_accounts 110 | 111 | 112 | def update_subscription_for_repo(repo, subscription_id=None): 113 | """ 114 | :param onedrive_client.od_repo.OneDriveLocalRepository repo: 115 | :param str | None subscription_id: 116 | :return onedrivesdk.Subscription | None: 117 | """ 118 | if webhook_server and webhook_worker: 119 | task = update_subscriptions.UpdateSubscriptionTask(repo, task_pool, webhook_worker, subscription_id) 120 | subscription = task.handle() 121 | if subscription: 122 | context.loop.call_later(int(context.config['webhook_renew_interval_sec'] * 0.75), 123 | update_subscription_for_repo, repo, subscription.id) 124 | gc.collect() 125 | return subscription 126 | return None 127 | 128 | 129 | def gen_start_repo_tasks(all_accounts): 130 | """ 131 | :param dict[str, [onedrive_client.od_repo.OneDriveLocalRepository]] all_accounts: 132 | """ 133 | if task_pool.outstanding_task_count == 0: 134 | for repo in itertools.chain.from_iterable(all_accounts.values()): 135 | task_pool.add_task(start_repo.StartRepositoryTask(repo, task_pool)) 136 | logging.info('Scheduled sync task for Drive %s of account %s.', repo.drive.id, repo.account_id) 137 | if update_subscription_for_repo(repo) is None: 138 | logging.warning('Failed to create webhook. Will deep sync again in %d sec.', 139 | context.config['scan_interval_sec']) 140 | context.loop.call_later(context.config['scan_interval_sec'], 141 | gen_start_repo_tasks, all_accounts) 142 | else: 143 | logging.info('Will use webhook to trigger sync events.') 144 | 145 | 146 | def delete_temp_files(all_accounts): 147 | """ 148 | Delete all onedrive_client temporary files from repository. 149 | :param dict[str, [onedrive_client.od_repo.OneDriveLocalRepository]] all_accounts: 150 | :return: 151 | """ 152 | logging.info('Sweeping onedrive_client temporary files from local repositories.') 153 | for repo in itertools.chain.from_iterable(all_accounts.values()): 154 | if os.path.isdir(repo.local_root): 155 | subprocess.call(('find', repo.local_root, '-type', 'f', 156 | '-name', repo.path_filter.get_temp_name('*'), '-delete')) 157 | 158 | 159 | def repo_updated_callback(repo): 160 | if task_pool and task_pool.outstanding_task_count == 0: 161 | item_request = repo.authenticator.client.item(drive=repo.drive.id, path='/') 162 | task_pool.add_task(merge_dir.MergeDirectoryTask( 163 | repo=repo, task_pool=task_pool, rel_path='', item_request=item_request, 164 | assume_remote_unchanged=True, parent_remote_unchanged=False)) 165 | logging.info('Added task to check delta update for Drive %s.', repo.drive.id) 166 | else: 167 | logging.error('Uninitialized task pool reference.') 168 | 169 | 170 | @click.command(cls=daemonocle.cli.DaemonCLI, 171 | daemon_params={ 172 | 'uid': context.user_uid, 173 | 'pidfile': pidfile, 174 | # 'detach': False, 175 | 'shutdown_callback': shutdown_callback, 176 | 'workdir': os.getcwd(), 177 | 'stop_timeout': 60, 178 | }) 179 | def main(): 180 | gc.enable() 181 | 182 | # When debugging, print to stdout. 183 | if '--debug' in sys.argv: 184 | context.loop.set_debug(True) 185 | context.set_logger(min_level=logging.DEBUG, path=None) 186 | else: 187 | context.set_logger(min_level=logging.INFO, path=context.config['logfile_path']) 188 | 189 | if context.config['start_delay_sec'] > 0: 190 | logging.info('Wait for %d seconds before starting.', context.config['start_delay_sec']) 191 | import time 192 | time.sleep(context.config['start_delay_sec']) 193 | 194 | # Initialize account information. 195 | all_accounts = get_repo_table(context) 196 | delete_temp_files(all_accounts) 197 | 198 | # Start task pool and task worker. 199 | init_task_pool_and_workers() 200 | 201 | # Start webhook. 202 | init_webhook() 203 | 204 | context.watcher = LocalRepositoryWatcher(task_pool=task_pool, loop=context.loop) 205 | 206 | try: 207 | context.loop.call_soon(gen_start_repo_tasks, all_accounts) 208 | context.loop.run_forever() 209 | finally: 210 | context.loop.close() 211 | 212 | 213 | if __name__ == '__main__': 214 | main() 215 | -------------------------------------------------------------------------------- /onedrive_client/od_models/__init__.py: -------------------------------------------------------------------------------- 1 | from onedrive_client.od_models import ( 2 | account_profile, 3 | bidict, 4 | drive_config, 5 | path_filter, 6 | pretty_api, 7 | webhook_notification, 8 | ) 9 | 10 | 11 | __all__ = [ 12 | 'account_profile', 13 | 'account_profile_business', 14 | 'bidict', 15 | 'drive_config', 16 | 'path_filter', 17 | 'pretty_api', 18 | 'webhook_notification', 19 | ] 20 | -------------------------------------------------------------------------------- /onedrive_client/od_models/account_profile.py: -------------------------------------------------------------------------------- 1 | 2 | class AccountTypes: 3 | PERSONAL = 0 4 | BUSINESS = 1 5 | 6 | 7 | class OneDriveAccount: 8 | 9 | def __init__(self, data): 10 | self.data = data 11 | 12 | @property 13 | def account_id(self): 14 | return self.data['id'] 15 | 16 | @property 17 | def account_name(self): 18 | return self.data['name'] 19 | 20 | @property 21 | def account_firstname(self): 22 | return self.data['first_name'] 23 | 24 | @property 25 | def account_lastname(self): 26 | return self.data['last_name'] 27 | 28 | @property 29 | def account_email(self): 30 | return self.data['emails'] 31 | 32 | def get_account(self): 33 | if self.data['account_type'] == AccountTypes.BUSINESS: 34 | return OneDriveAccountBusiness(self.data) 35 | else: 36 | return OneDriveAccountPersonal(self.data) 37 | 38 | 39 | class OneDriveAccountPersonal(OneDriveAccount): 40 | 41 | def __init__(self, data): 42 | super().__init__(data) 43 | self.account_type = AccountTypes.PERSONAL 44 | 45 | @property 46 | def account_email(self): 47 | return super().account_email['account'] 48 | 49 | def get_account(self): 50 | raise AttributeError("'OneDriveAccountPersonal' object has no attribute 'get_account'") 51 | 52 | 53 | class OneDriveAccountBusiness(OneDriveAccount): 54 | 55 | def __init__(self, data): 56 | super().__init__(data) 57 | self.account_type = AccountTypes.BUSINESS 58 | 59 | @property 60 | def account_root_folder(self): 61 | return self.data['webUrl'] 62 | 63 | @property 64 | def tenant(self): 65 | site = self.account_root_folder 66 | return site[8:site.find('-my.sharepoint.com/')] # 8 is the len of 'https://' 67 | 68 | @property 69 | def endpoint(self): 70 | return 'https://' + self.tenant + '-my.sharepoint.com/' 71 | 72 | def get_account(self): 73 | raise AttributeError("'OneDriveAccountPersonal' object has no attribute 'get_account'") 74 | -------------------------------------------------------------------------------- /onedrive_client/od_models/bidict.py: -------------------------------------------------------------------------------- 1 | try: 2 | from bidict import loosebidict 3 | except ImportError: 4 | import bidict 5 | class loosebidict(bidict.bidict): 6 | on_dup_val = bidict.OVERWRITE 7 | -------------------------------------------------------------------------------- /onedrive_client/od_models/dict_guard/__init__.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | from onedrive_client.od_models.dict_guard import exceptions 4 | 5 | 6 | class DictEntryTypes: 7 | INT = 'integer' 8 | STR = 'string' 9 | 10 | 11 | class StringSubTypes: 12 | FILE = 'file' 13 | 14 | 15 | def _test_bool_option(schema, opt_key): 16 | return opt_key in schema and schema[opt_key] is True 17 | 18 | 19 | def _test_str_subtype_file(key, value, schema): 20 | if not os.path.exists(value): 21 | if _test_bool_option(schema, 'create_if_missing'): 22 | with open(value, 'w'): 23 | pass 24 | else: 25 | raise exceptions.PathDoesNotExist(key, value) 26 | elif not os.path.isfile(value): 27 | raise exceptions.PathIsNotFile(key, value) 28 | elif 'permission' in schema: 29 | with open(value, schema['permission']): 30 | pass 31 | 32 | 33 | class GuardedDict: 34 | 35 | def __init__(self, config_dict, config_schema_dict): 36 | self.config_dict = config_dict 37 | self.config_schema_dict = config_schema_dict 38 | 39 | def set_int(self, key, value, schema): 40 | try: 41 | value = int(value) 42 | except ValueError: 43 | raise exceptions.IntValueRequired(key, value) 44 | if 'minimum' in schema and value < schema['minimum']: 45 | raise exceptions.IntValueBelowMinimum(key, value, schema['minimum']) 46 | if 'maximum' in schema and value > schema['maximum']: 47 | raise exceptions.IntValueAboveMaximum(key, value, schema['maximum']) 48 | self.config_dict[key] = value 49 | 50 | def set_str_subtype(self, key, value, schema): 51 | if schema['subtype'] == StringSubTypes.FILE: 52 | if _test_bool_option(schema, 'to_abspath'): 53 | value = os.path.abspath(value) 54 | _test_str_subtype_file(key, value, schema) 55 | else: 56 | raise exceptions.UnsupportedSchemaType(schema['subtype'], schema) 57 | self.config_dict[key] = value 58 | 59 | def set_str(self, key, value, schema): 60 | if not isinstance(value, str): 61 | value = str(value) 62 | if _test_bool_option(schema, 'allow_empty') and value == '': 63 | self.config_dict[key] = '' 64 | return 65 | if 'starts_with' in schema and not value.startswith(schema['starts_with']): 66 | raise exceptions.StringNotStartsWith(key, value, schema['starts_with']) 67 | if 'choices' in schema and value not in schema['choices']: 68 | raise exceptions.StringInvalidChoice(key, value, schema['choices']) 69 | if 'subtype' in schema: 70 | return self.set_str_subtype(key, value, schema) 71 | self.config_dict[key] = value 72 | 73 | def __setitem__(self, key, value): 74 | if key not in self.config_schema_dict: 75 | raise exceptions.DictGuardKeyError(key) 76 | schema = self.config_schema_dict[key] 77 | if schema['type'] == DictEntryTypes.INT: 78 | return self.set_int(key, value, schema) 79 | elif schema['type'] == DictEntryTypes.STR: 80 | return self.set_str(key, value, schema) 81 | else: 82 | raise exceptions.UnsupportedSchemaType(schema['type'], schema) 83 | 84 | 85 | class SchemaValidator: 86 | 87 | def __init__(self, config_schema_dict): 88 | self.config_schema_dict = config_schema_dict 89 | 90 | def validate(self): 91 | for key, spec in self.config_schema_dict.items(): 92 | if not isinstance(spec, dict): 93 | raise TypeError('Schema for key "%s" must be of dict type.' % key) 94 | if 'type' not in spec: 95 | raise KeyError('Required field "type" is missing in key "%s".' % key) 96 | elif spec['type'] not in (DictEntryTypes.INT, DictEntryTypes.STR): 97 | raise ValueError('Unsupported type of key "%s": "%s".' % (key, spec['type'])) 98 | -------------------------------------------------------------------------------- /onedrive_client/od_models/dict_guard/exceptions.py: -------------------------------------------------------------------------------- 1 | class DictGuardError(Exception): 2 | def __init__(self, *args, **kwargs): 3 | super().__init__(args, kwargs) 4 | 5 | 6 | class DictGuardSchemaError(DictGuardError): 7 | def __init__(self, schema): 8 | super().__init__() 9 | self.bad_schema = schema 10 | 11 | 12 | class UnsupportedSchemaType(DictGuardSchemaError): 13 | def __init__(self, schema_type, schema): 14 | super().__init__(schema) 15 | self.bad_schema_type = schema_type 16 | 17 | 18 | class DictGuardKeyError(DictGuardError): 19 | def __init__(self, key): 20 | self.key = key 21 | 22 | 23 | class DictGuardValueError(DictGuardError): 24 | def __init__(self, key, value): 25 | super().__init__() 26 | self.key = key 27 | self.value = value 28 | 29 | 30 | class IntValueRequired(DictGuardValueError): 31 | pass 32 | 33 | 34 | class IntValueBelowMinimum(DictGuardValueError): 35 | def __init__(self, key, value, minimum): 36 | super().__init__(key, value) 37 | self.minimum = minimum 38 | 39 | 40 | class IntValueAboveMaximum(DictGuardValueError): 41 | def __init__(self, key, value, maximum): 42 | super().__init__(key, value) 43 | self.maximum = maximum 44 | 45 | 46 | class StringNotStartsWith(DictGuardValueError): 47 | def __init__(self, key, value, expected_starts_with): 48 | super().__init__(key, value) 49 | self.expected_starts_with = expected_starts_with 50 | 51 | 52 | class StringInvalidChoice(DictGuardValueError): 53 | def __init__(self, key, value, choices_allowed): 54 | super().__init__(key, value) 55 | self.choices_allowed = choices_allowed 56 | 57 | 58 | class PathDoesNotExist(DictGuardValueError): 59 | pass 60 | 61 | 62 | class PathIsNotFile(DictGuardValueError): 63 | pass 64 | -------------------------------------------------------------------------------- /onedrive_client/od_models/drive_config.py: -------------------------------------------------------------------------------- 1 | from collections import namedtuple 2 | 3 | 4 | LocalDriveConfig = namedtuple('LocalDriveConfig', ('drive_id', 'account_id', 'ignorefile_path', 'localroot_path')) 5 | -------------------------------------------------------------------------------- /onedrive_client/od_models/path_filter.py: -------------------------------------------------------------------------------- 1 | import zgitignore 2 | 3 | 4 | class PathFilter(zgitignore.ZgitIgnore): 5 | """ 6 | PathFilter parses a gitignore-like file to an ignore list, and then allows for other components to query if a 7 | specific path should be ignored. 8 | """ 9 | 10 | TMP_PREFIX = '.' 11 | TMP_SUFFIX = '.odtemp!' 12 | 13 | def __init__(self, rules): 14 | """ 15 | Initialize the filter with a list of (case-INsensitive) gitignore rules. 16 | :param [str] rules: List of gitignore rules. 17 | """ 18 | super().__init__(rules, ignore_case=True) 19 | self.add_patterns((self.TMP_PREFIX + '*' + self.TMP_SUFFIX, '*[<>?*:"|]*', '.*')) 20 | 21 | def add_rules(self, rules): 22 | """ 23 | Add a new (case-INsensitive) gitignore rule. 24 | :param [str] rules: The rule to add. 25 | """ 26 | self.add_patterns(rules) 27 | 28 | def should_ignore(self, path, is_dir=False): 29 | """ 30 | Determine if a path should be ignored. 31 | :param str path: Path relative to repository root. 32 | :param True | False is_dir: Whether or not the path is a folder. 33 | :return True | False: Whether or not the path should be ignored. 34 | """ 35 | if path[-1] == '/': 36 | is_dir = True 37 | return self.is_ignored(path, is_directory=is_dir) 38 | 39 | @classmethod 40 | def get_temp_name(cls, name): 41 | return cls.TMP_PREFIX + name + cls.TMP_SUFFIX 42 | -------------------------------------------------------------------------------- /onedrive_client/od_models/pretty_api.py: -------------------------------------------------------------------------------- 1 | """ 2 | Some formatter functions for onedrive-python-sdk API objects. 3 | """ 4 | 5 | 6 | def pretty_print_bytes(size, precision=2): 7 | suffixes = ['B', 'KB', 'MB', 'GB', 'TB', 'PB'] 8 | index = 0 9 | while size >= 1024: 10 | index += 1 # increment the index of the suffix 11 | size /= 1024.0 # apply the division 12 | return "%.*f %s" % (precision, size, suffixes[index]) 13 | -------------------------------------------------------------------------------- /onedrive_client/od_models/webhook_notification.py: -------------------------------------------------------------------------------- 1 | """ 2 | webhook_notification.py 3 | 4 | Implementation of the datatypes used in OneDrive webhook notification, which is absent 5 | from official OneDrive Python SDK. 6 | 7 | :copyright: (c) Xiangyu Bu 8 | :license: MIT 9 | """ 10 | 11 | from onedrive_client import od_dateutils 12 | 13 | 14 | class WebhookNotification: 15 | 16 | """ https://dev.onedrive.com/resources/webhookNotifiation.htm """ 17 | 18 | def __init__(self, prop_dict): 19 | self._prop_dict = prop_dict 20 | 21 | @property 22 | def context(self): 23 | """ 24 | :return str | None: 25 | An optional string value that is passed back in the notification message for this subscription. 26 | """ 27 | try: 28 | return self._prop_dict['context'] 29 | except KeyError: 30 | return None 31 | 32 | @property 33 | def expiration_datetime(self): 34 | """ 35 | :return arrow.Arrow: The date and time when the subscription will expire if not updated or renewed. 36 | """ 37 | return od_dateutils.str_to_datetime(self._prop_dict['expirationDateTime']) 38 | 39 | @property 40 | def resource(self): 41 | """ 42 | :return str: URL to the item where the subscription is registered. 43 | """ 44 | return self._prop_dict['resource'] 45 | 46 | @property 47 | def subscription_id(self): 48 | """ 49 | :return str: The unique identifier for the subscription resource. 50 | """ 51 | return self._prop_dict['subscriptionId'] 52 | 53 | @property 54 | def tenant_id(self): 55 | """ 56 | :return str: 57 | Unique identifier for the tenant which generated this notification. 58 | This is only returned for OneDrive for Business and SharePoint. 59 | """ 60 | try: 61 | return self._prop_dict['tenantId'] 62 | except KeyError: 63 | return None 64 | 65 | @property 66 | def user_id(self): 67 | """ 68 | :return str: Unique identifier for the drive which generated this notification. 69 | """ 70 | return self._prop_dict['userId'] 71 | -------------------------------------------------------------------------------- /onedrive_client/od_repo.py: -------------------------------------------------------------------------------- 1 | """ 2 | od_repo.py 3 | Core component for local file management and tracking. 4 | :copyright: (c) Xiangyu Bu 5 | :license: MIT 6 | """ 7 | 8 | import atexit 9 | import logging 10 | import sqlite3 11 | import threading 12 | from datetime import datetime 13 | from contextlib import closing 14 | 15 | 16 | from onedrive_client import get_resource as _get_resource 17 | from onedrive_client.od_models.path_filter import PathFilter as _PathFilter 18 | from onedrive_client.od_api_helper import get_item_modified_datetime, get_item_created_datetime 19 | from onedrive_client.od_dateutils import str_to_datetime, datetime_to_str 20 | 21 | 22 | class ItemRecord: 23 | def __init__(self, row): 24 | self.item_id, self.type, self.item_name, self.parent_id, self.parent_path, self.e_tag, self.c_tag, \ 25 | self.size, self.size_local, self.created_time, self.modified_time, self.status, self.sha1_hash, \ 26 | self.record_time_str = row 27 | self.created_time = str_to_datetime(self.created_time) 28 | self.modified_time = str_to_datetime(self.modified_time) 29 | 30 | 31 | class ItemRecordType: 32 | FOLDER = 0 33 | FILE = 1 34 | 35 | 36 | class ItemRecordStatus: 37 | OK = 0 38 | 39 | 40 | class RepositoryType: 41 | PERSONAL = 0 42 | BUSINESS = 1 43 | 44 | 45 | def get_drive_db_path(config_dir, drive_id): 46 | return config_dir + '/items_' + drive_id + '.sqlite3' 47 | 48 | 49 | class OneDriveLocalRepository: 50 | SESSION_EXPIRE_THRESHOLD_SEC = 120 51 | 52 | def __init__(self, context, authenticator, drive, drive_config): 53 | """ 54 | :param onedrive_client.od_context.UserContext context: 55 | :param onedrive_client.od_auth.OneDriveAuthenticator authenticator: 56 | :param onedrivesdk.model.drive.Drive drive: 57 | :param onedrive_client.od_models.drive_config.LocalDriveConfig drive_config: 58 | """ 59 | self.context = context 60 | self.authenticator = authenticator 61 | self.drive = drive 62 | self.account_id = drive_config.account_id 63 | self.local_root = drive_config.localroot_path 64 | self.type = RepositoryType.BUSINESS if drive.drive_type == 'business' else RepositoryType.PERSONAL 65 | self._lock = threading.Lock() 66 | self._init_path_filter(ignore_file=drive_config.ignorefile_path) 67 | self._init_item_store() 68 | self.refresh_session() 69 | 70 | @property 71 | def _item_store_path(self): 72 | return get_drive_db_path(self.context.config_dir, self.drive.id) 73 | 74 | def _init_path_filter(self, ignore_file): 75 | try: 76 | with open(ignore_file, 'r') as f: 77 | rules = set(f.read().splitlines(keepends=False)) 78 | except OSError as e: 79 | logging.error('Failed to load ignore list file "%s": %s', ignore_file, e) 80 | rules = set() 81 | self.path_filter = _PathFilter(rules) 82 | 83 | def _init_item_store(self): 84 | self._conn = sqlite3.connect(self._item_store_path, check_same_thread=False) 85 | self._conn.execute(_get_resource('data/items_db.sql', pkg_name='onedrive_client')) 86 | atexit.register(self.close) 87 | 88 | def refresh_session(self): 89 | logging.debug('Refreshing repository session.') 90 | self.authenticator.refresh_session(self.account_id) 91 | logging.info('Session for account %s will expire in %d seconds.', 92 | self.account_id, self.authenticator.session_expires_in_sec) 93 | if self.context.loop: 94 | t = self.authenticator.session_expires_in_sec - self.SESSION_EXPIRE_THRESHOLD_SEC 95 | logging.debug('Will refresh session in %d seconds.', t) 96 | self.context.loop.call_later(t, self.refresh_session) 97 | 98 | def close(self): 99 | logging.debug('Closing database "%s".', self._item_store_path) 100 | self._conn.close() 101 | 102 | def get_item_by_path(self, item_name, parent_relpath): 103 | """ 104 | Fetch a record form database. Return None if not found. 105 | :param str item_name: 106 | :param str parent_relpath: 107 | :return ItemRecord | None: 108 | """ 109 | with self._lock: 110 | q = self._conn.execute('SELECT id, type, name, parent_id, parent_path, etag, ctag, size, size_local, ' 111 | 'created_time, modified_time, status, sha1_hash, record_time FROM items ' 112 | 'WHERE name=? AND parent_path=? LIMIT 1', (item_name, parent_relpath)) 113 | rec = q.fetchone() 114 | return ItemRecord(rec) if rec else None 115 | 116 | def get_immediate_children_of_dir(self, relpath): 117 | """ 118 | :param str relpath: 119 | :return dict(str, ItemRecord): 120 | """ 121 | with self._lock: 122 | q = self._conn.execute('SELECT id, type, name, parent_id, parent_path, etag, ctag, size, size_local, ' 123 | 'created_time, modified_time, status, sha1_hash, record_time FROM items ' 124 | 'WHERE parent_path=?', (relpath,)) 125 | return {rec[2]: ItemRecord(rec) for rec in q.fetchall() if rec} 126 | 127 | def delete_item(self, item_name, parent_relpath, is_folder=False): 128 | """ 129 | Delete the specified item from database. If it is a directory, then also delete all its children items. 130 | :param str item_name: Name of the item. 131 | :param str parent_relpath: Relative path of its parent item. 132 | :param True | False is_folder: True to indicate that the item is a folder (delete all children). 133 | """ 134 | with self._lock, self._conn, closing(self._conn.cursor()) as cursor: 135 | if is_folder: 136 | item_relpath = parent_relpath + '/' + item_name 137 | cursor.execute('DELETE FROM items WHERE parent_path=? OR parent_path LIKE ?', 138 | (item_relpath, item_relpath + '/%')) 139 | cursor.execute('DELETE FROM items WHERE parent_path=? AND name=?', (parent_relpath, item_name)) 140 | 141 | def move_item(self, item_name, parent_relpath, new_name, new_parent_relpath, is_folder=False): 142 | """ 143 | :param str item_name: Name of the item. 144 | :param str parent_relpath: Relative path of its parent item. 145 | :param str new_name: Name of the item. 146 | :param str new_parent_relpath: Relative path of its parent item. 147 | :param True | False is_folder: True to indicate that the item is a folder (delete all children). 148 | """ 149 | with self._lock, self._conn, closing(self._conn.cursor()) as cursor: 150 | if is_folder: 151 | item_relpath = parent_relpath + '/' + item_name 152 | cursor.execute('UPDATE items SET parent_path=? || substr(parent_path, ?) ' 153 | 'WHERE parent_path=? OR parent_path LIKE ?', 154 | (new_parent_relpath + '/' + new_name, len(item_relpath) + 1, 155 | item_relpath, item_relpath + '/%')) 156 | cursor.execute('UPDATE items SET parent_path=?, name=? WHERE parent_path=? AND name=?', 157 | (new_parent_relpath, new_name, parent_relpath, item_name)) 158 | 159 | def update_item(self, item, parent_relpath, size_local=0, status=ItemRecordStatus.OK): 160 | """ 161 | :param onedrivesdk.model.item.Item item: 162 | :param str parent_relpath: 163 | :param int size_local: 164 | :param int status: 165 | """ 166 | sha1_hash = None 167 | file_facet = item.file 168 | if file_facet: 169 | item_type = ItemRecordType.FILE 170 | hash_facet = file_facet.hashes 171 | if hash_facet: 172 | sha1_hash = hash_facet.sha1_hash 173 | elif item.folder: 174 | item_type = ItemRecordType.FOLDER 175 | else: 176 | raise ValueError('Unknown type of item "%s (%s)".' % (item.name, item.id)) 177 | parent_reference = item.parent_reference 178 | modified_time, _ = get_item_modified_datetime(item) 179 | modified_time_str = datetime_to_str(modified_time) 180 | created_time_str = datetime_to_str(get_item_created_datetime(item)) 181 | with self._lock, self._conn: 182 | self._conn.execute( 183 | 'INSERT OR REPLACE INTO items (id, type, name, parent_id, parent_path, etag, ' 184 | 'ctag, size, size_local, created_time, modified_time, status, sha1_hash, record_time)' 185 | ' VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)', 186 | (item.id, item_type, item.name, parent_reference.id, parent_relpath, item.e_tag, item.c_tag, 187 | item.size, size_local, created_time_str, modified_time_str, status, sha1_hash, 188 | str(datetime.utcnow().isoformat()) + 'Z')) 189 | -------------------------------------------------------------------------------- /onedrive_client/od_stringutils.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | 4 | def get_filename_with_incremented_count(filename): 5 | """ 6 | Get a new filename with a count value appended. The scheme follows the policy for name.conflictBehavior=rename. 7 | See https://dev.onedrive.com/items/create.htm. 8 | :param str filename: 9 | :return str: 10 | """ 11 | name, ext = os.path.splitext(filename) 12 | if ' ' in name: 13 | orig_name, count = name.rsplit(' ', maxsplit=1) 14 | if count.isdigit() and count[0] != '0': 15 | count = int(count) + 1 16 | return orig_name + ' ' + str(count) + ext 17 | return name + ' 1' + ext 18 | -------------------------------------------------------------------------------- /onedrive_client/od_task.py: -------------------------------------------------------------------------------- 1 | """ 2 | od_task.py 3 | Core component for OneDrive server-client interaction management. 4 | :copyright: (c) Xiangyu Bu 5 | :license: MIT 6 | """ 7 | 8 | import logging 9 | import threading 10 | 11 | 12 | class TaskPool: 13 | """ 14 | An in-memory storage for od_tasks. 15 | 16 | Some notes: 17 | (1) Tried to let worker threads and inotify watcher communicate by reading/writing a "working path set" but 18 | because workers tend to delete path before watcher can read it. 19 | """ 20 | 21 | def __init__(self): 22 | self.tasks_by_path = {} 23 | self.queued_tasks = [] 24 | self.semaphore = threading.Semaphore(0) 25 | self._lock = threading.Lock() 26 | 27 | def close(self, n=1): 28 | for _ in range(n): 29 | self.semaphore.release() 30 | 31 | def add_task(self, task): 32 | """ 33 | Add a task to internal storage. It will not add if there is already a task on the path. 34 | :param onedrive_client.tasks.base.TaskBase task: The task to add. 35 | """ 36 | logging.debug('Adding task %s...' % task) 37 | with self._lock: 38 | if task.local_abspath in self.tasks_by_path: 39 | return False 40 | self.queued_tasks.append(task) 41 | self.tasks_by_path[task.local_abspath] = task 42 | self.semaphore.release() 43 | return True 44 | 45 | def pop_task(self): 46 | """ 47 | Pop the oldest task. It's required that the caller first acquire the semaphore. 48 | :return onedrive_client.od_tasks.base.TaskBase | None: The first qualified task, or None. 49 | """ 50 | # logging.debug('Getting task...') 51 | with self._lock: 52 | ret = None 53 | if len(self.queued_tasks): 54 | ret = self.queued_tasks.pop(0) 55 | del self.tasks_by_path[ret.local_abspath] 56 | return ret 57 | 58 | @property 59 | def outstanding_task_count(self): 60 | with self._lock: 61 | return len(self.queued_tasks) 62 | 63 | def has_pending_task(self, local_abspath): 64 | with self._lock: 65 | if local_abspath in self.tasks_by_path: 66 | return self.tasks_by_path[local_abspath] 67 | return False 68 | 69 | def occupy_path(self, local_abspath, task): 70 | """ 71 | Record a task in progress on a local path so that duplicate tasks can be avoided. 72 | :param str local_abspath: 73 | :param onedrive_client.od_tasks.base.TaskBase | None task: The task working on the path. None to blacklist the path. 74 | :return onedrive_client.od_tasks.base.TaskBase | None: 75 | """ 76 | with self._lock: 77 | if local_abspath not in self.tasks_by_path: 78 | self.tasks_by_path[local_abspath] = task 79 | return task 80 | else: 81 | return self.tasks_by_path[local_abspath] 82 | 83 | def release_path(self, local_abspath): 84 | with self._lock: 85 | del self.tasks_by_path[local_abspath] 86 | 87 | def remove_children_tasks(self, local_parent_path): 88 | p = local_parent_path + '/' 89 | with self._lock: 90 | for t in self.queued_tasks[:]: 91 | if t.local_abspath.startswith(p) or t.local_abspath == local_parent_path: 92 | self.queued_tasks.remove(t) 93 | del self.tasks_by_path[t.local_abspath] 94 | -------------------------------------------------------------------------------- /onedrive_client/od_tasks/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/derrix060/onedriveClient/9bd1aba2d1fd7933780b535f4875f6428083d66b/onedrive_client/od_tasks/__init__.py -------------------------------------------------------------------------------- /onedrive_client/od_tasks/base.py: -------------------------------------------------------------------------------- 1 | class TaskBase: 2 | 3 | def __init__(self, repo, task_pool): 4 | """ 5 | :param onedrive_client.od_repo.OneDriveLocalRepository | None repo: 6 | :param onedrive_client.od_task.TaskPool task_pool: 7 | """ 8 | self.repo = repo 9 | self.task_pool = task_pool 10 | 11 | @property 12 | def local_abspath(self): 13 | return self._local_abspath 14 | 15 | # noinspection PyAttributeOutsideInit 16 | @local_abspath.setter 17 | def local_abspath(self, path): 18 | self._local_abspath = path 19 | 20 | def handle(self): 21 | raise NotImplementedError('Subclass should override this stub.') 22 | -------------------------------------------------------------------------------- /onedrive_client/od_tasks/delete_item.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | import onedrivesdk.error 4 | 5 | from onedrive_client.od_tasks import update_item_base 6 | from onedrive_client import od_api_helper 7 | 8 | 9 | class DeleteRemoteItemTask(update_item_base.UpdateItemTaskBase): 10 | 11 | def __init__(self, repo, task_pool, parent_relpath, item_name, item_id=None, is_folder=False): 12 | """ 13 | :param onedrive_client.od_repo.OneDriveLocalRepository repo: 14 | :param onedrive_client.od_task.TaskPool task_pool: 15 | :param str parent_relpath: 16 | :param str item_name: 17 | :param str | None item_id: 18 | :param True | False is_folder: 19 | """ 20 | super().__init__(repo=repo, task_pool=task_pool, parent_relpath=parent_relpath, 21 | item_name=item_name, item_id=item_id, is_folder=is_folder) 22 | 23 | def __repr__(self): 24 | return type(self).__name__ + '(%s, is_folder=%s)' % (self.local_abspath, self.is_folder) 25 | 26 | def handle(self): 27 | logging.info('Deleting remote item "%s".', self.rel_path) 28 | item_request = self.get_item_request() 29 | try: 30 | od_api_helper.item_request_call(self.repo, item_request.delete) 31 | self.repo.delete_item(self.item_name, self.parent_relpath, self.is_folder) 32 | logging.info('Deleted remote item "%s".', self.rel_path) 33 | return True 34 | except (onedrivesdk.error.OneDriveError, OSError) as e: 35 | logging.error('Error deleting item "%s": %s.', self.rel_path, e) 36 | return False 37 | -------------------------------------------------------------------------------- /onedrive_client/od_tasks/download_file.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import os 3 | 4 | import onedrivesdk.error 5 | 6 | from onedrive_client.od_tasks import base 7 | from onedrive_client import fix_owner_and_timestamp 8 | from onedrive_client.od_api_helper import get_item_modified_datetime 9 | from onedrive_client.od_api_helper import item_request_call 10 | from onedrive_client.od_dateutils import datetime_to_timestamp 11 | from onedrive_client.od_hashutils import sha1_value 12 | 13 | 14 | class DownloadFileTask(base.TaskBase): 15 | 16 | def __init__(self, repo, task_pool, remote_item, parent_relpath): 17 | """ 18 | :param onedrive_client.od_repo.OneDriveLocalRepository repo: 19 | :param onedrive_client.od_task.TaskPool task_pool: 20 | :param onedrivesdk.model.item.Item remote_item: 21 | :param str parent_relpath: 22 | """ 23 | super().__init__(repo, task_pool) 24 | self.remote_item = remote_item 25 | self.parent_relpath = parent_relpath 26 | self.local_abspath = repo.local_root + parent_relpath + '/' + remote_item.name 27 | 28 | def __repr__(self): 29 | return type(self).__name__ + '(%s)' % self.local_abspath 30 | 31 | def handle(self): 32 | logging.info('Downloading file "%s" to "%s".', self.remote_item.id, self.local_abspath) 33 | try: 34 | tmp_name = self.repo.path_filter.get_temp_name(self.remote_item.name) 35 | tmp_path = self.repo.local_root + self.parent_relpath + '/' + tmp_name 36 | item_request = self.repo.authenticator.client.item(drive=self.repo.drive.id, id=self.remote_item.id) 37 | item_mtime, item_mtime_editable = get_item_modified_datetime(self.remote_item) 38 | item_request_call(self.repo, item_request.download, tmp_path) 39 | hashes = self.remote_item.file.hashes 40 | if hashes is None or hashes.sha1_hash is None or hashes.sha1_hash == sha1_value(tmp_path): 41 | item_size_local = os.path.getsize(tmp_path) 42 | os.rename(tmp_path, self.local_abspath) 43 | fix_owner_and_timestamp(self.local_abspath, self.repo.context.user_uid, 44 | datetime_to_timestamp(item_mtime)) 45 | self.repo.update_item(self.remote_item, self.parent_relpath, item_size_local) 46 | logging.info('Finished downloading item "%s".', self.remote_item.id) 47 | return True 48 | else: 49 | # We assumed server's SHA-1 value is always correct -- might not be true. 50 | logging.error('Hash mismatch for downloaded file "%s".', self.local_abspath) 51 | os.remove(tmp_path) 52 | except (onedrivesdk.error.OneDriveError, OSError) as e: 53 | logging.error('Error when downloading file "%s": %s.', self.remote_item.id, e) 54 | return False 55 | -------------------------------------------------------------------------------- /onedrive_client/od_tasks/move_item.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import os 3 | 4 | import onedrivesdk.error 5 | from onedrivesdk import Item, ItemReference 6 | 7 | from onedrive_client.od_tasks import update_mtime 8 | from onedrive_client.od_api_helper import item_request_call 9 | 10 | 11 | class MoveItemTask(update_mtime.UpdateTimestampTask): 12 | 13 | def __init__(self, repo, task_pool, parent_relpath, item_name, 14 | new_parent_relpath=None, new_name=None, item_id=None, is_folder=False): 15 | """ 16 | :param onedrive_client.od_repo.OneDriveLocalRepository repo: 17 | :param onedrive_client.od_task.TaskPool task_pool: 18 | :param str parent_relpath: 19 | :param str item_name: 20 | :param str | None new_parent_relpath: 21 | :param str | None new_name: 22 | :param str | None item_id: 23 | :param True | False is_folder: 24 | """ 25 | super().__init__(repo=repo, task_pool=task_pool, parent_relpath=parent_relpath, 26 | item_name=item_name, item_id=item_id, is_folder=is_folder) 27 | if new_parent_relpath is None and new_name is None: 28 | raise ValueError('New parent directory or name cannot both be None in MoveItemTask.') 29 | if new_parent_relpath is None: 30 | new_parent_relpath = parent_relpath 31 | if new_name is None: 32 | new_name = item_name 33 | if new_parent_relpath == parent_relpath and new_name == item_name: 34 | raise ValueError('Old path and new path are the same. No move is needed.') 35 | self.new_parent_relpath = new_parent_relpath 36 | self.new_name = new_name 37 | self.new_relpath = new_parent_relpath + '/' + new_name 38 | self.new_local_abspath = self.repo.local_root + self.new_relpath 39 | 40 | def _get_new_item(self): 41 | item = Item() 42 | if self.new_parent_relpath != self.parent_relpath: 43 | ref = ItemReference() 44 | # Refer to https://dev.onedrive.com/items/move.htm for Move API request. 45 | ref.path = '/drives/' + self.repo.drive.id + '/root:' 46 | if self.new_parent_relpath != '': 47 | ref.path += self.new_parent_relpath 48 | item.parent_reference = ref 49 | if self.new_name != self.item_name: 50 | item.name = self.new_name 51 | return item 52 | 53 | def __repr__(self): 54 | return type(self).__name__ + '(from=%s, to=%s, is_folder=%s)' % ( 55 | self.rel_path, self.new_relpath, self.is_folder) 56 | 57 | def handle(self): 58 | logging.info('Moving item "%s" to "%s".', self.rel_path, self.new_relpath) 59 | 60 | # The routine assumes that the directory to save the new path exists remotely. 61 | item_request = self.get_item_request() 62 | try: 63 | item_stat = os.stat(self.new_local_abspath) 64 | item = item_request_call(self.repo, item_request.update, self._get_new_item()) 65 | # TODO: update all records or rebuild records after deletion? 66 | # self.repo.delete_item(self.item_name, self.parent_relpath, self.is_folder) 67 | self.repo.move_item(item_name=self.item_name, parent_relpath=self.parent_relpath, 68 | new_name=self.new_name, new_parent_relpath=self.new_parent_relpath, 69 | is_folder=self.is_folder) 70 | self.update_timestamp_and_record(item, item_stat) 71 | return True 72 | except (onedrivesdk.error.OneDriveError, OSError) as e: 73 | logging.error('Error moving item "%s" to "%s": %s.', self.rel_path, self.new_relpath, e) 74 | return False 75 | -------------------------------------------------------------------------------- /onedrive_client/od_tasks/start_repo.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import os 3 | 4 | from onedrive_client.od_tasks import ( 5 | base, 6 | merge_dir, 7 | ) 8 | 9 | 10 | class StartRepositoryTask(base.TaskBase): 11 | """A simple task that bootstraps the syncing process of a Drive. 12 | It checks if the root path is a directory, and if so, create a task to merge the remote root with local root. 13 | """ 14 | 15 | def __init__(self, repo, task_pool): 16 | """ 17 | :param onedrive_client.od_repo.OneDriveLocalRepository repo: 18 | :param onedrive_client.od_task.TaskPool task_pool: 19 | """ 20 | super().__init__(repo, task_pool) 21 | self.local_abspath = repo.local_root 22 | 23 | def __repr__(self): 24 | return type(self).__name__ + '(drive=' + self.repo.drive.id + ')' 25 | 26 | def handle(self): 27 | try: 28 | if os.path.isdir(self.repo.local_root): 29 | # And add a recursive merge task to task queue. 30 | item_request = self.repo.authenticator.client.item(drive=self.repo.drive.id, path='/') 31 | self.task_pool.add_task(merge_dir.MergeDirectoryTask(self.repo, self.task_pool, '', item_request)) 32 | else: 33 | raise OSError('Local root of Drive %s does not exist or is not a directory. Please check "%s".' % 34 | (self.repo.drive.id, self.repo.local_root)) 35 | except OSError as e: 36 | logging.error('Error: %s', e) 37 | 38 | 39 | # class ApplyLatestDeltaTask(StartRepositoryTask): 40 | # 41 | # TOKEN_LATEST = 'latest' 42 | # 43 | # def handle(self): 44 | # try: 45 | # logging.debug('Checking delta for Drive %s.', self.repo.drive.id) 46 | # item_request = self.repo.authenticator.client.item(drive=self.repo.drive.id, path='/') 47 | # delta_collection = item_request.delta(token=self.TOKEN_LATEST).get() 48 | # for item in delta_collection: 49 | # print(item.name) 50 | # logging.info('Delta token: %s.', delta_collection.token) 51 | # import json 52 | # with open('delta_latest.json', 'a') as f: 53 | # json.dump(delta_collection.__dict__, f, sort_keys=True, indent=4, separators=(',', ': ')) 54 | # f.write('\n') 55 | # delta_collection_2 = item_request.delta(token=delta_collection.token).get() 56 | # for item in delta_collection_2: 57 | # print(item.name) 58 | # logging.info('Delta token 2: %s.', delta_collection_2.token) 59 | # with open('delta_latest_succ.json', 'a') as f: 60 | # json.dump(delta_collection_2.__dict__, f, sort_keys=True, indent=4, separators=(',', ': ')) 61 | # f.write('\n') 62 | # except: 63 | # raise 64 | -------------------------------------------------------------------------------- /onedrive_client/od_tasks/update_item_base.py: -------------------------------------------------------------------------------- 1 | from onedrive_client.od_tasks import base 2 | 3 | 4 | class UpdateItemTaskBase(base.TaskBase): 5 | 6 | def __init__(self, repo, task_pool, parent_relpath, item_name, item_id=None, is_folder=False): 7 | """ 8 | :param onedrive_client.od_repo.OneDriveLocalRepository repo: 9 | :param onedrive_client.od_task.TaskPool task_pool: 10 | :param str parent_relpath: 11 | :param str item_name: 12 | :param str | None item_id: 13 | :param True | False is_folder: 14 | """ 15 | super().__init__(repo, task_pool) 16 | self.parent_relpath = parent_relpath 17 | self.item_name = item_name 18 | self.rel_path = parent_relpath + '/' + item_name 19 | self.item_id = item_id 20 | self.is_folder = is_folder 21 | self.local_abspath = repo.local_root + self.rel_path 22 | 23 | def get_item_request(self): 24 | if self.item_id is not None: 25 | return self.repo.authenticator.client.item(drive=self.repo.drive.id, id=self.item_id) 26 | else: 27 | return self.repo.authenticator.client.item(drive=self.repo.drive.id, path=self.rel_path) 28 | 29 | def handle(self): 30 | raise NotImplementedError() 31 | -------------------------------------------------------------------------------- /onedrive_client/od_tasks/update_mtime.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import os 3 | from datetime import datetime 4 | 5 | import onedrivesdk.error 6 | from onedrivesdk import Item, FileSystemInfo 7 | 8 | from onedrive_client.od_tasks import update_item_base 9 | from onedrive_client import fix_owner_and_timestamp 10 | from onedrive_client.od_api_helper import ( 11 | get_item_modified_datetime, 12 | item_request_call, 13 | ) 14 | from onedrive_client.od_dateutils import datetime_to_timestamp 15 | 16 | 17 | class UpdateTimestampTask(update_item_base.UpdateItemTaskBase): 18 | 19 | def __init__(self, repo, task_pool, parent_relpath, item_name, item_id=None, is_folder=False): 20 | """ 21 | :param onedrive_client.od_repo.OneDriveLocalRepository repo: 22 | :param onedrive_client.od_task.TaskPool task_pool: 23 | :param str parent_relpath: 24 | :param str item_name: 25 | """ 26 | super().__init__(repo, task_pool, parent_relpath, item_name, item_id, is_folder) 27 | 28 | def __repr__(self): 29 | return type(self).__name__ + '(%s)' % self.local_abspath 30 | 31 | def update_timestamp_and_record(self, new_item, item_local_stat): 32 | remote_mtime, remote_mtime_w = get_item_modified_datetime(new_item) 33 | if not remote_mtime_w: 34 | # last_modified_datetime attribute is not modifiable in OneDrive server. Update local mtime. 35 | fix_owner_and_timestamp(self.local_abspath, self.repo.context.user_uid, 36 | datetime_to_timestamp(remote_mtime)) 37 | else: 38 | file_system_info = FileSystemInfo() 39 | file_system_info.last_modified_date_time = datetime.utcfromtimestamp(item_local_stat.st_mtime) 40 | updated_item = Item() 41 | updated_item.file_system_info = file_system_info 42 | item_request = self.repo.authenticator.client.item(drive=self.repo.drive.id, id=new_item.id) 43 | new_item = item_request_call(self.repo, item_request.update, updated_item) 44 | self.repo.update_item(new_item, self.parent_relpath, item_local_stat.st_size) 45 | 46 | def handle(self): 47 | logging.info('Updating timestamp for file "%s".', self.local_abspath) 48 | try: 49 | if not os.path.isfile(self.local_abspath): 50 | logging.warning('Local path "%s" is no longer a file. Cannot update timestamp.', self.local_abspath) 51 | return False 52 | 53 | item = item_request_call(self.repo, self.get_item_request().get) 54 | item_stat = os.stat(self.local_abspath) 55 | self.update_timestamp_and_record(item, item_stat) 56 | logging.info('Finished updating timestamp for file "%s".', self.local_abspath) 57 | return True 58 | except (onedrivesdk.error.OneDriveError, OSError) as e: 59 | logging.error('Error updating timestamp for file "%s": %s.', self.local_abspath, e) 60 | return False 61 | -------------------------------------------------------------------------------- /onedrive_client/od_tasks/update_subscriptions.py: -------------------------------------------------------------------------------- 1 | from datetime import datetime, timedelta 2 | import logging 3 | 4 | import onedrivesdk 5 | import onedrivesdk.error 6 | 7 | from onedrive_client.od_tasks import base 8 | from onedrive_client import od_api_helper 9 | 10 | 11 | class UpdateSubscriptionTask(base.TaskBase): 12 | 13 | def __init__(self, repo, task_pool, webhook_worker, subscription_id=None): 14 | """ 15 | :param onedrive_client.od_repo.OneDriveLocalRepository repo: 16 | :param onedrive_client.od_task.TaskPool | None task_pool: 17 | :param onedrive_client.od_webhook.WebhookWorkerThread webhook_worker: 18 | :param str | None subscription_id: 19 | """ 20 | super().__init__(repo, task_pool) 21 | self.webhook_worker = webhook_worker 22 | self.subscription_id = subscription_id 23 | 24 | def handle(self): 25 | logging.info('Updating webhook for Drive %s.', self.repo.drive.id) 26 | item_request = self.repo.authenticator.client.item(drive=self.repo.drive.id, path='/') 27 | expiration_time = datetime.utcnow() + timedelta(seconds=self.repo.context.config['webhook_renew_interval_sec']) 28 | try: 29 | if self.subscription_id is None: 30 | subscription = od_api_helper.create_subscription( 31 | item_request, self.repo, self.webhook_worker.webhook_url, expiration_time) 32 | else: 33 | subscription = onedrivesdk.Subscription() 34 | subscription.id = self.subscription_id 35 | subscription.notification_url = self.webhook_worker.webhook_url 36 | subscription.expiration_date_time = expiration_time 37 | subscription = od_api_helper.item_request_call( 38 | self.repo, item_request.subscriptions[self.subscription_id].update, subscription) 39 | self.webhook_worker.add_subscription(subscription, self.repo) 40 | logging.info('Webhook for Drive %s updated.', self.repo.drive.id) 41 | return subscription 42 | except onedrivesdk.error.OneDriveError as e: 43 | logging.error('Error: %s', e) 44 | return None 45 | -------------------------------------------------------------------------------- /onedrive_client/od_tasks/upload_file.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import os 3 | 4 | import onedrivesdk 5 | import onedrivesdk.error 6 | 7 | from onedrive_client.od_tasks import update_mtime 8 | from onedrive_client.od_api_helper import item_request_call 9 | 10 | 11 | class UploadFileTask(update_mtime.UpdateTimestampTask): 12 | 13 | # If file is smaller than this size (in Bytes) use HTTP PUT method to upload. Otherwise upload in chunks 14 | # using Session API (https://dev.onedrive.com/items/upload_large_files.htm). 15 | PUT_FILE_SIZE_THRESHOLD_BYTES = 10 << 20 16 | 17 | def __init__(self, repo, task_pool, parent_dir_request, parent_relpath, item_name): 18 | """ 19 | :param onedrive_client.od_repo.OneDriveLocalRepository repo: 20 | :param onedrive_client.od_task.TaskPool task_pool: 21 | :param onedrivesdk.request.item_request_builder.ItemRequestBuilder parent_dir_request: 22 | :param str parent_relpath: 23 | :param str item_name: 24 | """ 25 | super().__init__(repo, task_pool, parent_relpath, item_name) 26 | self.parent_dir_request = parent_dir_request 27 | 28 | def __repr__(self): 29 | return type(self).__name__ + '(%s)' % self.local_abspath 30 | 31 | def update_progress(self, curr_part, total_part): 32 | if curr_part == total_part: 33 | logging.debug('All %d parts of file "%s" have been uploaded.', total_part, self.local_abspath) 34 | else: 35 | logging.debug('Uploading file "%s": Part %d / %d.', self.local_abspath, curr_part + 1, total_part) 36 | 37 | def handle(self): 38 | logging.info('Uploading file "%s" to OneDrive.', self.local_abspath) 39 | occupy_task = self.task_pool.occupy_path(self.local_abspath, self) 40 | if occupy_task is not self: 41 | logging.warning('Cannot upload "%s" because %s.', self.local_abspath, 42 | "path is blacklisted" if occupy_task is None else str(occupy_task) + ' is in progress') 43 | return False 44 | try: 45 | item_stat = os.stat(self.local_abspath) 46 | if item_stat.st_size < self.PUT_FILE_SIZE_THRESHOLD_BYTES: 47 | item_request = self.parent_dir_request.children[self.item_name] 48 | returned_item = item_request_call(self.repo, item_request.upload, self.local_abspath) 49 | if returned_item is None: 50 | logging.warning('Upload API did not return metadata of remote item for "%s". ' 51 | 'Make an explicit request.', self.local_abspath) 52 | returned_item = item_request_call(self.repo, item_request.get) 53 | else: 54 | logging.info('Uploading large file "%s" in chunks of 10MB.', self.local_abspath) 55 | item_request = self.repo.authenticator.client.item(drive=self.repo.drive.id, path=self.rel_path) 56 | returned_item = item_request_call(self.repo, item_request.upload_async, 57 | local_path=self.local_abspath, upload_status=self.update_progress) 58 | if not isinstance(returned_item, onedrivesdk.Item): 59 | if hasattr(returned_item, '_prop_dict'): 60 | returned_item = onedrivesdk.Item(returned_item._prop_dict) 61 | else: 62 | returned_item = item_request_call(self.repo, item_request.get) 63 | self.update_timestamp_and_record(returned_item, item_stat) 64 | self.task_pool.release_path(self.local_abspath) 65 | logging.info('Finished uploading file "%s".', self.local_abspath) 66 | return True 67 | except (onedrivesdk.error.OneDriveError, OSError) as e: 68 | logging.error('Error uploading file "%s": %s.', self.local_abspath, e) 69 | # TODO: what if quota is exceeded? 70 | if (isinstance(e, onedrivesdk.error.OneDriveError) and 71 | e.code == onedrivesdk.error.ErrorCode.MalwareDetected): 72 | logging.warning('File "%s" was detected as malware by OneDrive. ' 73 | 'Do not upload during program session.', self.local_abspath) 74 | self.task_pool.occupy_path(self.local_abspath, None) 75 | return False 76 | self.task_pool.release_path(self.local_abspath) 77 | return False 78 | -------------------------------------------------------------------------------- /onedrive_client/od_threads.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import threading 3 | 4 | 5 | class TaskWorkerThread(threading.Thread): 6 | 7 | def __init__(self, name, task_pool): 8 | """ 9 | :param onedrive_client.od_task.TaskPool task_pool: 10 | """ 11 | super().__init__(name=name, daemon=False) 12 | self.task_pool = task_pool 13 | self._running = True 14 | 15 | def stop(self): 16 | self._running = False 17 | 18 | def run(self): 19 | logging.debug('Started.') 20 | while self._running: 21 | # logging.debug('Getting semaphore.') 22 | self.task_pool.semaphore.acquire() 23 | # logging.debug('Got semaphore.') 24 | if not self._running: 25 | break 26 | task = self.task_pool.pop_task() 27 | if task is not None: 28 | logging.debug('Got task %s.', task) 29 | task.handle() 30 | logging.info('Stopped.') 31 | -------------------------------------------------------------------------------- /onedrive_client/od_webhook.py: -------------------------------------------------------------------------------- 1 | import http.server 2 | import http.client 3 | import json 4 | import logging 5 | import os 6 | import threading 7 | import queue 8 | import urllib.parse 9 | 10 | from onedrive_client.od_models.webhook_notification import WebhookNotification 11 | 12 | try: 13 | JSONDecodeError = json.JSONDecodeError 14 | except AttributeError: 15 | JSONDecodeError = ValueError 16 | 17 | 18 | def get_webhook_server(context): 19 | """ 20 | :param onedrive_client.od_context.UserContext context: 21 | """ 22 | if context.config['webhook_type'] == 'direct': 23 | from onedrive_client.od_webhooks.http_server import WebhookConfig, WebhookListener 24 | wh_config = WebhookConfig(host=context.config['webhook_host'], port=context.config['webhook_port']) 25 | elif context.config['webhook_type'] == 'ngrok': 26 | from onedrive_client.od_webhooks.ngrok_server import WebhookConfig, WebhookListener 27 | ngrok_config_file = context.config_dir + '/' + context.DEFAULT_NGROK_CONF_FILENAME 28 | if not os.path.isfile(ngrok_config_file): 29 | ngrok_config_file = None 30 | wh_config = WebhookConfig(port=context.config['webhook_port'], ngrok_config_path=ngrok_config_file) 31 | else: 32 | raise ValueError('Unsupported webhook type: "%s".' % context.config['webhook_type']) 33 | return WebhookListener(wh_config, OneDriveWebhookHandler) 34 | 35 | 36 | def parse_notification_body(body): 37 | try: 38 | decoded_body = body.decode('utf-8-sig') 39 | except ValueError: 40 | decoded_body = body.decode('utf-8') 41 | 42 | try: 43 | data = json.loads(decoded_body) 44 | try: 45 | subscription_ids = set([WebhookNotification(v).subscription_id for v in data['value']]) 46 | except KeyError: 47 | subscription_ids = (WebhookNotification(data).subscription_id,) 48 | return subscription_ids 49 | except (UnicodeError, ValueError, JSONDecodeError, KeyError) as e: 50 | logging.error(e) 51 | except Exception as e: 52 | logging.error(e) 53 | return None 54 | 55 | 56 | class WebhookWorkerThread(threading.Thread): 57 | 58 | def __init__(self, webhook_url, callback_func, action_delay_sec=60): 59 | super().__init__(name='WebhookWorker', daemon=True) 60 | self.webhook_url = webhook_url 61 | self.callback_func = callback_func 62 | self.action_delay_sec = action_delay_sec 63 | self._raw_input_queue = queue.Queue() 64 | self._registered_subscriptions = dict() 65 | 66 | def queue_input(self, raw_bytes): 67 | self._raw_input_queue.put(raw_bytes, block=False) 68 | 69 | def add_subscription(self, subscription, repo): 70 | """ 71 | :param onedrivesdk.Subscription subscription: 72 | :param onedrive_client.od_repo.OneDriveLocalRepository repo: 73 | """ 74 | self._registered_subscriptions[subscription.id] = repo 75 | logging.debug('Subscribed to root updates of drive %s. Subscription ID: %s.', 76 | repo.drive.id, subscription.id) 77 | 78 | def schedule_callback(self, subscription_ids): 79 | for subscription_id in subscription_ids: 80 | if subscription_id in self._registered_subscriptions: 81 | repo = self._registered_subscriptions[subscription_id] 82 | self.callback_func(repo) 83 | else: 84 | logging.error('Unknown subscription ID "%s".', subscription_id) 85 | 86 | @staticmethod 87 | def parse_and_update_set(body, set_buffer): 88 | subscription_ids = parse_notification_body(body) 89 | if subscription_ids is not None: 90 | set_buffer.update(subscription_ids) 91 | 92 | def run(self): 93 | subscription_ids_buf = set() 94 | logging.debug('Started.') 95 | while True: 96 | raw_bytes = self._raw_input_queue.get() 97 | self._raw_input_queue.task_done() 98 | self.parse_and_update_set(raw_bytes, subscription_ids_buf) 99 | del raw_bytes 100 | try: 101 | while True: 102 | more_bytes = self._raw_input_queue.get(block=True, timeout=self.action_delay_sec) 103 | self._raw_input_queue.task_done() 104 | self.parse_and_update_set(more_bytes, subscription_ids_buf) 105 | del more_bytes 106 | except queue.Empty: 107 | pass 108 | self.schedule_callback(subscription_ids_buf) 109 | subscription_ids_buf.clear() 110 | 111 | 112 | class OneDriveWebhookHandler(http.server.BaseHTTPRequestHandler): 113 | 114 | VALIDATION_REQUEST_QUERY = 'validationtoken' 115 | 116 | protocol_version = 'HTTP/1.1' 117 | 118 | def echo(self, s): 119 | """ 120 | :param str s: 121 | """ 122 | s = s.encode('utf-8') 123 | self.send_response(http.client.OK) 124 | self.send_header('Content-Type', 'text/plain') 125 | self.send_header('Content-Length', str(len(s))) 126 | self.end_headers() 127 | self.wfile.write(s) 128 | 129 | # noinspection PyPep8Naming 130 | def do_POST(self): 131 | url = urllib.parse.urlparse(self.path) 132 | 133 | # Some basic validation. 134 | if url.path != '/' + self.server.session_token: 135 | return self.send_error(http.client.UNAUTHORIZED) 136 | 137 | # Handle webhook validation request. 138 | query = urllib.parse.parse_qs(url.query) 139 | if self.VALIDATION_REQUEST_QUERY in query and len(query) == 1: 140 | return self.echo(query[self.VALIDATION_REQUEST_QUERY][0]) 141 | 142 | # Handle notifications. 143 | content_length = int(self.headers.get('Content-Length', 0)) 144 | body = self.rfile.read(content_length) 145 | self.send_response(http.client.OK) 146 | self.send_header('Content-Type', 'text/plain') 147 | self.send_header('Content-Length', '0') 148 | self.end_headers() 149 | logging.info(self.raw_requestline) 150 | self.server.worker_thread.queue_input(body) 151 | -------------------------------------------------------------------------------- /onedrive_client/od_webhooks/__init__.py: -------------------------------------------------------------------------------- 1 | DEFAULT_WEBHOOK_TYPE = 'ngrok' 2 | -------------------------------------------------------------------------------- /onedrive_client/od_webhooks/http_server.py: -------------------------------------------------------------------------------- 1 | """ 2 | http_server.py 3 | 4 | A webhook listener that directly accepts notifications from OneDrive server. 5 | """ 6 | 7 | import logging 8 | import os 9 | import random 10 | import string 11 | import threading 12 | import http.server 13 | import ssl 14 | 15 | import requests 16 | 17 | 18 | def resolve_public_ip(): 19 | response = requests.get('https://api.ipify.org/?format=json').json() 20 | return response['ip'] 21 | 22 | 23 | def gen_random_token(): 24 | return ''.join(random.sample(string.ascii_letters + string.digits, random.randrange(6, 12))) 25 | 26 | 27 | # TODO: OneDrive only accepts HTTPS webhook. 28 | 29 | 30 | class WebhookConfig: 31 | 32 | def __init__(self, host='', port=0): 33 | https_keyfile = os.getenv('WEBHOOK_KEY_FILE') 34 | https_certfile = os.getenv('WEBHOOK_CERT_FILE') 35 | if isinstance(https_keyfile, str) and isinstance(https_certfile, str): 36 | self.use_https = True 37 | self.https_keyfile = https_keyfile 38 | self.https_certfile = https_certfile 39 | else: 40 | self.use_https = False 41 | self.host = host 42 | self.port = port 43 | 44 | 45 | class WebhookHTTPServer(http.server.HTTPServer): 46 | 47 | def init_props(self): 48 | self._session_token = gen_random_token() 49 | self.worker_thread = None 50 | 51 | @property 52 | def session_token(self): 53 | return self._session_token 54 | 55 | 56 | class WebhookListener(threading.Thread): 57 | 58 | def __init__(self, config, handler_class): 59 | super().__init__(name='Webhook', daemon=False) 60 | self.config = config 61 | if self.config.use_https: 62 | self.server = WebhookHTTPServer(('', self.config.port), handler_class) 63 | self.server.socket = ssl.wrap_socket(self.server.socket.socket, 64 | keyfile=config.https_keyfile, 65 | certfile=config.https_certfile, 66 | server_side=True) 67 | else: 68 | self.server = WebhookHTTPServer(('', self.config.port), handler_class) 69 | self.server.init_props() 70 | 71 | @property 72 | def webhook_url(self): 73 | if not hasattr(self, '_webhook_url'): 74 | self.hostname = self.config.host if self.config.host != '' else resolve_public_ip() 75 | self._webhook_url = '%s://%s:%d/%s' % ( 76 | 'https' if self.config.use_https else 'http', 77 | self.hostname, self.server.server_port, self.server.session_token) 78 | return self._webhook_url 79 | 80 | def set_worker(self, worker): 81 | self.server.worker_thread = worker 82 | 83 | def stop(self): 84 | self.server.shutdown() 85 | 86 | def run(self): 87 | logging.info('Webhook server listening on %s.', self.webhook_url) 88 | self.server.serve_forever() 89 | logging.info('Webhook server stopped.') 90 | -------------------------------------------------------------------------------- /onedrive_client/od_webhooks/ngrok_server.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import os 3 | import subprocess 4 | import time 5 | import shutil 6 | 7 | import psutil 8 | import requests 9 | 10 | from onedrive_client.od_webhooks import http_server 11 | 12 | 13 | class WebhookConfig(http_server.WebhookConfig): 14 | 15 | def __init__(self, port=0, ngrok_config_path=None): 16 | super().__init__(port=port) 17 | self.ngrok_path = os.getenv('NGROK', 'ngrok') 18 | ngrok_config_path = os.getenv('NGROK_CONFIG_FILE', ngrok_config_path) 19 | if isinstance(ngrok_config_path, str): 20 | self.ngrok_config_path = ngrok_config_path 21 | 22 | 23 | def _append_cmd_arg(config, prop, arg, cmd): 24 | if hasattr(config, prop): 25 | cmd.append(arg) 26 | cmd.append(getattr(config, prop)) 27 | 28 | 29 | class WebhookListener(http_server.WebhookListener): 30 | 31 | POLL_TUNNELS_MAX_TRIES = 30 32 | 33 | def __init__(self, config, handler_class): 34 | super().__init__(config, handler_class) 35 | if shutil.which(config.ngrok_path) is None: 36 | raise RuntimeError('Did not find ngrok executable "%s".' % config.ngrok_path) 37 | cmd = [config.ngrok_path, 'http', str(self.server.server_port), '-bind-tls=true'] 38 | _append_cmd_arg(config, 'ngrok_config_path', '--config', cmd) 39 | self._start_ngrok_process(cmd) 40 | self._read_ngrok_tunnels() 41 | 42 | @property 43 | def ngrok_api_url(self): 44 | return self._api_url 45 | 46 | @property 47 | def webhook_url(self): 48 | return self._webhook_url 49 | 50 | def stop(self): 51 | try: 52 | self.ngrok_proc.terminate() 53 | self.ngrok_proc.wait(timeout=1) 54 | except subprocess.TimeoutExpired: 55 | self.ngrok_proc.kill() 56 | super().stop() 57 | 58 | def run(self): 59 | logging.info('Local webhook server listening on port %d.', self.server.server_port) 60 | super().run() 61 | 62 | def _start_ngrok_process(self, cmd): 63 | try: 64 | self.ngrok_proc = subprocess.Popen(cmd, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) 65 | self.ngrok_proc.wait(timeout=1) 66 | logging.critical('ngrok process (pid %d) terminated early with return code %d. Command: "%s".', 67 | self.ngrok_proc.pid, self.ngrok_proc.returncode, ' '.join(cmd)) 68 | raise RuntimeError('Ngrok process exited early.') 69 | except subprocess.TimeoutExpired: 70 | pass 71 | 72 | def _find_ngrok_inspection_port(self): 73 | """ 74 | :return (str, int): 75 | """ 76 | for c in psutil.Process(self.ngrok_proc.pid).connections(): 77 | if c.laddr[0] == '127.0.0.1' and c.raddr == () and c.status == psutil.CONN_LISTEN: 78 | return c.laddr 79 | raise RuntimeError('Did not find API interface of ngrok.') 80 | 81 | def _read_ngrok_tunnels(self): 82 | webhook_urls = dict() 83 | self._api_url = 'http://%s:%d/api' % self._find_ngrok_inspection_port() 84 | logging.info('Local ngrok API url: %s', self._api_url) 85 | for _ in range(0, self.POLL_TUNNELS_MAX_TRIES): 86 | try: 87 | data = requests.get(self._api_url + '/tunnels').json() 88 | if 'tunnels' not in data or len(data['tunnels']) == 0: 89 | raise ValueError('ngrok API did not return any tunnel.') 90 | for tunnel in data['tunnels']: 91 | if tunnel['config']['addr'].endswith(':' + str(self.server.server_port)): 92 | webhook_urls[tunnel['proto']] = tunnel['public_url'] 93 | break 94 | except (requests.ConnectionError, ValueError) as e: 95 | logging.error('Error reading ngrok API: %s. Retry in 1sec.', e) 96 | time.sleep(1) 97 | if 'https' in webhook_urls: 98 | self._webhook_url = webhook_urls['https'] + '/' + self.server.session_token 99 | else: 100 | raise RuntimeError('Did not receive any HTTPS tunnel from ngrok API.') 101 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | arrow==0.12.1 2 | asn1crypto==0.24.0 3 | bidict==0.13.1 4 | certifi==2018.1.18 5 | cffi==1.11.5 6 | chardet==3.0.4 7 | click==6.7 8 | colorama==0.3.9 9 | cryptography==2.1.4 10 | daemonocle==1.0.1 11 | idna==2.6 12 | inotify-simple==1.1.7 13 | keyring==11.0.0 14 | onedrivesdk==1.1.8 15 | psutil==5.4.3 16 | pycparser==2.18 17 | python-dateutil==2.7.0 18 | PyYAML==3.13 19 | requests==2.18.4 20 | SecretStorage==2.3.1 21 | Send2Trash==1.5.0 22 | six==1.11.0 23 | tabulate==0.8.2 24 | urllib3==1.22 25 | zgitignore==0.8.0 26 | keyrings.alt==3.0 27 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [pycodestyle] 2 | ignore = E401, E402 3 | max-line-length = 120 4 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | """ 4 | onedrive-d 5 | A Microsoft OneDrive client for Linux. 6 | :copyright: (c) Xiangyu Bu 7 | :license: MIT 8 | """ 9 | 10 | import sys 11 | from setuptools import setup, find_packages 12 | 13 | from onedrive_client import __author__, __email__, __homepage__, __project__, __version__ 14 | 15 | 16 | with open('requirements.txt', 'r') as f: 17 | install_requires = f.readlines() 18 | 19 | test_requires = [ 20 | 'requests-mock', 21 | ] 22 | 23 | with open('README.md', 'r') as f: 24 | readme = f.read() 25 | 26 | python_version = sys.version_info 27 | 28 | if python_version < (3, 4): 29 | raise Exception('%s %s only supports Python 3.4 and newer.' % (__project__, __version__)) 30 | 31 | if python_version < (3, 5): 32 | install_requires.append('dbus-python') 33 | 34 | setup( 35 | name=__project__, 36 | version=__version__, 37 | author=__author__, 38 | author_email=__email__, 39 | url=__homepage__, 40 | description='A Microsoft OneDrive client for Linux written in Python 3.', 41 | license='MIT', 42 | long_description=readme, 43 | packages=find_packages(exclude=["*.tests", "*.tests.*", "tests.*", "tests"]), 44 | include_package_data=True, 45 | package_data={ 46 | 'onedrive_client': ['data/*', 'lang/*'] 47 | }, 48 | package_dir={'onedrive_client': 'onedrive_client'}, 49 | entry_points={ 50 | 'console_scripts': [ 51 | 'onedrive-client = onedrive_client.od_main:main', 52 | 'onedrive-client-pref = onedrive_client.od_pref:main' 53 | ], 54 | 'gui_scripts': [] 55 | }, 56 | install_requires=install_requires, 57 | tests_require=test_requires, 58 | test_suite='tests', 59 | zip_safe=False 60 | ) 61 | -------------------------------------------------------------------------------- /sideci.yml: -------------------------------------------------------------------------------- 1 | flake8: 2 | version: 3 3 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- 1 | import os 2 | import keyring.backend 3 | 4 | 5 | class MockKeyring(keyring.backend.KeyringBackend): 6 | 7 | priority = 1 8 | 9 | def __init__(self): 10 | super().__init__() 11 | self.passwords = dict() 12 | 13 | def set_password(self, service, username, password): 14 | self.passwords[service + '.' + username] = password 15 | 16 | def get_password(self, service, username): 17 | return self.passwords[service + '.' + username] 18 | 19 | def delete_password(self, service, username): 20 | del self.passwords[service + '.' + username] 21 | 22 | 23 | if os.getenv('MOCK_KEYRING') is not None: 24 | keyring.set_keyring(MockKeyring()) 25 | -------------------------------------------------------------------------------- /tests/data/drive_config_item.json: -------------------------------------------------------------------------------- 1 | { 2 | "account_id": "acct_id", 3 | "drive_id": "drive_id", 4 | "ignorefile_path": "/home/xb/.config/onedrive_client/ignore_v2.txt", 5 | "localroot_path": "/home/foobar/OneDrive" 6 | } 7 | -------------------------------------------------------------------------------- /tests/data/drive_response.json: -------------------------------------------------------------------------------- 1 | { 2 | "id": "0123456789abc", 3 | "driveType": "personal", 4 | "owner": { 5 | "user": { 6 | "id": "12391913bac", 7 | "displayName": "Ryan Gregg" 8 | } 9 | }, 10 | "quota": { 11 | "total": 1024000, 12 | "used": 514000, 13 | "remaining": 1010112, 14 | "deleted": 0, 15 | "state": "normal" 16 | } 17 | } -------------------------------------------------------------------------------- /tests/data/folder_child_item.json: -------------------------------------------------------------------------------- 1 | { 2 | "createdBy": { 3 | "user": { 4 | "id": "xybu_id", 5 | "displayName": "Xiangyu Bu" 6 | }, 7 | "application": { 8 | "id": "4010c916", 9 | "displayName": "onedrive-d" 10 | } 11 | }, 12 | "id": "xybu_id!339", 13 | "fileSystemInfo": { 14 | "createdDateTime": "2015-02-23T19:02:31.433Z", 15 | "lastModifiedDateTime": "2015-02-23T19:02:31.433Z" 16 | }, 17 | "eTag": "aNTNCRUFBRjA1NjgyNjg4MiEzMzkuMA", 18 | "cTag": "aYzo1M0JFQUFGMDU2ODI2ODgyITMzOS4yNTY", 19 | "createdDateTime": "2015-02-23T19:02:31.433Z", 20 | "lastModifiedBy": { 21 | "user": { 22 | "id": "xybu_id", 23 | "displayName": "Xiangyu Bu" 24 | }, 25 | "application": { 26 | "id": "4010c916", 27 | "displayName": "onedrive-d" 28 | } 29 | }, 30 | "parentReference": { 31 | "driveId": "xybu_id", 32 | "id": "xybu_id!105", 33 | "path": "/drive/root:/Public" 34 | }, 35 | "webUrl": "https://onedrive.live.com/redir?resid=xybu_id!339", 36 | "@content.downloadUrl": "https://what/no", 37 | "size": 7632, 38 | "file": { 39 | "hashes": { 40 | "sha1Hash": "84DCA3C4C2A464F21963F4EBAEFBD0042094D286", 41 | "crc32Hash": "8A7A50C4" 42 | }, 43 | "mimeType": "text/plain" 44 | }, 45 | "name": "LICENSE", 46 | "lastModifiedDateTime": "2015-02-23T19:02:31.433Z" 47 | } -------------------------------------------------------------------------------- /tests/data/folder_item.json: -------------------------------------------------------------------------------- 1 | { 2 | "specialFolder": { 3 | "name": "public" 4 | }, 5 | "lastModifiedBy": { 6 | "user": { 7 | "displayName": "Xiangyu Bu", 8 | "id": "xybu_id" 9 | } 10 | }, 11 | "folder": { 12 | "childCount": 2 13 | }, 14 | "id": "xybu_id!105", 15 | "parentReference": { 16 | "id": "xybu_id!103", 17 | "driveId": "xybu_id", 18 | "path": "/drive/root:" 19 | }, 20 | "size": 7632, 21 | "cTag": "adDo1M0JFQUFGMDU2ODI2ODgyITEwNS42MzU2MTIwODg0MjMwMDAwMDA", 22 | "createdBy": { 23 | "user": { 24 | "displayName": "Xiangyu Bu", 25 | "id": "xybu_id" 26 | } 27 | }, 28 | "name": "Public", 29 | "lastModifiedDateTime": "2015-03-06T03:20:42.3Z", 30 | "createdDateTime": "2013-11-25T03:54:33.86Z", 31 | "eTag": "aNTNCRUFBRjA1NjgyNjg4MiExMDUuMQ", 32 | "webUrl": "https://onedrive.live.com/redir?resid=xybu_id!105", 33 | "fileSystemInfo": { 34 | "lastModifiedDateTime": "2013-11-25T03:54:33.86Z", 35 | "createdDateTime": "2013-11-25T03:54:33.86Z" 36 | } 37 | } -------------------------------------------------------------------------------- /tests/data/ignore_list.txt: -------------------------------------------------------------------------------- 1 | # A sample ignore list. 2 | 3 | # Paths relative to root 4 | # 'foo' can be either a file or a dir under repository root 5 | /foo 6 | # 'bar' must be a dir under root repository 7 | /bar/ 8 | 9 | # General rules 10 | # All files or dirs ending with ".swp" should be ignored. 11 | *.swp 12 | .ignore 13 | # All directories called "BUILD" (case-insensitive) should be ignored. 14 | BUILD/ 15 | 16 | # Path-specific rules are all relative to repository root 17 | path/to/ignore/file.txt 18 | 19 | # Negation rules to specify what MUST be included 20 | path-ignored/** 21 | !path-ignored/content 22 | 23 | # The following should not be treated as a comment 24 | # As a note, zgitignore does not support pattern '\#*#'. 25 | # The following is a workaround. 26 | [#]*[#] 27 | 28 | /Documents/**/resume.txt 29 | -------------------------------------------------------------------------------- /tests/data/image_item.json: -------------------------------------------------------------------------------- 1 | { 2 | "@content.downloadUrl": "http://public-sn3302.files.1drv.com/y2pcT7OaUEExF7EHOlpTjCE55mIUoiX7H3sx1ff6I-nP35XUTBqZlnkh9FJhWb_pf9sZ7LEpEchvDznIbQig0hWBeidpwFkOqSKCwQylisarN6T0ecAeMvantizBUzM2PA1", 3 | "createdDateTime": "2014-10-31T03:37:04.72Z", 4 | "lastModifiedDateTime": "2014-05-06T07:08:09.33Z", 5 | "cTag": "aYzpENDY0OEYwNkM5MUQ5RDNEITU0OTI3LjI1Ng", 6 | "eTag": "aRDQ2NDhGMDZDOTFEOUQzRCE1NDkyNy4w", 7 | "id": "D4648F06C91D9D3D!54927", 8 | "createdBy": { 9 | "user": { 10 | "displayName": "xybu", 11 | "id": "abc123" 12 | } 13 | }, 14 | "lastModifiedBy": { 15 | "user": { 16 | "displayName": "daron spektor", 17 | "id": "d4648f06c91d9d3d" 18 | } 19 | }, 20 | "name": "BritishShorthair.jpg", 21 | "description": "foobar!", 22 | "size": 45, 23 | "image": { 24 | "height": 398, 25 | "width": 273 26 | }, 27 | "parentReference": { 28 | "id": "53BEAAF056826882!103", 29 | "driveId": "53beaaf056826882", 30 | "path": "/drive/root:" 31 | }, 32 | "file": { 33 | "hashes": { 34 | "crc32Hash": "omY5NA==", 35 | "sha1Hash": "wmgPQ6jrSeMX7JP1XmstQEGM2fc=" 36 | }, 37 | "mimeType": "image/jpeg" 38 | }, 39 | "webUrl": "http://foo/bar/baz" 40 | } -------------------------------------------------------------------------------- /tests/data/list_drives.json: -------------------------------------------------------------------------------- 1 | { 2 | "value":[ 3 | { 4 | "quota":{ 5 | "remaining":29772095467, 6 | "used":5661384725, 7 | "total":35433480192, 8 | "state":"normal", 9 | "deleted":716108357, 10 | "storagePlans":{ 11 | "upgradeAvailable":true 12 | } 13 | }, 14 | "status":{ 15 | "state":"active" 16 | }, 17 | "owner":{ 18 | "user":{ 19 | "displayName":"Xiangyu Bu", 20 | "id":"d781730a6d2c6611" 21 | } 22 | }, 23 | "id":"xb_drive_id", 24 | "driveType":"personal" 25 | }, 26 | { 27 | "id":"xybu_id", 28 | "driveType":"personal", 29 | "owner":{ 30 | "user":{ 31 | "id":"713d61", 32 | "displayName":"Ryan Gregg" 33 | } 34 | }, 35 | "status":{ 36 | "state":"active" 37 | }, 38 | "quota":{ 39 | "total":1024000, 40 | "used":514000, 41 | "remaining":1010112, 42 | "deleted":0, 43 | "state":"normal" 44 | } 45 | }, 46 | { 47 | "id":"0123456789abc", 48 | "owner":{ 49 | "user":{ 50 | "id":"713d61", 51 | "displayName":"Ryan Gregg" 52 | } 53 | }, 54 | "quota":{ 55 | "total":10240000, 56 | "used":514133, 57 | "remaining":1010112, 58 | "deleted":0, 59 | "state":"normal" 60 | }, 61 | "status":{ 62 | "state":"active" 63 | }, 64 | "driveType":"personal" 65 | } 66 | ] 67 | } -------------------------------------------------------------------------------- /tests/data/me_profile_business_response.json: -------------------------------------------------------------------------------- 1 | { 2 | "@odata.context": "https://testtenant-my.sharepoint.com/_api/v1.0/$metadata#files/$entity", 3 | "@odata.editLink": "me/files/01EHFUASDHKL", 4 | "@odata.id": "https://testtenant-my.sharepoint.com/_api/v1.0/me/files/01EHFUASDHKL", 5 | "@odata.type": "#Microsoft.FileServices.Folder", 6 | "account_type": 1, 7 | "childCount": 4, 8 | "createdBy": null, 9 | "dateTimeCreated": "2015-09-13T22:12:52Z", 10 | "dateTimeLastModified": "2018-03-15T15:38:46Z", 11 | "eTag": null, 12 | "emails": "first.last@tenant.com", 13 | "first_name": "first", 14 | "id": "01EHFUASDHKL", 15 | "lastModifiedBy": null, 16 | "last_name": "last", 17 | "name": "first midle last", 18 | "parentReference": null, 19 | "refresh_token": "eyJ0eXAiOiJKV1QiLCJub25...", 20 | "size": 0, 21 | "type": "Folder", 22 | "webUrl": "https://testtenant-my.sharepoint.com/personal/first_last_tenant_com/Documents" 23 | } -------------------------------------------------------------------------------- /tests/data/me_profile_response.json: -------------------------------------------------------------------------------- 1 | { 2 | "first_name": "Xiangyu", 3 | "last_name": "Bu", 4 | "id": "713d61", 5 | "gender": null, 6 | "link": "https://profile.live.com/", 7 | "emails": { 8 | "account": "foo@bar.com", 9 | "preferred": "bar@baz.com", 10 | "business": "xb@personal", 11 | "personal": "xb@business" 12 | }, 13 | "updated_time": "2016-12-19T20:13:12+0000", 14 | "name": "Xiangyu Bu", 15 | "locale": "zh_CN", 16 | "account_type": 0 17 | } -------------------------------------------------------------------------------- /tests/data/quota_response.json: -------------------------------------------------------------------------------- 1 | { 2 | "total": 1000, 3 | "used": 100, 4 | "remaining": 900, 5 | "deleted": 500, 6 | "state": "normal" 7 | } 8 | -------------------------------------------------------------------------------- /tests/data/sample_config_schema.json: -------------------------------------------------------------------------------- 1 | { 2 | "webhook_port": { 3 | "type": "integer", 4 | "minimum": 1, 5 | "maximum": 65536, 6 | "@default_value": 8080, 7 | "description": "@lang['config.webhook_port.desc']" 8 | }, 9 | "logfile_path": { 10 | "type": "string", 11 | "subtype": "file", 12 | "to_abspath": true, 13 | "create_if_missing": true, 14 | "allow_empty": true, 15 | "permissions": "w", 16 | "@default_value": "/var/log/onedrive_client.log", 17 | "description": "@lang['config.logfile_path.desc']" 18 | }, 19 | "webhook_type": { 20 | "type": "string", 21 | "choices": ["direct", "ngrok"], 22 | "@default_value": "direct", 23 | "description": "@lang['config.webhook_type.desc']" 24 | }, 25 | "webhook_host": { 26 | "type": "string", 27 | "@default_value": "haha", 28 | "description": "@lang['config.webhook_host.desc']" 29 | }, 30 | "https_url": { 31 | "type": "string", 32 | "allow_empty": true, 33 | "@default_value": "https://foo/bar", 34 | "starts_with": "https://" 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /tests/data/session_response.json: -------------------------------------------------------------------------------- 1 | { 2 | "token_type":"bearer", 3 | "expires_in": 3600, 4 | "scope": "wl.basic onedrive.readwrite wl.offline_access", 5 | "access_token":"EwCo...AA==", 6 | "refresh_token":"eyJh...9323" 7 | } -------------------------------------------------------------------------------- /tests/data/subfolder_item.json: -------------------------------------------------------------------------------- 1 | { 2 | "createdDateTime": "2015-08-17T14:29:20.86Z", 3 | "lastModifiedBy": { 4 | "application": { 5 | "id": "4010c916", 6 | "displayName": "onedrive-d" 7 | }, 8 | "user": { 9 | "id": "xybu_id", 10 | "displayName": "Xiangyu Bu" 11 | } 12 | }, 13 | "parentReference": { 14 | "driveId": "xybu_id", 15 | "id": "xybu_id!103", 16 | "path": "/drive/root:" 17 | }, 18 | "createdBy": { 19 | "application": { 20 | "id": "4010c916", 21 | "displayName": "onedrive-d" 22 | }, 23 | "user": { 24 | "id": "xybu_id", 25 | "displayName": "Xiangyu Bu" 26 | } 27 | }, 28 | "fileSystemInfo": { 29 | "createdDateTime": "2015-08-17T14:29:20.86Z", 30 | "lastModifiedDateTime": "2015-08-17T14:29:20.86Z" 31 | }, 32 | "id": "xybu_id!376", 33 | "@odata.context": "https://api.onedrive.com/v1.0/$metadata#drives('me')/items('root')/children/$entity", 34 | "cTag": "adDo1M0JFQUFGMDU2ODI2ODgyITM3Ni42MzU3NTQxODU2MDg2MDAwMDA", 35 | "folder": { 36 | "childCount": 0 37 | }, 38 | "size": 0, 39 | "eTag": "aNTNCRUFBRjA1NjgyNjg4MiEzNzYuMA", 40 | "name": "foo 2", 41 | "lastModifiedDateTime": "2015-08-17T14:29:20.86Z", 42 | "webUrl": "https://onedrive.live.com/redir?resid=xybu_id!376" 43 | } -------------------------------------------------------------------------------- /tests/data/subscription_response.json: -------------------------------------------------------------------------------- 1 | { 2 | "clientState": "optional string", 3 | "createdBy": { "@odata.type": "oneDrive.identitySet" }, 4 | "expirationDateTime": "2015-02-23T19:02:31.433Z", 5 | "id": "string", 6 | "notificationUrl": "url", 7 | "resource": "relativePath" 8 | } -------------------------------------------------------------------------------- /tests/data/webhook_notification.json: -------------------------------------------------------------------------------- 1 | { 2 | "context": "string optional", 3 | "expirationDateTime": "2015-02-23T19:02:31.433Z", 4 | "resource": "r", 5 | "subscriptionId": "s", 6 | "tenantId": "t", 7 | "userId": "u" 8 | } -------------------------------------------------------------------------------- /tests/test_api_helper.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | try: 3 | from unittest import mock 4 | except ImportError: 5 | import mock 6 | 7 | import arrow 8 | import requests 9 | from onedrivesdk import Item, FileSystemInfo, error 10 | 11 | from onedrive_client import od_api_helper 12 | 13 | 14 | class TestApiHelper(unittest.TestCase): 15 | 16 | SAMPLE_ARROW_OBJ = arrow.utcnow() 17 | 18 | def dummy_api_call(self, excep): 19 | if self.count == 0: 20 | self.count = 1 21 | raise excep 22 | elif self.count > 1: 23 | raise ValueError('Dummy counter exceeds expected value 1.') 24 | 25 | def setUp(self): 26 | self.count = 0 27 | self.item = Item() 28 | 29 | def test_get_item_modified_datetime_modifiable(self): 30 | fs = FileSystemInfo() 31 | fs.last_modified_date_time = self.SAMPLE_ARROW_OBJ.datetime 32 | self.item.file_system_info = fs 33 | t, w = od_api_helper.get_item_modified_datetime(self.item) 34 | self.assertTrue(w, 'fileSystemInfo.lastModifiedDateTime should be modifiable.') 35 | self.assertEqual(self.SAMPLE_ARROW_OBJ, t) 36 | 37 | def test_get_item_modified_datetime_unmodifiable(self): 38 | self.item.last_modified_date_time = self.SAMPLE_ARROW_OBJ.datetime 39 | t, w = od_api_helper.get_item_modified_datetime(self.item) 40 | self.assertFalse(w, 'lastModifiedDateTime should be immutable.') 41 | self.assertEqual(self.SAMPLE_ARROW_OBJ, t) 42 | 43 | def test_get_item_created_datetime(self): 44 | self.item.created_date_time = self.SAMPLE_ARROW_OBJ.datetime 45 | self.assertEqual(self.SAMPLE_ARROW_OBJ, od_api_helper.get_item_created_datetime(self.item)) 46 | 47 | @mock.patch('time.sleep') 48 | def test_item_request_call_on_connection_error(self, mock_sleep): 49 | od_api_helper.item_request_call(None, self.dummy_api_call, requests.ConnectionError()) 50 | mock_sleep.assert_called_once_with(od_api_helper.THROTTLE_PAUSE_SEC) 51 | 52 | def test_item_request_call_on_unauthorized_error(self): 53 | account_id = 'dummy_acct' 54 | mock_repo = mock.MagicMock(account_id=account_id, 55 | **{'authenticator.refresh_session.return_value': 0, 'other.side_effect': KeyError}) 56 | od_api_helper.item_request_call( 57 | mock_repo, self.dummy_api_call, 58 | error.OneDriveError(prop_dict={'code': error.ErrorCode.Unauthenticated, 'message': 'dummy'}, 59 | status_code=requests.codes.unauthorized)) 60 | self.assertEqual(1, len(mock_repo.mock_calls)) 61 | name, args, _ = mock_repo.method_calls[0] 62 | self.assertEqual('authenticator.refresh_session', name) 63 | self.assertEqual((account_id,), args) 64 | 65 | 66 | if __name__ == '__main__': 67 | unittest.main() 68 | -------------------------------------------------------------------------------- /tests/test_api_session.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | 3 | import keyring 4 | 5 | from onedrive_client import od_api_session, od_auth 6 | 7 | 8 | class OneDriveAPISession(unittest.TestCase): 9 | 10 | def setUp(self): 11 | self.session = od_api_session.OneDriveAPISession( 12 | token_type='code', expires_in=100, scope_string=' '.join(od_auth.OneDriveAuthenticator.APP_SCOPES), 13 | access_token='abc', client_id='client_id', auth_server_url='https://foo/bar', redirect_uri='https://baz', 14 | refresh_token='hehe', client_secret='no_secret') 15 | 16 | def test_expires_in(self): 17 | self.assertLess(self.session.expires_in_sec, 100) 18 | 19 | def test_save_and_load(self): 20 | keydict = {self.session.SESSION_ARG_KEYNAME: 'mock_key'} 21 | self.session.save_session(**keydict) 22 | session = od_api_session.OneDriveAPISession.load_session(**keydict) 23 | self.assertEqual(self.session.token_type, session.token_type) 24 | self.assertEqual(self.session.scope, session.scope) 25 | self.assertEqual(self.session.access_token, session.access_token) 26 | self.assertEqual(self.session.client_id, session.client_id) 27 | self.assertEqual(self.session.client_secret, session.client_secret) 28 | self.assertEqual(self.session.refresh_token, session.refresh_token) 29 | keyring.delete_password(od_api_session.OneDriveAPISession.KEYRING_SERVICE_NAME, 'mock_key') 30 | -------------------------------------------------------------------------------- /tests/test_auth.py: -------------------------------------------------------------------------------- 1 | import json 2 | import os 3 | import unittest 4 | 5 | from onedrivesdk.helpers.http_provider_with_proxy import HttpProviderWithProxy 6 | 7 | from onedrive_client import get_resource, od_auth, od_api_session 8 | 9 | 10 | def get_sample_authenticator(): 11 | auth = od_auth.OneDriveAuthenticator() 12 | session_params = json.loads(get_resource('data/session_response.json', pkg_name='tests')) 13 | session_params['token_type'] = 'code' 14 | session_params['client_id'] = auth.APP_CLIENT_ID 15 | session_params['scope_string'] = session_params['scope'] 16 | session_params['redirect_uri'] = auth.APP_REDIRECT_URL 17 | session_params['auth_server_url'] = 'https://localhost/auth' 18 | del session_params['scope'] 19 | auth.client.auth_provider._session = od_api_session.OneDriveAPISession(**session_params) 20 | auth.refresh_session = lambda x: None 21 | return auth 22 | 23 | 24 | class TestOneDriveAuthenticator(unittest.TestCase): 25 | 26 | def test_get_auth_url(self): 27 | authenticator = od_auth.OneDriveAuthenticator() 28 | self.assertIsInstance(authenticator.get_auth_url(), str) 29 | 30 | def test_get_proxies(self): 31 | expected = 'http://foo/bar' 32 | for k in ('http_proxy', 'HTTP_PROXY', 'https_proxy', 'HTTPS_PROXY'): 33 | os.environ[k] = expected 34 | authenticator = od_auth.OneDriveAuthenticator() 35 | self.assertIsInstance( 36 | authenticator.client.http_provider, HttpProviderWithProxy) 37 | key = k.split('_')[0].lower() 38 | self.assertEqual( 39 | authenticator.client.http_provider.proxies[key], expected) 40 | del os.environ[k] 41 | 42 | 43 | if __name__ == '__main__': 44 | unittest.main() 45 | -------------------------------------------------------------------------------- /tests/test_context.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | import unittest 3 | 4 | from onedrive_client import od_context 5 | 6 | 7 | def get_sample_context(): 8 | return od_context.UserContext(loop=asyncio.get_event_loop()) 9 | 10 | 11 | class TestUserContext(unittest.TestCase): 12 | 13 | def setUp(self): 14 | self.ctx = get_sample_context() 15 | 16 | def test_get_login_username(self): 17 | self.assertIsInstance(od_context.get_login_username(), str) 18 | 19 | 20 | if __name__ == '__main__': 21 | unittest.main() 22 | -------------------------------------------------------------------------------- /tests/test_dateutils.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | from datetime import timezone, timedelta 3 | 4 | import arrow 5 | 6 | from onedrive_client import od_dateutils 7 | 8 | 9 | class TestDateUtils(unittest.TestCase): 10 | 11 | DT_OFFSET = timedelta(hours=1) 12 | DT_STR = '2015-08-06T18:45:20.260000Z' 13 | DT_TS = 1438886720.26 14 | DT_OBJ = arrow.Arrow(year=2015, month=8, day=6, hour=18, minute=45, second=20, microsecond=260000) 15 | DT_UTC_OBJ = DT_OBJ.replace(tzinfo=timezone.utc) 16 | DT_NONUTC_OBJ = DT_UTC_OBJ.replace(tzinfo=timezone(offset=DT_OFFSET)) 17 | 18 | def test_str_to_datetime(self): 19 | self.assertEqual(self.DT_UTC_OBJ, od_dateutils.str_to_datetime(self.DT_STR)) 20 | 21 | def test_datetime_to_str(self): 22 | self.assertEqual(self.DT_STR, od_dateutils.datetime_to_str(self.DT_UTC_OBJ)) 23 | 24 | def test_datetime_to_timestamp_explicit_utc(self): 25 | ts = od_dateutils.datetime_to_timestamp(self.DT_UTC_OBJ) 26 | self.assertEqual(self.DT_UTC_OBJ.float_timestamp, ts) 27 | self.assertEqual(self.DT_TS, ts) 28 | 29 | def test_datetime_to_timestamp_implicit_utc(self): 30 | """ 31 | onedrivesdk-python returns datetime objects that do not have tzinfo but onedrive_client wants it to be explicit. 32 | This test case check if the machine and program can handle implicit UTC. 33 | """ 34 | self.assertEqual(self.DT_UTC_OBJ.float_timestamp, od_dateutils.datetime_to_timestamp(self.DT_OBJ)) 35 | 36 | def test_datetime_to_timestamp_non_utc(self): 37 | ts = od_dateutils.datetime_to_timestamp(self.DT_NONUTC_OBJ) 38 | self.assertEqual(self.DT_UTC_OBJ.float_timestamp - self.DT_OFFSET.total_seconds(), ts) 39 | self.assertEqual(self.DT_NONUTC_OBJ.float_timestamp, ts) 40 | 41 | def test_diff_timestamps(self): 42 | self.assertTrue(od_dateutils.diff_timestamps(1, 2) < 0) 43 | self.assertTrue(od_dateutils.diff_timestamps(2, 1) > 0) 44 | self.assertTrue(od_dateutils.diff_timestamps(1.005, 1.007) == 0) 45 | 46 | 47 | if __name__ == '__main__': 48 | unittest.main() 49 | -------------------------------------------------------------------------------- /tests/test_dict_guard.py: -------------------------------------------------------------------------------- 1 | import json 2 | import os 3 | import tempfile 4 | import unittest 5 | 6 | from onedrive_client import get_resource 7 | from onedrive_client.od_models.dict_guard import GuardedDict, DictEntryTypes, SchemaValidator, exceptions 8 | 9 | 10 | class TestDictGuard(unittest.TestCase): 11 | 12 | def setUp(self): 13 | self.schema = json.loads(get_resource('data/sample_config_schema.json', pkg_name='tests')) 14 | self.config_dict = {k: v['@default_value'] for k, v in self.schema.items()} 15 | self.config_guard = GuardedDict(self.config_dict, self.schema) 16 | 17 | def _test_update_value(self, k, v): 18 | old_value = self.config_dict[k] 19 | self.assertNotEqual(old_value, v) 20 | self.config_guard[k] = v 21 | if self.schema[k]['type'] == DictEntryTypes.STR: 22 | v = str(v) 23 | self.assertEqual(v, self.config_dict[k]) 24 | 25 | def test_set_arbitrary_key(self): 26 | ex_raised = False 27 | try: 28 | self.config_guard['whatever123'] = 4 29 | except exceptions.DictGuardKeyError as e: 30 | self.assertEqual('whatever123', e.key) 31 | ex_raised = True 32 | self.assertTrue(ex_raised) 33 | 34 | def test_set_int(self): 35 | self._test_update_value('webhook_port', self.schema['webhook_port']['minimum']) 36 | 37 | def test_set_str(self): 38 | self._test_update_value('webhook_host', 'blah') 39 | self._test_update_value('webhook_host', 123123123) 40 | self._test_update_value('webhook_type', 'ngrok') 41 | self._test_update_value('https_url', 'https://www.facebook.com') 42 | 43 | def test_set_str_to_int(self): 44 | ex_raised = False 45 | try: 46 | self.config_guard['webhook_port'] = 'bar' 47 | except exceptions.IntValueRequired as e: 48 | self.assertEqual('webhook_port', e.key) 49 | ex_raised = True 50 | self.assertTrue(ex_raised) 51 | self.assertEqual(self.schema['webhook_port']['@default_value'], self.config_dict['webhook_port']) 52 | 53 | def _test_set_int_boundary(self, k, v, attr, ex_type): 54 | ex_raised = False 55 | try: 56 | self.config_guard[k] = v 57 | except ex_type as e: 58 | self.assertEqual(k, e.key) 59 | self.assertEqual(v, e.value) 60 | self.assertEqual(self.schema[k][attr], getattr(e, attr)) 61 | ex_raised = True 62 | self.assertTrue(ex_raised) 63 | self.assertEqual(self.schema[k]['@default_value'], self.config_dict[k]) 64 | 65 | def test_set_int_below_min(self): 66 | self._test_set_int_boundary( 67 | 'webhook_port', self.schema['webhook_port']['minimum'] - 1, 'minimum', exceptions.IntValueBelowMinimum) 68 | 69 | def test_set_int_above_max(self): 70 | self._test_set_int_boundary( 71 | 'webhook_port', self.schema['webhook_port']['maximum'] + 1, 'maximum', exceptions.IntValueAboveMaximum) 72 | 73 | def test_set_str_out_of_choice(self): 74 | ex_raised = False 75 | choices = self.schema['webhook_type']['choices'] 76 | self.assertNotIn('uuu', choices) 77 | try: 78 | self.config_guard['webhook_type'] = 'uuu' 79 | except exceptions.StringInvalidChoice as e: 80 | self.assertEqual('webhook_type', e.key) 81 | self.assertEqual('uuu', e.value) 82 | self.assertEqual(self.schema['webhook_type']['choices'], e.choices_allowed) 83 | ex_raised = True 84 | self.assertTrue(ex_raised) 85 | self.assertEqual(self.schema['webhook_type']['@default_value'], self.config_dict['webhook_type']) 86 | 87 | def test_set_str_empty(self): 88 | self._test_update_value('logfile_path', '') 89 | 90 | def test_set_str_create_file_if_missing(self): 91 | with tempfile.TemporaryDirectory() as td: 92 | path = td + '/' + 'test' 93 | self.config_guard['logfile_path'] = path 94 | self.assertTrue(os.path.isfile(path)) 95 | self.assertEqual(self.config_dict['logfile_path'], path) 96 | 97 | def _test_set_str_file_with_val(self, val, excep): 98 | ex_raised = False 99 | try: 100 | self.config_guard['logfile_path'] = val 101 | except excep as e: 102 | self.assertEqual('logfile_path', e.key) 103 | self.assertEqual(val, e.value) 104 | ex_raised = True 105 | self.assertTrue(ex_raised) 106 | self.assertEqual(self.schema['logfile_path']['@default_value'], self.config_dict['logfile_path']) 107 | 108 | def test_set_str_with_non_file_path(self): 109 | self._test_set_str_file_with_val('/', exceptions.PathIsNotFile) 110 | del self.schema['logfile_path']['create_if_missing'] 111 | self._test_set_str_file_with_val('/foo/bar/baz', exceptions.PathDoesNotExist) 112 | 113 | def test_set_str_with_permission_denied(self): 114 | ex_raised = False 115 | self.schema['logfile_path']['permission'] = 'w' 116 | try: 117 | self.config_guard['logfile_path'] = '/proc/1/stat' 118 | except OSError: 119 | ex_raised = True 120 | self.assertTrue(ex_raised) 121 | self.assertEqual(self.schema['logfile_path']['@default_value'], self.config_dict['logfile_path']) 122 | 123 | def test_str_with_permission_allowed(self): 124 | self.schema['logfile_path']['permission'] = 'r' 125 | self.config_guard['logfile_path'] = '/proc/1/stat' 126 | self.assertEqual('/proc/1/stat', self.config_dict['logfile_path']) 127 | 128 | def test_str_not_starts_with(self): 129 | ex_raised = False 130 | try: 131 | self.config_guard['https_url'] = 'http://www.facebook.com' 132 | except exceptions.StringNotStartsWith as e: 133 | self.assertEqual('https_url', e.key) 134 | self.assertEqual('http://www.facebook.com', e.value) 135 | ex_raised = True 136 | self.assertTrue(ex_raised) 137 | self.assertEqual(self.schema['https_url']['@default_value'], self.config_dict['https_url']) 138 | self.config_guard['https_url'] = '' 139 | self.assertEqual('', self.config_dict['https_url']) 140 | 141 | 142 | class TestConfigSchema(unittest.TestCase): 143 | 144 | def test_config_schema(self): 145 | curr_config_schema = json.loads(get_resource('data/config_schema.json', pkg_name='onedrive_client')) 146 | SchemaValidator(curr_config_schema).validate() 147 | 148 | 149 | if __name__ == '__main__': 150 | unittest.main() 151 | -------------------------------------------------------------------------------- /tests/test_hashutils.py: -------------------------------------------------------------------------------- 1 | import tempfile 2 | import unittest 3 | 4 | from onedrivesdk.model.item import Item 5 | from onedrive_client import od_hashutils 6 | 7 | 8 | class TestHashUtils(unittest.TestCase): 9 | 10 | TEST_CASES = [ 11 | (b'Hello world!\n', '47a013e660d408619d894b20806b1d5086aab03b'.upper()), 12 | (b'Purdue University', 'a68b0321ef0a3ec9e7ffa2de22d70f79ac3e4dda'.upper()) 13 | ] 14 | 15 | def setUp(self): 16 | self.TEST_FILES = [] 17 | for t in self.TEST_CASES: 18 | tmpfile = tempfile.NamedTemporaryFile() 19 | tmpfile.write(t[0]) 20 | tmpfile.flush() 21 | tmpfile.seek(0) 22 | self.TEST_FILES.append(tmpfile) 23 | 24 | def tearDown(self): 25 | for f in self.TEST_FILES: 26 | f.close() 27 | 28 | def test_hash(self): 29 | for i, (data, sha1) in enumerate(self.TEST_CASES): 30 | tmpname = self.TEST_FILES[i].name 31 | sha1_hash = od_hashutils.sha1_value(tmpname) 32 | self.assertEqual(sha1, sha1_hash) 33 | 34 | def _mock_item(self, sha1_hash=None): 35 | prop_dict = dict() 36 | if sha1_hash: 37 | prop_dict['sha1Hash'] = sha1_hash 38 | item = Item(prop_dict={'file': {'hashes': prop_dict}}) 39 | self.assertEqual(sha1_hash, item.file.hashes.sha1_hash) 40 | return item 41 | 42 | def test_hash_match(self): 43 | tmpname = self.TEST_FILES[0].name 44 | self.assertTrue( 45 | od_hashutils.hash_match(tmpname, self._mock_item(sha1_hash=self.TEST_CASES[0][1])), 46 | 'hash_match() should return True when only SHA1 hash is present and correct.') 47 | self.assertFalse(od_hashutils.hash_match(tmpname, self._mock_item(sha1_hash='BAR'))) 48 | self.assertFalse( 49 | od_hashutils.hash_match(tmpname, self._mock_item()), 50 | 'hash_match() should return False when SHA1 hash is missing.') 51 | 52 | 53 | if __name__ == '__main__': 54 | unittest.main() 55 | -------------------------------------------------------------------------------- /tests/test_main_cli.py: -------------------------------------------------------------------------------- 1 | import tempfile 2 | import unittest 3 | 4 | import click 5 | 6 | from onedrive_client import od_main 7 | 8 | 9 | class TestMainCLI(unittest.TestCase): 10 | 11 | def setUp(self): 12 | self.tempdir = tempfile.TemporaryDirectory() 13 | click.get_app_dir = lambda x: self.tempdir.name + '/' + x 14 | od_main.context = od_main.load_context() 15 | od_main.context._create_config_dir_if_missing() 16 | 17 | def tearDown(self): 18 | self.tempdir.cleanup() 19 | 20 | def test_init_and_shutdown_task_workers(self): 21 | od_main.init_task_pool_and_workers() 22 | od_main.shutdown_workers() 23 | 24 | 25 | if __name__ == '__main__': 26 | unittest.main() 27 | -------------------------------------------------------------------------------- /tests/test_models.py: -------------------------------------------------------------------------------- 1 | import json 2 | import os 3 | import sys 4 | import unittest 5 | 6 | import onedrivesdk 7 | 8 | from onedrive_client import get_resource, od_models, od_dateutils 9 | 10 | 11 | def get_sample_drive(): 12 | drive_response = json.loads(get_resource('data/drive_response.json', 'tests')) 13 | return onedrivesdk.Drive(drive_response) 14 | 15 | 16 | def get_sample_drive_config(): 17 | drive_dict = json.loads(get_resource('data/drive_config_item.json', 'tests')) 18 | drive_dict['ignorefile_path'] = os.path.join( 19 | os.path.dirname(sys.modules['tests'].__file__), 'data/ignore_list.txt') 20 | return drive_dict, od_models.drive_config.LocalDriveConfig(**drive_dict) 21 | 22 | 23 | class TestAccountPersonalProfile(unittest.TestCase): 24 | 25 | def setUp(self): 26 | self.data = json.loads(get_resource('data/me_profile_response.json', pkg_name='tests')) 27 | self.account = od_models.account_profile.OneDriveAccount(self.data).get_account() 28 | 29 | def test_properties(self): 30 | self.assertEqual(self.data['id'], self.account.account_id) 31 | self.assertEqual(self.data['name'], self.account.account_name) 32 | self.assertEqual(self.data['emails']['account'], self.account.account_email) 33 | self.assertEqual(self.data['first_name'], self.account.account_firstname) 34 | self.assertEqual(self.data['last_name'], self.account.account_lastname) 35 | self.assertEqual(od_models.account_profile.AccountTypes.PERSONAL, self.account.account_type) 36 | with self.assertRaises(AttributeError): 37 | self.account.get_account() 38 | 39 | def test_to_string(self): 40 | self.assertIsInstance(str(self.account), str) 41 | 42 | 43 | class TestAccountBusinessProfile(unittest.TestCase): 44 | 45 | def setUp(self): 46 | self.data = json.loads(get_resource('data/me_profile_business_response.json', pkg_name='tests')) 47 | self.account = od_models.account_profile.OneDriveAccount(self.data).get_account() 48 | 49 | def test_properties(self): 50 | self.assertEqual(self.data['id'], self.account.account_id) 51 | self.assertEqual(self.data['name'], self.account.account_name) 52 | self.assertEqual(self.data['emails'], self.account.account_email) 53 | self.assertEqual(self.data['first_name'], self.account.account_firstname) 54 | self.assertEqual(self.data['last_name'], self.account.account_lastname) 55 | self.assertEqual(od_models.account_profile.AccountTypes.BUSINESS, self.account.account_type) 56 | self.assertEqual(self.data['webUrl'], self.account.account_root_folder) 57 | self.assertIn(self.account.tenant, self.account.endpoint) 58 | self.assertIn(self.account.endpoint, self.account.account_root_folder) 59 | with self.assertRaises(AttributeError): 60 | self.account.get_account() 61 | 62 | def test_to_string(self): 63 | self.assertIsInstance(str(self.account), str) 64 | 65 | 66 | class TestPathFilter(unittest.TestCase): 67 | def setUp(self): 68 | self.rules = get_resource('data/ignore_list.txt', pkg_name='tests').splitlines() 69 | self.filter = od_models.path_filter.PathFilter(self.rules) 70 | 71 | def assert_cases(self, cases): 72 | """ 73 | Test a batch of cases. 74 | :param [(str, True | False, True | False)] cases: List of tuples (path, is_dir, answer). 75 | """ 76 | for c in cases: 77 | path, is_dir, answer = c 78 | self.assertEqual(answer, self.filter.should_ignore(path, is_dir), str(c) + ' failed.') 79 | 80 | def test_add_rules(self): 81 | r = '/i_am_new_rule' 82 | self.assertFalse(self.filter.should_ignore(r)) 83 | self.filter.add_rules([r]) 84 | self.assertTrue(self.filter.should_ignore(r)) 85 | 86 | def test_hardcoded_cases(self): 87 | cases = [ 88 | ('/.hehe', True, True), 89 | ('/' + self.filter.get_temp_name('hello.txt'), False, True), 90 | ('/he?he', False, True) 91 | ] 92 | self.assert_cases(cases) 93 | 94 | def test_ignore_in_root(self): 95 | # The following rules also test dir-only ignores. 96 | cases = [ 97 | ('/foo', True, True), 98 | ('/foo', False, True), 99 | ('/bar', True, True), 100 | ('/bar', False, False), 101 | ('/a/foo', False, False) 102 | ] 103 | self.assert_cases(cases) 104 | 105 | def test_ignore_general(self): 106 | cases = [ 107 | ('/.swp', False, True), 108 | ('/a.swp', False, True), 109 | ('/hello/world.swp', False, True), 110 | ('/.ignore', False, True), 111 | ('/baz/.ignore', True, True), 112 | ('/baz/dont.ignore', False, False), 113 | ('/build', True, True), # This rule tests case-insensitiveness 114 | ('/tmp/build', True, True) # because the rule is "BUILD/" 115 | ] 116 | self.assert_cases(cases) 117 | 118 | def test_ignore_path(self): 119 | cases = [ 120 | ('/path/to/ignore/file.txt', False, True), # If the rule specifies a path, it is 121 | ('/oops/path/to/ignore/file.txt', False, False), # relative to repository root. 122 | ('/path/to/dont_ignore/file.txt', False, False) 123 | ] 124 | self.assert_cases(cases) 125 | 126 | def test_negations(self): 127 | cases = [ 128 | ('/path-ignored/file', False, True), # Files under this dir should be ignored. 129 | ('/path-ignored/content', False, False) # This file is explicitly negated from ignore. 130 | ] 131 | self.assert_cases(cases) 132 | 133 | def test_special_patterns(self): 134 | self.assert_cases([ 135 | ('/#test#', False, True), 136 | ('/Documents/xb/old/resume.txt', False, True) # Test rule containing "**". 137 | ]) 138 | 139 | def test_auto_correction(self): 140 | cases = [ 141 | ('/bar/', False, True) # path indicates dir but is_dir says the contrary 142 | ] 143 | self.assert_cases(cases) 144 | 145 | 146 | class TestPrettyApi(unittest.TestCase): 147 | 148 | def test_pretty_print_bytes(self): 149 | self.assertEqual('0.000 B', od_models.pretty_api.pretty_print_bytes(size=0, precision=3)) 150 | self.assertEqual('1.00 KB', od_models.pretty_api.pretty_print_bytes(size=1025, precision=2)) 151 | self.assertEqual('1.0 MB', od_models.pretty_api.pretty_print_bytes(size=1048576, precision=1)) 152 | self.assertEqual('1.50 GB', od_models.pretty_api.pretty_print_bytes(size=1610612736, precision=2)) 153 | 154 | 155 | class TestDriveConfig(unittest.TestCase): 156 | 157 | def setUp(self): 158 | self.drive_dict, self.drive_config = get_sample_drive_config() 159 | 160 | def test_properties(self): 161 | self.assertEqual(self.drive_dict['account_id'], self.drive_config.account_id) 162 | self.assertEqual(self.drive_dict['drive_id'], self.drive_config.drive_id) 163 | self.assertEqual(self.drive_dict['ignorefile_path'], self.drive_config.ignorefile_path) 164 | self.assertEqual(self.drive_dict['localroot_path'], self.drive_config.localroot_path) 165 | 166 | 167 | class TestWebhookNotification(unittest.TestCase): 168 | 169 | def setUp(self): 170 | self.data = json.loads(get_resource('data/webhook_notification.json', 'tests')) 171 | 172 | def test_properties(self): 173 | notification = od_models.webhook_notification.WebhookNotification(self.data) 174 | self.assertEqual(self.data['context'], notification.context) 175 | self.assertEqual(self.data['resource'], notification.resource) 176 | self.assertEqual(self.data['subscriptionId'], notification.subscription_id) 177 | self.assertEqual(self.data['tenantId'], notification.tenant_id) 178 | self.assertEqual(self.data['userId'], notification.user_id) 179 | self.assertEqual( 180 | od_dateutils.str_to_datetime(self.data['expirationDateTime']), notification.expiration_datetime) 181 | 182 | def _assert_missing_property_none(self, data_key, property_key): 183 | del self.data[data_key] 184 | notification = od_models.webhook_notification.WebhookNotification(self.data) 185 | self.assertIsNone(getattr(notification, property_key)) 186 | 187 | def test_properties_missing_tenant_id(self): 188 | self._assert_missing_property_none('tenantId', 'tenant_id') 189 | 190 | def test_properties_missing_context(self): 191 | self._assert_missing_property_none('context', 'context') 192 | 193 | 194 | class TestBidict(unittest.TestCase): 195 | """ 196 | Bidict is used in od_watcher for fd -> (repo, file_path) mapping. 197 | """ 198 | def test_use(self): 199 | d = od_models.bidict.loosebidict() 200 | key, val = (123, 'abc') 201 | d[key] = val 202 | # Test ordinary operations. 203 | self.assertIn(key, d) 204 | self.assertEqual(val, d[key]) 205 | # Test inv operations. 206 | self.assertIn(val, d.inv) 207 | self.assertEqual(key, d.inv[val]) 208 | # Test inv removal. 209 | popped_key = d.inv.pop(val) 210 | self.assertNotIn(key, d) 211 | self.assertNotIn(val, d.inv) 212 | self.assertEqual(key, popped_key) 213 | 214 | 215 | if __name__ == '__main__': 216 | unittest.main() 217 | -------------------------------------------------------------------------------- /tests/test_pref_cli.py: -------------------------------------------------------------------------------- 1 | import json 2 | import os 3 | import re 4 | import tempfile 5 | import unittest 6 | 7 | import click 8 | import onedrivesdk 9 | import requests_mock 10 | 11 | from onedrive_client import get_resource, od_pref, od_repo 12 | 13 | 14 | class TestPrefCLI(unittest.TestCase): 15 | 16 | def setUp(self): 17 | self.tempdir = tempfile.TemporaryDirectory() 18 | click.get_app_dir = lambda x: self.tempdir.name + '/' + x 19 | od_pref.context = od_pref.load_context() 20 | od_pref.context._create_config_dir_if_missing() 21 | 22 | def tearDown(self): 23 | self.tempdir.cleanup() 24 | 25 | def test_config_set_valid(self): 26 | for arg in (('webhook_type', 'direct'), ('webhook_port', 0), ('logfile_path', self.tempdir.name + '/test.log')): 27 | try: 28 | od_pref.set_config(args=[str(v) for v in arg]) 29 | except SystemExit: 30 | pass 31 | context = od_pref.load_context() 32 | self.assertEqual(arg[1], context.config[arg[0]]) 33 | 34 | def test_config_set_invalid_str(self): 35 | for arg in (('webhook_type', 'whatever'), ('logfile_path', '/'), 36 | ('webhook_port', 70000), ('num_workers', 0)): 37 | try: 38 | od_pref.set_config(args=[str(v) for v in arg]) 39 | except SystemExit: 40 | pass 41 | context = od_pref.load_context() 42 | self.assertNotEqual(arg[1], context.config[arg[0]]) 43 | 44 | def test_config_set_key_typo(self): 45 | try: 46 | od_pref.set_config(args=['webhook_typo', 'whatever']) 47 | except SystemExit: 48 | pass 49 | context = od_pref.load_context() 50 | self.assertNotIn('webhook_typo', context.config) 51 | 52 | def test_config_print(self): 53 | try: 54 | od_pref.print_config(args=()) 55 | except SystemExit: 56 | pass 57 | 58 | def _call_authenticate_account(self, mock, code, args): 59 | profile = json.loads(get_resource('data/me_profile_response.json', pkg_name='tests')) 60 | 61 | def callback_auth(request, context): 62 | self.assertIn('code=' + code, request.text) 63 | context.status_code = 200 64 | return json.loads(get_resource('data/session_response.json', pkg_name='tests')) 65 | 66 | def callback_profile(request, context): 67 | context.status_code = 200 68 | return profile 69 | mock.post(re.compile('//login\.live\.com\.*'), json=callback_auth) 70 | mock.get('https://apis.live.net/v5.0/me', json=callback_profile) 71 | try: 72 | od_pref.authenticate_account(args=args) 73 | except SystemExit as e: 74 | if e.code == 0: 75 | pass 76 | context = od_pref.load_context() 77 | self.assertIsNotNone(context.get_account(profile['id'])) 78 | 79 | def test_authenticate_account_with_code(self): 80 | with requests_mock.Mocker() as mock: 81 | self._call_authenticate_account(mock=mock, code='foobar_code', args=('--code', 'foobar_code')) 82 | 83 | def test_authenticate_account_with_url(self): 84 | url = 'https://login.live.com/oauth20_desktop.srf?code=foobar_code&lc=1033' 85 | click.prompt = lambda x, type=str: url 86 | with requests_mock.Mocker() as mock: 87 | self._call_authenticate_account(mock=mock, code='foobar_code', args=()) 88 | 89 | def test_list_account(self): 90 | try: 91 | od_pref.list_accounts(args=()) 92 | except SystemExit as e: 93 | if e.code == 0: 94 | pass 95 | 96 | def test_quota_short_str(self): 97 | quota = onedrivesdk.Quota(json.loads(get_resource('data/quota_response.json', 'tests'))) 98 | self.assertIsInstance(od_pref.quota_short_str(quota), str) 99 | 100 | def _setup_list_drive_mock(self, mock): 101 | all_drives = json.loads(get_resource('data/list_drives.json', pkg_name='tests')) 102 | mock.register_uri('GET', 'https://api.onedrive.com/v1.0/drives', 103 | [{'status_code': 200, 'json': {'value': all_drives['value'][0:-1]}}, 104 | {'status_code': 200, 'json': {'value': all_drives['value'][-1:]}}]) 105 | mock.post('https://login.live.com/oauth20_token.srf', 106 | json=json.loads(get_resource('data/session_response.json', pkg_name='tests'))) 107 | 108 | @requests_mock.mock() 109 | def test_list_drives(self, mock): 110 | self._setup_list_drive_mock(mock) 111 | try: 112 | od_pref.list_drives(args=()) 113 | except SystemExit as e: 114 | if e.code == 0: 115 | pass 116 | 117 | @requests_mock.mock() 118 | def test_add_drive(self, mock): 119 | drive_id = 'xb_drive_id' 120 | self._setup_list_drive_mock(mock) 121 | tmp_local_repo = tempfile.TemporaryDirectory() 122 | try: 123 | od_pref.set_drive( 124 | args=('--drive-id', drive_id, '--email', 'xybu92@live.com', '--local-root', tmp_local_repo.name)) 125 | self.assertTrue(os.path.isfile(od_repo.get_drive_db_path(od_pref.context.config_dir, drive_id))) 126 | except SystemExit as e: 127 | if e.code == 0: 128 | pass 129 | 130 | @requests_mock.mock() 131 | def test_del_drive(self, mock): 132 | drive_id = 'xb_drive_id' 133 | self.test_add_drive() 134 | try: 135 | od_pref.delete_drive(args=('--drive-id', drive_id, '--yes')) 136 | self.assertFalse(os.path.exists(od_repo.get_drive_db_path(od_pref.context.config_dir, drive_id))) 137 | except SystemExit as e: 138 | if e.code == 0: 139 | pass 140 | 141 | 142 | if __name__ == '__main__': 143 | unittest.main() 144 | -------------------------------------------------------------------------------- /tests/test_repo.py: -------------------------------------------------------------------------------- 1 | import json 2 | import tempfile 3 | import unittest 4 | try: 5 | from unittest import mock 6 | except ImportError: 7 | import mock 8 | 9 | import onedrivesdk 10 | 11 | from onedrive_client import od_context, od_repo, od_api_helper, get_resource 12 | from tests.test_auth import get_sample_authenticator 13 | from tests.test_models import get_sample_drive, get_sample_drive_config 14 | 15 | 16 | def get_sample_repo(): 17 | temp_config_dir = tempfile.TemporaryDirectory() 18 | temp_repo_dir = tempfile.TemporaryDirectory() 19 | ctx = mock.MagicMock(spec=od_context.UserContext, 20 | config=od_context.UserContext.DEFAULT_CONFIG, 21 | config_dir=temp_config_dir.name, 22 | host_name='hostname', loop=None) 23 | auth = get_sample_authenticator() 24 | drive = get_sample_drive() 25 | drive_dict, drive_config = get_sample_drive_config() 26 | drive_dict['localroot_path'] = temp_repo_dir.name 27 | drive_config = drive_config._replace(localroot_path=temp_repo_dir.name) 28 | repo = od_repo.OneDriveLocalRepository(ctx, auth, drive, drive_config) 29 | return temp_config_dir, temp_repo_dir, drive_config, repo 30 | 31 | 32 | class TestOneDriveLocalRepository(unittest.TestCase): 33 | 34 | def setUp(self): 35 | self.temp_config_dir, self.temp_repo_dir, self.drive_config, self.repo = get_sample_repo() 36 | self.root_folder_item = onedrivesdk.Item(json.loads(get_resource('data/folder_item.json', pkg_name='tests'))) 37 | self.root_subfolder_item = onedrivesdk.Item(json.loads( 38 | get_resource('data/subfolder_item.json', pkg_name='tests'))) 39 | self.root_child_item = onedrivesdk.Item(json.loads( 40 | get_resource('data/folder_child_item.json', pkg_name='tests'))) 41 | self.image_item = onedrivesdk.Item(json.loads(get_resource('data/image_item.json', pkg_name='tests'))) 42 | self._add_all_items() 43 | 44 | def tearDown(self): 45 | self.temp_config_dir.cleanup() 46 | self.temp_repo_dir.cleanup() 47 | 48 | def _add_all_items(self): 49 | self.repo.update_item(self.root_folder_item, '', 0) 50 | self.repo.update_item(self.root_subfolder_item, '/' + self.root_folder_item.name, 0) 51 | self.repo.update_item(self.root_child_item, '/' + self.root_folder_item.name, self.root_child_item.size) 52 | self.repo.update_item(self.image_item, '', self.image_item.size) 53 | 54 | def _check_item_props(self, item, record, expected_type=od_repo.ItemRecordType.FILE): 55 | """ 56 | :param onedrivesdk.Item item: 57 | :param onedrive_client.od_repo.ItemRecord record: 58 | """ 59 | self.assertIsInstance(record, od_repo.ItemRecord) 60 | self.assertEqual(item.id, record.item_id) 61 | self.assertEqual(item.name, record.item_name) 62 | self.assertEqual(item.c_tag, record.c_tag) 63 | self.assertEqual(item.parent_reference.id, record.parent_id) 64 | self.assertEqual(od_api_helper.get_item_created_datetime(item), record.created_time) 65 | mtime, w = od_api_helper.get_item_modified_datetime(item) 66 | self.assertEqual(mtime, record.modified_time) 67 | self.assertEqual(expected_type, record.type) 68 | if expected_type == od_repo.ItemRecordType.FILE: 69 | self.assertEqual(item.size, record.size) 70 | self.assertEqual(item.file.hashes.sha1_hash, record.sha1_hash) 71 | 72 | def test_properties(self): 73 | self.assertEqual(self.drive_config.localroot_path, self.repo.local_root) 74 | self.assertEqual(self.drive_config.account_id, self.drive_config.account_id) 75 | 76 | def test_add_get_items(self): 77 | root_folder_item = self.repo.get_item_by_path(self.root_folder_item.name, '') 78 | self._check_item_props(self.root_folder_item, root_folder_item, od_repo.ItemRecordType.FOLDER) 79 | root_child_item = self.repo.get_item_by_path(self.root_child_item.name, '/' + self.root_folder_item.name) 80 | self._check_item_props(self.root_child_item, root_child_item, od_repo.ItemRecordType.FILE) 81 | 82 | def test_delete_folder(self): 83 | self.repo.delete_item(item_name=self.root_folder_item.name, parent_relpath='', is_folder=True) 84 | self.assertIsNone(self.repo.get_item_by_path(self.root_folder_item.name, '')) 85 | self.assertIsNone(self.repo.get_item_by_path(self.root_child_item.name, '/' + self.root_folder_item.name)) 86 | self.assertIsNone(self.repo.get_item_by_path(self.root_subfolder_item.name, '/' + self.root_folder_item.name)) 87 | self._check_item_props( 88 | self.image_item, self.repo.get_item_by_path(self.image_item.name, ''), od_repo.ItemRecordType.FILE) 89 | 90 | def test_delete_file(self): 91 | self.repo.delete_item(item_name=self.image_item.name, parent_relpath='', is_folder=False) 92 | self.assertIsNone(self.repo.get_item_by_path(self.image_item.name, '')) 93 | self.test_add_get_items() 94 | 95 | def test_move_item_down(self): 96 | self.repo.move_item(item_name=self.root_folder_item.name, parent_relpath='', 97 | new_name='Public2', new_parent_relpath='/Test', is_folder=True) 98 | self.assertIsNone(self.repo.get_item_by_path(self.root_folder_item.name, '')) 99 | self.assertIsNone(self.repo.get_item_by_path(self.root_child_item.name, '/' + self.root_folder_item.name)) 100 | self.assertIsNone(self.repo.get_item_by_path(self.root_subfolder_item.name, '/' + self.root_folder_item.name)) 101 | self.root_folder_item.name = 'Public2' 102 | root_folder_item = self.repo.get_item_by_path('Public2', '/Test') 103 | self._check_item_props(self.root_folder_item, root_folder_item, od_repo.ItemRecordType.FOLDER) 104 | root_child_item = self.repo.get_item_by_path(self.root_child_item.name, '/Test/Public2') 105 | self._check_item_props(self.root_child_item, root_child_item, od_repo.ItemRecordType.FILE) 106 | root_subfolder_item = self.repo.get_item_by_path(self.root_subfolder_item.name, '/Test/Public2') 107 | self._check_item_props(self.root_subfolder_item, root_subfolder_item, od_repo.ItemRecordType.FOLDER) 108 | self._check_item_props( 109 | self.image_item, self.repo.get_item_by_path(self.image_item.name, ''), od_repo.ItemRecordType.FILE) 110 | 111 | def test_move_item_up(self): 112 | self.repo.move_item(item_name=self.image_item.name, parent_relpath='', 113 | new_name=self.image_item.name, new_parent_relpath='/Public/foo 2', is_folder=False) 114 | self.repo.move_item(item_name=self.root_subfolder_item.name, parent_relpath='/' + self.root_folder_item.name, 115 | new_name=self.root_subfolder_item.name, new_parent_relpath='', is_folder=True) 116 | self.assertIsNone(self.repo.get_item_by_path(self.image_item.name, '')) 117 | self.assertIsNone(self.repo.get_item_by_path(self.root_subfolder_item.name, '/Public')) 118 | self._check_item_props( 119 | self.root_subfolder_item, self.repo.get_item_by_path(self.root_subfolder_item.name, ''), 120 | od_repo.ItemRecordType.FOLDER) 121 | self._check_item_props( 122 | self.image_item, self.repo.get_item_by_path(self.image_item.name, '/foo 2'), od_repo.ItemRecordType.FILE) 123 | 124 | def _check_immediate_children(self, relpath, expected_records): 125 | records = self.repo.get_immediate_children_of_dir(relpath) 126 | self.assertEqual(len(expected_records), len(records)) 127 | for r in expected_records: 128 | self.assertIn(r.name, records) 129 | expected_type = od_repo.ItemRecordType.FOLDER if r.folder else od_repo.ItemRecordType.FILE 130 | self._check_item_props(r, records[r.name], expected_type=expected_type) 131 | 132 | def test_get_immediate_children_of_root(self): 133 | self._check_immediate_children('', (self.image_item, self.root_folder_item)) 134 | 135 | def test_get_immediate_children(self): 136 | self._check_immediate_children('/' + self.root_folder_item.name, 137 | (self.root_child_item, self.root_subfolder_item)) 138 | 139 | 140 | if __name__ == '__main__': 141 | unittest.main() 142 | -------------------------------------------------------------------------------- /tests/test_stringutils.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | 3 | from onedrive_client import od_stringutils 4 | 5 | 6 | class TestStringUtils(unittest.TestCase): 7 | 8 | INCREMENTED_FILE_NAMES = (('Folder', 'Folder 1'), ('Folder 1', 'Folder 2'), 9 | ('file.txt', 'file 1.txt'), ('file 1.txt', 'file 2.txt'), 10 | ('Folder 0', 'Folder 0 1')) 11 | 12 | def test_get_filename_with_incremented_count(self): 13 | for orig, exp in self.INCREMENTED_FILE_NAMES: 14 | self.assertEqual(exp, od_stringutils.get_filename_with_incremented_count(orig)) 15 | 16 | 17 | if __name__ == '__main__': 18 | unittest.main() 19 | -------------------------------------------------------------------------------- /tests/test_task_pool.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | 3 | from onedrive_client import od_task 4 | from onedrive_client.od_tasks.base import TaskBase 5 | 6 | 7 | class TestTaskPool(unittest.TestCase): 8 | 9 | def _get_dummy_task(self, local_abspath=None): 10 | t = TaskBase(repo=None, task_pool=self.task_pool) 11 | t.local_abspath = local_abspath 12 | return t 13 | 14 | def setUp(self): 15 | self.task_pool = od_task.TaskPool() 16 | 17 | def test_add_pop(self): 18 | ts = [self._get_dummy_task(local_abspath='/1'), self._get_dummy_task(local_abspath='/2')] 19 | for i, t in enumerate(ts): 20 | self.assertEqual(i, self.task_pool.outstanding_task_count) 21 | self.assertTrue(self.task_pool.add_task(t)) 22 | self.assertEqual(2, self.task_pool.outstanding_task_count) 23 | self.assertFalse(self.task_pool.add_task(self._get_dummy_task(local_abspath='/1'))) 24 | for i, t in enumerate(ts): 25 | self.assertIs(t, self.task_pool.pop_task()) 26 | self.assertEqual(len(ts) - i - 1, self.task_pool.outstanding_task_count) 27 | self.assertEqual(0, self.task_pool.outstanding_task_count) 28 | 29 | def test_has_pending_task(self): 30 | task = self._get_dummy_task(local_abspath='/foo/bar') 31 | self.assertIs(self.task_pool.has_pending_task('/foo/bar'), False) 32 | self.task_pool.add_task(task) 33 | self.assertIs(self.task_pool.has_pending_task('/foo/bar'), task) 34 | for s in ('/foo/ba', '/foo/barz', '/foo/bar/baz', '/foo'): 35 | self.assertIs(self.task_pool.has_pending_task(s), False) 36 | 37 | def test_occupy_release_path(self): 38 | task = self._get_dummy_task(local_abspath='/foo/bar') 39 | self.assertIs(self.task_pool.has_pending_task('/foo/bar'), False) 40 | self.assertIs(self.task_pool.occupy_path(task.local_abspath, task), task) 41 | self.task_pool.release_path(task.local_abspath) 42 | self.assertIs(self.task_pool.has_pending_task('/foo/bar'), False) 43 | 44 | def test_occupy_failure(self): 45 | self.assertIsNone(self.task_pool.occupy_path('/foo/bar', None)) 46 | task = self._get_dummy_task(local_abspath='/foo/bar') 47 | self.assertIs(self.task_pool.occupy_path(task.local_abspath, task), None) 48 | 49 | def test_remove_children_tasks(self): 50 | for s in ('/foo', '/foo2', '/foo/bar', '/foo2/bar', '/foo/bar/baz'): 51 | self.task_pool.add_task(self._get_dummy_task(local_abspath=s)) 52 | self.task_pool.remove_children_tasks(local_parent_path='/foo') 53 | self.assertEqual(2, self.task_pool.outstanding_task_count) 54 | self.assertEqual('/foo2', self.task_pool.pop_task().local_abspath) 55 | self.assertEqual('/foo2/bar', self.task_pool.pop_task().local_abspath) 56 | 57 | 58 | if __name__ == '__main__': 59 | unittest.main() 60 | -------------------------------------------------------------------------------- /tests/test_tasks.py: -------------------------------------------------------------------------------- 1 | import json 2 | import os 3 | import unittest 4 | 5 | import onedrivesdk 6 | import requests_mock 7 | 8 | from onedrive_client import get_resource, od_task, od_webhook 9 | from onedrive_client.od_tasks.base import TaskBase 10 | from onedrive_client.od_tasks.start_repo import StartRepositoryTask 11 | from onedrive_client.od_tasks.update_subscriptions import UpdateSubscriptionTask 12 | import onedrive_client.od_tasks.merge_dir as merge_dir 13 | 14 | from tests.test_repo import get_sample_repo 15 | 16 | 17 | class TestTaskBase(unittest.TestCase): 18 | 19 | def test_task_base(self): 20 | p = '/home/xb/123' 21 | base = TaskBase(repo=None, task_pool=od_task.TaskPool()) 22 | base.local_abspath = p 23 | self.assertEqual(p, base.local_abspath) 24 | 25 | 26 | class TasksTestCaseBase(unittest.TestCase): 27 | 28 | def setUp(self): 29 | self.task_pool = od_task.TaskPool() 30 | self.temp_config_dir, self.temp_repo_dir, self.drive_config, self.repo = get_sample_repo() 31 | 32 | def tearDown(self): 33 | self.temp_config_dir.cleanup() 34 | self.temp_repo_dir.cleanup() 35 | 36 | 37 | class TestStartRepositoryTask(TasksTestCaseBase): 38 | 39 | def test_handle(self): 40 | task = StartRepositoryTask(self.repo, self.task_pool) 41 | task.handle() 42 | self.assertEqual(1, self.task_pool.outstanding_task_count) 43 | 44 | 45 | class TestUpdateSubscriptionTask(TasksTestCaseBase): 46 | 47 | def setUp(self): 48 | super().setUp() 49 | self.webhook_worker = od_webhook.WebhookWorkerThread('https://localhost', callback_func=None) 50 | self.data = json.loads(get_resource('data/subscription_response.json', pkg_name='tests')) 51 | 52 | def assert_subscription(self, subscription): 53 | self.assertEqual(self.data['id'], subscription.id) 54 | self.assertEqual(self.data['resource'], subscription.resource) 55 | 56 | @requests_mock.mock() 57 | def test_handle_create(self, m): 58 | m.post('%sdrives/%s/root:/:/subscriptions' % (self.repo.authenticator.client.base_url, self.repo.drive.id), 59 | json=self.data) 60 | task = UpdateSubscriptionTask(self.repo, self.task_pool, self.webhook_worker) 61 | subscription = task.handle() 62 | self.assert_subscription(subscription) 63 | 64 | @requests_mock.mock() 65 | def test_handle_update(self, m): 66 | m.patch('%sdrives/%s/root:/:/subscriptions/%s' % ( 67 | self.repo.authenticator.client.base_url, self.repo.drive.id, self.data['id']), json=self.data) 68 | task = UpdateSubscriptionTask(self.repo, self.task_pool, self.webhook_worker, subscription_id=self.data['id']) 69 | subscription = task.handle() 70 | self.assert_subscription(subscription) 71 | 72 | 73 | class TestMergeDirTask(TasksTestCaseBase): 74 | 75 | def test_remote_dir_matches_record(self): 76 | item = onedrivesdk.Item(json.loads(get_resource('data/folder_item.json', pkg_name='tests'))) 77 | self.repo.update_item(item, '', size_local=0) 78 | record = self.repo.get_item_by_path(item.name, '') 79 | merge_dir.MergeDirectoryTask._remote_dir_matches_record(item, record) 80 | 81 | def _generate_random_files(self, filenames): 82 | for filename in filenames: 83 | with open(self.repo.local_root + '/' + filename, 'w') as f: 84 | f.write(filename) 85 | 86 | def test_list_local_names(self): 87 | self._generate_random_files(('foo.txt', 'Foo.txt', 'fOO.tXT', 'FoO 1.txt', 'fOo 2.txt')) 88 | task = merge_dir.MergeDirectoryTask(self.repo, self.task_pool, '', item_request=None) 89 | entries = task.list_local_names() 90 | for ent in entries: 91 | self.assertTrue(os.path.isfile(task.local_abspath + '/' + ent)) 92 | # All other files are properly renamed. 93 | self.assertEqual(5, len(set([ent.lower() for ent in entries]))) 94 | self.assertIn('FoO 1.txt', entries) 95 | self.assertIn('fOo 2.txt', entries) 96 | 97 | def test_rename_with_suffix(self): 98 | self._generate_random_files(('foo',)) 99 | merge_dir.rename_with_suffix(self.repo.local_root, 'foo', 'hostname') 100 | self.assertTrue(os.path.isfile(self.repo.local_root + '/foo (hostname)')) 101 | 102 | def test_rename_with_suffix_and_count(self): 103 | self._generate_random_files(('foo.txt', 'foo (hostname).txt', 'foo 2 (hostname).txt')) 104 | 105 | merge_dir.rename_with_suffix(self.repo.local_root, 'foo.txt', 'hostname') 106 | self.assertFalse(os.path.exists(self.repo.local_root + '/foo.txt')) 107 | self.assertTrue(os.path.isfile(self.repo.local_root + '/foo 1 (hostname).txt')) 108 | 109 | merge_dir.rename_with_suffix(self.repo.local_root, 'foo 1 (hostname).txt', 'hostname') 110 | self.assertFalse(os.path.exists(self.repo.local_root + '/foo 1 (hostname).txt')) 111 | self.assertTrue(os.path.isfile(self.repo.local_root + '/foo 3 (hostname).txt')) 112 | 113 | def test_get_os_stat(self): 114 | self.assertIsNone(merge_dir.get_os_stat('/foo/bar/baz/blah')) 115 | self.assertIsNotNone(merge_dir.get_os_stat('/')) 116 | 117 | 118 | if __name__ == '__main__': 119 | unittest.main() 120 | -------------------------------------------------------------------------------- /tests/test_threads.py: -------------------------------------------------------------------------------- 1 | import threading 2 | import unittest 3 | 4 | from onedrive_client import od_threads, od_task 5 | from onedrive_client.od_tasks.base import TaskBase 6 | 7 | 8 | class DummyTask(TaskBase): 9 | 10 | def __init__(self, sem, repo=None, task_pool=None): 11 | super().__init__(repo, task_pool) 12 | self.sem = sem 13 | self.local_abspath = '/Dummy' 14 | 15 | def handle(self): 16 | self.sem.release() 17 | 18 | 19 | class TestTaskWorkerThread(unittest.TestCase): 20 | 21 | def setUp(self): 22 | self.task_pool = od_task.TaskPool() 23 | 24 | def test_lifecycle(self): 25 | sem = threading.Semaphore(value=0) 26 | t = DummyTask(sem, None, self.task_pool) 27 | w = od_threads.TaskWorkerThread('DummyWorker', task_pool=self.task_pool) 28 | w.start() 29 | self.assertTrue(self.task_pool.add_task(t)) 30 | self.assertTrue(sem.acquire(timeout=5)) 31 | w.stop() 32 | self.task_pool.close(1) 33 | w.join(timeout=5) 34 | 35 | 36 | if __name__ == '__main__': 37 | unittest.main() 38 | -------------------------------------------------------------------------------- /tests/test_watcher.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | 3 | import inotify_simple 4 | 5 | from onedrive_client import od_task, od_watcher 6 | 7 | 8 | class TestLocalRepositoryWatcher(unittest.TestCase): 9 | 10 | def setUp(self): 11 | self.watcher = od_watcher.LocalRepositoryWatcher(od_task.TaskPool, None) 12 | 13 | def test_recognize_event_patterns(self): 14 | events = [inotify_simple.Event(wd=1, mask=inotify_simple.flags.MOVED_FROM, cookie=233, name='fromA'), 15 | inotify_simple.Event(wd=3, mask=inotify_simple.flags.MOVED_TO, cookie=233, name='toA'), 16 | inotify_simple.Event(wd=5, mask=inotify_simple.flags.MOVED_FROM, cookie=234, name='from'), 17 | inotify_simple.Event(wd=7, mask=inotify_simple.flags.MOVED_TO, cookie=235, name='to')] 18 | move_pairs, all_events = self.watcher._recognize_event_patterns(events) 19 | self.assertEqual(4, len(all_events)) 20 | self.assertIn(233, move_pairs) 21 | self.assertNotIn(234, move_pairs) 22 | self.assertNotIn(235, move_pairs) 23 | (ev_a, flags_a), (ev_b, flags_b) = move_pairs[233] 24 | self.assertEqual([inotify_simple.flags.MOVED_FROM], flags_a) 25 | self.assertEqual('fromA', ev_a.name) 26 | self.assertEqual([inotify_simple.flags.MOVED_TO], flags_b) 27 | self.assertEqual('toA', ev_b.name) 28 | self.assertEqual((ev_b, flags_b), all_events[1]) 29 | 30 | 31 | if __name__ == '__main__': 32 | unittest.main() 33 | -------------------------------------------------------------------------------- /tests/test_webhook_worker.py: -------------------------------------------------------------------------------- 1 | import json 2 | import threading 3 | import unittest 4 | 5 | import onedrivesdk 6 | 7 | from onedrive_client import get_resource, od_webhook 8 | 9 | from tests.test_repo import get_sample_repo 10 | 11 | 12 | class TestWebhookWorker(unittest.TestCase): 13 | 14 | def setUp(self): 15 | self.temp_config_dir, self.temp_repo_dir, self.drive_config, self.repo = get_sample_repo() 16 | self.worker = od_webhook.WebhookWorkerThread('https://localhost:12345', 17 | callback_func=self._dummy_webhook_callback, 18 | action_delay_sec=0) 19 | self.callback_called_sem = threading.Semaphore(value=0) 20 | self.callback_repos = [] 21 | self.callback_count = 0 22 | 23 | def tearDown(self): 24 | self.temp_config_dir.cleanup() 25 | self.temp_repo_dir.cleanup() 26 | 27 | def _dummy_webhook_callback(self, repo): 28 | self.callback_repos.append(repo) 29 | self.callback_count += 1 30 | self.callback_called_sem.release() 31 | 32 | def test_execution(self): 33 | self.worker.start() 34 | notification_data = json.loads(get_resource('data/webhook_notification.json', pkg_name='tests')) 35 | subscription = onedrivesdk.Subscription() 36 | subscription.id = notification_data['subscriptionId'] 37 | self.worker.add_subscription(subscription, self.repo) 38 | # Send a notification. 39 | self.worker.queue_input(json.dumps({'value': [notification_data]}).encode('utf-8')) 40 | # Duplicate notifications should be ignored. 41 | self.worker.queue_input(json.dumps(notification_data).encode('utf-8')) 42 | # Unknown subscriptions should be ignored. 43 | notification_data['subscriptionId'] = '233' 44 | self.worker.queue_input(json.dumps(notification_data).encode('utf-8')) 45 | self.assertTrue(self.callback_called_sem.acquire(timeout=3)) 46 | self.assertFalse(self.callback_called_sem.acquire(timeout=1)) 47 | self.assertEqual([self.repo], self.callback_repos) 48 | self.assertEqual(1, self.callback_count) 49 | 50 | 51 | if __name__ == '__main__': 52 | unittest.main() 53 | --------------------------------------------------------------------------------