├── cloudsync ├── py.typed ├── tests │ ├── __init__.py │ ├── fixtures │ │ ├── __init__.py │ │ ├── fake_api.py │ │ ├── mock_storage.py │ │ ├── mock_provider.py │ │ └── util.py │ ├── test_prov_unit.py │ ├── pytest.ini │ ├── test_notification.py │ ├── conftest.py │ ├── test_registry.py │ ├── test_storage.py │ ├── test_dropbox.py │ ├── test_cmd_main.py │ ├── test_resolve_file.py │ ├── test_cmd_debug.py │ ├── test_fs_provider.py │ ├── test_oauth_redir_server.py │ ├── test_longpoll.py │ ├── test_cmd_list.py │ ├── test_utils.py │ ├── test_cmd_sync.py │ ├── test_runnable.py │ ├── sync_notification_handler.py │ ├── test_oauth.py │ ├── test_box.py │ └── test_events.py ├── command │ ├── __init__.py │ ├── main.py │ ├── list.py │ ├── sync.py │ ├── debug.py │ └── utils.py ├── __main__.py ├── sync │ ├── __init__.py │ └── sqlite_storage.py ├── oauth │ ├── __init__.py │ ├── redir_server.py │ └── oauth_config.py ├── log.py ├── providers │ └── __init__.py ├── __init__.py ├── registry.py ├── types.py ├── exceptions.py ├── long_poll.py ├── notification.py ├── utils.py ├── runnable.py └── cs.py ├── docs ├── DEVELOP.md ├── README.md ├── _static │ ├── .placeholder │ └── logo.png ├── registry.rst ├── provider.rst ├── runnable.rst ├── requirements.txt ├── cs.rst ├── index.rst ├── oauth.rst ├── notification.rst ├── apiserver.rst ├── test.sh ├── Makefile └── conf.py ├── pytest.ini ├── .pep8 ├── box.token.enc ├── mypy.ini ├── mkdocs.yml ├── coverage.sh ├── .readthedocs.yml ├── vernum.cfg ├── .coveragerc-integ ├── package.json ├── requirements-dev.txt ├── codecov-integ.yml ├── codecov.yml ├── deploy.sh ├── .coveragerc ├── requirements.txt ├── azure-test.sh ├── test_providers.sh ├── azure-pipelines.yml ├── cleanall.py ├── DEVELOP.md ├── pyproject.toml ├── Makefile ├── check-deps.py ├── .travis.yml ├── .gitignore ├── verok.py ├── README.md └── LICENSE /cloudsync/py.typed: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /docs/DEVELOP.md: -------------------------------------------------------------------------------- 1 | ../DEVELOP.md -------------------------------------------------------------------------------- /docs/README.md: -------------------------------------------------------------------------------- 1 | ../README.md -------------------------------------------------------------------------------- /docs/_static/.placeholder: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /pytest.ini: -------------------------------------------------------------------------------- 1 | [pytest] 2 | markers = 3 | manual 4 | providers 5 | -------------------------------------------------------------------------------- /.pep8: -------------------------------------------------------------------------------- 1 | [pep8] 2 | max_line_length = 160 3 | ignore=E731,E226,E24,W50,W690 4 | -------------------------------------------------------------------------------- /box.token.enc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AtakamaLLC/cloudsync/HEAD/box.token.enc -------------------------------------------------------------------------------- /cloudsync/tests/__init__.py: -------------------------------------------------------------------------------- 1 | from .fixtures import * 2 | from .test_provider import * 3 | -------------------------------------------------------------------------------- /docs/_static/logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AtakamaLLC/cloudsync/HEAD/docs/_static/logo.png -------------------------------------------------------------------------------- /mypy.ini: -------------------------------------------------------------------------------- 1 | [mypy] 2 | ignore_missing_imports = True 3 | no_strict_optional = True 4 | check_untyped_defs = True -------------------------------------------------------------------------------- /mkdocs.yml: -------------------------------------------------------------------------------- 1 | site_name: cloudsync 2 | repo_url: https://github.com/AtakamaLLC/cloudsync 3 | nav: 4 | - Home: index.rst 5 | docs_dir: docs 6 | -------------------------------------------------------------------------------- /cloudsync/tests/fixtures/__init__.py: -------------------------------------------------------------------------------- 1 | from .util import * 2 | from .fake_api import * 3 | from .mock_provider import * 4 | from .mock_storage import * 5 | -------------------------------------------------------------------------------- /docs/registry.rst: -------------------------------------------------------------------------------- 1 | cloudsync.registry 2 | =============================================== 3 | 4 | .. automodule:: cloudsync.registry 5 | :members: 6 | -------------------------------------------------------------------------------- /coverage.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # upload to codecov 4 | 5 | ls .coverage 6 | 7 | codecov --env TRAVIS_OS_NAME || ( sleep 5 && codecov --env TRAVIS_OS_NAME ) 8 | -------------------------------------------------------------------------------- /.readthedocs.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | sphinx: 3 | configuration: docs/conf.py 4 | python: 5 | version: 3.6 6 | install: 7 | - requirements: docs/requirements.txt 8 | -------------------------------------------------------------------------------- /docs/provider.rst: -------------------------------------------------------------------------------- 1 | cloudsync.Provider 2 | =============================================== 3 | 4 | .. autoclass:: cloudsync.Provider 5 | :members: 6 | :undoc-members: 7 | -------------------------------------------------------------------------------- /docs/runnable.rst: -------------------------------------------------------------------------------- 1 | cloudsync.Runnable 2 | =============================================== 3 | 4 | .. autoclass:: cloudsync.Runnable 5 | :members: 6 | :undoc-members: 7 | -------------------------------------------------------------------------------- /docs/requirements.txt: -------------------------------------------------------------------------------- 1 | pystrict 2 | sphinx==2.2.1 3 | recommonmark==0.6.0 4 | docutils==0.17.1 5 | requests_oauthlib 6 | python-daemon 7 | 8 | # docutils 0.18 breaks sphinx 9 | docutils!=0.18 10 | -------------------------------------------------------------------------------- /vernum.cfg: -------------------------------------------------------------------------------- 1 | component_min = 3.0 2 | component_max = 3.0 3 | component_int_max = 65535.0 4 | allowed_labels = [ 5 | "a", 6 | "b", 7 | "dev" 8 | ] 9 | prerelease_uses_dash = false 10 | dev_uses_dot = false 11 | -------------------------------------------------------------------------------- /.coveragerc-integ: -------------------------------------------------------------------------------- 1 | [run] 2 | branch=True 3 | omit = 4 | cloudsync/tests/* 5 | 6 | [report] 7 | exclude_lines = 8 | pragma: no cover 9 | def __repr__ 10 | assert 11 | if __name__ == __main__: 12 | log.debug 13 | -------------------------------------------------------------------------------- /docs/cs.rst: -------------------------------------------------------------------------------- 1 | cloudsync.CloudSync 2 | =================== 3 | 4 | The main sync class, instance this to initiate a sync. 5 | 6 | .. autoclass:: cloudsync.CloudSync 7 | :members: 8 | :inherited-members: 9 | :undoc-members: 10 | -------------------------------------------------------------------------------- /cloudsync/command/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | Command-line processing package 3 | 4 | cloudsync can be run from the command line to performa one-time sync, 5 | or to run a daemon that keeps syncing. 6 | """ 7 | from .main import main 8 | from .utils import * 9 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "dependencies": { 3 | "remark-cli": "^7.0.1", 4 | "remark-preset-lint-recommended": "^3.0.3" 5 | }, 6 | "remarkConfig": { 7 | "plugins": [ 8 | "remark-preset-lint-recommended" 9 | ] 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /requirements-dev.txt: -------------------------------------------------------------------------------- 1 | pytest~=5.4 2 | pytest-xdist~=1.34 3 | pytest-profiling~=1.7 4 | pytest-repeat 5 | pytest-cov~=2.10 6 | pytest-timeout~=1.4 7 | pylint~=2.6.0 8 | mypy==0.770 9 | flaky>=3.6 10 | coverage 11 | codecov 12 | diff-cover 13 | toml 14 | -------------------------------------------------------------------------------- /cloudsync/tests/test_prov_unit.py: -------------------------------------------------------------------------------- 1 | from .fixtures import MockProvider 2 | 3 | 4 | def test_subpath(): 5 | m = MockProvider(True, False) 6 | x = "c:/Users\\Hello\\world.pptx" 7 | y = "c:/Users/hello" 8 | 9 | assert m.is_subpath(y, x) 10 | -------------------------------------------------------------------------------- /docs/index.rst: -------------------------------------------------------------------------------- 1 | cloudsync 2 | ********* 3 | 4 | .. toctree:: 5 | :maxdepth: 2 6 | 7 | Synopsis 8 | Developer Notes 9 | cs 10 | provider 11 | registry 12 | oauth 13 | notification 14 | runnable 15 | apiserver 16 | -------------------------------------------------------------------------------- /cloudsync/tests/pytest.ini: -------------------------------------------------------------------------------- 1 | [pytest] 2 | log_format = %(asctime)s p%(process)s {%(pathname)s:%(lineno)d} %(levelname)s: %(message)s 3 | log_date_format = %Y-%m-%d %H:%M:%S 4 | log_level = 5 5 | # dont change this to debug, it will log passing tests as debug 6 | log_cli_level = warning 7 | -------------------------------------------------------------------------------- /docs/oauth.rst: -------------------------------------------------------------------------------- 1 | cloudsync.oauth 2 | =============================================== 3 | 4 | .. autoclass:: cloudsync.oauth.OAuthConfig 5 | :members: 6 | :undoc-members: 7 | 8 | .. autoclass:: cloudsync.oauth.OAuthProviderInfo 9 | :members: 10 | :undoc-members: 11 | -------------------------------------------------------------------------------- /docs/notification.rst: -------------------------------------------------------------------------------- 1 | cloudsync.Notification 2 | =============================================== 3 | 4 | .. autoclass:: cloudsync.Notification 5 | :members: 6 | :undoc-members: 7 | 8 | .. autoclass:: cloudsync.NotificationType 9 | :members: 10 | :undoc-members: 11 | -------------------------------------------------------------------------------- /cloudsync/__main__.py: -------------------------------------------------------------------------------- 1 | """ 2 | Used when running cloudsync as a command-line utility 3 | """ 4 | 5 | from cloudsync.command import main 6 | 7 | if __name__ == "__main__": 8 | import sys 9 | import os 10 | sys.path.append(os.path.join(os.path.dirname(os.path.abspath(__file__)), "..")) 11 | main() 12 | -------------------------------------------------------------------------------- /cloudsync/sync/__init__.py: -------------------------------------------------------------------------------- 1 | __all__ = ['SyncManager', 'SyncState', 'SyncEntry', 'Storage', 'FILE', 'DIRECTORY', 'UNKNOWN', 'SqliteStorage', 2 | 'MISSING', 'TRASHED', 'EXISTS', 'UNKNOWN', 'LIKELY_TRASHED', 'OTHER_SIDE', 'CORRUPT'] 3 | 4 | from .manager import * 5 | from .state import * 6 | from .sqlite_storage import SqliteStorage 7 | -------------------------------------------------------------------------------- /codecov-integ.yml: -------------------------------------------------------------------------------- 1 | # integration doesn't require patch coverage 2 | # since oauth code is still hard to test 3 | 4 | codecov: 5 | require_ci_to_pass: no 6 | 7 | coverage: 8 | status: 9 | project: 10 | default: yes 11 | base: pr 12 | threshold: 1 13 | patch: 14 | default: no 15 | base: pr 16 | -------------------------------------------------------------------------------- /codecov.yml: -------------------------------------------------------------------------------- 1 | coverage: 2 | status: 3 | project: 4 | default: yes 5 | threshold: 0.5 6 | base: pr 7 | patch: 8 | default: yes 9 | threshold: 0.5 10 | base: pr 11 | ignore: 12 | - "cloudsync/providers/box.py" 13 | - "cloudsync/providers/dropbox.py" 14 | - "cloudsync/providers/filesystem.py" 15 | -------------------------------------------------------------------------------- /docs/apiserver.rst: -------------------------------------------------------------------------------- 1 | cloudsync.oauth.apiserver.ApiServer 2 | =============================================== 3 | 4 | .. autoclass:: cloudsync.oauth.apiserver.ApiServer 5 | :members: 6 | :undoc-members: 7 | 8 | 9 | .. autoclass:: cloudsync.oauth.apiserver.ApiError 10 | :members: 11 | :undoc-members: 12 | 13 | .. autofunction:: cloudsync.oauth.apiserver.api_route 14 | -------------------------------------------------------------------------------- /deploy.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -e 2 | 3 | pip install flit 4 | 5 | VERSION=$(./verok.py "$(git describe --abbrev=0 --tags)") 6 | 7 | echo ver $VERSION 8 | 9 | # if this is changed, also change the .gitignore script 10 | sed -i.bak "s/%VERSION%/$VERSION/" cloudsync/__init__.py 11 | 12 | flit publish 13 | 14 | # it's nice to put things back, because reasons 15 | mv cloudsync/__init__.py.bak cloudsync/__init__.py 16 | -------------------------------------------------------------------------------- /cloudsync/oauth/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | OAuth helpers for building new providers 3 | """ 4 | 5 | from typing import NamedTuple, List 6 | 7 | 8 | from .redir_server import * 9 | from .oauth_config import * 10 | 11 | 12 | class OAuthProviderInfo(NamedTuple): 13 | """ 14 | Providers can set their ._oauth_info protected member to one of these. 15 | """ 16 | auth_url: str 17 | token_url: str 18 | scopes: List[str] 19 | -------------------------------------------------------------------------------- /docs/test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -e 2 | 3 | cd $(git rev-parse --show-toplevel)/docs 4 | 5 | set -o xtrace 6 | 7 | pip install virtualenv 8 | virtualenv docenv 9 | . docenv/bin/activate 10 | pip install -r requirements.txt 11 | rm -rf _build 12 | 13 | make html > make.out 2>&1 || cat make.out 14 | 15 | grep 'WARNING' make.out && cat make.out && exit 1 16 | grep 'build succeeded' make.out && exit 0 17 | 18 | echo "unknown status" 19 | exit 1 20 | -------------------------------------------------------------------------------- /cloudsync/log.py: -------------------------------------------------------------------------------- 1 | """ 2 | Log initialization for cloudsync. Adds the 'TRACE' level to the logger, which 3 | only shows while unit testing cloudsync itself. 4 | """ 5 | 6 | # add TRACE named level, because libraries need it 7 | 8 | import logging 9 | logger = logging.getLogger(__package__) 10 | if isinstance(logging.getLevelName('TRACE'), str): 11 | logging.addLevelName(5, 'TRACE') 12 | 13 | # ses docs, this actually gets a number, because reasons 14 | TRACE = logging.getLevelName('TRACE') 15 | -------------------------------------------------------------------------------- /cloudsync/tests/test_notification.py: -------------------------------------------------------------------------------- 1 | import logging 2 | from unittest.mock import MagicMock 3 | from cloudsync.notification import Notification, NotificationManager, NotificationType, SourceEnum 4 | log = logging.getLogger(__name__) 5 | 6 | 7 | def test_notification(): 8 | handle_notification = MagicMock() 9 | nm = NotificationManager(evt_handler=handle_notification) 10 | nm.notify(Notification(SourceEnum.LOCAL, NotificationType.DISCONNECTED_ERROR, None)) 11 | nm.do() 12 | handle_notification.assert_called_once() 13 | -------------------------------------------------------------------------------- /.coveragerc: -------------------------------------------------------------------------------- 1 | [run] 2 | branch=True 3 | omit = 4 | cloudsync/providers/box.py 5 | cloudsync/providers/dropbox.py 6 | cloudsync/providers/filesystem.py 7 | cloudsync/tests/* 8 | 9 | [report] 10 | exclude_lines = 11 | pragma: no cover 12 | def __repr__ 13 | assert 14 | if __name__ == __main__: 15 | log.debug 16 | [.][.][.] 17 | raise NotImplementedError 18 | if __name__ == .__main__.: 19 | if typing.TYPE_CHECKING: 20 | if TYPE_CHECKING: 21 | @pytest.mark.manual 22 | @pytest.fixture 23 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | pystrict 2 | arrow~=0.17.0 3 | msgpack 4 | dataclasses 5 | requests_oauthlib 6 | requests 7 | xxhash 8 | 9 | # gdrive provider 10 | google-oauth 11 | google-auth-httplib2 12 | google-api-python-client 13 | 14 | # dropbox provider 15 | dropbox>=10.3.0, <11.0.0 16 | six>=1.12.0 17 | 18 | # box provider 19 | boxsdk[jwt]>=2.9.0 20 | 21 | # other providers we run tests for 22 | cloudsync-onedrive>=3.1.0 23 | cloudsync-gdrive>=2.0.0 24 | 25 | # command line 26 | python-daemon 27 | 28 | # fsprovider 29 | watchdog 30 | pywin32; sys.platform == 'win32' 31 | -------------------------------------------------------------------------------- /azure-test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -o errexit 4 | 5 | pip install pytest pytest-azurepipelines 6 | 7 | pytest --durations=1 --cov=cloudsync --cov-report=xml cloudsync/tests cloudsync/oauth/apiserver.py --timeout=300 & 8 | pid1=$! 9 | 10 | pytest --durations=1 --cov=cloudsync --cov-append --cov-report=xml cloudsync/tests/test_provider.py --provider=filesystem & 11 | pid2=$! 12 | 13 | echo Waiting for "cloudsync/tests cloudsync/oauth/apiserver.py" 14 | wait $pid1 15 | 16 | echo Waiting for "cloudsync/tests/test_provider.py --provider=filesystem" 17 | wait $pid2 18 | 19 | echo Done 20 | -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Minimal makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line. 5 | SPHINXOPTS = 6 | SPHINXBUILD = sphinx-build 7 | SOURCEDIR = . 8 | BUILDDIR = _build 9 | 10 | # Put it first so that "make" without argument is like "make help". 11 | help: 12 | @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 13 | 14 | .PHONY: help Makefile 15 | 16 | # Catch-all target: route all unknown targets to Sphinx using the new 17 | # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). 18 | %: Makefile 19 | @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 20 | -------------------------------------------------------------------------------- /cloudsync/tests/conftest.py: -------------------------------------------------------------------------------- 1 | import cloudsync 2 | 3 | from cloudsync.tests.fixtures import * # pylint: disable=unused-import, unused-wildcard-import, wildcard-import 4 | 5 | cloudsync.logger.setLevel("TRACE") 6 | 7 | 8 | def pytest_configure(config): 9 | config.addinivalue_line("markers", "manual") 10 | 11 | 12 | def pytest_runtest_setup(item): 13 | if 'manual' in item.keywords and not item.config.getoption("--manual"): 14 | pytest.skip("need --manual option to run this test") 15 | 16 | 17 | def pytest_addoption(parser): 18 | parser.addoption("--provider", action="append", default=[], help="provider(s) to run tests for") 19 | parser.addoption("--manual", action="store_true", default=False, help="run the manual tests") 20 | -------------------------------------------------------------------------------- /cloudsync/tests/test_registry.py: -------------------------------------------------------------------------------- 1 | import logging 2 | log = logging.getLogger(__name__) 3 | 4 | 5 | class Fake: 6 | def __init__(self, *args, **kwargs): 7 | if kwargs.get('test'): 8 | self.test = kwargs['test'] 9 | else: 10 | self.test = None 11 | log.error("Arguments to Fake provider init: args=%s kwargs=%s", args, kwargs) 12 | 13 | name = "fake" 14 | 15 | 16 | __cloudsync__ = Fake 17 | 18 | 19 | import cloudsync.registry as registry 20 | 21 | 22 | def test_registry_discover(): 23 | assert registry.get_provider("fake") == Fake 24 | provider = registry.create_provider("fake", test='testvalue') 25 | assert isinstance(provider, Fake) 26 | assert provider.test == 'testvalue' 27 | 28 | -------------------------------------------------------------------------------- /test_providers.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # allow coverage to include provider dirs when running integration tests 4 | cp codecov-integ.yml codecov.yml 5 | 6 | git fetch origin master 7 | git_result=$(git diff origin/master --name-only) 8 | if [ $? -eq 0 ]; then 9 | echo "git diff origin/master --name-only" 10 | echo "$git_result" 11 | else 12 | echo "git diff failed in detached state" 13 | exit 0 14 | fi 15 | 16 | if echo "$git_result" | grep -qE '(cloudsync/provider.py|cloudsync/providers/|test_provider)'; then 17 | pytest --cov=cloudsync --cov-append --cov-report=xml --cov-config=.coveragerc-integ --durations=0 -n=4 cloudsync/tests/test_provider.py --provider "$1" --timeout=600 18 | else 19 | echo "Skipping integration test because no provider.py|providers/ changes" 20 | fi 21 | -------------------------------------------------------------------------------- /azure-pipelines.yml: -------------------------------------------------------------------------------- 1 | # Python package 2 | # Create and test a Python package on multiple Python versions. 3 | # Add steps that analyze code, save the dist with the build record, publish to a PyPI-compatible index, and more: 4 | # https://docs.microsoft.com/azure/devops/pipelines/languages/python 5 | 6 | trigger: 7 | - master 8 | 9 | strategy: 10 | matrix: 11 | windows: 12 | imageName: 'windows-latest' 13 | 14 | pool: 15 | vmImage: $(imageName) 16 | 17 | steps: 18 | - script: | 19 | python -m pip install --upgrade pip 20 | pip install -r requirements.txt 21 | pip install -r requirements-dev.txt 22 | displayName: 'install dependencies' 23 | 24 | - script: bash azure-test.sh 25 | displayName: 'pytest' 26 | 27 | - script: bash codecov.sh 28 | displayName: 'upload coverage' 29 | env: 30 | CODECOV_TOKEN: $(CODECOV_TOKEN) 31 | condition: succeeded() 32 | -------------------------------------------------------------------------------- /cleanall.py: -------------------------------------------------------------------------------- 1 | import threading 2 | 3 | from cloudsync import * 4 | 5 | from cloudsync.providers import GDriveProvider, DropboxProvider, BoxProvider, OneDriveProvider 6 | 7 | gd = GDriveProvider.test_instance() 8 | db = DropboxProvider.test_instance() 9 | bx = BoxProvider.test_instance() 10 | od = OneDriveProvider.test_instance() 11 | 12 | threads = [] 13 | 14 | provs = [gd, db, bx, od] 15 | 16 | for prov in provs: 17 | print(prov._test_creds) 18 | prov.connect(prov._test_creds) 19 | 20 | def run(prov): 21 | ld = list(prov.listdir_path("/")) 22 | print(prov.name, "# folder count:", len(ld)) 23 | oid = prov.info_path("/").oid 24 | prov.rmtree(oid) 25 | print(prov.name, "# done") 26 | t = threading.Thread(target=lambda: run(prov), daemon=True) 27 | t.start() 28 | threads.append(t) 29 | 30 | for t in threads: 31 | t.join() 32 | -------------------------------------------------------------------------------- /DEVELOP.md: -------------------------------------------------------------------------------- 1 | ### Misc dev notes 2 | 3 | * Disconnect after token error 4 | 5 | Currently it's critical to "disconnect" every time a CloudTokenError is raised. 6 | 7 | There are some unit tests that check obvious cases and assert not connected. 8 | 9 | Otherwise the event loop won't try to reauthenticate .... since "connected" currently implies "connected and authenitcated". 10 | 11 | * Env Vars and Integration tests 12 | 13 | When performing integration tests with cloud providers, credentials will be needed. 14 | 15 | To encrypt env vars for this project, run: 16 | 17 | travis encrypt --pro "ENVVAR=VALUE" --add 18 | 19 | These vars will *only* be available when the pull req is from within the main repo's "organization". 20 | 21 | External pull reqs that affect provider logic should be carefully manually verified to ensure 22 | that they don't leak test tokens, merged to a feature branch, and resubmitted by a member, 23 | otherwise integration tests will never run for that branch. 24 | -------------------------------------------------------------------------------- /cloudsync/providers/__init__.py: -------------------------------------------------------------------------------- 1 | import importlib 2 | 3 | from cloudsync.registry import register_provider 4 | from .mock import MockProvider 5 | 6 | # optionally load support for supported providers 7 | # todo: eventually there will be some factory class 8 | 9 | 10 | def _local_import(class_name, module, short_name): 11 | try: 12 | mod = importlib.import_module(module, __name__) 13 | ret = getattr(mod, class_name) 14 | except Exception as e: 15 | _ex = e 16 | 17 | class Fake(MockProvider): 18 | name = short_name 19 | def __init__(self, *a, **k): # pylint: disable=super-init-not-called 20 | raise _ex 21 | 22 | # syntax errors and other stuff propagates immediately 23 | if not isinstance(e, ImportError): 24 | raise 25 | 26 | # other errors are allowed, unless you use it 27 | ret = Fake 28 | 29 | register_provider(ret) 30 | return ret 31 | 32 | 33 | DropboxProvider = _local_import("DropboxProvider", ".dropbox", "dropbox") 34 | FileSystemProvider = _local_import("FileSystemProvider", ".filesystem", "filesystem") 35 | BoxProvider = _local_import("BoxProvider", ".box", "box") 36 | -------------------------------------------------------------------------------- /cloudsync/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | 3 | cloudsync enables simple cloud file-level sync with a variety of cloud providers 4 | 5 | External modules: 6 | 7 | cloudsync.Event 8 | cloudsync.Provider 9 | cloudsync.Sync 10 | 11 | Example: 12 | 13 | import cloudsync 14 | 15 | # use directly 16 | prov = cloudsync.get_provider('gdrive') 17 | creds = prov.authenticate() 18 | prov.connect(creds) 19 | with open("file") as file: 20 | info = prov.create("/dest", file) 21 | 22 | print("id of /dest is %s, hash of /dest is %s" % (info.oid, info.hash)) 23 | 24 | # use as sync 25 | local = cloudsync.get_provider('file') 26 | cs = cloudsync.CloudSync((local, prov), "/home/stuff", "/stuff") 27 | 28 | # run forever 29 | cs.run() 30 | """ 31 | 32 | __version__ = "%VERSION%" 33 | 34 | from pystrict import strict, StrictError 35 | 36 | # must be imported before other cloudsync imports 37 | from .log import logger 38 | 39 | # import modules into top level for convenience 40 | from .exceptions import * 41 | from .provider import * 42 | from .event import * 43 | from .sync import * 44 | from .types import * 45 | from .cs import * 46 | from .long_poll import * 47 | from .registry import * 48 | from .notification import * 49 | from .oauth import OAuthConfig 50 | from .providers import * 51 | from .command import * 52 | from .smartsync import * 53 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | requires = ["setuptools", "flit"] 3 | build-backend = "flit.buildapi" 4 | 5 | [tool.flit.metadata] 6 | module = "cloudsync" 7 | author = "Atakama, LLC" 8 | author-email = "dev-support@atakama.com" 9 | home-page = "https://github.com/atakamallc/cloudsync" 10 | description-file="README.md" 11 | # MUST be from this list: https://pypi.org/pypi?%3Aaction=list_classifiers 12 | classifiers=["Intended Audience :: Developers", 13 | "License :: OSI Approved :: GNU Lesser General Public License v3 or later (LGPLv3+)", 14 | "Programming Language :: Python", 15 | "Topic :: Software Development :: Libraries :: Python Modules", 16 | ] 17 | requires = ["arrow~=0.17.0", "dataclasses; python_version < '3.7'", "pystrict", "msgpack", "requests_oauthlib", "python-daemon", "xxhash", "urllib3>=1.25.3", "watchdog", "pywin32; sys_platform == 'win32'"] 18 | requires-python = ">=3.6" 19 | 20 | [tool.flit.metadata.requires-extra] 21 | box = [ "boxsdk>=2.9.0", ] 22 | dropbox = [ "dropbox>=10.3.0", "six>=1.14.0"] 23 | boxcom = [ "boxsdk[jwt]", ] 24 | onedrive = [ "cloudsync-onedrive>=3.1.9", ] 25 | gdrive = [ "cloudsync-gdrive>=2.0.0", ] 26 | all = [ "cloudsync-gdrive>=2.0.0", "cloudsync-onedrive>=3.1.9", "boxsdk[jwt]", "dropbox>=10.3.0", "six>=1.14.0", "boxsdk>=2.9.0" ] 27 | 28 | [tool.flit.scripts] 29 | cloudsync = "cloudsync.command:main" 30 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | SHELL := /bin/bash 2 | PYTEST = pytest -rfE --cov=cloudsync --durations=1 -n=4 cloudsync/tests --tb=short --timeout=20 3 | 4 | ifeq ($(OS),Windows_NT) 5 | ENVBIN="scripts" 6 | else 7 | ENVBIN="bin" 8 | endif 9 | 10 | BASE := $(shell git merge-base HEAD origin/master) 11 | 12 | env: 13 | virtualenv env 14 | 15 | requirements: env 16 | . env/$(ENVBIN)/activate && pip install -r requirements-dev.txt 17 | . env/$(ENVBIN)/activate && pip install -r requirements.txt 18 | 19 | lint: lint-pylint lint-mypy lint-md lint-deps 20 | 21 | lint-pylint: 22 | pylint cloudsync --enable=duplicate-code --ignore tests 23 | 24 | lint-mypy: 25 | mypy cloudsync 26 | 27 | lint-md: ./node_modules/.bin/remark 28 | ./node_modules/.bin/remark -f docs/*.md *.md 29 | 30 | lint-deps: 31 | python check-deps.py 32 | 33 | test: test-py test-doc 34 | 35 | .coverage: $(shell find cloudsync -type f -name '*.py') 36 | $(PYTEST) 37 | 38 | test-doc: 39 | docs/test.sh 40 | 41 | test-py: 42 | $(PYTEST) 43 | 44 | coverage.xml: .coverage 45 | coverage xml 46 | 47 | coverage: coverage.xml 48 | diff-cover coverage.xml --compare-branch=$(BASE) 49 | 50 | format: 51 | autopep8 --in-place -r -j 8 cloudsync/ 52 | 53 | bumpver: 54 | ./bumpver.py 55 | 56 | ./node_modules/.bin/remark: 57 | npm install 58 | 59 | .PHONY: test test-py test-doc lint format bumpver env requirements coverage lint-md lint-deps 60 | -------------------------------------------------------------------------------- /check-deps.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | from typing import List, Set, Any, Optional, Tuple 3 | import sys 4 | import toml 5 | from pprint import pprint 6 | 7 | def _find_difference(left: Set[Any], right: Set[Any]) -> Optional[Tuple[Set[Any], Set[Any]]]: 8 | left_only = left - right 9 | right_only = right - left 10 | 11 | if left_only or right_only: 12 | return left_only, right_only 13 | 14 | return None 15 | 16 | 17 | def main() -> int: 18 | with open("pyproject.toml") as f: 19 | data = toml.load(f) 20 | 21 | requires_extra = data["tool"]["flit"]["metadata"]["requires-extra"] 22 | 23 | deps = set() 24 | 25 | for k, v in requires_extra.items(): 26 | if k == "all": 27 | continue 28 | 29 | deps.update(v) 30 | 31 | extra_all_deps = set(requires_extra["all"]) 32 | 33 | diff = _find_difference(deps, extra_all_deps) 34 | if diff: 35 | extras_only, all_only = diff 36 | print("Mismatched dependencies between individual extras and cloudsync[all]") 37 | print("Individual features require:", deps) 38 | print("[all] requires:", extra_all_deps) 39 | print() 40 | 41 | print("Missing from [all]:", extras_only) 42 | print("Only in [all]", all_only) 43 | 44 | return 1 45 | 46 | return 0 47 | 48 | 49 | 50 | if __name__ == "__main__": 51 | sys.exit(main()) 52 | -------------------------------------------------------------------------------- /cloudsync/tests/test_storage.py: -------------------------------------------------------------------------------- 1 | import os 2 | from typing import Dict 3 | 4 | import pytest 5 | 6 | from cloudsync import SqliteStorage 7 | from cloudsync.utils import NamedTemporaryFile 8 | 9 | from .fixtures import MockStorage 10 | 11 | @pytest.fixture(name="sqlite_store") 12 | def fixture_sqlite_storage(): 13 | f = NamedTemporaryFile(mode=None) 14 | s = SqliteStorage(f.name) 15 | yield s 16 | s.close() 17 | del f 18 | 19 | @pytest.fixture(name="mock_store") 20 | def fixture_mock_storage(): 21 | d: Dict[str, Dict[int, bytes]] = {} 22 | s = MockStorage(d) 23 | yield s 24 | s.close() 25 | 26 | @pytest.fixture(name="store", params=[0, 1], ids=["mock","sqlite"]) 27 | def fixture_storage(request, mock_store, sqlite_store): 28 | stores = (mock_store, sqlite_store) 29 | yield stores[request.param] 30 | 31 | def test_storage_update(store): 32 | eid = store.create("tag", b'bar') 33 | store.update("tag", b'baz', eid) 34 | store.delete("tag", eid) 35 | store.delete("tag", eid) 36 | assert store.read_all("tag") == {} 37 | 38 | def test_storage_read_multi(store): 39 | id1 = store.create("tag1", b'bar') 40 | id2 = store.create("tag2", b'baz') 41 | rall = store.read_all() 42 | assert rall == {"tag1":{id1:b'bar'}, "tag2":{id2:b'baz'}} 43 | 44 | 45 | def test_storage_close(store): 46 | store.close() 47 | if isinstance(store, SqliteStorage): 48 | # this only works on windows if the file is closed 49 | os.unlink(store._filename) 50 | # ok to close twice 51 | store.close() 52 | -------------------------------------------------------------------------------- /cloudsync/tests/test_dropbox.py: -------------------------------------------------------------------------------- 1 | # pylint: disable=protected-access,missing-docstring 2 | 3 | import logging 4 | import os 5 | 6 | from typing import Generator 7 | 8 | from unittest.mock import patch, Mock 9 | 10 | from cloudsync.providers import DropboxProvider 11 | 12 | import dropbox 13 | 14 | log = logging.getLogger(__name__) 15 | 16 | 17 | def test_rtmp(): 18 | rt = DropboxProvider._gen_rtmp("filename") 19 | assert DropboxProvider._is_rtmp(rt) 20 | assert not DropboxProvider._is_rtmp(".-(2304987239048234908239048") 21 | 22 | 23 | @patch("cloudsync.providers.dropbox._FolderIterator") 24 | def test_events(fi: Mock): 25 | db = DropboxProvider() 26 | 27 | chash = os.urandom(32).hex() 28 | 29 | def mock_iterate(*_a, **_kw) -> Generator[dropbox.files.Metadata, None, None]: 30 | yield dropbox.files.FileMetadata(name="YO.txt", id="id1", path_display="/YO.txt", path_lower="/yo.txt", content_hash=chash) 31 | tmpname = db._gen_rtmp("TMP") 32 | yield dropbox.files.FileMetadata(name=tmpname, id="id2", path_display="/" + tmpname, path_lower="/" + tmpname.lower(), content_hash=chash) 33 | 34 | fi.side_effect = mock_iterate 35 | 36 | evs = list(db._events(cursor=None, path="/")) 37 | 38 | ids = {} 39 | for ev in evs: 40 | ids[ev.oid] = ev 41 | 42 | assert "id1" in ids 43 | assert "id2" not in ids 44 | 45 | ev1 = ids["id1"] 46 | 47 | assert ev1.hash == chash 48 | assert ev1.path == "/YO.txt" 49 | assert ev1.exists 50 | assert ev1.mtime > 0 51 | 52 | log.info("evs %s", evs) 53 | -------------------------------------------------------------------------------- /cloudsync/command/main.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import argparse 3 | import logging 4 | import importlib 5 | import traceback 6 | from typing import Any 7 | 8 | from .utils import SubCmd 9 | 10 | log = logging.getLogger() 11 | 12 | def main(): 13 | """cloudsync command line main""" 14 | 15 | logging.basicConfig(format='%(asctime)s,%(msecs)d %(levelname)-8s [%(filename)s:%(lineno)d] %(message)s', 16 | datefmt='%Y-%m-%d:%H:%M:%S',) 17 | 18 | parser = argparse.ArgumentParser(description='cloudsync - monitor and sync between cloud providers') 19 | cmds = parser.add_subparsers(title="Commands") 20 | 21 | cmds.metavar = "Commands:" 22 | sub_cmds = ["debug", "sync", "list"] 23 | for sub_cmd in sub_cmds: 24 | module: Any = importlib.import_module(".." + sub_cmd, __name__) 25 | cmd: SubCmd = module.cmd_class(cmds) 26 | 27 | cmd.parser.add_argument('-v', '--verbose', help='More verbose logging', action="store_true") 28 | cmd.parser.set_defaults(func=cmd.run) 29 | 30 | args = argparse.Namespace(verbose=False, func=None) 31 | 32 | parser.parse_args(namespace=args) 33 | 34 | log.setLevel(logging.INFO) 35 | if args.verbose: 36 | log.setLevel(logging.DEBUG) 37 | log.debug("args %s", args.__dict__) 38 | 39 | if not args.func: 40 | parser.print_help(file=sys.stderr) 41 | sys.exit(1) 42 | 43 | try: 44 | args.func(args) 45 | except Exception as e: 46 | print("Error ", e, file=sys.stderr) 47 | if args.verbose: 48 | traceback.print_exc(None, sys.stderr) 49 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: python 2 | cache: 3 | pip: True 4 | 5 | sudo: false 6 | 7 | jobs: 8 | include: 9 | - name: lint 10 | stage: lint/unit 11 | script: 12 | - make -j4 lint 13 | - name: unit 14 | script: 15 | - pytest --durations=1 --cov=cloudsync --cov-report=xml -n=2 cloudsync/tests cloudsync/oauth/apiserver.py --timeout=300 16 | - ./docs/test.sh 17 | - ./test_providers.sh "filesystem" 18 | python: 19 | - '3.6' 20 | - '3.7' 21 | - stage: integ 22 | script: 23 | # only run tests if the interface or any implementations change 24 | # todo: move providers to their own individual repos/projects with a plugin model 25 | - ./test_providers.sh "gdrive,onedrive,box" 26 | python: 27 | - '3.6' 28 | - script: 29 | # only run tests if the interface or any implementations change 30 | # todo: move providers to their own individual repos/projects with a plugin model 31 | - ./test_providers.sh "dropbox,testodbiz,mock_oid_ci_ns,mock_path_cs" 32 | python: 33 | - '3.6' 34 | - stage: deploy 35 | if: type = push and tag =~ ^v 36 | python: 37 | - '3.6' 38 | script: 39 | - ./deploy.sh 40 | 41 | after_success: 42 | - ./coverage.sh 43 | 44 | branches: 45 | only: 46 | - master 47 | - /^v/ 48 | 49 | install: 50 | - pip install -r requirements-dev.txt 51 | - pip install -r requirements.txt 52 | - openssl aes-256-cbc -K $encrypted_ac820621abca_key -iv $encrypted_ac820621abca_iv -in box.token.enc -out box.token -d 53 | -------------------------------------------------------------------------------- /cloudsync/tests/test_cmd_main.py: -------------------------------------------------------------------------------- 1 | # pylint: disable=missing-docstring 2 | 3 | import sys 4 | import pytest 5 | from cloudsync.command.main import main 6 | 7 | 8 | def test_main(capsys): 9 | sys.argv = ["cloudsync", "debug", "--help"] 10 | 11 | ex = None 12 | try: 13 | main() 14 | except SystemExit as e: 15 | ex = e 16 | 17 | assert ex.code == 0 18 | 19 | rd = capsys.readouterr() 20 | 21 | assert "usage" in rd.out.lower() 22 | assert rd.err == "" 23 | 24 | @pytest.mark.parametrize("arg", [["badcommand"], []]) 25 | def test_main_badcmd(capsys, arg): 26 | sys.argv = ["cloudsync"] + arg 27 | 28 | ex = None 29 | try: 30 | main() 31 | except SystemExit as e: 32 | ex = e 33 | 34 | # raise an error 35 | assert ex.code > 0 36 | 37 | rd = capsys.readouterr() 38 | 39 | # show some usage 40 | assert "usage" in rd.err.lower() 41 | 42 | 43 | def test_main_disp(capsys): 44 | sys.argv = ["cloudsync", "debug"] 45 | 46 | ex = None 47 | try: 48 | main() 49 | except SystemExit as e: 50 | ex = e 51 | 52 | if ex: 53 | assert ex.code == 0 54 | 55 | rd = capsys.readouterr() 56 | 57 | assert rd.out == "" 58 | assert rd.err == "" 59 | 60 | 61 | def test_main_err(capsys): 62 | sys.argv = "cloudsync sync -v fozay:55 refo:66".split(" ") 63 | 64 | ex = None 65 | try: 66 | main() 67 | except SystemExit as e: 68 | ex = e 69 | 70 | if ex: 71 | assert ex.code > 0 72 | 73 | rd = capsys.readouterr() 74 | 75 | assert rd.err != "" 76 | # verbose logs a traceback on failz 77 | assert "aceback" in rd.err 78 | -------------------------------------------------------------------------------- /cloudsync/registry.py: -------------------------------------------------------------------------------- 1 | """ 2 | The registry maintains a map of provider classes by name. 3 | """ 4 | 5 | 6 | import sys 7 | from typing import List, Type 8 | import pkg_resources 9 | from cloudsync.provider import Provider 10 | 11 | __all__ = ["create_provider", "get_provider", "known_providers", "register_provider"] 12 | 13 | 14 | providers = {} 15 | 16 | 17 | def register_provider(prov: Type[Provider]): 18 | """Add a provider class to the registry""" 19 | providers[prov.name] = prov 20 | 21 | 22 | def discover_providers(): 23 | """Loop through imported modules, and autoregister providers, including plugins""" 24 | for m in sys.modules: 25 | mod = sys.modules[m] 26 | if hasattr(mod, "__cloudsync__"): 27 | if mod.__cloudsync__.name not in providers: # type: ignore 28 | register_provider(mod.__cloudsync__) # type: ignore 29 | 30 | for entry_point in pkg_resources.iter_entry_points('cloudsync.providers'): 31 | register_provider(entry_point.resolve()) 32 | 33 | 34 | def get_provider(name: str): 35 | """Get a provider class with the given name""" 36 | if name not in providers: 37 | discover_providers() 38 | 39 | if name not in providers: 40 | raise RuntimeError("%s not a registered provider, maybe you forgot to import cloudsync_%s" % (name, name)) 41 | 42 | return providers[name] 43 | 44 | 45 | def create_provider(name: str, *args, **kws) -> Provider: 46 | """Construct a provider instance""" 47 | return get_provider(name)(*args, **kws) 48 | 49 | 50 | def known_providers() -> List[str]: 51 | """List all known provider names, sorted order.""" 52 | discover_providers() 53 | return list(sorted(providers.keys())) 54 | -------------------------------------------------------------------------------- /cloudsync/types.py: -------------------------------------------------------------------------------- 1 | """ 2 | Base types for cloudsync 3 | """ 4 | from typing import Optional, Any 5 | from enum import Enum 6 | from dataclasses import dataclass 7 | 8 | 9 | # these are not really local or remote 10 | # but it's easier to reason about using these labels 11 | LOCAL = 0 12 | REMOTE = 1 13 | 14 | 15 | class OType(Enum): 16 | DIRECTORY = "dir" 17 | FILE = "file" 18 | NOTKNOWN = "trashed" 19 | 20 | 21 | class IgnoreReason(Enum): 22 | NONE = "none" 23 | DISCARDED = "discarded" 24 | CONFLICT = "conflict" 25 | TEMP_RENAME = "temp rename" 26 | IRRELEVANT = "irrelevant" 27 | 28 | 29 | DIRECTORY = OType.DIRECTORY 30 | FILE = OType.FILE 31 | NOTKNOWN = OType.NOTKNOWN # only allowed for deleted files! 32 | 33 | 34 | @dataclass # pylint: disable=too-many-instance-attributes 35 | class OInfo: 36 | """Base class for object returned by info_oid, info_path, create and listdir""" 37 | otype: OType # fsobject type (DIRECTORY or FILE) 38 | oid: str # fsobject id 39 | hash: Any # fsobject hash (better name: ohash) 40 | path: Optional[str] # path 41 | size: int = 0 # size of object in bytes 42 | name: Optional[str] = None # just the filename, without the path, when the full path is expensive 43 | mtime: Optional[float] = None # modification time 44 | shared: bool = False # file is shared by the cloud provider 45 | readonly: bool = False # file is readonly in the cloud 46 | custom: Optional[Any] = None # dict of provider specific information 47 | 48 | 49 | @dataclass 50 | class DirInfo(OInfo): 51 | pass 52 | -------------------------------------------------------------------------------- /cloudsync/exceptions.py: -------------------------------------------------------------------------------- 1 | """ 2 | This is the complete list of exceptions that should be thrown by providers. 3 | """ 4 | 5 | 6 | class CloudException(Exception): # largely treated as a temporary error with a heavy backoff 7 | def __init__(self, *args, original_exception=None): 8 | super().__init__(*args) 9 | self.original_exception = original_exception 10 | 11 | 12 | class CloudFileNotFoundError(CloudException): # ENOENT 13 | pass 14 | 15 | 16 | class CloudTemporaryError(CloudException): # 'keep trying to sync this file' 17 | pass 18 | 19 | 20 | class CloudFileNameError(CloudException): # 'stop syncing unless renamed' 21 | pass 22 | 23 | 24 | class CloudOutOfSpaceError(CloudTemporaryError): # ENOSPC 25 | pass 26 | 27 | 28 | class CloudRootMissingError(CloudException): # ENOENT, but treated differently! 29 | pass 30 | 31 | 32 | class CloudResourceModifiedError(CloudTemporaryError): # resource being updated changed since the caller last read it 33 | pass 34 | 35 | 36 | class CloudFileExistsError(CloudException): # EEXIST 37 | pass 38 | 39 | 40 | class CloudTokenError(CloudException): # 'creds don't work, refresh or reauth' 41 | pass 42 | 43 | 44 | class CloudDisconnectedError(CloudException): # 'reconnect plz' 45 | pass 46 | 47 | 48 | class CloudCursorError(CloudException): # 'cursor is invalid' 49 | pass 50 | 51 | 52 | class CloudNamespaceError(CloudException): # 'namespaces are not supported or the namespace is invalid' 53 | pass 54 | 55 | 56 | class CloudTooManyRetriesError(CloudException): # giving up on an operation after N unsuccessful attempts 57 | pass 58 | 59 | 60 | class CloudCorruptError(CloudException): # identifies a file that is not readable when downloaded 61 | pass 62 | -------------------------------------------------------------------------------- /cloudsync/tests/test_resolve_file.py: -------------------------------------------------------------------------------- 1 | # pylint: disable=missing-docstring 2 | 3 | import os 4 | import io 5 | 6 | from cloudsync.sync.manager import ResolveFile, SyncEntry, FILE, LOCAL 7 | 8 | from requests_toolbelt import MultipartEncoder 9 | from unittest.mock import MagicMock, patch 10 | import pytest 11 | 12 | 13 | @pytest.fixture(name="rfile") 14 | def _rfile(mock_provider, tmp_path): 15 | ent = SyncEntry(MagicMock(), FILE) 16 | 17 | info = mock_provider.create("/path", io.BytesIO(b"hello")) 18 | 19 | ent[LOCAL].path = info.path 20 | ent[LOCAL].oid = info.oid 21 | 22 | ent[LOCAL].temp_file = str(tmp_path / "tmp") 23 | 24 | rf = ResolveFile(ent[LOCAL], mock_provider) 25 | 26 | yield (rf, ent[LOCAL].temp_file) 27 | 28 | def test_rfile(rfile): 29 | rf, temp_file = rfile 30 | # this will raise an odd error of rf doesnt' support len(rf) 31 | 32 | MultipartEncoder( 33 | fields={'field0': 'value', 'field1': 'value', 34 | 'field2': ('filename', rf, 'text/plain')} 35 | ) 36 | 37 | assert open(temp_file).read() == "hello" 38 | 39 | def test_rfile_ops(rfile): 40 | rf, _ = rfile 41 | assert rf.read() == b"hello" 42 | assert rf.tell() == 5 43 | rf.seek(0, 0) 44 | assert rf.read() == b"hello" 45 | 46 | 47 | def test_rfile_fail(rfile): 48 | rf, temp_file = rfile 49 | 50 | with pytest.raises(ZeroDivisionError): 51 | with patch.object(rf.provider, "download", side_effect=lambda *a, **k: 4/0): 52 | rf.download() 53 | 54 | assert not os.path.exists(temp_file) 55 | 56 | 57 | def test_rfile_fail_badpath(mock_provider): 58 | ent = SyncEntry(MagicMock(), FILE) 59 | 60 | info = mock_provider.create("/path", io.BytesIO(b"hello")) 61 | 62 | ent[LOCAL].path = info.path 63 | ent[LOCAL].oid = info.oid 64 | 65 | ent[LOCAL].temp_file = str("/this/path/is/missing") 66 | 67 | rf = ResolveFile(ent[LOCAL], mock_provider) 68 | 69 | with pytest.raises(FileNotFoundError): 70 | with patch.object(rf.provider, "download", side_effect=lambda *a, **k: 4/0): 71 | rf.download() 72 | 73 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | #sphinx 2 | _build 3 | 4 | # don't check in tokens 5 | *.token 6 | 7 | # coverage annotations 8 | *,cover 9 | 10 | # backup file from sed in the deploy.sh script 11 | *.bak 12 | 13 | # Byte-compiled / optimized / DLL files 14 | __pycache__/ 15 | *.py[cod] 16 | *$py.class 17 | 18 | # python profiler 19 | prof/ 20 | 21 | # C extensions 22 | *.so 23 | 24 | # Distribution / packaging 25 | .Python 26 | build/ 27 | develop-eggs/ 28 | dist/ 29 | downloads/ 30 | eggs/ 31 | .eggs/ 32 | lib/ 33 | lib64/ 34 | parts/ 35 | sdist/ 36 | var/ 37 | wheels/ 38 | *.egg-info/ 39 | .installed.cfg 40 | *.egg 41 | MANIFEST 42 | 43 | # PyInstaller 44 | # Usually these files are written by a python script from a template 45 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 46 | *.manifest 47 | *.spec 48 | 49 | # Installer logs 50 | pip-log.txt 51 | pip-delete-this-directory.txt 52 | 53 | # Unit test / coverage reports 54 | htmlcov/ 55 | .tox/ 56 | .coverage 57 | .coverage.* 58 | .cache 59 | nosetests.xml 60 | coverage.xml 61 | *.cover 62 | .hypothesis/ 63 | .pytest_cache/ 64 | 65 | # Translations 66 | *.mo 67 | *.pot 68 | 69 | # Django stuff: 70 | *.log 71 | local_settings.py 72 | db.sqlite3 73 | 74 | # Flask stuff: 75 | instance/ 76 | .webassets-cache 77 | 78 | # Scrapy stuff: 79 | .scrapy 80 | 81 | # Sphinx documentation 82 | docs/_build/ 83 | 84 | # PyBuilder 85 | target/ 86 | 87 | # Node 88 | node_modules/ 89 | 90 | # Jupyter Notebook 91 | .ipynb_checkpoints 92 | 93 | # pyenv 94 | .python-version 95 | 96 | # celery beat schedule file 97 | celerybeat-schedule 98 | 99 | # SageMath parsed files 100 | *.sage.py 101 | 102 | # Environments 103 | .env 104 | .venv 105 | env/ 106 | venv/ 107 | ENV/ 108 | env.bak/ 109 | venv.bak/ 110 | 111 | # Spyder project settings 112 | .spyderproject 113 | .spyproject 114 | 115 | # Rope project settings 116 | .ropeproject 117 | 118 | # mkdocs documentation 119 | /site 120 | 121 | # mypy 122 | .mypy_cache/ 123 | 124 | # vim 125 | *.swp 126 | 127 | # PyCharm 128 | .idea 129 | 130 | # Visual Studio Code 131 | .vscode/ 132 | -------------------------------------------------------------------------------- /cloudsync/tests/test_cmd_debug.py: -------------------------------------------------------------------------------- 1 | # pylint: disable=missing-docstring 2 | 3 | import logging 4 | import json 5 | 6 | from unittest.mock import MagicMock 7 | 8 | import pytest 9 | 10 | from cloudsync.utils import NamedTemporaryFile 11 | from cloudsync.command.debug import DebugCmd 12 | 13 | from cloudsync import SqliteStorage, SyncState, LOCAL, FILE, IgnoreReason 14 | from .fixtures import MockProvider 15 | 16 | log = logging.getLogger() 17 | 18 | 19 | @pytest.mark.parametrize("arg_json", [True, False], ids=["json", "nojson"]) 20 | @pytest.mark.parametrize("arg_discard", [True, False], ids=["discarded", "nodiscarded"]) 21 | @pytest.mark.parametrize("arg_changed", [True, False], ids=["changed", "unchanged"]) 22 | def test_debug_mode(capsys, arg_json, arg_discard, arg_changed): 23 | providers = (MockProvider(False, False), MockProvider(False, False)) 24 | 25 | tf = NamedTemporaryFile(mode=None) 26 | try: 27 | storage = SqliteStorage(tf.name) 28 | state = SyncState(providers, storage, tag="whatever") 29 | state.update(LOCAL, FILE, path="123", oid="123", hash=b"123") 30 | state.update(LOCAL, FILE, path="456", oid="456", hash=b"456") 31 | state.update(LOCAL, FILE, path="789", oid="789", hash=b"789") 32 | state.lookup_oid(LOCAL, "456")[LOCAL].sync_path = "456" 33 | state.lookup_oid(LOCAL, "456").ignored = IgnoreReason.CONFLICT 34 | state.lookup_oid(LOCAL, "456")[LOCAL].changed = False 35 | 36 | state.storage_commit() 37 | 38 | args = MagicMock() 39 | 40 | args.state = tf.name 41 | args.json = arg_json 42 | args.discarded = arg_discard 43 | args.changed = arg_changed 44 | 45 | res = "" 46 | DebugCmd.run(args) 47 | res = capsys.readouterr().out 48 | 49 | assert "whatever" in res 50 | assert "123" in res 51 | 52 | if arg_json: 53 | log.info("json: %s", res) 54 | ret = json.loads(res) 55 | log.info("loaded: %s", ret) 56 | assert ret["whatever"] 57 | if arg_discard: 58 | if arg_changed: 59 | assert len(ret["whatever"]) == 2 60 | else: 61 | assert len(ret["whatever"]) == 3 62 | else: 63 | assert len(ret["whatever"]) == 2 64 | finally: 65 | storage.close() 66 | -------------------------------------------------------------------------------- /cloudsync/command/list.py: -------------------------------------------------------------------------------- 1 | import sys 2 | 3 | import logging 4 | import datetime 5 | 6 | from typing import Union 7 | 8 | from cloudsync.utils import debug_sig 9 | 10 | from .utils import CloudURI, SubCmd 11 | 12 | log = logging.getLogger() 13 | 14 | def sizeof_fmt(num, suffix='B'): 15 | for unit in ['', 'K', 'M', 'G', 'T', 'P', 'E', 'Z']: 16 | if abs(num) < 1024.0: 17 | return "%3.1f%s%s" % (num, unit, suffix) 18 | num /= 1024.0 19 | return "%.1f%s%s" % (num, 'Y', suffix) 20 | 21 | class ListCmd(SubCmd): 22 | """List command. Just connects and lists the files at a provider path. 23 | 24 | Useful for testing a connection. 25 | """ 26 | 27 | def __init__(self, cmds): 28 | """Command line args for list.""" 29 | 30 | super().__init__(cmds, 'list', help='List files at provider') 31 | self.parser.add_argument('prov', help='Provider uri') 32 | self.parser.add_argument('-l', "--long", help='Long listing', action='store_true') 33 | self.parser.add_argument('-n', "--namespaces", help='List namespaces', action='store_true') 34 | 35 | self.common_sync_args() 36 | 37 | @staticmethod 38 | def run(args): 39 | """Processes the 'list' command, which begins syncing two providers from the command line""" 40 | if args.quiet: 41 | log.setLevel(logging.ERROR) 42 | 43 | uri = CloudURI(args.prov) 44 | prov = uri.provider_instance(args) 45 | 46 | if args.namespaces: 47 | ns = prov.list_ns() 48 | if ns is None: 49 | print("Namspaces not supported.", sys.stderr) 50 | else: 51 | for n in prov.list_ns(): 52 | print(n) 53 | return 54 | 55 | for f in prov.listdir_path(uri.path): 56 | if args.long: 57 | print("%-40s %-20s %8s %s" % ("name", "time", "size", "oid")) 58 | print("%-40s %-20s %8s %s" % ("----", "----", "----", "---")) 59 | ftime: Union[str, float] = f.mtime or 0 60 | if not isinstance(ftime, str): 61 | mtime = datetime.datetime.fromtimestamp(ftime) 62 | ftime = datetime.datetime.strftime(mtime, "%Y%M%D %H:%M:%S") 63 | 64 | print("%-40s %-20s %8s %s" % (f.name, ftime, sizeof_fmt(f.size), debug_sig(f.oid))) 65 | else: 66 | print(f.name) 67 | 68 | cmd_class = ListCmd 69 | -------------------------------------------------------------------------------- /docs/conf.py: -------------------------------------------------------------------------------- 1 | # Configuration file for the Sphinx documentation builder. 2 | # 3 | # This file only contains a selection of the most common options. For a full 4 | # list see the documentation: 5 | # http://www.sphinx-doc.org/en/master/config 6 | 7 | # -- Path setup -------------------------------------------------------------- 8 | 9 | # If extensions (or modules to document with autodoc) are in another directory, 10 | # add these directories to sys.path here. If the directory is relative to the 11 | # documentation root, use os.path.abspath to make it absolute, like shown here. 12 | # 13 | import os 14 | import sys 15 | sys.path.insert(0, os.path.abspath('..')) 16 | 17 | 18 | # -- Project information ----------------------------------------------------- 19 | 20 | project = 'cloudsync' 21 | copyright = '2019, Atakama, LLc' 22 | author = 'Atakama, LLC' 23 | 24 | # The full version, including alpha/beta/rc tags 25 | release = os.popen('git describe --abbrev=0 --tags').read().strip() 26 | 27 | # -- General configuration --------------------------------------------------- 28 | 29 | # Add any Sphinx extension module names here, as strings. They can be 30 | # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom 31 | # ones. 32 | extensions = [ 33 | 'recommonmark', 34 | 'sphinx.ext.autodoc', 35 | 'sphinx.ext.coverage', 36 | ] 37 | 38 | autodoc_mock_imports = ['xxhash', 'arrow', 'dataclasses', 'msgpack', 'requests_oauthlib', 'pytest', '_pytest'] 39 | 40 | # Add any paths that contain templates here, relative to this directory. 41 | templates_path = ['_templates'] 42 | 43 | # List of patterns, relative to source directory, that match files and 44 | # directories to ignore when looking for source files. 45 | # This pattern also affects html_static_path and html_extra_path. 46 | exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store', 'docenv'] 47 | 48 | 49 | master_doc = 'index' 50 | 51 | # -- Options for HTML output ------------------------------------------------- 52 | 53 | # The theme to use for HTML and HTML Help pages. See the documentation for 54 | # a list of builtin themes. 55 | 56 | html_theme = 'default' 57 | html_logo = '_static/logo.png' 58 | 59 | html_theme_options = { 60 | } 61 | 62 | # Add any paths that contain custom static files (such as style sheets) here, 63 | # relative to this directory. They are copied after the builtin static files, 64 | # so a file named "default.css" will overwrite the builtin "default.css". 65 | html_static_path = ['_static'] 66 | -------------------------------------------------------------------------------- /cloudsync/command/sync.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | import logging 4 | 5 | try: 6 | import daemon 7 | except ImportError: 8 | daemon = None 9 | 10 | from cloudsync import CloudSync, SqliteStorage 11 | 12 | from .utils import CloudURI, get_providers, log, SubCmd 13 | 14 | class SyncCmd(SubCmd): 15 | """Sync subcommand: primary method of spinning up a sync.""" 16 | 17 | def __init__(self, cmds): 18 | """Command line args for sync.""" 19 | 20 | super().__init__(cmds, "sync", "Sync command") 21 | 22 | self.common_sync_args() 23 | 24 | self.parser.add_argument('src', help='Provider uri 1') 25 | self.parser.add_argument('dest', help='Provider uri 2') 26 | self.parser.add_argument('-o', '--onetime', help='Just walk/copy files once and exit', action="store_true") 27 | self.parser.add_argument('-D', '--daemon', help='Run in the background', action="store_true") 28 | default_state = os.path.expanduser("~/.config/cloudsync/state") 29 | self.parser.add_argument('-S', '--statedb', help='State file path', action="store", default=default_state) 30 | 31 | @staticmethod 32 | def run(args): 33 | """Processes the 'sync' command, which begins syncing two providers from the command line""" 34 | if args.quiet: 35 | log.setLevel(logging.ERROR) 36 | 37 | uris = (CloudURI(args.src), CloudURI(args.dest)) 38 | _provs = get_providers(args, uris) 39 | 40 | provs = (_provs[0], _provs[1]) 41 | roots = (uris[0].path, uris[1].path) 42 | provs[0].set_root(root_path=roots[0]) 43 | provs[1].set_root(root_path=roots[1]) 44 | 45 | storage = SqliteStorage(args.statedb) 46 | 47 | cs = CloudSync(provs, roots, storage=storage) 48 | 49 | # todo: providers should let cs know that cursors shouldn't be stored/used later 50 | for side, uri in enumerate(uris): 51 | if uri.method == "filesystem": 52 | cs.walk(side, uri.path) 53 | 54 | done = None 55 | if args.onetime: 56 | done = lambda: not cs.busy 57 | 58 | if args.daemon: 59 | if not daemon: 60 | raise NotImplementedError("daemon mode is not available") 61 | with daemon.DaemonContext(stderr=sys.stderr, stdout=sys.stdout): 62 | cs.start(until=done) 63 | cs.wait() 64 | else: 65 | cs.start(until=done) 66 | cs.wait() 67 | 68 | cmd_class = SyncCmd 69 | -------------------------------------------------------------------------------- /cloudsync/tests/fixtures/fake_api.py: -------------------------------------------------------------------------------- 1 | """ 2 | Fake api helpers for making mock provider apis 3 | """ 4 | 5 | import logging 6 | import threading 7 | from typing import Dict, List 8 | 9 | from cloudsync.oauth.apiserver import ApiServer, ApiError, api_route 10 | from cloudsync.oauth import OAuthProviderInfo, OAuthConfig 11 | 12 | __all__ = ["FakeApi", "fake_oauth_provider", 'ApiError', 'api_route'] 13 | 14 | log = logging.getLogger(__name__) 15 | 16 | 17 | class FakeApi(ApiServer): 18 | """ 19 | Fake api base class, inherit from this. 20 | 21 | - spins up on init 22 | - default handler for all urls returns empty dict 23 | - logs all calls 24 | """ 25 | 26 | def __init__(self): 27 | super().__init__("127.0.0.1", 0) 28 | self.calls: Dict[str, List] = {} 29 | threading.Thread(target=self.serve_forever, daemon=True).start() 30 | 31 | @api_route("/token") 32 | def __token(self, ctx, req): 33 | self.called("token", (ctx, req)) 34 | return { 35 | "token_type": "bearer", 36 | "refresh_token": "rtok", 37 | "access_token": "atok", 38 | "expires_in": 340, 39 | "scope": "yes", 40 | } 41 | 42 | @api_route(None) 43 | def __default(self, ctx, req): 44 | log.debug("url %s %s", ctx["REQUEST_METHOD"], ctx["PATH_INFO"]) 45 | self.called("default", (ctx, req)) 46 | return {} 47 | 48 | def called(self, name, args): 49 | """Call this to log calls to urls""" 50 | # todo, change to use mock call counts? 51 | log.debug("called %s", name) 52 | if name not in self.calls: 53 | self.calls[name] = [] 54 | self.calls[name].append((name, args)) 55 | 56 | 57 | def fake_oauth_provider(api_server, provider_class): 58 | """ 59 | Calling this returns an instance of the provider class with the oauth config set to the fake server. 60 | """ 61 | # TODO: shutting this down is slow, fix that 62 | # and then make this a fixture 63 | srv = api_server 64 | base_url = srv.uri() 65 | 66 | prov = provider_class(OAuthConfig(app_id="fakeappid", app_secret="fakesecret")) 67 | prov._oauth_info = OAuthProviderInfo( 68 | auth_url=base_url + "auth", 69 | token_url=base_url + "token", 70 | scopes=['whatever'], 71 | ) 72 | fake_creds = { 73 | "refresh_token": "rtok", 74 | "access_token": "atok", 75 | } 76 | prov.connect(fake_creds) 77 | return prov 78 | -------------------------------------------------------------------------------- /verok.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | from __future__ import print_function 3 | 4 | import sys 5 | import re 6 | 7 | COMPONENT_MIN = 3 8 | COMPONENT_MAX = 3 # not including allowed label 9 | COMPONENT_INT_MAX = 65535 10 | ALLOWED_LABELS = ('a', 'b', 'dev') # see pep, don't modify 11 | PRERELEASE_USES_DASH = False # 1.2.3-b5 or 1.2.3b5, pick one 12 | 13 | 14 | def verok(ver): 15 | try: 16 | return _verok(ver) 17 | except AssertionError as e: 18 | raise ValueError(str(e)) 19 | 20 | 21 | def _verok(ver): 22 | # cannot use packaging.version 23 | # because it allows malleable versions: 24 | # ie: 1.2.3b6 == 1.2.3-beta6, 1.02 == 1.2 25 | # which is not ok here 26 | 27 | ver = ver.strip() 28 | 29 | # in python land, dev tags have dots 30 | # but this crappy parser doesn't support them 31 | ver.replace(".dev", "dev") 32 | 33 | if ver[0] == "v": 34 | ver = ver[1:] 35 | 36 | tup = ver.split(".") 37 | 38 | assert len(tup) >= COMPONENT_MIN, "Not enough components" 39 | assert len(tup) <= COMPONENT_MAX, "Too many components" 40 | 41 | last = tup[len(tup) - 1] 42 | 43 | if re.search('[a-z]', last, re.I): 44 | label = None 45 | if PRERELEASE_USES_DASH and "-" not in last: 46 | assert False, "Dash required for release tag" 47 | 48 | if (not PRERELEASE_USES_DASH) and "-" in last: 49 | assert False, "No dash between version and release tag" 50 | 51 | for lab in sorted(ALLOWED_LABELS, key=lambda x: -len(x)): 52 | if PRERELEASE_USES_DASH: 53 | lab = "-" + lab 54 | 55 | if lab in last: 56 | label = lab 57 | v, relnum = last.split(label) 58 | tup = tup[:-1] + [v, relnum] 59 | break 60 | 61 | assert label, "Invalid release tag in last component '%s'" % last 62 | 63 | for i, e in enumerate(tup): 64 | try: 65 | ie = int(e) 66 | except ValueError: 67 | raise ValueError("Component '%s' invalid" % e) 68 | 69 | if str(ie) != e: 70 | raise ValueError("Component '%s' is not a simple integer" % e) 71 | 72 | if ie <= -1 or ie > COMPONENT_INT_MAX: 73 | raise ValueError("Component '%s' out of range" % ie) 74 | 75 | if tup == ['0'] * len(tup): 76 | raise ValueError("All components cannot be zero") 77 | 78 | return ver 79 | 80 | 81 | if __name__ == "__main__": 82 | try: 83 | ver = sys.argv[1] 84 | print(verok(ver)) 85 | except ValueError as e: 86 | print(e, file=sys.stderr) 87 | sys.exit(1) 88 | 89 | -------------------------------------------------------------------------------- /cloudsync/tests/test_fs_provider.py: -------------------------------------------------------------------------------- 1 | # pylint: disable=protected-access 2 | 3 | import os 4 | import logging 5 | import shutil 6 | import time 7 | from unittest.mock import patch 8 | 9 | import pytest 10 | 11 | from cloudsync.providers import FileSystemProvider 12 | from cloudsync.providers.filesystem import get_hash 13 | from watchdog import events as watchdog_events 14 | 15 | log = logging.getLogger(__name__) 16 | 17 | @pytest.fixture 18 | def fsp(): 19 | fsp = FileSystemProvider() 20 | fsp.namespace = fsp._test_namespace 21 | yield fsp 22 | shutil.rmtree(fsp.namespace_id) 23 | 24 | 25 | def test_fast_hash(fsp: FileSystemProvider, tmpdir): 26 | f = tmpdir / "file" 27 | 28 | f.write(b"hi"*2000) 29 | h1 = fsp._fast_hash_path(str(f)) 30 | mtime = f.stat().mtime 31 | 32 | #### mtime/data is the same 33 | with patch("cloudsync.providers.filesystem.get_hash", side_effect=get_hash) as m: 34 | h2 = fsp._fast_hash_path(str(f)) 35 | print("calls %s", m.mock_calls) 36 | # get-hash called once, on the subset of data only 37 | m.assert_called_once() 38 | 39 | assert h1 == h2 40 | 41 | #### mtime changed, so we re-hash 42 | os.utime(f, (time.time(), time.time())) 43 | 44 | with patch("cloudsync.providers.filesystem.get_hash", side_effect=get_hash) as m: 45 | h2 = fsp._fast_hash_path(str(f)) 46 | print("calls %s", m.mock_calls) 47 | # get-hash called twice ... re-get the fast hash, and then get the full hash 48 | if f.stat().mtime != mtime: 49 | assert len(m.mock_calls) == 2 50 | else: 51 | print("utime not supported in some vms") 52 | 53 | f.write(b"hi"*2000 + b"ho") 54 | h3 = fsp._fast_hash_path(str(f)) 55 | assert h3 != h2 56 | 57 | f.write(b"hi") 58 | with patch("cloudsync.providers.filesystem.get_hash", side_effect=get_hash) as m: 59 | h1 = fsp._fast_hash_path(str(f)) 60 | m.assert_called_once() 61 | 62 | assert h1 != h3 63 | 64 | 65 | def test_cursor_prune(fsp): 66 | fsp._event_window = 20 67 | cs1 = fsp.latest_cursor 68 | 69 | for i in range(100): 70 | ev = watchdog_events.FileCreatedEvent("/file%s" % i) 71 | fsp._on_any_event(ev) 72 | 73 | i = 0 74 | last = None 75 | cpos = None 76 | for ev in fsp.events(): 77 | if str(ev.oid) == str(fsp._test_namespace): 78 | log.debug("root oid skipped %s: event %s", i, ev.oid) 79 | continue 80 | log.debug("%s: event %s", i, ev.oid) 81 | cpos = ev.new_cursor 82 | i += 1 83 | last = ev 84 | assert last.oid == "/file99" 85 | assert fsp.current_cursor == cpos 86 | 87 | assert i == 100 88 | 89 | fsp.current_cursor = cs1 90 | 91 | i = 0 92 | for ev in fsp.events(): 93 | cpos = ev.new_cursor 94 | last = ev 95 | i += 1 96 | 97 | assert last.oid == "/file99" 98 | assert fsp.current_cursor == cpos 99 | 100 | assert i == 20 101 | -------------------------------------------------------------------------------- /cloudsync/tests/fixtures/mock_storage.py: -------------------------------------------------------------------------------- 1 | from threading import Lock 2 | from typing import Dict, Any, Tuple, Optional, Union, overload 3 | import logging 4 | from cloudsync import Storage, LOCAL, REMOTE 5 | 6 | log = logging.getLogger(__name__) 7 | 8 | 9 | class MockStorage(Storage): # Does not actually persist the data... but it's just a mock 10 | top_lock = Lock() 11 | lock_dict: Dict[str, Lock] = dict() 12 | 13 | def __init__(self, storage_dict: Dict[str, Dict[int, bytes]]): 14 | self.storage_dict = storage_dict 15 | self.cursor: int = 0 # the next eid 16 | 17 | def _get_internal_storage(self, tag: str) -> Tuple[Lock, Dict[int, bytes]]: 18 | with self.top_lock: 19 | lock: Lock = self.lock_dict.setdefault(tag, Lock()) 20 | return lock, self.storage_dict.setdefault(tag, dict()) 21 | 22 | def create(self, tag: str, serialization: bytes) -> Any: 23 | lock, storage = self._get_internal_storage(tag) 24 | with lock: 25 | current_index = self.cursor 26 | self.cursor += 1 27 | storage[current_index] = serialization 28 | return current_index 29 | 30 | def update(self, tag: str, serialization: bytes, eid: Any): 31 | lock, storage = self._get_internal_storage(tag) 32 | with lock: 33 | if eid not in storage: 34 | raise ValueError("id %s doesn't exist" % eid) 35 | storage[eid] = serialization 36 | return 1 37 | 38 | def delete(self, tag: str, eid: Any): 39 | lock, storage = self._get_internal_storage(tag) 40 | log.debug("deleting eid%s", eid) 41 | with lock: 42 | if eid not in storage: 43 | log.debug("ignoring delete: id %s doesn't exist", eid) 44 | return 45 | del storage[eid] 46 | 47 | @overload 48 | def read_all(self) -> Dict[str, Dict[Any, bytes]]: 49 | ... 50 | 51 | @overload 52 | def read_all(self, tag: str) -> Dict[Any, bytes]: 53 | ... 54 | 55 | def read_all(self, tag: str = None): 56 | if tag is not None: 57 | lock, storage = self._get_internal_storage(tag) 58 | with lock: 59 | ret: Dict[Any, bytes] = storage.copy() 60 | return ret 61 | else: 62 | ret_all: Dict[str, Dict[Any, bytes]] = {} 63 | with self.top_lock: 64 | tags = self.storage_dict.keys() 65 | for t in tags: 66 | lock, storage = self._get_internal_storage(t) 67 | for k, v in storage.items(): 68 | if t not in ret_all: 69 | ret_all[t] = {} 70 | ret_all[t][k] = v 71 | return ret_all 72 | 73 | def read(self, tag: str, eid: Any) -> Optional[bytes]: 74 | lock, storage = self._get_internal_storage(tag) 75 | with lock: 76 | if eid not in storage: 77 | raise ValueError("id %s doesn't exist" % eid) 78 | return storage[eid] 79 | 80 | def close(self): #pylint: disable=no-self-use 81 | pass 82 | 83 | -------------------------------------------------------------------------------- /cloudsync/tests/test_oauth_redir_server.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import time 3 | import threading 4 | import requests 5 | from unittest.mock import Mock, patch 6 | 7 | from cloudsync.oauth import OAuthRedirServer 8 | from cloudsync.oauth.apiserver import ApiServer 9 | from cloudsync.tests import RunUntilHelper 10 | 11 | log = logging.getLogger(__name__) 12 | 13 | def resp_gen(success: bool, error_msg: str) -> str: 14 | time.sleep(2) # We are making a biiiiiig response that takes way too long 15 | if success: 16 | return f'Success' 17 | else: 18 | return f'Failure: {error_msg}' 19 | 20 | 21 | shutdown_signal = threading.Event() 22 | server_close_calls = 0 23 | 24 | 25 | class EventApiServer(ApiServer): 26 | def __init__(self, *args, **kwargs): 27 | # This signal ensures that the test function only starts shutting down the oauth server after the request is 28 | # received by the OAuth server 29 | super().__init__(*args, **kwargs) 30 | 31 | def __call__(self, *args, **kwargs): 32 | shutdown_signal.set() 33 | return super().__call__(*args, **kwargs) 34 | 35 | def server_close(self): 36 | global server_close_calls 37 | server_close_calls += 1 38 | super().server_close() 39 | 40 | 41 | @patch('cloudsync.oauth.redir_server.ApiServer', EventApiServer) 42 | def test_oauth_redir_server(): 43 | srv = OAuthRedirServer(html_generator=resp_gen) 44 | on_success = Mock() 45 | on_failure = Mock() 46 | 47 | global server_close_calls 48 | server_close_calls = 0 49 | 50 | # should have no effect on server that is not yet running 51 | srv.server_close() 52 | assert server_close_calls == 0 53 | 54 | srv.run(on_success=on_success, on_failure=on_failure) 55 | port = srv.port() 56 | 57 | # should have no effect on server that is running but has not been shutdown yet 58 | srv.server_close() 59 | assert server_close_calls == 0 60 | 61 | def send_req(): 62 | res = requests.get(url=f'http://127.0.0.1:{port}/auth/', params={ 63 | 'state': ['-T_wMR7edzQAc8i3UiH3Fg=='], 64 | 'error_description': ['Long error descrption'], 65 | 'error': ['badman'] 66 | }) 67 | assert res.status_code == 200 68 | 69 | t = threading.Thread(target=send_req, daemon=True) 70 | t.start() 71 | 72 | shutdown_signal.wait() 73 | srv.shutdown() 74 | srv.server_close() 75 | assert server_close_calls == 1 76 | 77 | t.join(4) 78 | assert not t.is_alive() 79 | 80 | on_success.assert_not_called() 81 | on_failure.assert_called_once() 82 | 83 | 84 | def test_server_close(): 85 | api = ApiServer("127.0.0.1", 0) 86 | with patch.object(api, "_ApiServer__server") as server: 87 | # not started yet 88 | api.server_close() 89 | server.server_close.assert_not_called() 90 | 91 | # started, but not shutdown yet 92 | t = threading.Thread(target=api.serve_forever, daemon=True) 93 | t.start() 94 | RunUntilHelper.wait_until(lambda: api._ApiServer__started) # type: ignore 95 | api.server_close() 96 | server.server_close.assert_not_called() 97 | 98 | # started and shutdown 99 | api.shutdown() 100 | api.server_close() 101 | server.server_close.assert_called_once() 102 | -------------------------------------------------------------------------------- /cloudsync/tests/fixtures/mock_provider.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from cloudsync.registry import register_provider 4 | from cloudsync.providers.mock import MockProvider 5 | 6 | 7 | def mock_provider_instance(*args, **kws): 8 | prov = MockProvider(*args, **kws) 9 | prov.connect({"key": "val"}) 10 | return prov 11 | 12 | 13 | @pytest.fixture(name="mock_provider", params=[(False, True), (True, True)], ids=["mock_oid_cs", "mock_path_cs"]) 14 | def mock_provider_fixture(request): 15 | return mock_provider_instance(*request.param) 16 | 17 | 18 | @pytest.fixture(params=[(False, True), (True, True)], ids=["mock_oid_cs", "mock_path_cs"]) 19 | def mock_provider_generator(request): 20 | return lambda oid_is_path=None, case_sensitive=None: \ 21 | mock_provider_instance( 22 | request.param[0] if oid_is_path is None else oid_is_path, 23 | request.param[1] if case_sensitive is None else case_sensitive) 24 | 25 | 26 | def mock_provider_tuple_instance(local, remote): 27 | return ( 28 | mock_provider_instance(oid_is_path=local[0], case_sensitive=local[1], filter_events=local[2]), 29 | mock_provider_instance(oid_is_path=remote[0], case_sensitive=remote[1], filter_events=remote[2]) 30 | ) 31 | 32 | 33 | # parameterization: 34 | # - (oid-cs-unfiltered, oid-cs-unfiltered) 35 | # - (path-cs-unfiltered, oid-cs-filtered) 36 | @pytest.fixture(params=[((False, True, False), (False, True, False)), ((True, True, False), (False, True, True))], 37 | ids=["mock_oid_cs_unfiltered", "mock_path_cs_filtered"]) 38 | def mock_provider_tuple(request): 39 | return mock_provider_tuple_instance(request.param[0], request.param[1]) 40 | 41 | 42 | # parameterization: 43 | # - (oid-ci-unfiltered, oid-ci-unfiltered) 44 | # - (path-ci-unfiltered, oid-ci-filtered) 45 | @pytest.fixture(params=[((False, False, False), (False, False, False)), ((True, False, False), (False, False, True))], 46 | ids=["mock_oid_cs_unfiltered", "mock_path_cs_filtered"]) 47 | def mock_provider_tuple_ci(request): 48 | return mock_provider_tuple_instance(request.param[0], request.param[1]) 49 | 50 | 51 | @pytest.fixture 52 | def mock_provider_creator(): 53 | return mock_provider_instance 54 | 55 | 56 | # one of two default providers for test_provider.py tests 57 | class MockPathCs(MockProvider): 58 | name = "mock_path_cs" 59 | 60 | def __init__(self): 61 | super().__init__(oid_is_path=True, case_sensitive=True) 62 | 63 | 64 | class MockPathCi(MockProvider): 65 | name = "mock_path_ci" 66 | 67 | def __init__(self): 68 | super().__init__(oid_is_path=True, case_sensitive=False) 69 | 70 | 71 | class MockOidCs(MockProvider): 72 | name = "mock_oid_cs" 73 | 74 | def __init__(self): 75 | super().__init__(oid_is_path=False, case_sensitive=True) 76 | 77 | 78 | # one of two default providers for test_provider.py tests 79 | class MockOidCi(MockProvider): 80 | name = "mock_oid_ci" 81 | 82 | def __init__(self): 83 | super().__init__(oid_is_path=False, case_sensitive=False, use_ns=False) 84 | 85 | 86 | class MockOidCiNs(MockProvider): 87 | name = "mock_oid_ci_ns" 88 | 89 | def __init__(self): 90 | super().__init__(oid_is_path=False, case_sensitive=False, use_ns=True) 91 | 92 | 93 | register_provider(MockPathCs) 94 | register_provider(MockPathCi) 95 | register_provider(MockOidCs) 96 | register_provider(MockOidCi) 97 | register_provider(MockOidCiNs) 98 | -------------------------------------------------------------------------------- /cloudsync/command/debug.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import json 3 | from typing import Iterable 4 | 5 | from unittest.mock import MagicMock 6 | 7 | import msgpack 8 | 9 | from cloudsync.sync.sqlite_storage import SqliteStorage 10 | from cloudsync.sync.state import SyncEntry, SyncState 11 | 12 | from .utils import SubCmd 13 | 14 | log = logging.getLogger("cloudsync.command") 15 | 16 | class DebugCmd(SubCmd): 17 | """Debug subcommand""" 18 | 19 | def __init__(self, cmds): 20 | super().__init__(cmds, 'debug', help='Debug commands') 21 | self.parser.add_argument('-s', '--state', help='Debug state file', action="store") 22 | self.parser.add_argument('-c', '--changed', help='Only changed records', action="store_true") 23 | self.parser.add_argument('-d', '--discarded', help='Show discarded records', action="store_true") 24 | self.parser.add_argument('-j', '--json', help='Output as json', action="store_true") 25 | 26 | @staticmethod 27 | def run(args): 28 | """Implements the 'debug' command, mostly for diagnosing state databases""" 29 | if args.state: 30 | fake_state = MagicMock() 31 | fake_state._pretty_time = 0 # pylint: disable=protected-access 32 | 33 | if args.json: 34 | print("{") 35 | 36 | store = SqliteStorage(args.state) 37 | tags = set() 38 | for tag, _ in store.read_all().items(): 39 | tags.add(tag) 40 | 41 | first = True 42 | for tag in tags: 43 | logging.getLogger().setLevel(logging.CRITICAL) 44 | ss = SyncState((MagicMock(), MagicMock()), store, tag) 45 | logging.getLogger().setLevel(logging.INFO) 46 | if args.json: 47 | output_json_for_tag(args, ss, tag, first) 48 | else: 49 | if ss.get_all(discarded=args.discarded): 50 | print("****", tag, "****") 51 | print(ss.pretty_print()) 52 | 53 | if args.json: 54 | print("]") 55 | first = False 56 | 57 | if args.json: 58 | print("}") 59 | 60 | cmd_class = DebugCmd 61 | 62 | def to_jsonable(d): 63 | """Make something jsonable, for pretty printing reasons.""" 64 | r = d 65 | if type(d) is dict: 66 | r = {} 67 | for k, v in d.items(): 68 | r[k] = to_jsonable(v) 69 | elif type(d) is list: 70 | r = [] 71 | for v in d: 72 | r.append(to_jsonable(v)) 73 | elif type(d) is bytes: 74 | r = "bytes:" + d.hex() 75 | return r 76 | 77 | 78 | def output_json_for_tag(args, ss, tag, first): 79 | """Outputs json for a given state tag, for debugging only""" 80 | stuff: Iterable[SyncEntry] 81 | if args.changed: 82 | stuff = ss.changes 83 | else: 84 | stuff = ss.get_all(discarded=args.discarded) 85 | 86 | if not stuff: 87 | return 88 | 89 | print("," if not first else "", '"%s":[' % tag) 90 | ent_comma = "" 91 | se: SyncEntry 92 | for se in stuff: 93 | if not args.discarded: 94 | if se.is_conflicted or se.is_discarded: 95 | continue 96 | ser = se.serialize() 97 | d = msgpack.loads(ser, use_list=True, raw=False) 98 | 99 | d = to_jsonable(d) 100 | print(ent_comma, json.dumps(d)) 101 | ent_comma = "," 102 | 103 | -------------------------------------------------------------------------------- /cloudsync/tests/test_longpoll.py: -------------------------------------------------------------------------------- 1 | import time 2 | import threading 3 | import logging 4 | import pytest 5 | 6 | from cloudsync.long_poll import LongPollManager 7 | 8 | log = logging.getLogger(__name__) 9 | 10 | 11 | def wait_for(f, timeout): 12 | t = threading.Thread(target=f, daemon=True) 13 | t.start() 14 | t.join(timeout=timeout) 15 | if t.is_alive(): 16 | raise TimeoutError() 17 | 18 | 19 | class lpfix: 20 | """Longpoll fixture""" 21 | def __init__(self, *, q=None): 22 | self.orig_q = q or [1, 2, 3] 23 | self.q = self.orig_q.copy() 24 | self.has_stuff = True 25 | self.ready = threading.Event() 26 | self.raised = 0 27 | self.lpcalls = 0 28 | 29 | def sp(self): 30 | log.debug("queue: %s", self.q) 31 | for i in self.q: 32 | yield i 33 | self.q = [] 34 | 35 | def lp(self, timeout): 36 | self.lpcalls += 1 37 | self.ready.wait(timeout=timeout) 38 | if isinstance(self.has_stuff, Exception): 39 | self.raised += 1 40 | raise self.has_stuff 41 | return self.has_stuff 42 | 43 | 44 | @pytest.mark.parametrize("uses_cursor", [0, 1]) 45 | def test_lpsimple(uses_cursor): 46 | """Basic lp test""" 47 | lp = lpfix() 48 | 49 | man = LongPollManager(lp.sp, lp.lp, uses_cursor=uses_cursor) 50 | 51 | man.start() 52 | lp.ready.set() 53 | assert list(man()) == lp.orig_q 54 | assert lp.q == [] 55 | man.stop() 56 | 57 | 58 | @pytest.mark.parametrize("uses_cursor", [0, 1]) 59 | def test_lpwait(uses_cursor): 60 | """Wait forever cause False, exception is try it""" 61 | lp = lpfix() 62 | 63 | man = LongPollManager(lp.sp, lp.lp, uses_cursor=uses_cursor) 64 | man.long_poll_timeout = 0.01 65 | 66 | # this prevents it from ever returning 67 | lp.has_stuff = False 68 | 69 | man.start() 70 | 71 | if uses_cursor: 72 | # this should time-out, because it never has stuff 73 | with pytest.raises(TimeoutError): 74 | wait_for(lambda: log.debug("to: %s", list(man())), timeout=0.1) 75 | else: 76 | # we dont trust the return value from longpoll without cursors 77 | assert list(man()) == lp.orig_q 78 | man.stop() 79 | 80 | 81 | def test_cursor_mode(): 82 | """Wait forever cause False, exception is try it""" 83 | uses_cursor = 1 84 | 85 | lp = lpfix() 86 | 87 | if uses_cursor: 88 | # short polling is very very slow 89 | lp.sp = lambda: time.sleep(10) # type: ignore 90 | 91 | man = LongPollManager(lp.sp, lp.lp, uses_cursor=uses_cursor) 92 | man.long_poll_timeout = 1 93 | 94 | lp.has_stuff = True 95 | 96 | man.start() 97 | 98 | if uses_cursor: 99 | log.info("shouldn't long poll more than once, because short pollint takes too long") 100 | with pytest.raises(TimeoutError): 101 | wait_for(lambda: log.debug("to: %s", list(man())), timeout=0.1) 102 | assert lp.lpcalls == 1 103 | man.stop() 104 | 105 | 106 | @pytest.mark.parametrize("uses_cursor", [0, 1]) 107 | def test_lpex(uses_cursor): 108 | """Wait forever cause False, exception is try it""" 109 | lp = lpfix() 110 | 111 | man = LongPollManager(lp.sp, lp.lp, uses_cursor=uses_cursor) 112 | man.long_poll_timeout = 0.01 113 | man.start() 114 | 115 | lp.has_stuff = Exception("exceptions cause lp to be conservative") # type: ignore 116 | 117 | assert list(man()) == lp.orig_q 118 | assert lp.raised > 0 119 | 120 | with pytest.raises(TimeoutError): 121 | # give the runnable system time to process the exception 122 | man.wait(timeout=0.1) 123 | 124 | # lpman is backing off 125 | assert man.in_backoff > 0 126 | 127 | man.stop() 128 | -------------------------------------------------------------------------------- /cloudsync/tests/test_cmd_list.py: -------------------------------------------------------------------------------- 1 | # pylint: disable=missing-docstring 2 | 3 | import logging 4 | from unittest.mock import MagicMock, patch 5 | 6 | import pytest 7 | 8 | from cloudsync.tests.fixtures import MockProvider 9 | from cloudsync import CloudNamespaceError 10 | import cloudsync.command.list as csync 11 | 12 | log = logging.getLogger(__name__) 13 | 14 | @pytest.mark.parametrize("long", [True, False]) 15 | def test_list_basic(caplog, long): 16 | args = MagicMock() 17 | 18 | logging.getLogger().setLevel(logging.DEBUG) 19 | args.prov = "mock_oid_cs:/" 20 | args.quiet = False # log less, don't prompt for auth, get tokens from files or other commands 21 | args.verbose = True # log a lot (overrides quiet) 22 | args.daemon = False # don't keep running after i quit 23 | args.long = long 24 | args.namespaces = False 25 | 26 | with patch.object(MockProvider, "listdir_path") as mock: 27 | csync.ListCmd.run(args) 28 | mock.assert_called_once() 29 | 30 | logs = caplog.record_tuples 31 | 32 | assert any("connect mock" in t[2].lower() for t in logs) 33 | 34 | def test_list_ns(caplog, capsys): 35 | args = MagicMock() 36 | 37 | args.prov = "mock_oid_ci_ns@ns1:/" 38 | args.quiet = False # log less, don't prompt for auth, get tokens from files or other commands 39 | args.verbose = True # log a lot (overrides quiet) 40 | args.daemon = False # don't keep running after i quit 41 | args.long = False 42 | args.namespaces = True 43 | 44 | csync.ListCmd.run(args) 45 | 46 | logs = caplog.record_tuples 47 | out = capsys.readouterr().out 48 | 49 | log.debug("out %s", out) 50 | 51 | assert any("connect mock" in t[2].lower() for t in logs) 52 | 53 | # mock prov lists ns1, ns2 as namespaces 54 | 55 | assert "ns1" in out 56 | assert "ns2" in out 57 | 58 | 59 | def test_list_badns(): 60 | args = MagicMock() 61 | 62 | args.prov = "mock_oid_ci_ns@namespace:/" 63 | args.quiet = False # log less, don't prompt for auth, get tokens from files or other commands 64 | args.verbose = True # log a lot (overrides quiet) 65 | args.daemon = False # don't keep running after i quit 66 | args.long = False 67 | args.namespaces = True 68 | 69 | with pytest.raises(CloudNamespaceError): 70 | csync.ListCmd.run(args) 71 | 72 | def test_list_err(tmpdir): 73 | args = MagicMock() 74 | 75 | args.prov = "gdrive:/" 76 | args.quiet = False # log less, don't prompt for auth, get tokens from files or other commands 77 | args.verbose = True # log a lot (overrides quiet) 78 | args.daemon = False # don't keep running after i quit 79 | args.long = False 80 | args.namespaces = False 81 | 82 | badj = tmpdir / "bad" 83 | with open(badj, "w") as f: 84 | f.write("bad stuff") 85 | args.config = str(badj) 86 | with pytest.raises(ValueError): 87 | csync.ListCmd.run(args) 88 | args.config = "file not found" 89 | args.creds = str(badj) 90 | with pytest.raises(ValueError): 91 | csync.ListCmd.run(args) 92 | args.creds = "file not found" 93 | args.prov = "-dfjfhj::fdsf:/" 94 | with pytest.raises(ValueError): 95 | csync.ListCmd.run(args) 96 | 97 | 98 | @pytest.mark.parametrize("long", [True, False]) 99 | @pytest.mark.parametrize("fsname", ["file", "filesystem"]) 100 | def test_list_fs(capsys, long, fsname, tmpdir): 101 | args = MagicMock() 102 | 103 | # make one long to test size formatting code path 104 | (tmpdir / "foo").write("yo" * 2048) 105 | (tmpdir / "bar").write("yo") 106 | 107 | args.prov = "%s:%s" % (fsname, tmpdir) 108 | args.quiet = False # log less, don't prompt for auth, get tokens from files or other commands 109 | args.verbose = True # log a lot (overrides quiet) 110 | args.daemon = False # don't keep running after i quit 111 | args.long = long 112 | args.namespaces = False 113 | 114 | csync.ListCmd.run(args) 115 | 116 | out = capsys.readouterr().out 117 | 118 | assert "foo" in out 119 | assert "bar" in out 120 | 121 | # todo: fs prov shows size, mtime reliably! 122 | # if long: 123 | # assert "2K" in out 124 | -------------------------------------------------------------------------------- /cloudsync/tests/test_utils.py: -------------------------------------------------------------------------------- 1 | import os 2 | import pytest 3 | import time 4 | import logging 5 | 6 | from cloudsync.utils import debug_args, memoize, NamedTemporaryFile, debug_sig 7 | from cloudsync import OAUTH_CONFIG, generic_oauth_config 8 | from .fixtures import RunUntilHelper 9 | 10 | log = logging.getLogger(__name__) 11 | 12 | 13 | def test_debug_args(): 14 | res = debug_args([1, 2, 3]) 15 | assert res == [1, 2, 3] 16 | res = debug_args([1, 2, 3], {1: 2}, True) 17 | assert res == ([1, 2, 3], {1: 2}, True) 18 | res = debug_args({"k": b'0'*100}) 19 | assert res == {"k": b'0'*61 + b'...'} 20 | res = debug_args("str", {"k": b'0'*100}) 21 | assert res == ("str", {"k": b'0'*61 + b'...'}) 22 | 23 | 24 | def test_memoize1(): 25 | func = lambda *c: (c, os.urandom(32)) 26 | cached = memoize(func, 60) 27 | 28 | a = cached() 29 | assert cached.get() == a 30 | b = cached() 31 | # same vals 32 | assert a == b 33 | 34 | # clear test 35 | cached.clear() 36 | b = cached() 37 | assert a != b 38 | 39 | # with param test 40 | p1 = cached(32) 41 | 42 | assert p1[0] == (32,) 43 | assert p1 != b and p1 != a 44 | p2 = cached(32) 45 | p3 = cached(33) 46 | 47 | assert p1 == p2 48 | assert p3[0] == (33,) 49 | 50 | # clears z only 51 | cached.clear(32) 52 | p4 = cached(33) 53 | 54 | assert p3 == p4 55 | 56 | # zero param is still ok 57 | a = cached() 58 | assert a == b 59 | 60 | b = cached.get() 61 | assert a == b 62 | 63 | cached.clear() 64 | assert cached.get() is None 65 | 66 | cached.set(3, b=4, _value=44) 67 | assert cached.get(3, b=4) == 44 68 | assert cached(3, b=4) == 44 69 | 70 | 71 | def test_memoize2(): 72 | @memoize 73 | def fun(a): 74 | return a, os.urandom(32) 75 | 76 | x = fun(1) 77 | y = fun(1) 78 | assert x == y 79 | z = fun(2) 80 | assert z != x 81 | 82 | @memoize(expire_secs=3) 83 | def fun2(a): 84 | return a, os.urandom(32) 85 | 86 | x = fun2(1) 87 | y = fun2(1) 88 | assert x == y 89 | z = fun2(2) 90 | assert z != x 91 | 92 | 93 | def test_memoize3(): 94 | class Cls: 95 | @memoize 96 | def fun(self, a): 97 | return a, os.urandom(32) 98 | 99 | @memoize 100 | def fun2(self): 101 | return os.urandom(32) 102 | 103 | # different self's 104 | x = Cls().fun(1) 105 | y = Cls().fun(1) 106 | assert x != y 107 | 108 | c = Cls() 109 | x = c.fun(1) 110 | assert c.fun.get(1) == x 111 | assert c.fun.get(1) 112 | y = c.fun(1) 113 | assert x == y 114 | z = c.fun(2) 115 | assert z != x 116 | 117 | log.debug("no args test") 118 | m = c.fun2() 119 | assert m 120 | assert c.fun2.get() == m 121 | 122 | 123 | def test_alt_ntp(): 124 | x = NamedTemporaryFile() 125 | x.write(b"foo") 126 | x.seek(0) 127 | assert x.read() == b"foo" 128 | name = x.name 129 | assert os.path.exists(name) 130 | del x 131 | assert not os.path.exists(name) 132 | 133 | 134 | def test_alt_ntp_win(): 135 | x = NamedTemporaryFile(mode=None) 136 | name = x.name 137 | assert not os.path.exists(name) 138 | 139 | 140 | def test_alt_ntp_clos(): 141 | x = NamedTemporaryFile(mode=None) 142 | name = x.name 143 | with open(x.name, "w") as f: 144 | f.write("hi") 145 | del x 146 | assert not os.path.exists(name) 147 | 148 | 149 | def test_already_gone(): 150 | x = NamedTemporaryFile(mode=None) 151 | del x 152 | 153 | 154 | def test_debug_sig(): 155 | test_sig = debug_sig("Hello, world!") 156 | assert test_sig == '9YM' # debug_sig should be deterministic across runs 157 | 158 | 159 | def test_wait_until(): 160 | def found_callable_too_long(): 161 | time.sleep(0.3) 162 | return False 163 | 164 | with pytest.raises(TimeoutError): 165 | RunUntilHelper.wait_until(until=found_callable_too_long, timeout=0.1) 166 | 167 | 168 | def test_generic_oauth_config(): 169 | provider = list(OAUTH_CONFIG.keys())[0] 170 | test_config = generic_oauth_config(provider) 171 | assert test_config.app_id == OAUTH_CONFIG[provider]['id'] 172 | assert test_config.app_secret == OAUTH_CONFIG[provider]['secret'] 173 | -------------------------------------------------------------------------------- /cloudsync/long_poll.py: -------------------------------------------------------------------------------- 1 | import time 2 | import threading 3 | import logging 4 | from typing import Callable, Generator 5 | from cloudsync.runnable import Runnable 6 | from cloudsync.event import Event 7 | log = logging.getLogger(__name__) 8 | 9 | 10 | class LongPollManager(Runnable): 11 | """ 12 | Class for helping providers with long poll support avoid potential threading issues 13 | arising from long running api requests. 14 | """ 15 | long_poll_timeout = 120.0 16 | 17 | def __init__(self, short_poll: Callable[[], Generator[Event, None, None]], 18 | long_poll: Callable[[float], bool], 19 | short_poll_only=False, 20 | uses_cursor=False): 21 | self.__provider_events_pending = threading.Event() 22 | self.short_poll = short_poll 23 | self.long_poll = long_poll 24 | self.short_poll_only = short_poll_only 25 | self.min_backoff = 1.0 26 | self.max_backoff = 15 27 | self.last_set = time.monotonic() 28 | self.uses_cursor = uses_cursor 29 | self.got_events = threading.Event() 30 | log.debug("EVSET: set got_events") 31 | self.got_events.set() 32 | 33 | def __call__(self) -> Generator[Event, None, None]: 34 | log.debug("waiting for events") 35 | if not self.short_poll_only: 36 | self.__provider_events_pending.wait() 37 | log.debug("done waiting for events") 38 | self.__provider_events_pending.clear() 39 | has_items = True 40 | while has_items: 41 | has_items = False 42 | log.debug("about to short poll") 43 | generator = self.short_poll() 44 | if generator is not None: 45 | for event in self.short_poll(): 46 | log.debug("short poll returned an event, yielding %s", event) 47 | has_items = True 48 | yield event 49 | log.debug("EVSET: set got_events") 50 | self.got_events.set() 51 | 52 | def unblock(self): 53 | # clear all events... let stuff loop around once 54 | # usually done in response to rewinding the cursor, for tests 55 | self.__provider_events_pending.set() 56 | self.got_events.set() 57 | self.last_set = time.monotonic() 58 | 59 | def done(self): 60 | self.unblock() 61 | 62 | def do(self): # this is really "do_once" 63 | if self.short_poll_only: 64 | self.__provider_events_pending.set() 65 | self.interruptable_sleep(1) 66 | else: 67 | try: 68 | log.debug("about to long poll") 69 | # care should be taken to return "true" on timeouts for providers, like box that don't use cursors 70 | if self.uses_cursor: 71 | log.debug("wait for got_events") 72 | self.got_events.wait(timeout=self.long_poll_timeout) 73 | if self.stopped: 74 | return 75 | self.got_events.clear() 76 | assert not self.got_events.is_set() 77 | 78 | # if a cursor is not used, we never trust the results 79 | if self.long_poll(self.long_poll_timeout) or not self.uses_cursor: 80 | log.debug("LPSET: long poll finished, about to check events") 81 | self.__provider_events_pending.set() 82 | self.last_set = time.monotonic() 83 | log.debug("events check complete") 84 | else: 85 | log.debug("long poll finished, not checking events") 86 | 87 | except Exception as e: 88 | if self.last_set and (time.monotonic() > (self.last_set + self.long_poll_timeout)): 89 | # if we're getting exceptions from long_poll, still trigger a short poll after timeout seconds 90 | log.debug("LPSET: long poll exceptions, about to check events") 91 | self.__provider_events_pending.set() 92 | self.last_set = time.monotonic() 93 | log.exception('Unhandled exception during long poll %s', e) 94 | Runnable.backoff() 95 | 96 | def stop(self, forever=True, wait=False): 97 | # Don't wait for do() to finish, could wait for up to long_poll_timeout seconds 98 | self.unblock() 99 | super().stop(forever=forever, wait=wait) 100 | -------------------------------------------------------------------------------- /cloudsync/tests/fixtures/util.py: -------------------------------------------------------------------------------- 1 | from typing import NamedTuple, Union, Sequence, List, cast, Any, Tuple, Callable 2 | import time 3 | import logging 4 | 5 | from cloudsync.provider import Provider 6 | from cloudsync.runnable import time_helper 7 | from cloudsync import CloudFileNotFoundError, CloudDisconnectedError 8 | 9 | 10 | log = logging.getLogger(__name__) 11 | 12 | log.setLevel(logging.INFO) 13 | 14 | TIMEOUT = 4 15 | 16 | WaitForArg = Union[Tuple[int, str], 'WaitFor'] 17 | 18 | 19 | class WaitFor(NamedTuple): 20 | side: int = None 21 | path: str = None 22 | hash: bytes = None 23 | oid: str = None 24 | exists: bool = True 25 | 26 | @staticmethod 27 | def is_found(files: Sequence[WaitForArg], providers: Tuple[Provider, Provider], errs: List[str]): 28 | ok = True 29 | 30 | errs.clear() 31 | for f in files: 32 | if type(f) is tuple: 33 | info = WaitFor(side=f[0], path=f[1]) 34 | else: 35 | info = cast(WaitFor, f) 36 | 37 | try: 38 | other_info = providers[info.side].info_path(info.path) 39 | except CloudFileNotFoundError: 40 | other_info = None 41 | except CloudDisconnectedError: 42 | other_info = None 43 | 44 | if other_info is None: 45 | if info.exists is False: 46 | log.debug("waiting not exists %s", info.path) 47 | continue 48 | log.debug("waiting exists %s", info.path) 49 | errs.append("file not found %s" % info.path) 50 | ok = False 51 | break 52 | 53 | if info.exists is False: 54 | errs.append("file exists %s" % info.path) 55 | ok = False 56 | break 57 | 58 | if info.hash and info.hash != other_info.hash: 59 | log.debug("waiting hash %s", info.path) 60 | errs.append("mismatch hash %s" % info.path) 61 | ok = False 62 | break 63 | 64 | return ok 65 | 66 | 67 | class RunUntilHelper: 68 | default_timeout = 10 # seconds 69 | 70 | def run_until(self: Any, until, timeout=None, poll_time=0.1, exc=None): 71 | if timeout is None: 72 | timeout = self.default_timeout 73 | 74 | if not exc: 75 | exc = TimeoutError("Timed out waiting for %s" % str(until)) 76 | while not until(): 77 | timeout -= poll_time 78 | if timeout <= 0: 79 | log.debug("Cond %s returned False after waiting %.2f", until, timeout) 80 | raise exc 81 | self.do() 82 | time.sleep(poll_time) 83 | 84 | def run_until_clean(self: Any, timeout=TIMEOUT): 85 | # self.run(until=lambda: not self.busy, timeout=1) # older, SLIGHTLY slower version 86 | start = time.monotonic() 87 | busy = True # make sure we run the loop once -- necessary for smartsync tests 88 | while busy: 89 | if time.monotonic() - start > timeout: 90 | raise TimeoutError() 91 | self.do() 92 | busy = self.busy 93 | 94 | def run_until_found(self: Any, *files: WaitForArg, timeout=TIMEOUT): 95 | log.debug("running until found") 96 | 97 | errs: List[str] = [] 98 | found = lambda: WaitFor.is_found(files, self.providers, errs) 99 | 100 | self.run(timeout=timeout, until=found) 101 | 102 | if not found(): 103 | raise TimeoutError("timed out while waiting: %s" % errs) 104 | 105 | @classmethod 106 | def wait_until(cls, until, timeout=None, poll_time=0.1, exc=None): 107 | if timeout is None: 108 | timeout = cls.default_timeout 109 | 110 | if not exc: 111 | exc = TimeoutError("Timed out waiting for %s" % str(until)) 112 | while not until(): 113 | timeout -= poll_time 114 | if timeout <= 0: 115 | log.debug("Cond %s returned False after waiting %.2f", until, timeout) 116 | raise exc 117 | time.sleep(poll_time) 118 | 119 | def wait_until_found(self: Any, *files: WaitForArg, timeout=TIMEOUT): 120 | log.debug("waiting until found") 121 | 122 | errs: List[str] = [] 123 | found = lambda: WaitFor.is_found(files, self.providers, errs) 124 | 125 | try: 126 | self.wait_until(found) 127 | except TimeoutError: 128 | raise TimeoutError("timed out while waiting: %s" % errs) 129 | 130 | -------------------------------------------------------------------------------- /cloudsync/tests/test_cmd_sync.py: -------------------------------------------------------------------------------- 1 | # pylint: disable=protected-access, missing-docstring 2 | 3 | import os 4 | import sys 5 | import logging 6 | import importlib 7 | from tempfile import NamedTemporaryFile 8 | 9 | from unittest.mock import MagicMock, patch 10 | 11 | import pytest 12 | 13 | from cloudsync import get_provider 14 | from cloudsync.exceptions import CloudTokenError 15 | import cloudsync.command.sync as csync 16 | 17 | log = logging.getLogger(__name__) 18 | 19 | def test_cmd_sync_basic(caplog, tmpdir): 20 | args = MagicMock() 21 | 22 | args.src = "mock_oid_cs:/a" 23 | args.dest = "mock_path_cs:/b" 24 | args.quiet = False # log less, don't prompt for auth, get tokens from files or other commands 25 | args.verbose = True # log a lot (overrides quiet) 26 | args.daemon = False # don't keep running after i quit 27 | args.statedb = str(tmpdir / "storage") 28 | args.creds = "whatevs" 29 | 30 | csync.SyncCmd.run(args) 31 | logs = caplog.record_tuples 32 | assert any("initialized" in t[2].lower() for t in logs) 33 | 34 | 35 | @pytest.mark.parametrize("conf", ["with_conf", "no_conf"]) 36 | @pytest.mark.parametrize("creds", ["with_creds", "no_creds"]) 37 | @pytest.mark.parametrize("quiet", [True, False]) 38 | def test_cmd_sync_oauth(caplog, conf, creds, quiet, tmpdir): 39 | args = MagicMock() 40 | 41 | args.src = "mock_oid_cs:/a" 42 | args.dest = "gdrive:/b" 43 | args.quiet = quiet # log less, don't prompt for auth, get tokens from files or other commands 44 | args.verbose = True # log a lot (overrides quiet) 45 | args.daemon = False # don't keep running after i quit 46 | args.statedb = str(tmpdir / "storage") 47 | 48 | try: 49 | tf = NamedTemporaryFile(delete=False) 50 | tf.write(b'{"oauth":{"host":"localhost"}}') 51 | tf.flush() 52 | tf.close() 53 | 54 | tf2 = NamedTemporaryFile(delete=False) 55 | tf2.write(b'{"mock_oid_cs":{"fake":"creds"}}') 56 | tf2.flush() 57 | tf2.close() 58 | 59 | err: type = CloudTokenError 60 | 61 | if sys.platform == "linux": 62 | badname = "/not-allowed" 63 | plat_err = PermissionError 64 | else: 65 | badname = "x:/bad" 66 | plat_err = FileNotFoundError 67 | 68 | args.creds = tf2.name if creds == "with_creds" else badname 69 | 70 | if conf == "with_conf": 71 | args.config = tf.name 72 | else: 73 | args.config = "someconfigthatisnthere" 74 | 75 | log.info("start sync") 76 | 77 | if creds != "with_creds" and not args.quiet: 78 | err = plat_err 79 | 80 | with pytest.raises(err): 81 | called = 0 82 | 83 | def authen(self): 84 | nonlocal called 85 | self._oauth_config.creds_changed({"k":"v"}) 86 | called += 1 87 | return {"k":"v"} 88 | 89 | with patch.object(get_provider("gdrive"), "authenticate", authen): 90 | try: 91 | csync.SyncCmd.run(args) 92 | except CloudTokenError: 93 | if args.quiet: 94 | assert called == 0 95 | else: 96 | assert called == 1 97 | raise 98 | finally: 99 | os.unlink(tf.name) 100 | os.unlink(tf2.name) 101 | 102 | logs = caplog.record_tuples 103 | 104 | if err == CloudTokenError: 105 | assert any("connect gdrive" in t[2].lower() for t in logs) 106 | 107 | @pytest.mark.parametrize("daemon", ["with_daemon", "no_daemon"]) 108 | def test_cmd_sync_daemon(daemon, tmpdir): 109 | args = MagicMock() 110 | 111 | args.src = "mock_oid_cs:/a" 112 | args.dest = "mock_path_cs:/b" 113 | args.daemon = True # don't keep running after i quit 114 | args.statedb = str(tmpdir / "storage") 115 | 116 | if daemon == "with_daemon": 117 | # daemon module is available - forcibly 118 | dm = MagicMock() 119 | with patch.dict("sys.modules", {'daemon': dm}): 120 | importlib.reload(csync) 121 | csync.SyncCmd.run(args) 122 | dm.DaemonContext.assert_called_once() 123 | else: 124 | # daemon module is not available 125 | with patch.dict("sys.modules", {'daemon': None}): 126 | importlib.reload(csync) 127 | # import will fail here, which is ok 128 | with pytest.raises(NotImplementedError): 129 | csync.SyncCmd.run(args) 130 | -------------------------------------------------------------------------------- /cloudsync/sync/sqlite_storage.py: -------------------------------------------------------------------------------- 1 | from typing import Dict, Any, Optional, overload 2 | import logging 3 | import sqlite3 4 | from threading import Lock 5 | from .state import Storage 6 | 7 | log = logging.getLogger(__name__) 8 | 9 | 10 | class SqliteStorage(Storage): 11 | """ 12 | Local disk storage using sqlite. 13 | """ 14 | def __init__(self, filename: str): 15 | self._mutex = Lock() 16 | self._filename = filename 17 | self.db = None 18 | self.db = self.__db_connect() 19 | self._ensure_table_exists() 20 | 21 | def __db_connect(self): 22 | if self.db: 23 | self.close() 24 | 25 | self.db = sqlite3.connect(self._filename, 26 | uri=self._filename.startswith('file:'), 27 | check_same_thread=self._filename == ":memory:", 28 | timeout=5, 29 | isolation_level=None, 30 | ) 31 | return self.db 32 | 33 | def __db_execute(self, sql, parameters=()): 34 | # in python 3.6, this will randomly crash unless there's a mutex involved 35 | # it's not supposed to be a problem... but it is 36 | with self._mutex: 37 | try: 38 | retval = self.db.execute(sql, parameters) 39 | except sqlite3.OperationalError: 40 | self.__db_connect() # reconnect 41 | retval = self.db.execute(sql, parameters) 42 | return retval 43 | 44 | def _ensure_table_exists(self): 45 | self.__db_execute("PRAGMA journal_mode=WAL;") 46 | self.__db_execute("PRAGMA busy_timeout=5000;") 47 | 48 | # Not using AUTOINCREMENT: http://www.sqlitetutorial.net/sqlite-autoincrement/ 49 | self.__db_execute('CREATE TABLE IF NOT EXISTS cloud (id INTEGER PRIMARY KEY, ' 50 | 'tag TEXT NOT NULL, serialization BLOB)') 51 | self.__db_execute('CREATE INDEX IF NOT EXISTS cloud_tag_ix on cloud(tag)') 52 | self.__db_execute('CREATE INDEX IF NOT EXISTS cloud_id_ix on cloud(id)') 53 | 54 | def create(self, tag: str, serialization: bytes) -> Any: 55 | assert tag is not None 56 | db_cursor = self.__db_execute('INSERT INTO cloud (tag, serialization) VALUES (?, ?)', 57 | [tag, serialization]) 58 | eid = db_cursor.lastrowid 59 | return eid 60 | 61 | def update(self, tag: str, serialization: bytes, eid: Any) -> int: 62 | db_cursor = self.__db_execute('UPDATE cloud SET serialization = ? WHERE id = ? AND tag = ?', 63 | [serialization, eid, tag]) 64 | ret = db_cursor.rowcount 65 | if ret == 0: 66 | raise ValueError("id %s doesn't exist" % eid) 67 | return ret 68 | 69 | def delete(self, tag: str, eid: Any): 70 | db_cursor = self.__db_execute('DELETE FROM cloud WHERE id = ? AND tag = ?', 71 | [eid, tag]) 72 | if db_cursor.rowcount == 0: 73 | log.debug("ignoring delete: id %s doesn't exist", eid) 74 | return 75 | 76 | @overload 77 | def read_all(self) -> Dict[str, Dict[Any, bytes]]: 78 | ... 79 | 80 | @overload 81 | def read_all(self, tag: str) -> Dict[Any, bytes]: # pylint: disable=arguments-differ 82 | ... 83 | 84 | def read_all(self, tag: str = None): # pylint: disable=arguments-differ 85 | ret = {} 86 | if tag is not None: 87 | query = 'SELECT id, tag, serialization FROM cloud WHERE tag = ?' 88 | db_cursor = self.__db_execute(query, [tag]) 89 | else: 90 | query = 'SELECT id, tag, serialization FROM cloud' 91 | db_cursor = self.__db_execute(query) 92 | 93 | for row in db_cursor.fetchall(): 94 | eid, row_tag, row_serialization = row 95 | if tag is not None: 96 | ret[eid] = row_serialization 97 | else: 98 | if row_tag not in ret: 99 | ret[row_tag] = {} 100 | ret[row_tag][eid] = row_serialization 101 | return ret 102 | 103 | def read(self, tag: str, eid: Any) -> Optional[bytes]: 104 | db_cursor = self.__db_execute('SELECT serialization FROM cloud WHERE id = ? and tag = ?', [eid, tag]) 105 | for row in db_cursor.fetchall(): 106 | return row 107 | return None 108 | 109 | 110 | def close(self): 111 | try: 112 | self.db.close() 113 | except Exception: 114 | self.db = None 115 | 116 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [![Build Status](https://travis-ci.com/AtakamaLLC/cloudsync.svg?branch=master&token=WD7aozR2wQ3ePGe1QpA8)](https://travis-ci.com/AtakamaLLC/cloudsync) 2 | [![Code Coverage](https://codecov.io/gh/AtakamaLLC/cloudsync/branch/master/graph/badge.svg)](https://codecov.io/gh/AtakamaLLC/cloudsync) 3 | 4 | 5 | # cloudsync README 6 | 7 | Python Cloud Synchronization Library 8 | 9 | ## Installation 10 | 11 | ```bash 12 | pip install cloudsync 13 | 14 | # install provider support 15 | pip install cloudsync-gdrive 16 | ``` 17 | 18 | ## Links 19 | 20 | * [Documentation](https://atakama-llc-cloudsync.readthedocs-hosted.com/en/latest/) 21 | * [Source Code + Issue Tracker](https://github.com/AtakamaLLC/cloudsync) 22 | 23 | ## Command-line Example 24 | 25 | ```bash 26 | 27 | cloudsync sync --help 28 | 29 | cloudsync sync file:c:/users/me/documents gdrive:/mydocs 30 | 31 | # on linux you can pass -D for 'daemon mode', which will detatch and run in the background 32 | ``` 33 | ## Example of a single cloud provider integration 34 | 35 | ```python 36 | import cloudsync 37 | 38 | # Get a generic client_id and client_secret. Do not use this in production code. 39 | # For more information on getting your own client_id and client_secret, see README_OAUTH.md 40 | oauth_config = cloudsync.command.utils.generic_oauth_config('gdrive') 41 | 42 | # get an instance of the gdrive provider class 43 | provider = cloudsync.create_provider('gdrive', oauth_config=oauth_config) 44 | 45 | # Start the oauth process to login in to the cloud provider 46 | creds = provider.authenticate() 47 | 48 | # Use the credentials to connect to the cloud provider 49 | provider.connect(creds) 50 | 51 | # Perform cloud operations 52 | for entry in provider.listdir_path("/"): 53 | print(entry.path) 54 | ``` 55 | ## Example of a syncronization between two cloud providers 56 | 57 | ```python 58 | import cloudsync 59 | import tempfile 60 | import time 61 | 62 | # a little setup 63 | local_root = tempfile.mkdtemp() 64 | remote_root = "/cloudsync_test/" + time.strftime("%Y%m%d%H%M") 65 | provider_name = 'gdrive' 66 | print("syncronizing between %s locally and %s on %s" % (local_root, remote_root, provider_name)) 67 | 68 | # Get a generic client_id and client_secret. Do not use this in production code. 69 | # For more information on getting your own client_id and client_secret, see README_OAUTH.md 70 | cloud_oauth_config = cloudsync.command.utils.generic_oauth_config(provider_name) 71 | 72 | # get instances of the local file provider and cloud provider from the provider factory 73 | local = cloudsync.create_provider("filesystem") 74 | remote = cloudsync.create_provider(provider_name, oauth_config=cloud_oauth_config) 75 | 76 | # Authenticate with the remote provider using OAuth 77 | creds = remote.authenticate() 78 | 79 | # Connect with the credentials acquired by authenticating with the provider 80 | local.namespace = local_root # filesystem provider wants to know the root namespace before connecting 81 | local.connect(None) 82 | remote.connect(creds) 83 | 84 | # Create the folder on google drive to syncronize locally 85 | print("Creating folder %s on %s" % (remote_root, provider_name)) 86 | remote.mkdirs(remote_root) # provider.mkdirs() will automatically create any necessary parent folders 87 | 88 | # Specify which folders to syncronize 89 | sync_roots = (local_root, remote_root) 90 | 91 | # instantiate a new sync engine and start syncing 92 | sync = cloudsync.CloudSync((local, remote), roots=sync_roots) 93 | sync.start() 94 | 95 | # should sync this file as soon as it's noticed by watchdog 96 | local_hello_path = local.join(local_root, "hello.txt") 97 | print("Creating local file %s" % local_hello_path) 98 | with open(local_hello_path, "w") as f: 99 | f.write("hello") 100 | 101 | # note remote.join, NOT os.path.join... Gets the path separator correct 102 | remote_hello_path = remote.join(remote_root, "hello.txt") 103 | 104 | # wait for sync to upload the new file to the cloud 105 | while not remote.exists_path(remote_hello_path): 106 | time.sleep(1) 107 | 108 | remote_hello_info = remote.info_path(remote_hello_path) 109 | 110 | # rename in the cloud 111 | local_goodbye_path = local.join(local_root, "goodbye.txt") 112 | remote_goodbye_path = remote.join(remote_root, "goodbye.txt") 113 | print("renaming %s to %s on %s" % (remote_hello_path, remote_goodbye_path, provider_name)) 114 | remote.rename(remote_hello_info.oid, remote_goodbye_path) # rename refers to the file to rename by oid 115 | 116 | # wait for sync to cause the file to get renamed locally 117 | while not local.exists_path(local_goodbye_path): 118 | time.sleep(1) 119 | 120 | print("synced") 121 | 122 | sync.stop(forever=True) 123 | local.disconnect() 124 | remote.disconnect() 125 | ``` 126 | -------------------------------------------------------------------------------- /cloudsync/notification.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import typing 3 | import queue 4 | import enum 5 | from dataclasses import dataclass 6 | from cloudsync.runnable import Runnable 7 | import cloudsync.types as typ 8 | import cloudsync.exceptions as ex 9 | 10 | 11 | log = logging.getLogger(__name__) 12 | __all__ = ["Notification", "NotificationType", "NotificationManager", "SourceEnum"] 13 | 14 | 15 | # pylint: disable=multiple-statements 16 | class NotificationType(enum.Enum): 17 | """These types roughly correspond to the exceptions defined in exceptions.py""" 18 | STARTED = 'started' ; """Sync has started""" 19 | CONNECTED = 'connected' ; """Provider has connected""" 20 | STOPPED = 'stopped' ; """Sync engine was stopped""" 21 | FILE_NAME_ERROR = 'file_name_error' ; """File name is invalid, and is being ignored""" 22 | OUT_OF_SPACE_ERROR = 'out_of_space_error' ; """Sync is halted because one provider is out of space""" 23 | DISCONNECTED_ERROR = 'disconnected_error' ; """Provider was disconnected""" 24 | NAMESPACE_ERROR = 'namespace_error' ; """Specified namespace is invalid/unavailable (could be auth issue)""" 25 | ROOT_MISSING_ERROR = 'root_missing_error' ; """Root of cloud sync is missing, and will not be created""" 26 | TEMPORARY_ERROR = 'temporary_error' ; """Upload failure, or other temp error that will be retried.""" 27 | SYNC_DISCARDED = 'sync_discarded' ; """Sync discarded a file due to path translation failure""" 28 | SYNC_SMART_UNSYNCED = 'sync_smart_skipped' ; """SmartSync file has not been requested, and therefore skipped during sync""" 29 | SYNC_CORRUPT_IGNORED = 'sync_corrupt_ignored' ; """a corrupt file has been deleted or renamed, and the delete/rename was dropped""" 30 | 31 | 32 | class SourceEnum(enum.Enum): 33 | """Local and remote, probably belongs in types.py""" 34 | LOCAL = typ.LOCAL 35 | REMOTE = typ.REMOTE 36 | SYNC = 2 37 | 38 | 39 | @dataclass 40 | class Notification: 41 | """Notification from cloudsync about something.""" 42 | source: SourceEnum ; """Source of the event as defined in the SourceEnum""" 43 | ntype: NotificationType ; """Type of notification as defined by the enum""" 44 | path: typing.Optional[str] ; """Path to the file in question, if applicable""" 45 | # pylint: enable=multiple-statements 46 | 47 | 48 | class NotificationManager(Runnable): 49 | """Service that receives notifications in a queue, and calls a handler for each.""" 50 | def __init__(self, evt_handler: typing.Callable[[Notification], None]): 51 | self.__queue: queue.Queue = queue.Queue() 52 | self.__handler: typing.Callable = evt_handler 53 | 54 | def do(self): 55 | log.debug("Looking for a notification") 56 | if self._run_until: 57 | try: 58 | e = self.__queue.get(timeout=0.1) 59 | except queue.Empty: 60 | return 61 | else: 62 | e = self.__queue.get() 63 | try: 64 | log.debug("Processing a notification: %s", e) 65 | if e is not None: 66 | self.__handler(e) 67 | else: 68 | # None event == stop 69 | if not self.stopped: 70 | super().stop(forever=False) 71 | except Exception: 72 | log.exception("Error while handling a notification: %s", e) 73 | 74 | def notify_from_exception(self, source: SourceEnum, e: ex.CloudException, path: typing.Optional[str] = None): 75 | """Insert notification into the queue based on exception information.""" 76 | 77 | log.debug("notify from exception %s, %s : %s", source, repr(e), path) 78 | 79 | if isinstance(e, ex.CloudDisconnectedError): 80 | self.notify(Notification(source, NotificationType.DISCONNECTED_ERROR, path)) 81 | elif isinstance(e, ex.CloudOutOfSpaceError): 82 | self.notify(Notification(source, NotificationType.OUT_OF_SPACE_ERROR, path)) 83 | elif isinstance(e, ex.CloudFileNameError): 84 | self.notify(Notification(source, NotificationType.FILE_NAME_ERROR, path)) 85 | elif isinstance(e, ex.CloudNamespaceError): 86 | self.notify(Notification(source, NotificationType.NAMESPACE_ERROR, path)) 87 | elif isinstance(e, ex.CloudRootMissingError): 88 | self.notify(Notification(source, NotificationType.ROOT_MISSING_ERROR, path)) 89 | elif isinstance(e, ex.CloudTemporaryError): 90 | self.notify(Notification(source, NotificationType.TEMPORARY_ERROR, path)) 91 | else: 92 | log.debug("Encountered a cloud exception: %s (type %s)", e, type(e)) 93 | 94 | def notify(self, e: Notification): 95 | """Add notification to the queue""" 96 | self.__queue.put(e) 97 | 98 | def stop(self, forever=True, wait=True): 99 | """Stop the server""" 100 | self.__queue.put(None) 101 | super().stop(forever=forever, wait=wait) 102 | -------------------------------------------------------------------------------- /cloudsync/tests/test_runnable.py: -------------------------------------------------------------------------------- 1 | import time 2 | import threading 3 | import logging 4 | import pytest 5 | from threading import Barrier 6 | 7 | from cloudsync import Runnable 8 | from cloudsync.tests.fixtures.util import RunUntilHelper 9 | 10 | log = logging.getLogger(__name__) 11 | 12 | 13 | def test_runnable(): 14 | class TestRunTestRunnable(Runnable): 15 | def __init__(self): 16 | self.cleaned = False 17 | self.called = 0 18 | 19 | def do(self): 20 | self.called += 1 21 | 22 | def done(self): 23 | self.cleaned = True 24 | 25 | testrun = TestRunTestRunnable() 26 | 27 | with pytest.raises(TimeoutError): 28 | testrun.run(timeout=0.02, sleep=0.001) 29 | 30 | assert testrun.called 31 | 32 | testrun.called = 0 33 | 34 | testrun.run(until=lambda: testrun.called == 1) 35 | 36 | assert testrun.called == 1 37 | 38 | thread = threading.Thread(target=testrun.run, kwargs={"timeout": 10}) 39 | thread.start() 40 | while not testrun.started: 41 | time.sleep(0.1) 42 | testrun.stop(forever=False) 43 | thread.join(timeout=1) 44 | assert testrun.stopped 45 | assert not thread.is_alive() 46 | 47 | assert testrun.stopped == 1 48 | 49 | testrun.called = 0 50 | testrun.start(until=lambda: testrun.called > 0) 51 | testrun.wait(timeout=2) 52 | 53 | 54 | def test_timeout(): 55 | do_barrier_start = Barrier(2) 56 | do_barrier_end = Barrier(2) 57 | 58 | class TestRunTestTimeout(Runnable): 59 | def __init__(self): 60 | self.cleaned = False 61 | self.called = 0 62 | 63 | def do(self): 64 | do_barrier_start.wait(10) # tell the test we're started 65 | do_barrier_end.wait(10) # hang here until the test is done testing 66 | self.called += 1 67 | 68 | def done(self): 69 | self.cleaned = True 70 | 71 | testrun = TestRunTestTimeout() 72 | testrun.start() 73 | do_barrier_start.wait(10) # hang here until runnable gets into do() 74 | with pytest.raises(TimeoutError): 75 | testrun.wait(timeout=.01) 76 | 77 | # clean up 78 | do_barrier_end.wait(10) # release the do loop from prison 79 | testrun.stop(wait=10) 80 | RunUntilHelper.wait_until(until=lambda: testrun.called, timeout=10) # make sure do dropped out 81 | RunUntilHelper.wait_until(until=lambda: testrun.stopped, timeout=10) # make sure the thread is done 82 | RunUntilHelper.wait_until(until=lambda: testrun.cleaned, timeout=10) # make sure done was called 83 | 84 | 85 | def test_start_exceptions(): 86 | class TestRunTestStartExceptions(Runnable): 87 | def __init__(self): 88 | self.cleaned = False 89 | self.called = 0 90 | 91 | def do(self): 92 | self.called += 1 93 | time.sleep(1) 94 | 95 | def done(self): 96 | self.cleaned = True 97 | 98 | testrun = TestRunTestStartExceptions() 99 | testrun.start() 100 | RunUntilHelper.wait_until(lambda: testrun.started) 101 | 102 | with pytest.raises(RuntimeError): 103 | testrun.start() 104 | 105 | testrun.stop(forever=False) 106 | testrun.start() 107 | testrun.stop(forever=True) 108 | with pytest.raises(RuntimeError): 109 | testrun.start() 110 | 111 | 112 | def test_no_wait_stop(): 113 | do_barrier_start = Barrier(2) 114 | do_barrier_end = Barrier(2) 115 | 116 | class TestRunTestNoWaitStop(Runnable): 117 | def __init__(self): 118 | self.cleaned = False 119 | self.called = 0 120 | 121 | def do(self): 122 | do_barrier_start.wait(10) # tell the test we're started 123 | do_barrier_end.wait(10) # hang here until the test is done testing 124 | self.called += 1 125 | 126 | def done(self): 127 | self.cleaned = True 128 | 129 | testrun = TestRunTestNoWaitStop() 130 | testrun.start() 131 | do_barrier_start.wait(10) # hang here until runnable gets into do() 132 | 133 | assert testrun.called == 0 134 | testrun.stop(wait=False) 135 | assert testrun.called == 0 136 | 137 | # the test framework chokes when the service stops after the test is complete and stop tries to log 138 | do_barrier_end.wait(10) # release the do loop from prison 139 | RunUntilHelper.wait_until(until=lambda: testrun.called, timeout=10) # make sure do dropped out 140 | RunUntilHelper.wait_until(until=lambda: testrun.stopped, timeout=10) # make sure the thread is done 141 | RunUntilHelper.wait_until(until=lambda: testrun.cleaned, timeout=10) # make sure done was called 142 | 143 | 144 | def test_runnable_wake(): 145 | class TestRunTestRunnableWake(Runnable): 146 | def __init__(self): 147 | self.cleaned = False 148 | self.called = 0 149 | 150 | def do(self): 151 | self.called += 1 152 | 153 | def done(self): 154 | self.cleaned = True 155 | 156 | testrun = TestRunTestRunnableWake() 157 | 158 | # noop 159 | log.info("noop") 160 | testrun.wake() 161 | 162 | # this will sleep for a long time, doing nothing 163 | thread = threading.Thread(target=testrun.run, kwargs={"sleep": 10, "timeout": 10}) 164 | thread.start() 165 | 166 | # called once at start, then sleepz 167 | while testrun.called == 0: 168 | time.sleep(0.1) 169 | assert testrun.called == 1 170 | 171 | while not testrun.started: 172 | time.sleep(0.1) 173 | 174 | # now sleeping for 10 secs, doing nothing 175 | assert testrun.called == 1 176 | 177 | log.info("wake") 178 | # wake it up right away 179 | testrun.wake() 180 | while testrun.called == 1: 181 | time.sleep(0.1) 182 | assert testrun.called == 2 183 | 184 | testrun.stop() 185 | thread.join(timeout=2) 186 | assert not thread.is_alive() 187 | -------------------------------------------------------------------------------- /cloudsync/tests/sync_notification_handler.py: -------------------------------------------------------------------------------- 1 | from typing import Optional, Tuple, List, Union 2 | import logging 3 | 4 | from cloudsync import CloudSync 5 | from cloudsync.notification import Notification, NotificationType, SourceEnum 6 | from cloudsync.types import LOCAL, REMOTE, OInfo 7 | from cloudsync.tests.fixtures import RunUntilHelper 8 | 9 | log = logging.getLogger(__name__) 10 | 11 | 12 | class SyncNotificationHandler: 13 | """ Class that allows tests or other consumers to know when SyncManager chooses to not sync a file """ 14 | def __init__(self, csync: CloudSync): 15 | self.skipped_paths: set = set() 16 | self.discarded_paths: set = set() 17 | self.corrupt_paths: set = set() 18 | self.csync = csync 19 | 20 | def handle_notification(self, notification: Notification): 21 | """ implementation of callback that logs when files are discarded or skipped by SmartSync """ 22 | n = notification 23 | 24 | if n.ntype == NotificationType.SYNC_SMART_UNSYNCED and n.source == SourceEnum.LOCAL: # pragma: no cover 25 | return # only interested in REMOTE events, because smartsync operates primarily remotely 26 | 27 | if n.ntype == NotificationType.SYNC_SMART_UNSYNCED: 28 | self.skipped_paths.add(n.path) 29 | elif n.ntype == NotificationType.SYNC_DISCARDED: 30 | self.discarded_paths.add(n.path) 31 | elif n.ntype == NotificationType.SYNC_CORRUPT_IGNORED: 32 | self.corrupt_paths.add(n.path) 33 | 34 | @staticmethod 35 | def _path_in(path, paths, provider): 36 | """ Checks if path is in path_dict, the keys of which are remote paths """ 37 | for candidate in paths: 38 | if provider.paths_match(path, candidate): 39 | return True 40 | return False 41 | 42 | def _is_synced(self, side, path, hash_str): 43 | info: OInfo = self.csync.providers[side].info_path(path) 44 | return info and (not hash_str or info.hash == hash_str) 45 | 46 | def clear_sync_state(self): 47 | """ Resets the log of synced/skipped paths """ 48 | self.skipped_paths = set() 49 | self.discarded_paths = set() 50 | self.corrupt_paths = set() 51 | 52 | def check_sync_state( # pylint: disable=too-many-branches 53 | self, 54 | *, 55 | remote_paths: Optional[Union[List[str], List[Tuple[str, str]]]] = None, # tuple is (path, hash) 56 | local_paths: Optional[Union[List[str], List[Tuple[str, str]]]] = None, 57 | skipped_paths: Optional[Union[List[str], List[Tuple[str, str]]]] = None, 58 | discarded_paths: Optional[Union[List[str], List[Tuple[str, int]]]] = None, # tuple is (path, side) 59 | quiet=False 60 | ): 61 | """ Returns True if synced_paths have synced and skipped_paths have explicitly been skipped """ 62 | if not (remote_paths or local_paths or skipped_paths or discarded_paths): 63 | raise ValueError("Specify remote_paths or local_paths or skipped_paths or discarded_paths") 64 | 65 | retval = True 66 | for path in remote_paths or []: 67 | hash_str = None 68 | if isinstance(path, tuple): 69 | path, hash_str = path 70 | if not self._is_synced(REMOTE, path, hash_str): 71 | if not quiet: 72 | log.error("%s not synced remotely", path) 73 | retval = False 74 | 75 | for path in local_paths or []: 76 | hash_str = None 77 | if isinstance(path, tuple): 78 | path, hash_str = path 79 | if not self._is_synced(LOCAL, path, hash_str): 80 | if not quiet: 81 | log.error("%s not synced locally", path) 82 | retval = False 83 | 84 | for path in skipped_paths or []: 85 | if isinstance(path, tuple): 86 | path, _ = path 87 | if not self._path_in(path, self.skipped_paths, self.csync.providers[REMOTE]): 88 | if not quiet: 89 | log.error("%s not found in skipped paths %s", path, self.skipped_paths) 90 | retval = False 91 | 92 | for path2 in discarded_paths or []: 93 | side = REMOTE 94 | if isinstance(path2, tuple): 95 | path2, side = path2 96 | if not self._path_in(path2, self.discarded_paths, self.csync.providers[side]): 97 | if not quiet: 98 | log.error("%s not found in discarded paths %s", path2, self.discarded_paths) 99 | retval = False 100 | 101 | return retval 102 | 103 | def wait_sync_state(self, 104 | *, 105 | remote_paths: Optional[Union[List[str], List[Tuple[str, str]]]] = None, 106 | local_paths: Optional[Union[List[str], List[Tuple[str, str]]]] = None, 107 | skipped_paths: Optional[Union[List[str], List[Tuple[str, str]]]] = None, 108 | discarded_paths: Optional[Union[List[str], List[Tuple[str, int]]]] = None, 109 | timeout=20, 110 | poll_time=0.25, 111 | exc=None): 112 | """ Waits for when synced paths have been synced and skipped paths have been explicitly skipped """ 113 | if not (remote_paths or local_paths or skipped_paths or discarded_paths): 114 | raise ValueError("Specify remote_paths or local_paths or skipped_paths or discarded_paths") 115 | try: 116 | RunUntilHelper.wait_until( 117 | until=lambda: self.check_sync_state( 118 | remote_paths=remote_paths, 119 | local_paths=local_paths, 120 | skipped_paths=skipped_paths, 121 | discarded_paths=discarded_paths, 122 | quiet=True), 123 | timeout=timeout, 124 | poll_time=poll_time, 125 | exc=exc 126 | ) 127 | except Exception: 128 | # one last check, and also log what is missing 129 | if not self.check_sync_state( 130 | remote_paths=remote_paths, local_paths=local_paths, skipped_paths=skipped_paths, quiet=False 131 | ): 132 | raise 133 | -------------------------------------------------------------------------------- /cloudsync/tests/test_oauth.py: -------------------------------------------------------------------------------- 1 | import os 2 | import logging 3 | import threading 4 | from unittest.mock import patch 5 | import requests 6 | import pytest 7 | 8 | from cloudsync.oauth import OAuthConfig, OAuthError, OAuthProviderInfo, OAuthRedirServer 9 | from cloudsync.oauth.apiserver import ApiServer, api_route 10 | from cloudsync.exceptions import CloudTokenError 11 | from .fixtures import MockProvider 12 | 13 | log = logging.getLogger(__name__) 14 | 15 | 16 | class TokenServer(ApiServer): 17 | @api_route("/token") 18 | def token(self, ctx, req): 19 | return { 20 | "token_type": "bearer", 21 | "refresh_token": "r1", 22 | "access_token": "a1", 23 | "expires_in": 340 24 | } 25 | 26 | 27 | def x_test_oauth(): 28 | OAuthRedirServer.SHUFFLE_PORTS = False 29 | os.environ['OAUTHLIB_INSECURE_TRANSPORT'] = '1' 30 | 31 | t = TokenServer("127.0.0.1", 0) 32 | threading.Thread(target=t.serve_forever, daemon=True).start() 33 | 34 | auth_url = t.uri("/auth") 35 | token_url = t.uri("/token") 36 | 37 | log.debug("start auth") 38 | o = OAuthConfig(app_id="foo", app_secret="bar", port_range=(54045, 54099), host_name="localhost") 39 | o.start_auth(auth_url) 40 | requests.get(o.redirect_uri, params={"code": "cody"}) 41 | log.debug("wait auth") 42 | res = o.wait_auth(token_url=token_url, timeout=5) 43 | 44 | assert res.refresh_token == "r1" 45 | assert res.expires_in == 340 46 | 47 | o.start_auth(auth_url) 48 | requests.get(o.redirect_uri, params={"error": "erry"}) 49 | with pytest.raises(OAuthError): 50 | res = o.wait_auth(token_url=token_url, timeout=5) 51 | 52 | 53 | @patch('webbrowser.open') 54 | def test_oauth_threaded(wb): 55 | thread_count = 4 56 | threads = [] 57 | tpass = 0 58 | tfail = [] 59 | for _ in range(thread_count): 60 | def trap(): 61 | try: 62 | nonlocal tpass 63 | x_test_oauth() 64 | tpass += 1 65 | except Exception as e: 66 | tfail.append(e) 67 | t = threading.Thread(target=trap, daemon=True) 68 | t.start() 69 | threads.append(t) 70 | 71 | for t in threads: 72 | t.join(timeout=10) 73 | assert not t.is_alive() 74 | 75 | log.debug("errs %s", tfail) 76 | assert tpass == 4 77 | 78 | 79 | @patch('webbrowser.open') 80 | def test_oauth_refresh(wb): 81 | os.environ['OAUTHLIB_INSECURE_TRANSPORT'] = '1' 82 | 83 | t = TokenServer("127.0.0.1", 0) 84 | threading.Thread(target=t.serve_forever, daemon=True).start() 85 | 86 | token_url = t.uri("/token") 87 | 88 | o = OAuthConfig(app_id="foo", app_secret="bar") 89 | res = o.refresh(token_url, "token", ["scope"]) 90 | 91 | assert res.refresh_token == "r1" 92 | assert res.expires_in == 340 93 | 94 | 95 | @patch('webbrowser.open') 96 | def test_oauth_interrupt(wb): 97 | os.environ['OAUTHLIB_INSECURE_TRANSPORT'] = '1' 98 | 99 | t = TokenServer("127.0.0.1", 0) 100 | threading.Thread(target=t.serve_forever, daemon=True).start() 101 | 102 | auth_url = t.uri("/auth") 103 | token_url = t.uri("/token") 104 | 105 | o = OAuthConfig(app_id="foo", app_secret="bar", port_range=(54045, 54099), host_name="localhost") 106 | o.start_auth(auth_url) 107 | wb.assert_called_once() 108 | o.shutdown() 109 | with pytest.raises(OAuthError): 110 | o.wait_auth(token_url=token_url, timeout=5) 111 | 112 | 113 | @patch('webbrowser.open') 114 | def test_oauth_defaults(wb): 115 | 116 | # when CI testing, oauth providers stick tokens, ids, and secrets in the environment 117 | os.environ["TEST_APP_ID"] = "123" 118 | os.environ["TEST_APP_SECRET"] = "456" 119 | os.environ["TEST_TOKEN"] = "ABC|DEF" 120 | 121 | t = TokenServer("127.0.0.1", 0) 122 | threading.Thread(target=t.serve_forever, daemon=True).start() 123 | 124 | # here's an oauth provider 125 | class Prov(MockProvider): 126 | name = "TEST" 127 | 128 | def __init__(self, oc: OAuthConfig): 129 | self._oauth_config = oc 130 | _oauth_info = OAuthProviderInfo( # signal's oauth mode 131 | auth_url=t.uri("/auth"), 132 | token_url=t.uri("/token"), 133 | scopes=[], 134 | ) 135 | 136 | inst = Prov.test_instance() 137 | assert inst._oauth_config.app_id == "123" 138 | assert inst._oauth_config.app_secret == "456" 139 | assert inst._test_creds in [{"refresh_token": "ABC"}, {"refresh_token": "DEF"}] 140 | 141 | # actually test the instance 142 | creds = None 143 | creds_ex = None 144 | os.environ['OAUTHLIB_INSECURE_TRANSPORT'] = '1' 145 | 146 | # this is a blocking function, set an event when creds are found 147 | event = threading.Event() 148 | 149 | def auth(): 150 | nonlocal creds 151 | nonlocal creds_ex 152 | try: 153 | creds = inst.authenticate() 154 | event.set() 155 | except Exception as e: 156 | creds_ex = e 157 | raise 158 | threading.Thread(target=auth, daemon=True).start() 159 | 160 | while True: 161 | try: 162 | wb.assert_called_once() 163 | # pretend user clicked ok 164 | requests.get(inst._oauth_config.redirect_uri, params={"code": "cody"}) 165 | break 166 | except AssertionError: 167 | # webbrowser not launched yet... 168 | pass 169 | 170 | # click received, wait for token 171 | event.wait() 172 | assert creds 173 | 174 | log.debug("test interrupt") 175 | creds = None 176 | th = threading.Thread(target=auth, daemon=True) 177 | th.start() 178 | while True: 179 | try: 180 | assert wb.call_count == 2 181 | inst.interrupt_auth() 182 | break 183 | except AssertionError: 184 | # webbrowser not launched yet... 185 | pass 186 | th.join() 187 | 188 | assert creds is None 189 | assert type(creds_ex) is CloudTokenError 190 | 191 | 192 | def test_err(): 193 | with pytest.raises(OAuthError): 194 | OAuthConfig(app_id=None, app_secret="secret").start_auth("whatever") 195 | 196 | with pytest.raises(OAuthError): 197 | OAuthConfig(app_id="id", app_secret=None).start_auth("whatever") 198 | 199 | with pytest.raises(OAuthError): 200 | OAuthConfig(app_id="id", app_secret="secret").start_auth(None) 201 | -------------------------------------------------------------------------------- /cloudsync/utils.py: -------------------------------------------------------------------------------- 1 | """ 2 | The ubuquitous "misc utilities" file required in every library 3 | """ 4 | 5 | import os, tempfile 6 | from typing import IO 7 | 8 | import logging 9 | import time 10 | import functools 11 | 12 | from base64 import b64encode 13 | from typing import Any, List, Dict, Callable, cast 14 | import xxhash 15 | 16 | log = logging.getLogger(__name__) 17 | 18 | 19 | MAX_DEBUG_STR = 64 20 | 21 | 22 | def _debug_arg(val: Any): 23 | ret: Any = val 24 | if isinstance(val, dict): 25 | r: Dict[Any, Any] = {} 26 | for k, v in val.items(): 27 | r[k] = _debug_arg(v) 28 | ret = r 29 | elif isinstance(val, str): 30 | if len(val) > 64: 31 | ret = val[0:61] + "..." 32 | elif isinstance(val, bytes): 33 | if len(val) > 64: 34 | ret = val[0:61] + b"..." 35 | else: 36 | try: 37 | rlist: List[Any] = [] 38 | for v in iter(val): 39 | rlist.append(_debug_arg(v)) 40 | ret = rlist 41 | except TypeError: 42 | pass 43 | return ret 44 | 45 | 46 | def debug_args(*stuff: Any): 47 | """ 48 | Use this when logging stuff that might be too long. It truncates them. 49 | """ 50 | if log.isEnabledFor(logging.DEBUG): 51 | r = _debug_arg(stuff) 52 | if len(r) == 1: 53 | return r[0] 54 | return tuple(r) 55 | if len(stuff) == 1: 56 | return "N/A" 57 | return tuple(["N/A"] * len(stuff)) 58 | 59 | 60 | # useful for converting oids and pointer numbers into digestible nonces 61 | def debug_sig(t: Any, size: int = 3) -> str: 62 | """ 63 | Useful for converting oids and pointer numbers into short digestible nonces 64 | """ 65 | if not t: 66 | return "0" 67 | th = xxhash.xxh64() 68 | th.update(str(t)) 69 | return b64encode(th.digest()).decode("utf8")[0:size] 70 | 71 | 72 | class memoize(): 73 | """ Very simple memoize wrapper 74 | 75 | function decorator: cache lives globally 76 | method decorator: cache lives inside `obj_instance.__memoize_cache` 77 | """ 78 | 79 | def __init__(self, func: Callable[..., Any] = None, expire_secs: float = 0, obj=None, cache: Dict[Any, Any] = None): 80 | self.func = func 81 | self.expire_secs = expire_secs 82 | self.cache = cache 83 | if cache is None: 84 | self.cache = {} 85 | if self.func is not None: 86 | functools.update_wrapper(self, func) 87 | self.obj = obj 88 | 89 | def __get__(self, obj, objtype=None): 90 | if obj is None: 91 | # does this ever happen? 92 | return self.func 93 | 94 | if type(self.cache) is str: 95 | # user specified name of a property that contains the cache dictionary 96 | cache = getattr(obj, cast(str, self.cache)) 97 | else: 98 | # inject cache into the instance, so it doesn't live beyond the scope of the instance 99 | # without this, memoizing can cause serious unexpected memory leaks 100 | try: 101 | cache = obj.__memoize_cache # pylint: disable=protected-access 102 | except AttributeError: 103 | try: 104 | cache = obj.__memoize_cache = {} 105 | except Exception as e: 106 | # some objects don't work with injection 107 | log.warning("cannot inject cache: '%s', ensure object is a singleton, or pass a cache in!", e) 108 | cache = self.cache 109 | 110 | return memoize(self.func, expire_secs=self.expire_secs, cache=cache, obj=obj) 111 | 112 | def __call__(self, *args, **kwargs): 113 | if self.func is None: 114 | # this was used as a function style decorator 115 | # there should be no kwargs 116 | assert not kwargs 117 | func = args[0] 118 | return memoize(func, expire_secs=self.expire_secs, cache=self.cache) 119 | 120 | if self.obj is not None: 121 | args = (self.obj, *args) 122 | 123 | key = (args, tuple(sorted(kwargs.items()))) 124 | cur_time = time.monotonic() 125 | 126 | if key in self.cache: 127 | (cresult, ctime) = self.cache[key] 128 | if not self.expire_secs or cur_time < (ctime + self.expire_secs): 129 | return cresult 130 | 131 | result = self.func(*args, **kwargs) 132 | self.cache[key] = (result, cur_time) 133 | return result 134 | 135 | def clear(self, *args, **kwargs): 136 | if self.obj is not None: 137 | args = (self.obj, *args) 138 | key = (args, tuple(sorted(kwargs.items()))) 139 | self.cache.pop(key, None) 140 | 141 | def get(self, *args, **kwargs): 142 | if self.obj is not None: 143 | args = (self.obj, *args) 144 | key = (args, tuple(sorted(kwargs.items()))) 145 | if key in self.cache: 146 | return self.cache[key][0] 147 | return None 148 | 149 | def set(self, *args, _value, **kwargs): 150 | if self.obj is not None: 151 | args = (self.obj, *args) 152 | key = (args, tuple(sorted(kwargs.items()))) 153 | self.cache[key] = (_value, time.monotonic()) 154 | 155 | 156 | # from https://gist.github.com/earonesty/a052ce176e99d5a659472d0dab6ea361 157 | # windows compatible temp files 158 | 159 | class TemporaryFile: 160 | """ 161 | File-like for NamedTemporaryFile 162 | """ 163 | def __init__(self, name, io, delete): 164 | self.name = name 165 | self.__io = io 166 | self.__delete = delete 167 | 168 | def __getattr__(self, k): 169 | return getattr(self.__io, k) 170 | 171 | def __del__(self): 172 | """ 173 | Delete on going out of scope. This isn't safe, but it ususally works. 174 | """ 175 | if self.__delete: 176 | if self.__io: 177 | self.__io.close() 178 | try: 179 | os.unlink(self.name) 180 | except FileNotFoundError: 181 | pass 182 | 183 | 184 | def NamedTemporaryFile(mode='w+b', bufsize=-1, suffix='', prefix='tmp', dir=None, delete=True): # pylint: disable=redefined-builtin 185 | """ 186 | Windows compatible temp files. 187 | """ 188 | if not dir: 189 | dir = tempfile.gettempdir() 190 | name = os.path.join(dir, prefix + os.urandom(32).hex() + suffix) 191 | if mode is None: 192 | return TemporaryFile(name, None, delete) 193 | fh: IO = open(name, "w+b", bufsize) 194 | if mode != "w+b": 195 | fh.close() 196 | fh = open(name, mode) 197 | return TemporaryFile(name, fh, delete) 198 | 199 | 200 | -------------------------------------------------------------------------------- /cloudsync/oauth/redir_server.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import sys 3 | import random 4 | import threading 5 | import errno 6 | from typing import Callable, Any, Optional, Tuple 7 | # from src import config 8 | from .apiserver import ApiServer 9 | 10 | log = logging.getLogger(__name__) 11 | 12 | __all__ = ['OAuthRedirServer'] 13 | 14 | # todo: use https://requests-oauthlib.readthedocs.io/en/latest/oauth2_workflow.html#web-application-flow 15 | # then provide tools to that provider-writers don't have to do much to get their app-specific oauth to work other than 16 | # providing the resource name, auth url and any app-specific parameters 17 | 18 | 19 | def _is_windows(): 20 | return sys.platform in ("win32", "cygwin") 21 | 22 | 23 | class OAuthRedirServer: # pylint: disable=too-many-instance-attributes 24 | """ 25 | Locally running OAuth redirect server for desktop authentication 26 | 27 | from cloudsync.oauth import OAuthRedirServer 28 | """ 29 | SHUFFLE_PORTS: bool = True 30 | 31 | def __init__(self, *, html_generator: Callable[[bool, str], str] = None, 32 | port_range: Tuple[int, int] = None, 33 | host_name: str = None): 34 | """ 35 | Redirect web server instance 36 | 37 | Args: 38 | html_generator: Function that returns the string to show the user. 39 | port_range: List of allowable ports 40 | host_name: Host name to use (default: 127.0.0.1) 41 | """ 42 | self.__html_response_generator = html_generator 43 | self.__port_range = port_range 44 | 45 | # generally 127.0.0.1 is better than "localhost", since it cannot 46 | # be accidentally or malicuously overridden in a config file 47 | # however, some providers (Onedrive) do not allow it 48 | 49 | self.__host_name = host_name or "127.0.0.1" 50 | 51 | self.__on_success: Optional[Callable[[Any], None]] = None 52 | self.__on_failure: Optional[Callable[[str], None]] = None 53 | self.__api_server: Optional[ApiServer] = None 54 | self.__thread: Optional[threading.Thread] = None 55 | self.__running = False 56 | self.event = threading.Event() 57 | self.success_code: str = None 58 | self.failure_info: str = None 59 | 60 | @property 61 | def running(self): 62 | return self.__running 63 | 64 | def run(self, on_success: Callable[[Any], None], on_failure: Callable[[str], None]): 65 | """ 66 | Starts the server, and autodiscovers the port it will be binding to. 67 | """ 68 | if self.__running: 69 | raise RuntimeError('OAuth server was run() twice') 70 | self.__on_success = on_success 71 | self.__on_failure = on_failure 72 | 73 | self.success_code = None 74 | self.failure_info = None 75 | self.event.clear() 76 | log.debug('Creating oauth redir server') 77 | self.__running = True 78 | if self.__port_range: 79 | (port_min, port_max) = self.__port_range 80 | # Some providers (Dropbox, Onedrive) don't allow us to just use localhost 81 | # redirect. For these providers, we define a range of 82 | # host_name:(port_min, port_max) as valid redir URLs 83 | ports = list(range(port_min, port_max)) 84 | 85 | # generally this is faster, but it can make testing falsely more forgiving 86 | # so expose this for tests 87 | if self.SHUFFLE_PORTS: 88 | random.shuffle(ports) 89 | 90 | for port in ports: 91 | try: 92 | self.__api_server = ApiServer('127.0.0.1', port) 93 | break 94 | except OSError: 95 | pass 96 | else: 97 | self.__api_server = ApiServer('127.0.0.1', 0) 98 | if not self.__api_server: 99 | raise OSError(errno.EADDRINUSE, "Unable to open any port in range %s-%s" % (port_min, (port_max))) 100 | 101 | self.__api_server.add_route('/', self._auth_redir_success, content_type='text/html') 102 | self.__api_server.add_route('/auth', self._auth_redir_success, content_type='text/html') 103 | self.__api_server.add_route('/favicon.ico', lambda s, x, y: "", content_type='text/html') 104 | 105 | self.__thread = threading.Thread(target=self.__api_server.serve_forever, 106 | daemon=True) 107 | self.__thread.start() 108 | log.info('Listening on %s', self.uri()) 109 | 110 | def _auth_redir_success(self, _env, info): 111 | err = "" 112 | if info and ('error' in info or 'error_description' in info): 113 | log.debug("auth error") 114 | err = info['error'] if 'error' in info else \ 115 | info['error_description'][0] 116 | if isinstance(err, list): 117 | err = err[0] 118 | self.failure_info = err 119 | if self.__on_failure: 120 | self.__on_failure(err) 121 | return self.auth_failure(err) 122 | try: 123 | log.debug("auth success") 124 | self.success_code = info["code"][0] 125 | if self.__on_success: 126 | self.__on_success(info) 127 | except Exception as e: 128 | log.exception('Failed to authenticate') 129 | err = 'Unknown error: %s' % e 130 | self.failure_info = err 131 | 132 | return self.auth_failure(err) if err else self.auth_success() 133 | 134 | def auth_success(self): 135 | self.event.set() 136 | if self.__html_response_generator: 137 | return self.__html_response_generator(True, '') 138 | return "OAuth Success" 139 | 140 | def auth_failure(self, msg): 141 | self.event.set() 142 | if self.__html_response_generator: 143 | return self.__html_response_generator(False, msg) 144 | return "OAuth Failure:" + msg 145 | 146 | def shutdown(self): 147 | """Abandon any waiting oauths and shut down the server""" 148 | self.event.set() 149 | if self.__api_server and self.__running: 150 | try: 151 | self.__api_server.shutdown() 152 | except Exception: 153 | log.exception("failed to shutdown") 154 | self.__running = False 155 | self.__on_success = None 156 | self.__on_failure = None 157 | self.__thread = None 158 | 159 | def server_close(self): 160 | """Closes the server and joins all threads""" 161 | if self.__api_server and not self.__running: 162 | try: 163 | self.__api_server.server_close() 164 | except Exception: # pragma: no cover 165 | log.exception("failed to close server") 166 | 167 | def wait(self, timeout=None): 168 | """Wait for oauth response""" 169 | self.event.wait(timeout=timeout) 170 | 171 | def uri(self): 172 | """Return the base url for this server""" 173 | if not self.__api_server: 174 | return None 175 | return self.__api_server.uri("/", self.__host_name) 176 | 177 | def port(self): 178 | """Port number for this server""" 179 | if not self.__api_server: 180 | return None 181 | return self.__api_server.port() 182 | 183 | -------------------------------------------------------------------------------- /cloudsync/command/utils.py: -------------------------------------------------------------------------------- 1 | import os 2 | import re 3 | import logging 4 | import json 5 | import argparse 6 | import abc 7 | from typing import Optional 8 | 9 | from cloudsync import get_provider, known_providers, OAuthConfig, Provider, Creds, CloudNamespaceError 10 | 11 | log = logging.getLogger() 12 | 13 | PROVIDER_ALIASES = { 14 | "file" : "filesystem", 15 | } 16 | 17 | def cli_providers(): 18 | return sorted(p for p in known_providers() if not p.startswith("test") and not p.startswith("mock_")) 19 | 20 | class SubCmd(abc.ABC): 21 | """Base class for sub commands""" 22 | 23 | def __init__(self, main, name, help): # pylint: disable=redefined-builtin 24 | self.parser: argparse.ArgumentParser = main.add_parser( 25 | name, 26 | help=help, 27 | formatter_class=argparse.ArgumentDefaultsHelpFormatter) 28 | 29 | def common_sync_args(self): 30 | """Add common sync config args.""" 31 | default_config = os.path.expanduser("~/.config/cloudsync/config") 32 | default_creds = os.path.expanduser("~/.config/cloudsync/creds") 33 | 34 | self.parser.add_argument('-C', '--config', help='Read config from file', action="store", default=default_config) 35 | # toto: support two schemes: file:/path, keyring:cloudsync 36 | self.parser.add_argument('-R', '--creds', help='Read/save creds from file', action="store", default=default_creds) 37 | self.parser.add_argument('-q', '--quiet', help='Quiet mode, no interactive auth', action="store_true") 38 | 39 | self.parser.epilog = f""" 40 | Supported providers: {cli_providers()} 41 | """ 42 | 43 | @abc.abstractmethod 44 | def run(self, args: argparse.Namespace): 45 | ... 46 | 47 | class FauxURI: # pylint: disable=too-few-public-methods 48 | """ 49 | Represents a faux-URI passed on the command line. 50 | 51 | For example: foo:bar 52 | """ 53 | def __init__(self, uri): 54 | (method, path) = ('file', uri) 55 | if ':' in uri: 56 | # this enhrines that provider names must be 2 or more characters 57 | # and it protects us from seeing c:/whatever as a provider 58 | m = re.match(r"([^:]{2,}):(.*)", uri) 59 | if m: 60 | (method, path) = m.groups() 61 | 62 | self.method = method 63 | self.path = path 64 | 65 | class CloudURI(FauxURI): # pylint: disable=too-few-public-methods 66 | """ 67 | Represents a faux-cloud-URI passed on the command line. 68 | 69 | For example: gdrive:/path/to/file 70 | onedrive@team/namespace:path/to/file 71 | """ 72 | def __init__(self, uri): 73 | super().__init__(uri) 74 | 75 | self.namespace = "" 76 | namespace = re.match(r"(.*)@(.*)", self.method) 77 | if namespace: 78 | (self.method, self.namespace) = namespace.groups() 79 | 80 | if self.method in PROVIDER_ALIASES: 81 | self.method = PROVIDER_ALIASES[self.method] 82 | if self.method not in known_providers(): 83 | raise ValueError("Unknown provider %s, try pip install cloudsync[%s] or pip install cloudsync-%s" % (self.method, self.method, self.method)) 84 | self.provider_type = get_provider(self.method) 85 | 86 | def _set_namespace(self, provider: Provider): 87 | if provider.name == "filesystem": 88 | # todo: this is a crappy hack because we don't initialze providers with the root oid or path 89 | # once that's done, this can go away 90 | provider.namespace_id = self.path 91 | self.path = "/" 92 | elif self.namespace: 93 | # lookup namespace by name 94 | namespace = next((ns for ns in provider.list_ns() if ns.name == self.namespace), None) 95 | if not namespace: 96 | raise CloudNamespaceError(f"unknown namespace: {self.namespace}") 97 | provider.namespace = namespace 98 | 99 | def provider_instance(self, args, *, connect=True) -> Provider: 100 | """Given command-line args, construct a provider object, and connect it.""" 101 | 102 | cls = self.provider_type 103 | 104 | creds = None 105 | 106 | if cls.uses_oauth(): 107 | oc = get_oauth_config(args, cls.name, args.creds) 108 | log.debug("init %s oauth", cls.name) 109 | prov = cls(oc) 110 | creds = oc.get_creds() 111 | if not args.quiet and not creds and connect: 112 | creds = prov.authenticate() 113 | 114 | if creds: 115 | prov.set_creds(creds) 116 | 117 | if connect: 118 | prov.connect(creds) 119 | else: 120 | prov = cls() 121 | if cls.name.startswith("mock_"): 122 | prov.set_creds({"fake" : "creds"}) 123 | prov.reconnect() 124 | 125 | self._set_namespace(prov) 126 | return prov 127 | 128 | 129 | # TOOD: better documentation on how to get your own one of these 130 | OAUTH_CONFIG = { 131 | "gdrive": { 132 | "id": "918922786103-f842erh49vb7jecl9oo4b5g4gm1eka6v.apps.googleusercontent.com", 133 | "secret": "F2CdO5YTzX6TfKGlOMDbV1WS", 134 | }, 135 | "onedrive": { 136 | "id": "797a365f-772d-421f-a3fe-7b55ab6defa4", 137 | "secret": "", 138 | } 139 | } 140 | 141 | _config = None 142 | 143 | 144 | def config(args): 145 | """The global config singleton, parsed from ~/.config/cloudsync""" 146 | global _config # pylint: disable=global-statement 147 | if _config is None: 148 | try: 149 | log.debug("config : %s", args.config) 150 | _config = json.load(open(args.config, "r")) 151 | except FileNotFoundError: 152 | log.debug("config not used: %s", args.config) 153 | _config = {} 154 | return _config 155 | 156 | 157 | class CliOAuthConfig(OAuthConfig): 158 | """OAuth config for command line. Writes to local creds file or keyring.""" 159 | def __init__(self, *ar, prov_name, save_uri, **kw): 160 | self.save: Optional[FauxURI] = save_uri and FauxURI(save_uri) 161 | self.prov = prov_name 162 | self.creds: Creds = {} 163 | 164 | if self.save: 165 | if self.save.method != "file": 166 | raise ValueError("Unsupported creds save method: %s" % self.save.method) 167 | 168 | try: 169 | log.debug("load creds %s", self.save.path) 170 | with open(self.save.path) as f: 171 | self.creds = json.load(f) 172 | except FileNotFoundError: 173 | pass 174 | 175 | super().__init__(*ar, **kw) 176 | 177 | def get_creds(self): 178 | if not self.save: 179 | return None 180 | 181 | return self.creds.get(self.prov, None) 182 | 183 | def creds_changed(self, creds): 184 | if not self.save: 185 | super().creds_changed(creds) 186 | return 187 | 188 | try: 189 | was = os.umask(0o77) 190 | self.creds.update({self.prov: creds}) 191 | os.makedirs(os.path.dirname(self.save.path), mode=0o700, exist_ok=True) 192 | with open(self.save.path, "w") as f: 193 | json.dump(self.creds, f) 194 | finally: 195 | os.umask(was) 196 | 197 | 198 | def generic_oauth_config(name): 199 | return get_oauth_config(None, name, None) 200 | 201 | 202 | def get_oauth_config(args, name, save_uri): 203 | """Reads oauth config from the global config singleton, or uses the defaults""" 204 | if args: 205 | top = config(args).get("oauth", {}) 206 | oauth = top.get(name, {}) 207 | else: 208 | oauth = {} 209 | default = OAUTH_CONFIG.get(name, {}) 210 | 211 | for k in ["id", "secret", "host", "ports"]: 212 | oauth[k] = oauth.get(k, default.get(k, None)) 213 | 214 | return CliOAuthConfig(prov_name=name, save_uri=save_uri, app_id=oauth["id"], app_secret=oauth["secret"], 215 | host_name=oauth["host"], port_range=oauth["ports"]) 216 | 217 | 218 | def get_providers(args, uris): 219 | """Given command args and a pair of CloudURI objects, return provider objects.""" 220 | _provs = [] 221 | for uri in uris: 222 | prov = uri.provider_instance(args) 223 | _provs.append(prov) 224 | return _provs 225 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | GNU LESSER GENERAL PUBLIC LICENSE 2 | 3 | 'cloudsync' is Copyright (C) 2019 Atakama, LLC 4 | 5 | For commercial licensing and support, please contact: info@atakama.com 6 | 7 | Atakama LLC is distributing this library in the hope that it will be useful, 8 | but WITHOUT ANY WARRANTY; without even the implied warranty of 9 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 10 | 11 | See the GNU Lesser General Public License version 3 (below) for more 12 | details and the limitations around any commercial use of the files distributed 13 | with this library. 14 | 15 | ------------------------------------------------------------------------- 16 | 17 | GNU LESSER GENERAL PUBLIC LICENSE 18 | Version 3, 29 June 2007 19 | 20 | Copyright © 2007 Free Software Foundation, Inc. 21 | Everyone is permitted to copy and distribute verbatim copies of this 22 | licensedocument, but changing it is not allowed. 23 | 24 | This version of the GNU Lesser General Public License incorporates 25 | the terms and conditions of version 3 of the GNU General Public 26 | License, supplemented by the additional permissions listed below. 27 | 28 | 0. Additional Definitions. 29 | 30 | As used herein, “this License” refers to version 3 of the GNU Lesser 31 | General Public License, and the “GNU GPL” refers to version 3 of the 32 | GNU General Public License. 33 | 34 | “The Library” refers to a covered work governed by this License, 35 | other than an Application or a Combined Work as defined below. 36 | 37 | An “Application” is any work that makes use of an interface provided 38 | by the Library, but which is not otherwise based on the Library. 39 | Defining a subclass of a class defined by the Library is deemed a mode 40 | of using an interface provided by the Library. 41 | 42 | A “Combined Work” is a work produced by combining or linking an 43 | Application with the Library. The particular version of the Library 44 | with which the Combined Work was made is also called the “Linked 45 | Version”. 46 | 47 | The “Minimal Corresponding Source” for a Combined Work means the 48 | Corresponding Source for the Combined Work, excluding any source code 49 | for portions of the Combined Work that, considered in isolation, are 50 | based on the Application, and not on the Linked Version. 51 | 52 | The “Corresponding Application Code” for a Combined Work means the 53 | object code and/or source code for the Application, including any data 54 | and utility programs needed for reproducing the Combined Work from the 55 | Application, but excluding the System Libraries of the Combined Work. 56 | 57 | 1. Exception to Section 3 of the GNU GPL. 58 | 59 | You may convey a covered work under sections 3 and 4 of this License 60 | without being bound by section 3 of the GNU GPL. 61 | 62 | 2. Conveying Modified Versions. 63 | 64 | If you modify a copy of the Library, and, in your modifications, a 65 | facility refers to a function or data to be supplied by an Application 66 | that uses the facility (other than as an argument passed when the 67 | facility is invoked), then you may convey a copy of the modified 68 | version: 69 | 70 | a) under this License, provided that you make a good faith effort 71 | to ensure that, in the event an Application does not supply the 72 | function or data, the facility still operates, and performs 73 | whatever part of its purpose remains meaningful, or 74 | 75 | b) under the GNU GPL, with none of the additional permissions of 76 | this License applicable to that copy. 77 | 78 | 3. Object Code Incorporating Material from Library Header Files. 79 | 80 | The object code form of an Application may incorporate material from 81 | a header file that is part of the Library. You may convey such object 82 | code under terms of your choice, provided that, if the incorporated 83 | material is not limited to numerical parameters, data structure 84 | layouts and accessors, or small macros, inline functions and templates 85 | (ten or fewer lines in length), you do both of the following: 86 | 87 | a) Give prominent notice with each copy of the object code that 88 | the Library is used in it and that the Library and its use are 89 | covered by this License. 90 | 91 | b) Accompany the object code with a copy of the GNU GPL and this 92 | license document. 93 | 94 | 4. Combined Works. 95 | 96 | You may convey a Combined Work under terms of your choice that, taken 97 | together, effectively do not restrict modification of the portions of 98 | the Library contained in the Combined Work and reverse engineering for 99 | debugging such modifications, if you also do each of the following: 100 | 101 | a) Give prominent notice with each copy of the Combined Work that 102 | the Library is used in it and that the Library and its use are 103 | covered by this License. 104 | 105 | b) Accompany the Combined Work with a copy of the GNU GPL and this 106 | license document. 107 | 108 | c) For a Combined Work that displays copyright notices during 109 | execution, include the copyright notice for the Library among 110 | these notices, as well as a reference directing the user to the 111 | copies of the GNU GPL and this license document. 112 | 113 | d) Do one of the following: 114 | 115 | 0) Convey the Minimal Corresponding Source under the terms of 116 | this License, and the Corresponding Application Code in a form 117 | suitable for, and under terms that permit, the user to 118 | recombine or relink the Application with a modified version of 119 | the Linked Version to produce a modified Combined Work, in the 120 | manner specified by section 6 of the GNU GPL for conveying 121 | Corresponding Source. 122 | 123 | 1) Use a suitable shared library mechanism for linking with 124 | the Library. A suitable mechanism is one that (a) uses at run 125 | time a copy of the Library already present on the user's 126 | computer system, and (b) will operate properly with a modified 127 | version of the Library that is interface-compatible with the 128 | Linked Version. 129 | 130 | e) Provide Installation Information, but only if you would 131 | otherwise be required to provide such information under section 6 132 | of the GNU GPL, and only to the extent that such information is 133 | necessary to install and execute a modified version of the 134 | Combined Work produced by recombining or relinking the Application 135 | with a modified version of the Linked Version. (If you use option 136 | 4d0, the Installation Information must accompany the Minimal 137 | Corresponding Source and Corresponding Application Code. If you 138 | use option 4d1, you must provide the Installation Information in 139 | the manner specified by section 6 of the GNU GPL for conveying 140 | Corresponding Source.) 141 | 142 | 5. Combined Libraries. 143 | 144 | You may place library facilities that are a work based on the Library 145 | side by side in a single library together with other library 146 | facilities that are not Applications and are not covered by this 147 | License, and convey such a combined library under terms of your 148 | choice, if you do both of the following: 149 | 150 | a) Accompany the combined library with a copy of the same work 151 | based on the Library, uncombined with any other library 152 | facilities, conveyed under the terms of this License. 153 | 154 | b) Give prominent notice with the combined library that part of 155 | it is a work based on the Library, and explaining where to find 156 | the accompanying uncombined form of the same work. 157 | 158 | 6. Revised Versions of the GNU Lesser General Public License. 159 | 160 | The Free Software Foundation may publish revised and/or new versions 161 | of the GNU Lesser General Public License from time to time. Such new 162 | versions will be similar in spirit to the present version, but may 163 | differ in detail to address new problems or concerns. 164 | 165 | Each version is given a distinguishing version number. If the Library 166 | as you received it specifies that a certain numbered version of the 167 | GNU Lesser General Public License “or any later version” applies to 168 | it, you have the option of following the terms and conditions either 169 | of that published version or of any later version published by the 170 | Free Software Foundation. If the Library as you received it does not 171 | specify a version number of the GNU Lesser General Public License, 172 | you may choose any version of the GNU Lesser General Public License 173 | ever published by the Free Software Foundation. 174 | 175 | If the Library as you received it specifies that a proxy can decide 176 | whether future versions of the GNU Lesser General Public License shall 177 | apply, that proxy's public statement of acceptance of any version is 178 | permanent authorization for you to choose that version for the Library. 179 | -------------------------------------------------------------------------------- /cloudsync/runnable.py: -------------------------------------------------------------------------------- 1 | """ 2 | Generic 'runnable' abstract base class. 3 | 4 | All cloudsync services inherit from this, instead of implementing their own 5 | thread management. 6 | """ 7 | 8 | import time 9 | 10 | from abc import ABC, abstractmethod 11 | 12 | import threading 13 | import logging 14 | from typing import List 15 | 16 | log = logging.getLogger(__name__) 17 | 18 | 19 | def time_helper(timeout, sleep=None, multiply=1): 20 | """ 21 | Simple generator that yields every `sleep` seconds, and stops after `timeout` seconds 22 | """ 23 | forever = not timeout 24 | end = forever or time.monotonic() + timeout 25 | while forever or end >= time.monotonic(): 26 | yield True 27 | if sleep is not None: 28 | time.sleep(sleep) 29 | sleep = sleep * multiply 30 | raise TimeoutError() 31 | 32 | 33 | class _BackoffError(Exception): 34 | pass 35 | 36 | 37 | class Runnable(ABC): # pylint: disable=too-many-instance-attributes 38 | """ 39 | Abstract base class for a runnable service. 40 | 41 | User needs to override and implement the "do" method. 42 | """ 43 | # pylint: disable=multiple-statements 44 | min_backoff = 0.01 ; """Min backoff time in seconds""" 45 | max_backoff = 1.0 ; """Max backoff time in seconds""" 46 | mult_backoff = 2.0 ; """Backoff multiplier""" 47 | in_backoff = 0.0 ; """Current backoff seconds, 0.0 if not in a backoff state""" 48 | service_name = None ; """The name of this runnable service, defaults to the class name""" 49 | # pylint: enable=multiple-statements 50 | 51 | __thread = None 52 | __shutdown = False 53 | __interrupt: threading.Event = None 54 | __stopped = False 55 | __stopping = False 56 | __log: logging.Logger = None 57 | __clear_on_success: bool = True 58 | _run_until = None 59 | 60 | @property 61 | def stopped(self): 62 | """Set when you call stop(), causes the services to drop out.""" 63 | return self.__stopped or self.__shutdown 64 | 65 | def interruptable_sleep(self, secs): 66 | """Call this instead of sleep, so the service can be interrupted""" 67 | if self.__interrupt and self.__interrupt.wait(secs): 68 | self.__interrupt.clear() 69 | 70 | def __increment_backoff(self): 71 | self.in_backoff = min(self.max_backoff, max(self.in_backoff * self.mult_backoff, self.min_backoff)) 72 | 73 | def run(self, *, timeout=None, until=None, sleep=0.001): # pylint: disable=too-many-branches 74 | """ 75 | Calls do in a loop. 76 | 77 | Args: 78 | timeout: stop calling do after secs 79 | until: lambda returns bool 80 | sleep: seconds 81 | 82 | 83 | If an unhandled exception occurs, backoff sleep will occur. 84 | """ 85 | self._run_until = until 86 | 87 | if self.service_name is None: 88 | self.service_name = self.__class__.__name__ 89 | 90 | self.__log = logging.getLogger(__name__ + "." + self.service_name) 91 | log.debug("running %s", self.service_name) 92 | 93 | # ordering of these two prevents race condition if you start/stop quickly 94 | # see `def started` 95 | self.__interrupt = threading.Event() 96 | self.__stopped = False 97 | 98 | try: 99 | for _ in time_helper(timeout): 100 | if self.__stopping or self.__shutdown: 101 | break 102 | 103 | try: 104 | self.__clear_on_success = True 105 | self.do() 106 | if self.__clear_on_success and self.in_backoff > 0: 107 | self.in_backoff = 0 108 | log.debug("%s: clear backoff", self.service_name) 109 | except _BackoffError: 110 | self.__increment_backoff() 111 | log.debug("%s: backing off %s", self.service_name, self.in_backoff) 112 | except Exception: 113 | self.__increment_backoff() 114 | log.exception("unhandled exception in %s", self.service_name) 115 | except BaseException: 116 | self.__increment_backoff() 117 | log.exception("very serious exception in %s", self.service_name) 118 | 119 | if self.__stopping or self.__shutdown or (until is not None and until()): 120 | break 121 | 122 | if self.in_backoff > 0: 123 | log.debug("%s: backoff sleep %s", self.service_name, self.in_backoff) 124 | self.interruptable_sleep(self.in_backoff) 125 | else: 126 | self.interruptable_sleep(sleep) 127 | finally: 128 | # clear started flag 129 | self.__stopping = False 130 | self.__stopped = True 131 | self.__interrupt = None 132 | 133 | # if logging "stopping %s" fails during a test with a "ValueError: I/O operation on closed file" 134 | # then an instance of Runnable was permitted to run beyond the end of the test. The test log should show 135 | # the service_name, which should identify which service was left around. Make sure that the service 136 | # is really stopped before dropping out of the test, by catching the call to done(), and when done() 137 | # is called, then you know there won't be any more logging from the service. 138 | log.debug("stopping %s", self.service_name) 139 | 140 | if self.__shutdown: 141 | self.done() 142 | 143 | @property 144 | def started(self): 145 | """ 146 | True if the service has been started and has not finished stopping 147 | """ 148 | return self.__interrupt is not None 149 | 150 | @staticmethod 151 | def backoff(): 152 | """ 153 | Raises an exception, interrupting the durrent do() call, and sleeping for backoff seconds. 154 | """ 155 | raise _BackoffError() 156 | 157 | def nothing_happened(self): 158 | """ 159 | Sets a "nothing happened" flag. This will cause backoff to remain the same, even on success. 160 | """ 161 | self.__clear_on_success = False 162 | 163 | def wake(self): 164 | """ 165 | Wake up, if do was sleeping, and do things right away. 166 | """ 167 | if self.__interrupt is None: 168 | log.warning("not running, wake ignored") 169 | return 170 | self.__interrupt.set() 171 | 172 | def start(self, *, daemon=True, **kwargs): 173 | """ 174 | Start a thread, kwargs are passed to run() 175 | """ 176 | if self.service_name is None: 177 | self.service_name = self.__class__.__name__ 178 | if self.__shutdown: 179 | raise RuntimeError("Service was stopped, create a new instance to run.") 180 | if self.__thread and self.__thread.is_alive(): 181 | self.__thread.join(timeout=1) # give the old thread a chance to die 182 | if self.__thread and self.__thread.is_alive(): 183 | raise RuntimeError("Service already started") 184 | self.__stopping = False 185 | self.__thread = threading.Thread(target=self.run, kwargs=kwargs, daemon=daemon, name=self.service_name) 186 | self.__thread.name = self.service_name 187 | self.__thread.start() 188 | 189 | @abstractmethod 190 | def do(self): 191 | """ 192 | Override this to do something in a loop. 193 | """ 194 | ... 195 | 196 | def stop(self, forever=True, wait=True): 197 | """ 198 | Stop the service, allowing any do() to complete first. 199 | """ 200 | self.__stopping = True 201 | self.wake() 202 | self.__shutdown = forever 203 | thread = self.__thread # otherwise race condition -- self.__thread can change value in another thread 204 | if thread: 205 | if threading.current_thread() != thread: 206 | if wait: 207 | self.wait() 208 | 209 | @staticmethod 210 | def stop_all(runnables: List["Runnable"], forever: bool = True, wait: bool = True): 211 | """ 212 | Convenience function for stopping multiple Runnables efficiently. 213 | """ 214 | log.info("stop_all: forever=%s wait=%s", forever, wait) 215 | for run in runnables: 216 | # signal all runnables to stop before waiting on any of them 217 | run.stop(forever=forever, wait=False) 218 | if wait: 219 | for run in runnables: 220 | run.wait() 221 | 222 | def done(self): 223 | """ 224 | Cleanup code goes here. This is called when a service is stopped. 225 | """ 226 | 227 | def wait(self, timeout=None): 228 | """ 229 | Wait for the service to stop. 230 | """ 231 | thread = self.__thread # otherwise race condition -- self.__thread can change value in another thread 232 | if thread and threading.current_thread() != thread: 233 | thread.join(timeout=timeout) 234 | if thread and thread.is_alive(): 235 | raise TimeoutError() 236 | return True 237 | else: 238 | return False 239 | -------------------------------------------------------------------------------- /cloudsync/oauth/oauth_config.py: -------------------------------------------------------------------------------- 1 | import os 2 | import logging 3 | from typing import Optional, Tuple, Any 4 | import webbrowser 5 | from oauthlib.oauth2 import OAuth2Error 6 | from requests_oauthlib import OAuth2Session 7 | 8 | from .redir_server import OAuthRedirServer 9 | 10 | __all__ = ["OAuthConfig", "OAuthToken", "OAuthError"] 11 | 12 | # don't log tokens 13 | logging.getLogger("requests_oauthlib").setLevel(logging.INFO) 14 | 15 | log = logging.getLogger(__name__) 16 | 17 | OAuthError = OAuth2Error 18 | 19 | # this class delibarately not strict, since it can contain provider-specific configuration 20 | # applications can derive from this class and provide appropriate defaults 21 | 22 | 23 | class OAuthToken: # pylint: disable=too-few-public-methods 24 | """ 25 | Just a class representation of the oauth2 standard token. 26 | 27 | See: https://www.oauth.com/oauth2-servers/access-tokens/access-token-response/ 28 | """ 29 | def __init__(self, data=None, **kwargs): 30 | if data is None: 31 | data = kwargs 32 | self.access_token = data["access_token"] 33 | self.token_type = data["token_type"] 34 | self.expires_in = data.get("expires_in") 35 | self.refresh_token = data.get("refresh_token") 36 | self.scope = data.get("scope") 37 | 38 | 39 | class OAuthConfig: 40 | """ 41 | Required argument for providers that return True to uses_oauth. 42 | 43 | Args: 44 | app_id: also known as "client id", provided for your application by the cloud provider 45 | app_secret: also known as "client secret", provided for your application by the cloud provider 46 | manual_mode: set to True, if you don't intend to use the redirect server 47 | redirect_server: a server that, at a minimum, supports the uri() command, probably should just change this to uri() 48 | port_range: the range of valid ports for your registered app (some providers burden you with this) 49 | host_name: defaults to 127.0.0.1 50 | """ 51 | def __init__(self, *, app_id: str, app_secret: str, 52 | manual_mode: bool = False, 53 | redirect_server: Optional[OAuthRedirServer] = None, 54 | port_range: Tuple[int, int] = None, 55 | host_name: str = None): 56 | """ 57 | There are two ways to create an OAuthConfig object: by providing a OAuthRedirServer or by providing the 58 | success and failure callbacks, as well as the port and host configs 59 | :param app_id 60 | :param app_secret 61 | :param manual_mode 62 | :param redirect_server (if none, one will be created for you) 63 | :param port_range (defaults to 'any port') 64 | :param host_name (defaults to 127.0.0.1) 65 | """ 66 | 67 | # Ideally, provider-specific endpoints and behaviors are controlled by the provider code 68 | # Consumer-specific settings are initialized here 69 | # So far, only app id's and redir endpoints seem to be necessary 70 | 71 | self.app_id = app_id 72 | self.app_secret = app_secret 73 | self.manual_mode = manual_mode 74 | self.authorization_url = None 75 | self._session: OAuth2Session = None 76 | self._token: OAuthToken = None 77 | 78 | self._redirect_server = redirect_server 79 | 80 | if manual_mode and self._redirect_server: 81 | raise ValueError('Cannot use both manual mode and an oauth server') 82 | 83 | if port_range and self._redirect_server: 84 | raise ValueError('If providing a server, no need to set port range') 85 | 86 | if not self.manual_mode and not self._redirect_server: 87 | self._redirect_server = OAuthRedirServer(html_generator=self._gen_html_response, 88 | port_range=port_range, host_name=host_name) 89 | 90 | def start_auth(self, auth_url, scope=None, **kwargs): 91 | """ 92 | Call this if you want oauth to be handled for you 93 | This starts a server, pops a browser. 94 | Do some stuff, then follow with wait_auth() to wait 95 | """ 96 | log.debug("appid %s", self.app_id) 97 | if self.app_id is None: 98 | raise OAuthError("app id None") 99 | if self.app_secret is None: 100 | raise OAuthError("app secret is None") 101 | if auth_url is None: 102 | raise OAuthError("auth url bad") 103 | os.environ["OAUTHLIB_RELAX_TOKEN_SCOPE"] = "1" 104 | self.start_server() 105 | self._session = OAuth2Session(client_id=self.app_id, scope=scope, redirect_uri=self.redirect_uri, **kwargs) 106 | self.authorization_url, _unused_state = self._session.authorization_url(auth_url) 107 | log.debug("start oauth url %s, redir %s, appid %s", self.authorization_url, self.redirect_uri, self.app_id) 108 | webbrowser.open(self.authorization_url) 109 | 110 | def wait_auth(self, token_url, timeout=None, **kwargs): 111 | """ 112 | Returns an OAuthToken object, or raises a OAuthError 113 | """ 114 | assert self._session 115 | try: 116 | if not self.wait_success(timeout): 117 | if self.failure_info: 118 | raise OAuthError(self.failure_info) 119 | raise OAuthError("Oauth interrupted") 120 | 121 | self._token = OAuthToken(self._session.fetch_token(token_url, 122 | include_client_id=True, 123 | client_secret=self.app_secret, 124 | code=self.success_code, 125 | timeout=60, 126 | **kwargs)) 127 | self._token_changed() 128 | return self._token 129 | finally: 130 | self.shutdown() 131 | 132 | def refresh(self, refresh_url, token=None, scope=None, **extra): 133 | """ 134 | Given a refresh url (often the same as token_url), will refresh the token 135 | Call this when your provider raises an exception implying your token has expired 136 | Or, you could just call it before the expiration 137 | """ 138 | assert self._session or scope 139 | if not self._session and scope: 140 | self._session = OAuth2Session(client_id=self.app_id, scope=scope, redirect_uri=self.redirect_uri) 141 | extra["client_id"] = self.app_id 142 | extra["client_secret"] = self.app_secret 143 | extra["timeout"] = 60 144 | if isinstance(token, OAuthToken): 145 | token = token.refresh_token 146 | self._token = OAuthToken(self._session.refresh_token(refresh_url, refresh_token=token, **extra)) 147 | if self._token.refresh_token != token: 148 | self._token_changed() 149 | return self._token 150 | 151 | @property 152 | def success_code(self): 153 | return self._redirect_server.success_code 154 | 155 | @property 156 | def failure_info(self): 157 | return self._redirect_server.failure_info 158 | 159 | def start_server(self, *, on_success=None, on_failure=None): 160 | """ 161 | Start the redirect server in a thread 162 | """ 163 | assert self._redirect_server 164 | self._redirect_server.run(on_success=on_success, on_failure=on_failure) 165 | 166 | def wait_success(self, timeout=None): 167 | """ 168 | Wait for the redirect server, return true if it succeeded 169 | Shut down the server 170 | """ 171 | assert self._redirect_server 172 | try: 173 | self._redirect_server.wait(timeout=timeout) 174 | return bool(self._redirect_server.success_code) 175 | finally: 176 | self.shutdown() 177 | 178 | def shutdown(self): 179 | """ 180 | Stop the redirect server, and interrupt/fail any ongoing oauth 181 | """ 182 | assert self._redirect_server 183 | self._redirect_server.shutdown() 184 | 185 | def server_close(self): 186 | """ 187 | Close the redirect server and join all threads 188 | """ 189 | assert self._redirect_server 190 | self._redirect_server.server_close() 191 | 192 | @property 193 | def redirect_uri(self) -> str: 194 | """ 195 | Get the redirect server's uri 196 | """ 197 | if self._redirect_server is None: 198 | return None 199 | return self._redirect_server.uri() 200 | 201 | def _gen_html_response(self, success: bool, err_msg: str): 202 | if success: 203 | return self.success_message() 204 | else: 205 | return self.failure_message(err_msg) 206 | 207 | def _token_changed(self): 208 | # this could just be a setter 209 | creds = {"refresh_token": self._token.refresh_token, "access_token": self._token.access_token} 210 | self.creds_changed(creds) 211 | 212 | def creds_changed(self, creds: Any): # pylint: disable=unused-argument, no-self-use 213 | """Override this to save creds on refresh""" 214 | log.warning("creds will not be saved, implement OAuthConfig.creds_changed to save it.") 215 | 216 | # override this to make a nicer message on success 217 | @staticmethod 218 | def success_message() -> str: 219 | return 'OAuth succeeded!' 220 | 221 | # override this to make a nicer message on failure 222 | @staticmethod 223 | def failure_message(error_str: str) -> str: 224 | return 'OAuth failed: {}'.format(error_str) 225 | -------------------------------------------------------------------------------- /cloudsync/tests/test_box.py: -------------------------------------------------------------------------------- 1 | import os 2 | import io 3 | import threading 4 | import logging 5 | from typing import Dict, List 6 | from unittest.mock import patch 7 | 8 | import pytest 9 | 10 | from cloudsync.exceptions import CloudTokenError 11 | from cloudsync.providers import BoxProvider 12 | from cloudsync.oauth import OAuthConfig, OAuthProviderInfo 13 | from cloudsync.oauth.apiserver import ApiServer, ApiError, api_route 14 | 15 | from .fixtures import FakeApi, fake_oauth_provider 16 | 17 | log = logging.getLogger(__name__) 18 | 19 | 20 | class FakeBoxApi(FakeApi): 21 | @api_route("/users/me") 22 | def upload(self, ctx, req): 23 | self.called("users/me", (ctx, req)) 24 | return {'address': '', 25 | 'avatar_url': 'https://app.box.com/api/avatar/large/8506151483', 26 | 'created_at': '2019-05-29T08:35:19-07:00', 27 | 'id': '8506151483', 28 | 'job_title': '', 29 | 'language': 'en', 30 | 'login': 'AutomationUser_813890_GmcM3Cohcy@boxdevedition.com', 31 | 'max_upload_size': 5368709120, 32 | 'modified_at': '2019-12-12T05:13:29-08:00', 33 | 'name': 'Atakama JWT', 34 | 'notification_email': [], 35 | 'phone': '', 36 | 'space_amount': 10737418240, 37 | 'space_used': 5551503, 38 | 'status': 'active', 39 | 'timezone': 'America/Los_Angeles', 40 | 'type': 'user'} 41 | 42 | @api_route("/folders/0/items") 43 | def folder_items(self, ctx, req): 44 | return { 45 | 'entries': 46 | [{'etag': '0', 47 | 'id': '95401994626', 48 | 'name': '0109d27be3d76224f640e6076c77184d', 49 | 'sequence_id': '0', 50 | 'content_modified_at': '2019-12-12T06:48:48-08:00', 51 | 'type': 'folder'}, 52 | {'etag': '0', 53 | 'id': '95382018330', 54 | 'name': '037c2561c96ec54635d50f71ae13ab72', 55 | 'sequence_id': '0', 56 | 'content_modified_at': '2019-12-12T06:48:48-08:00', 57 | 'type': 'folder'}, 58 | ], 59 | 'limit': 1000, 60 | 'offset': 0, 61 | 'order': [{'by': 'type', 'direction': 'ASC'}, 62 | {'by': 'name', 'direction': 'ASC'}], 63 | 'total_count': 2} 64 | 65 | @api_route("/folders/") 66 | def folders(self, ctx, req): 67 | if ctx.get("REQUEST_METHOD") == "POST": 68 | self.called("mkdir", (ctx, req)) 69 | return {'content_created_at': '2019-12-12T06:48:48-08:00', 70 | 'content_modified_at': '2019-12-12T06:48:48-08:00', 71 | 'created_at': '2019-12-12T06:48:48-08:00', 72 | 'created_by': {'id': '8506151483', 73 | 'login': 'AutomationUser_813890_GmcM3Cohcy@boxdevedition.com', 74 | 'name': 'Atakama JWT', 75 | 'type': 'user'}, 76 | 'description': '', 77 | 'etag': '0', 78 | 'folder_upload_email': None, 79 | 'id': '96120809690', 80 | 'item_collection': {'entries': [], 81 | 'limit': 100, 82 | 'offset': 0, 83 | 'order': [{'by': 'type', 'direction': 'ASC'}, 84 | {'by': 'name', 'direction': 'ASC'}], 85 | 'total_count': 0}, 86 | 'item_status': 'active', 87 | 'modified_at': '2019-12-12T06:48:48-08:00', 88 | 'modified_by': {'id': '8506151483', 89 | 'login': 'AutomationUser_813890_GmcM3Cohcy@boxdevedition.com', 90 | 'name': 'Atakama JWT', 91 | 'type': 'user'}, 92 | 'name': 'c09bf978eab751234c418e6ff06a43bd(.dest', 93 | 'owned_by': {'id': '8506151483', 94 | 'login': 'AutomationUser_813890_GmcM3Cohcy@boxdevedition.com', 95 | 'name': 'Atakama JWT', 96 | 'type': 'user'}, 97 | 'parent': {'etag': '0', 98 | 'id': '96128905139', 99 | 'name': '0274d8039a0277f56f489352011d9f2f', 100 | 'sequence_id': '0', 101 | 'type': 'folder'}, 102 | 'path_collection': {'entries': [{'etag': None, 103 | 'id': '0', 104 | 'name': 'All Files', 105 | 'sequence_id': None, 106 | 'type': 'folder'}, 107 | {'etag': '0', 108 | 'id': '96128905139', 109 | 'name': '0274d8039a0277f56f489352011d9f2f', 110 | 'sequence_id': '0', 111 | 'type': 'folder'}], 112 | 'total_count': 2}, 113 | 'purged_at': None, 114 | 'sequence_id': '0', 115 | 'shared_link': None, 116 | 'size': 0, 117 | 'trashed_at': None, 118 | 'type': 'folder'} 119 | 120 | self.called("folders", (ctx, req)) 121 | return {'content_created_at': None, 122 | 'content_modified_at': None, 123 | 'created_at': None, 124 | 'created_by': {'id': '', 'login': '', 'name': '', 'type': 'user'}, 125 | 'description': '', 126 | 'etag': None, 127 | 'folder_upload_email': None, 128 | 'id': '0', 129 | 'item_collection': 130 | {'entries': 131 | [ 132 | {'etag': '0', 133 | 'id': '95401994626', 134 | 'name': '0109d27be3d76224f640e6076c77184d', 135 | 'sequence_id': '0', 136 | 'type': 'folder'}, 137 | ], 138 | 'limit': 100, 139 | 'offset': 0, 140 | 'order': [{'by': 'type', 'direction': 'ASC'}, 141 | {'by': 'name', 'direction': 'ASC'}], 142 | 'total_count': 1}, 143 | 'item_status': 'active', 144 | 'modified_at': None, 145 | 'modified_by': {'id': '8506151483', 146 | 'login': 'AutomationUser_813890_GmcM3Cohcy@boxdevedition.com', 147 | 'name': 'Atakama JWT', 148 | 'type': 'user'}, 149 | 'name': 'All Files', 150 | 'owned_by': {'id': '8506151483', 151 | 'login': 'AutomationUser_813890_GmcM3Cohcy@boxdevedition.com', 152 | 'name': 'Atakama JWT', 153 | 'type': 'user'}, 154 | 'parent': None, 155 | 'path_collection': {'entries': [], 'total_count': 0}, 156 | 'purged_at': None, 157 | 'sequence_id': None, 158 | 'shared_link': None, 159 | 'size': 5551503, 160 | 'trashed_at': None, 161 | 'type': 'folder'} 162 | 163 | @api_route("/upload/files/") 164 | def upload_files(self, ctx, req): 165 | self.called("upload/files", (ctx, req)) 166 | return {'entries': [{'content_created_at': '2019-12-12T05:13:57-08:00', 167 | 'content_modified_at': '2019-12-12T05:13:57-08:00', 168 | 'created_at': '2019-12-12T05:13:57-08:00', 169 | 'created_by': {'id': '8506151483', 170 | 'login': 'AutomationUser_813890_GmcM3Cohcy@boxdevedition.com', 171 | 'name': 'Atakama JWT', 172 | 'type': 'user'}, 173 | 'description': '', 174 | 'etag': '0', 175 | 'file_version': {'id': '609837449506', 176 | 'sha1': '85c185b43850ed22c99570b7c04a1e6c9d12ad7d', 177 | 'type': 'file_version'}, 178 | 'id': '575144701906', 179 | 'item_status': 'active', 180 | 'modified_at': '2019-12-12T05:13:57-08:00', 181 | 'modified_by': {'id': '8506151483', 182 | 'login': 'AutomationUser_813890_GmcM3Cohcy@boxdevedition.com', 183 | 'name': 'Atakama JWT', 184 | 'type': 'user'}, 185 | 'name': '7075e7dbd6c7bb49da2b74ab60efde68(.dest', 186 | 'owned_by': {'id': '8506151483', 187 | 'login': 'AutomationUser_813890_GmcM3Cohcy@boxdevedition.com', 188 | 'name': 'Atakama JWT', 189 | 'type': 'user'}, 190 | 'parent': {'etag': '0', 191 | 'id': '96100489030', 192 | 'name': 'd49d35bdfb91cee9ccc1581dde986866', 193 | 'sequence_id': '0', 194 | 'type': 'folder'}, 195 | 'path_collection': {'entries': [{'etag': None, 196 | 'id': '0', 197 | 'name': 'All Files', 198 | 'sequence_id': None, 199 | 'type': 'folder'}, 200 | {'etag': '0', 201 | 'id': '96100489030', 202 | 'name': 'd49d35bdfb91cee9ccc1581dde986866', 203 | 'sequence_id': '0', 204 | 'type': 'folder'}], 205 | 'total_count': 2}, 206 | 'purged_at': None, 207 | 'sequence_id': '0', 208 | 'sha1': '85c185b43850ed22c99570b7c04a1e6c9d12ad7d', 209 | 'shared_link': None, 210 | 'size': 32, 211 | 'trashed_at': None, 212 | 'type': 'file'}], 213 | 'total_count': 1} 214 | 215 | 216 | def fake_prov(): 217 | # TODO: shutting this down is slow, fix that and then fix all tests using the api server to shut down, or use fixtures or something 218 | srv = FakeBoxApi() 219 | base_url = srv.uri() 220 | 221 | class API(object): 222 | """Configuration object containing the URLs for the Box API.""" 223 | BASE_API_URL = base_url.rstrip("/") 224 | UPLOAD_URL = base_url + "upload" 225 | OAUTH2_API_URL = base_url + "oauth" 226 | OAUTH2_AUTHORIZE_URL = base_url + "oauth/auth" 227 | MAX_RETRY_ATTEMPTS = 1 228 | 229 | with patch("boxsdk.config.API", API): 230 | prov = fake_oauth_provider(srv, BoxProvider) 231 | assert srv.calls["users/me"] 232 | return srv, prov 233 | 234 | def test_upload(): 235 | srv, prov = fake_prov() 236 | prov.large_file_size = 10 237 | prov.create("/small", io.BytesIO(b'123')) 238 | assert srv.calls["upload/files"] 239 | prov.disconnect() 240 | 241 | def test_mkdir(): 242 | srv, prov = fake_prov() 243 | log.info("calls %s", list(srv.calls.keys())) 244 | prov.mkdir("/dir") 245 | assert srv.calls["mkdir"] 246 | prov.disconnect() 247 | 248 | def test_nocred(): 249 | srv, prov = fake_prov() 250 | with pytest.raises(CloudTokenError): 251 | prov.disconnect() 252 | prov.connect(None) 253 | 254 | -------------------------------------------------------------------------------- /cloudsync/tests/test_events.py: -------------------------------------------------------------------------------- 1 | # pylint: disable=protected-access,too-many-lines,missing-docstring,logging-format-interpolation,too-many-statements,too-many-locals 2 | import time 3 | from io import BytesIO 4 | 5 | import pytest 6 | from pystrict import strict 7 | 8 | from cloudsync import ( 9 | exceptions, 10 | EventManager, 11 | Event, 12 | SyncState, 13 | LOCAL, 14 | CloudTokenError, 15 | DIRECTORY, 16 | CloudRootMissingError, 17 | CloudCursorError, FILE, 18 | ) 19 | from unittest.mock import patch, MagicMock 20 | import logging 21 | log = logging.getLogger(__name__) 22 | 23 | 24 | @strict 25 | class EventManagerWithCounter(EventManager): 26 | def __init__(self, *args, **kwargs): 27 | self.event_count = 0 28 | self.stop_after = -1 29 | super().__init__(*args, **kwargs) 30 | 31 | def _process_event(self, *args, **kwargs): 32 | super()._process_event(*args, **kwargs) 33 | self.event_count += 1 34 | if self.event_count == self.stop_after: 35 | self.stop() 36 | 37 | 38 | def create_event_manager(provider_generator, root_path, event_manager_type=EventManager): 39 | provider = provider_generator() 40 | state = SyncState((provider, provider), shuffle=True) 41 | if provider.oid_is_path: 42 | root_oid = provider.mkdirs(root_path) if root_path else None 43 | provider.set_root(root_oid=root_oid) 44 | event_manager = event_manager_type(provider, state, LOCAL, reauth=MagicMock(), root_oid=root_oid) 45 | else: 46 | provider.set_root(root_path=root_path) 47 | event_manager = event_manager_type(provider, state, LOCAL, reauth=MagicMock(), root_path=root_path) 48 | event_manager._drain() 49 | return event_manager 50 | 51 | 52 | @pytest.fixture(name="manager") 53 | def fixture_manager(mock_provider_generator): 54 | # TODO extend this to take any provider 55 | ret = create_event_manager(mock_provider_generator, "/root") 56 | yield ret 57 | ret.stop() 58 | 59 | 60 | @pytest.fixture(name="rootless_manager") 61 | def fixture_rootless_manager(mock_provider_generator): 62 | # TODO extend this to take any provider 63 | ret = create_event_manager(mock_provider_generator, None) 64 | yield ret 65 | ret.stop() 66 | 67 | @pytest.fixture(name="event_counter") 68 | def fixture_event_counter(mock_provider_generator): 69 | ret = create_event_manager(mock_provider_generator, "/root", EventManagerWithCounter) 70 | yield ret 71 | ret.stop() 72 | 73 | 74 | def make_event(): 75 | return Event(FILE, "oid", "path", "hash", exists=True) 76 | 77 | 78 | def test_event_basic(manager): 79 | provider = manager.provider 80 | state = manager.state 81 | create_path = provider.join(provider._root_path, "dest") 82 | info = provider.create(create_path, BytesIO(b'hello')) 83 | 84 | assert not state.lookup_path(LOCAL, create_path) 85 | 86 | oid = None 87 | 88 | # this is normally a blocking function that runs forever 89 | def done(): 90 | nonlocal oid 91 | states = state.get_all() 92 | if states: 93 | oid = list(states)[0][LOCAL].oid 94 | return state.lookup_oid(LOCAL, oid) 95 | return False 96 | 97 | # loop the sync until the file is found 98 | manager.run(timeout=1, until=done) 99 | 100 | assert oid 101 | 102 | info = provider.info_oid(oid) 103 | 104 | assert info.path == create_path 105 | 106 | 107 | def test_events_shutdown_event_shouldnt_process(manager): 108 | manager.start(sleep=.3) 109 | try: 110 | time.sleep(0.1) # give it a chance to finish the first do() and get into the sleep before we create event 111 | provider = manager.provider 112 | provider.create("/dest", BytesIO(b'hello')) 113 | manager.stop() 114 | time.sleep(.4) 115 | try: 116 | manager.provider.events().__next__() 117 | except exceptions.CloudDisconnectedError: 118 | pass 119 | finally: 120 | manager.stop() 121 | 122 | 123 | def test_events_shutdown_force_process_event(manager): 124 | manager.start(sleep=.3) 125 | try: 126 | time.sleep(0.1) # give it a chance to finish the first do() and get into the sleep before we create event 127 | provider = manager.provider 128 | provider.create("/dest", BytesIO(b'hello')) 129 | manager.stop() 130 | time.sleep(.4) 131 | assert provider.latest_cursor > provider.current_cursor 132 | manager.do() 133 | try: 134 | manager.provider.events().__next__() 135 | assert False 136 | except StopIteration: 137 | pass 138 | finally: 139 | manager.stop() 140 | 141 | 142 | def test_events_stop_walk_processing(event_counter): 143 | with patch.object(event_counter.provider, "walk_oid", lambda x: [make_event()] * 10): 144 | with patch.object(event_counter.provider, "events", lambda: [make_event()] * 10): 145 | event_counter.stop_after = 5 146 | event_counter._do_unsafe() 147 | # walk event processing is interruptable 148 | assert event_counter.event_count == 5 149 | 150 | 151 | def test_events_stop_queue_processing(event_counter): 152 | with patch.object(event_counter.provider, "walk_oid", lambda x: [make_event()] * 10): 153 | with patch.object(event_counter.provider, "events", lambda: [make_event()] * 10): 154 | event_counter._queue = [(None, True)] * 10 155 | event_counter.stop_after = 15 156 | event_counter._do_unsafe() 157 | # queue event processing is not interruptable - stop before the first provider event 158 | assert event_counter.event_count == 20 159 | 160 | 161 | def test_events_stop_provider_processing(event_counter): 162 | event_counter.need_walk = False 163 | with patch.object(event_counter.provider, "events", lambda: [make_event()] * 10): 164 | event_counter.stop_after = 5 165 | event_counter._do_unsafe() 166 | # provider event processing is interruptible 167 | assert event_counter.event_count == 5 168 | 169 | 170 | def test_events_no_stop(event_counter): 171 | with patch.object(event_counter.provider, "events", lambda: [make_event()] * 6): 172 | with patch.object(event_counter.provider, "walk_oid", lambda x: [make_event()] * 7): 173 | # _process_event() is is resilient to invalid events 174 | event_counter._queue = [(None, True)] * 8 175 | event_counter._do_unsafe() 176 | assert event_counter.event_count == 6 + 7 + 8 177 | 178 | 179 | def test_backoff(manager): 180 | try: 181 | provider = manager.provider 182 | provider.create("/dest", BytesIO(b'hello')) 183 | provider.disconnect() 184 | 185 | called = 0 186 | 187 | def fail_to_connect(creds): 188 | nonlocal called 189 | called = True 190 | raise CloudTokenError() 191 | 192 | with patch.object(provider, "connect", fail_to_connect): 193 | manager.start(until=lambda: called, timeout=1) 194 | manager.wait() 195 | 196 | assert manager.in_backoff 197 | prev_backoff = manager.in_backoff 198 | 199 | called = 0 200 | with patch.object(provider, "connect", fail_to_connect): 201 | manager.start(until=lambda: called, timeout=1) 202 | manager.wait() 203 | assert manager.in_backoff > prev_backoff 204 | 205 | assert manager.reauthenticate.call_count > 0 206 | 207 | manager.start(until=lambda: provider.connected, timeout=1) 208 | manager.wait() 209 | finally: 210 | manager.stop() 211 | 212 | @pytest.mark.parametrize("mode", ["root", "no-root"]) 213 | def test_event_provider_contract(manager, rootless_manager, mode): 214 | if mode == "no-root": 215 | manager.done() 216 | manager = rootless_manager 217 | 218 | prov = manager.provider 219 | 220 | with pytest.raises(ValueError): 221 | # do not reuse provider while another EventManager is actively using it 222 | manager = EventManager(prov, MagicMock(), LOCAL) 223 | 224 | # ok to reuse provider once the other EventManager is done with it 225 | manager.done() 226 | manager = EventManager(prov, MagicMock(), LOCAL, root_path=prov._root_path, root_oid=prov._root_oid) 227 | 228 | manager.done() 229 | if mode == "root": 230 | assert prov.info_path(manager._root_path) 231 | assert prov.info_oid(manager._root_oid) 232 | assert prov.root_path == manager._root_path 233 | assert prov.root_oid == manager._root_oid 234 | else: 235 | foo = prov.create("/foo", BytesIO(b"oo")) 236 | prov.mkdir("/bar") 237 | bar = prov.info_path("/bar") 238 | with pytest.raises(CloudRootMissingError): 239 | prov.set_root(root_oid="does-not-exist") 240 | with pytest.raises(CloudRootMissingError): 241 | # not a directory oid 242 | prov.set_root(root_oid=foo.oid) 243 | with pytest.raises(CloudRootMissingError): 244 | # oid/path mismatch 245 | prov.set_root(root_path="/not-bar", root_oid=bar.oid) 246 | with pytest.raises(CloudRootMissingError): 247 | # not a directory path 248 | prov.set_root(root_path="/foo") 249 | # provider creates folder if it does not already exist 250 | prov.set_root(root_path="/new-folder") 251 | assert prov.info_path("/new-folder") 252 | manager._drain() 253 | 254 | manager.done() 255 | manager = EventManager(prov, MagicMock(), LOCAL, root_path=prov._root_path, root_oid=prov._root_oid) 256 | assert not manager.busy 257 | prov.mkdir("/busy-test") 258 | assert manager.busy 259 | manager.done() 260 | 261 | def raise_root_missing_error(): 262 | raise CloudRootMissingError("unrooted") 263 | 264 | notify = MagicMock() 265 | manager = EventManager(prov, MagicMock(), LOCAL, notify, root_path=prov._root_path, root_oid=prov._root_oid) 266 | with patch.object(manager, "_reconnect_if_needed", raise_root_missing_error): 267 | with pytest.raises(Exception): 268 | # _BackoffError 269 | manager.do() 270 | notify.notify_from_exception.assert_called_once() 271 | assert not manager._root_validated 272 | 273 | with patch.object(manager, "_validate_root", raise_root_missing_error): 274 | with pytest.raises(Exception): 275 | # _BackoffError 276 | manager.do() 277 | notify.notify_from_exception.assert_called_once() 278 | assert not manager._root_validated 279 | 280 | prov.connection_id = None 281 | with pytest.raises(ValueError): 282 | # connection id is required 283 | manager = EventManager(prov, MagicMock(), LOCAL, root_path=prov._root_path, root_oid=prov._root_oid) 284 | 285 | 286 | def test_event_root_change(manager): 287 | # root renamed 288 | with pytest.raises(CloudRootMissingError): 289 | event = Event(DIRECTORY, manager._root_oid, "/renamed", "hash-1", True) 290 | manager._notify_on_root_change_event(event) 291 | if manager.provider.oid_is_path: 292 | with pytest.raises(CloudRootMissingError): 293 | event = Event(DIRECTORY, "/renamed", "", "hash-1", True, prior_oid=f"{manager._root_path}") 294 | manager._notify_on_root_change_event(event) 295 | # root deleted 296 | with pytest.raises(CloudRootMissingError): 297 | event = Event(DIRECTORY, manager._root_oid, "", "hash-1", False) 298 | event.accurate = True 299 | manager._notify_on_root_change_event(event) 300 | 301 | # root still present ... but inaccurate event arrives 302 | event = Event(DIRECTORY, manager._root_oid, "", "hash-1", False) 303 | event.accurate = False 304 | # no error 305 | manager._notify_on_root_change_event(event) 306 | 307 | 308 | def test_event_cursor_error(manager): 309 | manager.need_walk = False 310 | 311 | with patch.object(manager.provider, "events", side_effect=CloudCursorError): 312 | with pytest.raises(Exception): 313 | # _BackoffError 314 | manager.do() 315 | assert manager.need_walk 316 | 317 | 318 | def test_event_no_info_oid_calls(manager): 319 | manager.need_walk = False 320 | oid1 = manager.provider.create("/file1", BytesIO(b'hello')).oid 321 | oid2 = manager.provider.create("/file2", BytesIO(b'hello')).oid 322 | 323 | def done(): 324 | return manager.state.lookup_oid(manager.side, oid1) and manager.state.lookup_oid(manager.side, oid2) 325 | 326 | with patch.object(manager.provider, "info_oid", side_effect=manager.provider.info_oid) as api: 327 | 328 | # run until both oids are in the change set 329 | manager.run(timeout=1, until=done) 330 | 331 | # path is missing for oid providers 332 | if not manager.provider.oid_is_path: 333 | assert not manager.state.lookup_oid(manager.side, oid1)[manager.side].path 334 | assert not manager.state.lookup_oid(manager.side, oid2)[manager.side].path 335 | 336 | # info_oid() not called 337 | api.assert_not_called() 338 | -------------------------------------------------------------------------------- /cloudsync/cs.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | from typing import Optional, Tuple, IO, Any, Type, TYPE_CHECKING 4 | 5 | from pystrict import strict 6 | 7 | from .sync import SyncManager, SyncState, Storage 8 | from .runnable import Runnable 9 | from .event import EventManager 10 | from .provider import Provider 11 | from .log import TRACE 12 | from .utils import debug_sig 13 | from .notification import NotificationManager, Notification, NotificationType, SourceEnum 14 | if TYPE_CHECKING: # pragma: no cover 15 | from .smartsync import SmartSyncState, SmartSyncManager 16 | 17 | log = logging.getLogger(__name__) 18 | 19 | 20 | @strict # pylint: disable=too-many-instance-attributes 21 | class CloudSync(Runnable): 22 | """ 23 | The main synchronization class used. 24 | """ 25 | def __init__(self, # pylint: disable=too-many-arguments 26 | providers: Tuple[Provider, Provider], 27 | roots: Optional[Tuple[str, str]] = None, 28 | storage: Optional[Storage] = None, 29 | sleep: Optional[Tuple[float, float]] = None, 30 | root_oids: Optional[Tuple[str, str]] = None, 31 | state_class: Type = SyncState, 32 | smgr_class: Type = SyncManager, 33 | emgr_class: Type = EventManager 34 | ): 35 | 36 | """ 37 | Construct a new synchronizer between two providers. 38 | 39 | Args: 40 | providers: Two connected, authenticated providers. 41 | roots: The folder to synchronize 42 | storage: The back end storage mechanism for long-running sync information. 43 | sleep: The amount of time to sleep between event processing loops for each provider. 44 | Defaults to the provider's default_sleep value. 45 | 46 | When run, will receive events from each provider, and use those events to trigger 47 | copying files from one provider to the other. 48 | 49 | Conflicts (changes made on both sides) are handled by renaming files, unless py::meth::`resolve_conflict` 50 | is overridden. 51 | 52 | The first time a sync starts up, it checks storage. If event cursors are invalid, or a walk has never 53 | been done, then the sync engine will trigger a walk of all files. 54 | 55 | File names are translated between both sides using the `translate` function, which can be overriden 56 | to deal with incompatible naming conventions, special character translation, etc. By default, 57 | invalid names are not synced, and notifications about this are sent to py::method::`handle_notification`. 58 | """ 59 | if not roots and self.translate == CloudSync.translate: # pylint: disable=comparison-with-callable 60 | raise ValueError("Either override the translate() function, or pass in a pair of roots") 61 | 62 | self.providers = providers 63 | self.roots = roots 64 | 65 | if sleep is None: 66 | sleep = (providers[0].default_sleep, providers[1].default_sleep) 67 | self.sleep = sleep 68 | 69 | self.nmgr = NotificationManager(lambda n: self.handle_notification(n)) # pylint: disable=unnecessary-lambda 70 | 71 | # The tag for the SyncState will isolate the state of a pair of providers along with the sync roots 72 | 73 | # by using a lambda here, tests can inject functions into cs.prioritize, and they will get passed through 74 | state = state_class(providers, storage, tag=self.storage_label(), shuffle=False, 75 | prioritize=lambda *a: self.prioritize(*a), nmgr=self.nmgr) # pylint: disable=unnecessary-lambda 76 | 77 | smgr = smgr_class(state, providers, lambda *a, **kw: self.translate(*a, **kw), # pylint: disable=unnecessary-lambda 78 | self.resolve_conflict, self.nmgr, sleep=sleep, root_paths=roots, root_oids=root_oids) 79 | 80 | # for tests, make these accessible 81 | self.state: 'SmartSyncState' = state 82 | self.smgr: 'SmartSyncManager' = smgr 83 | 84 | # the label for each event manager will isolate the cursor to the provider/login combo for that side 85 | event_root_paths: Tuple[Optional[str], Optional[str]] = roots or (None, None) 86 | event_root_oids: Tuple[Optional[str], Optional[str]] = root_oids or (None, None) 87 | 88 | self.emgrs = ( 89 | emgr_class(smgr.providers[0], state, 0, self.nmgr, root_path=event_root_paths[0], 90 | reauth=lambda: self.authenticate(0), root_oid=event_root_oids[0]), 91 | emgr_class(smgr.providers[1], state, 1, self.nmgr, root_path=event_root_paths[1], 92 | reauth=lambda: self.authenticate(1), root_oid=event_root_oids[1]) 93 | ) 94 | 95 | self._runnables = [self.smgr, *self.emgrs, self.nmgr] 96 | log.info("initialized sync: %s, manager: %s", self.storage_label(), debug_sig(id(smgr))) 97 | 98 | def forget(self): 99 | """ 100 | Forget and discard state information, and drop any events in the queue. This will trigger a walk. 101 | """ 102 | self.state.forget() 103 | self.emgrs[0].forget() 104 | self.emgrs[1].forget() 105 | 106 | def set_need_walk(self, side, need_walk=True): 107 | self.emgrs[side].need_walk=need_walk 108 | 109 | @property 110 | def aging(self) -> float: 111 | """float: The number of seconds to wait before syncing a file. 112 | 113 | Reduces storage provider traffic at the expense of increased conflict risk. 114 | 115 | Default is based on the max(provider.default_sleep) value 116 | """ 117 | return self.smgr.aging 118 | 119 | @aging.setter 120 | def aging(self, secs: float): 121 | self.smgr.aging = secs 122 | 123 | def storage_label(self): 124 | """ 125 | Returns: 126 | str: a unique label representing this paired, translated sync 127 | 128 | Override this if if you are re-using storage and are using a rootless translate. 129 | """ 130 | 131 | # if you're using a pure translate, and not roots, you don't have to override the storage label 132 | # just don't resuse storage for the same pair of providers 133 | 134 | roots = self.roots or ('?', '?') 135 | assert self.providers[0].connection_id is not None 136 | assert self.providers[1].connection_id is not None 137 | return f"{self.providers[0].name}:{self.providers[0].connection_id}:{roots[0]}."\ 138 | f"{self.providers[1].name}:{self.providers[1].connection_id}:{roots[1]}" 139 | 140 | def walk(self, side=None, root=None, recursive=True): 141 | """Manually run a walk on a provider, causing a single-direction sync.""" 142 | roots = self.roots or ('/', '/') 143 | if root is not None and side is None: 144 | # a root without a side makes no sense (which root ?) 145 | raise ValueError("If you specify a root, you need to specify which side") 146 | 147 | for index, provider in enumerate(self.providers): 148 | if side is not None and index != side: 149 | continue 150 | 151 | path = root 152 | if path is None: 153 | path = roots[index] 154 | 155 | # todo: this should not be called here, and instead, we should queue the walk itself 156 | for event in provider.walk(path, recursive=recursive): 157 | self.emgrs[index].queue(event, from_walk=True) 158 | 159 | def authenticate(self, side: int): # pylint: disable=unused-argument, no-self-use 160 | """Override this method to change (re)authentication 161 | 162 | Default is to call provider[side].authenticate() 163 | 164 | Args: 165 | side: either 0 (LOCAL) or 1 (REMOTE) 166 | 167 | """ 168 | self.providers[side].connect(self.providers[side].authenticate()) 169 | 170 | def prioritize(self, side: int, path: str): # pylint: disable=unused-argument, no-self-use 171 | """Override this method to change the sync priority 172 | 173 | Default priority is 0 174 | Negative values happen first 175 | Positive values happen later 176 | 177 | Args: 178 | side: either 0 (LOCAL) or 1 (REMOTE) 179 | path: a path value in the (side) provider 180 | 181 | """ 182 | return 0 183 | 184 | def translate(self, side: int, path: str): 185 | """Override this method to translate between local and remote paths 186 | 187 | By default uses `self.roots` to strip the path provided, and 188 | join the result to the root of the other side. 189 | 190 | If `self.roots` is None, this function must be overridden. 191 | 192 | Example: 193 | translate(REMOTE, "/home/full/local/path.txt") -> "/cloud/path.txt" 194 | 195 | Args: 196 | side: either 0 (LOCAL) or 1 (REMOTE) 197 | path: a path valid in the (1-side) provider 198 | 199 | Returns: 200 | The path, valid for the provider[side], or None to mean "don't sync" 201 | """ 202 | if not self.roots: 203 | raise ValueError("Override translate function or provide root paths") 204 | 205 | relative = self.providers[1-side].is_subpath(self.roots[1-side], path) 206 | if not relative: 207 | log.log(TRACE, "%s is not subpath of %s", path, self.roots[1-side]) 208 | return None 209 | return self.providers[side].join(self.roots[side], relative) 210 | 211 | def resolve_conflict(self, f1: IO, f2: IO) -> Tuple[Any, bool]: # pylint: disable=no-self-use, unused-argument 212 | """Override this method to handle conflict resolution of files 213 | 214 | Note: 215 | - f1 and f2 are file-likes that will block on read, and can possibly pull data from the network, internet, etc 216 | - f1 and f2 also support the .path property to get a relative path to the file 217 | - f1 and f2 also support the .side property 218 | 219 | Returns: 220 | A tuple of (result, keep) or None, meaning there is no good resolution 221 | result is one of: 222 | - A "merged" file-like which should be used as the data to replace both f1/f2 with 223 | - One of f1 or f2, which is selected as the correct version 224 | keep is true if we want to keep the old version of the file around as a .conflicted file, else False 225 | """ 226 | return None 227 | 228 | @property 229 | def change_count(self): 230 | """ 231 | Number of relevant changes to be processed. 232 | """ 233 | return self.smgr.change_count 234 | 235 | @property 236 | def busy(self): 237 | """ 238 | True if there are any changes or events to be processed 239 | """ 240 | return self.smgr.busy or self.emgrs[0].busy or self.emgrs[1].busy 241 | 242 | def start(self, *, daemon=True, **kwargs): 243 | """ 244 | Starts the cloudsync service. 245 | """ 246 | log.debug("starting sync: %s", self.storage_label()) 247 | self.nmgr.notify(Notification(SourceEnum.SYNC, NotificationType.STARTED, None)) 248 | self.smgr.start(daemon=daemon, sleep=0.1, **kwargs) 249 | self.emgrs[0].start(daemon=daemon, sleep=self.sleep[0], **kwargs) 250 | self.emgrs[1].start(daemon=daemon, sleep=self.sleep[1], **kwargs) 251 | self.nmgr.start(daemon=daemon, **kwargs) 252 | 253 | def stop(self, forever=True, wait=True): 254 | """ 255 | Stops the cloudsync service. 256 | 257 | Args: 258 | forever: If false is passed, then handles are left open for a future start. Generally used for tests only. 259 | wait: If false, manager threads are signaled to stop, but are NOT joined 260 | """ 261 | log.info("stopping sync: %s", self.storage_label()) 262 | self.stop_all(self._runnables, forever, wait) 263 | 264 | # for tests, make this manually runnable 265 | # This method is NEVER called in production, it is only called in tests! 266 | # Notice that the start() method is overridden in this class, and prevents do() from being called 267 | # by starting threads for all the managers, which get their do() methods called, but never this class! BEWARE 268 | def do(self): 269 | """ 270 | One loop of sync, used for *tests only*. 271 | 272 | This randomly chooses to process local events, remote events or local syncs. 273 | """ 274 | # imports are in the body of this test-only function 275 | import random # pylint: disable=import-outside-toplevel 276 | mgrs = [*self.emgrs, self.smgr] 277 | random.shuffle(mgrs) 278 | # TODO: log the order of operations here, in case the test fails. 279 | # we could use this info to reproduce the failure on a dev machine more easily 280 | # self.test_mgr_order.append(order_of(mgrs)) 281 | caught = None 282 | for m in mgrs: 283 | try: 284 | m.do() 285 | except Exception as e: 286 | log.error("exception in %s : %s", m.service_name, repr(e)) 287 | caught = e 288 | if caught is not None: 289 | raise caught 290 | 291 | def done(self): 292 | """ 293 | Called at shutdown, override if you need some shutdown code. 294 | """ 295 | for mgr in self._runnables: 296 | mgr.done() 297 | 298 | def wait(self, timeout=None): 299 | """ 300 | Wait for all threads. 301 | 302 | Will wait forever, unless stop() is called or timeout is specified. 303 | """ 304 | for mgr in self._runnables: 305 | mgr.wait(timeout=timeout) 306 | 307 | def handle_notification(self, notification: Notification): 308 | """ 309 | Override to receive notifications during sync processing. 310 | 311 | Args: 312 | notification: Information about errors, or other sync events. 313 | """ 314 | --------------------------------------------------------------------------------