├── mergin ├── test │ ├── __init__.py │ ├── test_data │ │ ├── test.qgs │ │ ├── test.txt │ │ ├── test3.txt │ │ ├── test_dir │ │ │ ├── test2.txt │ │ │ └── modified_1_geom.gpkg │ │ ├── base.gpkg │ │ ├── two_tables.gpkg │ │ ├── inserted_1_A.gpkg │ │ ├── inserted_1_B.gpkg │ │ ├── two_tables_1_A.gpkg │ │ ├── two_tables_drop.gpkg │ │ ├── inserted_1_A_mod.gpkg │ │ ├── old_metadata.json │ │ └── new_metadata.json │ ├── test_missing_basefile_pull │ │ ├── test.qgs │ │ └── base.gpkg │ ├── test_empty_file_in_subdir │ │ ├── subdir │ │ │ └── empty.txt │ │ └── nonempty.txt │ ├── modified_schema │ │ ├── modified_schema.gpkg │ │ └── modified_schema.gpkg-wal │ ├── sqlite_con.py │ └── test_push_logging.py ├── __init__.py ├── version.py ├── common.py ├── editor.py ├── utils.py ├── report.py ├── client_push.py ├── cli.py ├── client_pull.py └── merginproject.py ├── examples ├── 03_projects_assets │ ├── team_a.png │ ├── team_b.png │ ├── Ready_to_survey_trees_team_A.gpkg │ └── Ready_to_survey_trees_team_B.gpkg ├── 02_sync_assets │ ├── edit_tree_health.jpg │ └── synchronized_trees.jpg ├── 01_users_assets │ └── users.csv ├── README.md ├── 02_sync.ipynb ├── 03_projects.ipynb └── 01_users.ipynb ├── scripts ├── check_all.bash ├── update_version.bash └── update_version.py ├── .gitignore ├── .github └── workflows │ ├── code_style.yml │ ├── autotests.yml │ └── python_packages.yml ├── LICENSE ├── setup.py ├── README.md └── CHANGELOG.md /mergin/test/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /mergin/test/test_data/test.qgs: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /mergin/test/test_data/test.txt: -------------------------------------------------------------------------------- 1 | Some content. -------------------------------------------------------------------------------- /mergin/test/test_data/test3.txt: -------------------------------------------------------------------------------- 1 | Some content. -------------------------------------------------------------------------------- /mergin/test/test_missing_basefile_pull/test.qgs: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /mergin/test/test_data/test_dir/test2.txt: -------------------------------------------------------------------------------- 1 | To be renamed. -------------------------------------------------------------------------------- /mergin/test/test_empty_file_in_subdir/subdir/empty.txt: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /mergin/test/test_empty_file_in_subdir/nonempty.txt: -------------------------------------------------------------------------------- 1 | hello Mergin 2 | -------------------------------------------------------------------------------- /mergin/test/test_data/base.gpkg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MerginMaps/python-api-client/HEAD/mergin/test/test_data/base.gpkg -------------------------------------------------------------------------------- /mergin/test/test_data/two_tables.gpkg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MerginMaps/python-api-client/HEAD/mergin/test/test_data/two_tables.gpkg -------------------------------------------------------------------------------- /examples/03_projects_assets/team_a.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MerginMaps/python-api-client/HEAD/examples/03_projects_assets/team_a.png -------------------------------------------------------------------------------- /examples/03_projects_assets/team_b.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MerginMaps/python-api-client/HEAD/examples/03_projects_assets/team_b.png -------------------------------------------------------------------------------- /mergin/test/test_data/inserted_1_A.gpkg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MerginMaps/python-api-client/HEAD/mergin/test/test_data/inserted_1_A.gpkg -------------------------------------------------------------------------------- /mergin/test/test_data/inserted_1_B.gpkg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MerginMaps/python-api-client/HEAD/mergin/test/test_data/inserted_1_B.gpkg -------------------------------------------------------------------------------- /mergin/test/test_data/two_tables_1_A.gpkg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MerginMaps/python-api-client/HEAD/mergin/test/test_data/two_tables_1_A.gpkg -------------------------------------------------------------------------------- /mergin/test/test_data/two_tables_drop.gpkg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MerginMaps/python-api-client/HEAD/mergin/test/test_data/two_tables_drop.gpkg -------------------------------------------------------------------------------- /examples/02_sync_assets/edit_tree_health.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MerginMaps/python-api-client/HEAD/examples/02_sync_assets/edit_tree_health.jpg -------------------------------------------------------------------------------- /mergin/test/test_data/inserted_1_A_mod.gpkg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MerginMaps/python-api-client/HEAD/mergin/test/test_data/inserted_1_A_mod.gpkg -------------------------------------------------------------------------------- /scripts/check_all.bash: -------------------------------------------------------------------------------- 1 | DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" 2 | PWD=`pwd` 3 | cd $DIR 4 | black -l 120 $DIR/../mergin 5 | cd $PWD 6 | -------------------------------------------------------------------------------- /examples/02_sync_assets/synchronized_trees.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MerginMaps/python-api-client/HEAD/examples/02_sync_assets/synchronized_trees.jpg -------------------------------------------------------------------------------- /mergin/test/modified_schema/modified_schema.gpkg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MerginMaps/python-api-client/HEAD/mergin/test/modified_schema/modified_schema.gpkg -------------------------------------------------------------------------------- /mergin/test/test_missing_basefile_pull/base.gpkg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MerginMaps/python-api-client/HEAD/mergin/test/test_missing_basefile_pull/base.gpkg -------------------------------------------------------------------------------- /mergin/test/modified_schema/modified_schema.gpkg-wal: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MerginMaps/python-api-client/HEAD/mergin/test/modified_schema/modified_schema.gpkg-wal -------------------------------------------------------------------------------- /mergin/test/test_data/test_dir/modified_1_geom.gpkg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MerginMaps/python-api-client/HEAD/mergin/test/test_data/test_dir/modified_1_geom.gpkg -------------------------------------------------------------------------------- /examples/03_projects_assets/Ready_to_survey_trees_team_A.gpkg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MerginMaps/python-api-client/HEAD/examples/03_projects_assets/Ready_to_survey_trees_team_A.gpkg -------------------------------------------------------------------------------- /examples/03_projects_assets/Ready_to_survey_trees_team_B.gpkg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MerginMaps/python-api-client/HEAD/examples/03_projects_assets/Ready_to_survey_trees_team_B.gpkg -------------------------------------------------------------------------------- /mergin/test/test_data/old_metadata.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "test_plugin/test_project_metadata", 3 | "version": "v0", 4 | "project_id": "effeca08-ef22-4fc1-b620-5261c6a081eb", 5 | "files": [] 6 | } 7 | -------------------------------------------------------------------------------- /scripts/update_version.bash: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" 3 | VER=$1 4 | 5 | python3 $DIR/update_version.py --version $VER 6 | 7 | 8 | 9 | -------------------------------------------------------------------------------- /examples/01_users_assets/users.csv: -------------------------------------------------------------------------------- 1 | username,email 2 | jdoe,jdoe@example.com 3 | asmith,asmith@example.com 4 | bwilliams,bwilliams@example.com 5 | cjohnson,cjohnson@example.com 6 | omartin,omartin@example.com -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | venv* 2 | __pycache__/ 3 | *.py[co] 4 | *.egg 5 | *.egg-info 6 | dist 7 | *build 8 | *.whl 9 | .idea 10 | .coverage 11 | htmlcov 12 | .pytest_cache 13 | deps 14 | venv 15 | .vscode/settings.json 16 | -------------------------------------------------------------------------------- /mergin/__init__.py: -------------------------------------------------------------------------------- 1 | from . import common 2 | 3 | from .client import MerginClient 4 | from .common import ClientError, LoginError 5 | from .merginproject import MerginProject, InvalidProject 6 | from .version import __version__ 7 | -------------------------------------------------------------------------------- /mergin/version.py: -------------------------------------------------------------------------------- 1 | # The version is also stored in ../setup.py 2 | __version__ = "0.11.0" 3 | 4 | # There seems to be no single nice way to keep version info just in one place: 5 | # https://packaging.python.org/guides/single-sourcing-package-version/ 6 | -------------------------------------------------------------------------------- /.github/workflows/code_style.yml: -------------------------------------------------------------------------------- 1 | name: Code Style 2 | 3 | on: [push] 4 | 5 | jobs: 6 | code_style_python: 7 | runs-on: ubuntu-latest 8 | steps: 9 | - uses: actions/checkout@v2 10 | - uses: psf/black@stable 11 | with: 12 | options: "--check --diff --verbose -l 120" 13 | src: "./mergin" -------------------------------------------------------------------------------- /mergin/test/sqlite_con.py: -------------------------------------------------------------------------------- 1 | """ 2 | A script to be used with AnotherSqliteConn in test_client.py to simulate 3 | an open connection to a sqlite3 database from a different process. 4 | """ 5 | 6 | import sqlite3 7 | import sys 8 | 9 | con = sqlite3.connect(sys.argv[1]) 10 | cursor = con.cursor() 11 | while True: 12 | cmd = input() 13 | sys.stderr.write(cmd + "\n") 14 | if cmd == "stop": 15 | break 16 | cursor.execute(cmd) 17 | -------------------------------------------------------------------------------- /mergin/test/test_data/new_metadata.json: -------------------------------------------------------------------------------- 1 | { 2 | "access": { 3 | "owners": [ 4 | 2 5 | ], 6 | "ownersnames": [ 7 | "test_plugin" 8 | ], 9 | "public": false, 10 | "readers": [ 11 | 2 12 | ], 13 | "readersnames": [ 14 | "test_plugin" 15 | ], 16 | "writers": [ 17 | 2 18 | ], 19 | "writersnames": [ 20 | "test_plugin" 21 | ] 22 | }, 23 | "created": "2023-10-16T09:17:27Z", 24 | "creator": 2, 25 | "disk_usage": 0, 26 | "files": [], 27 | "id": "a6f9d38c-e30d-49f2-bfc5-76495afdbf27", 28 | "name": "test_project_metadata", 29 | "namespace": "test_plugin", 30 | "permissions": { 31 | "delete": true, 32 | "update": true, 33 | "upload": true 34 | }, 35 | "removed_at": null, 36 | "removed_by": null, 37 | "role": "owner", 38 | "tags": [], 39 | "updated": "2023-10-16T09:17:27.345127", 40 | "uploads": [], 41 | "version": "v0", 42 | "workspace_id": 18 43 | } -------------------------------------------------------------------------------- /scripts/update_version.py: -------------------------------------------------------------------------------- 1 | import os 2 | import re 3 | import argparse 4 | 5 | 6 | def replace_in_file(filepath, regex, sub): 7 | with open(filepath, "r") as f: 8 | content = f.read() 9 | 10 | content_new = re.sub(regex, sub, content, flags=re.M) 11 | 12 | with open(filepath, "w") as f: 13 | f.write(content_new) 14 | 15 | 16 | dir_path = os.path.dirname(os.path.realpath(__file__)) 17 | parser = argparse.ArgumentParser() 18 | parser.add_argument("--version", help="version to replace") 19 | args = parser.parse_args() 20 | ver = args.version 21 | print("using version " + ver) 22 | 23 | about_file = os.path.join(dir_path, os.pardir, "mergin", "version.py") 24 | print("patching " + about_file) 25 | replace_in_file(about_file, '__version__\s=\s".*', '__version__ = "' + ver + '"') 26 | 27 | setup_file = os.path.join(dir_path, os.pardir, "setup.py") 28 | print("patching " + setup_file) 29 | replace_in_file(setup_file, "version='.*", "version='" + ver + "',") 30 | -------------------------------------------------------------------------------- /examples/README.md: -------------------------------------------------------------------------------- 1 | # Jupyter Notebooks examples gallery 2 | 3 | Here you can find some tailored Jupyter Notebooks examples on how to interact with Mergin Maps Python API client. 4 | 5 | These examples are split into scenarios in order to highlight some of the core capabilities of the Mergin Maps API. 6 | 7 | ## [Scenario 1](01_users.ipynb) - Users Management 8 | In this scenario you'll create some random users from an existing CSV into Mergin Maps, change user's `ROLE` on a `WORKSPACE` and `PROJECT` perspective as well remove a user from a specific project. 9 | 10 | ## [Scenario 2](02_sync.ipynb) - Synchronisation 11 | In this scenario you'll learn how to do basic synchronisation (`PUSH`, `PULL`) of projects with the Mergin Maps Python API client. 12 | 13 | ## [Scenario 3](03_projects.ipynb) - Projects Management 14 | In this scenario you'll learn how to manage projects with Mergin Maps Python API client, namely how to clone projects and how to separate team members in isolated projects from the cloned template. 15 | -------------------------------------------------------------------------------- /.github/workflows/autotests.yml: -------------------------------------------------------------------------------- 1 | name: Auto Tests 2 | on: [push] 3 | env: 4 | TEST_MERGIN_URL: https://app.dev.merginmaps.com/ 5 | TEST_API_USERNAME: test_plugin 6 | TEST_API_PASSWORD: ${{ secrets.MERGINTEST_API_PASSWORD }} 7 | TEST_API_USERNAME2: test_plugin2 8 | TEST_API_PASSWORD2: ${{ secrets.MERGINTEST_API_PASSWORD2 }} 9 | 10 | concurrency: 11 | group: ci-${{github.ref}}-autotests 12 | cancel-in-progress: true 13 | 14 | jobs: 15 | tests: 16 | runs-on: ubuntu-latest 17 | steps: 18 | - uses: actions/checkout@v2 19 | 20 | - uses: actions/setup-python@v2 21 | with: 22 | python-version: "3.8" 23 | 24 | - name: Install python package dependencies 25 | run: | 26 | python -m pip install --upgrade pip 27 | pip install python-dateutil pytz pytest pytest-cov pygeodiff coveralls 28 | 29 | - name: Run tests 30 | run: | 31 | pytest --cov=mergin --cov-report=lcov mergin/test/ 32 | 33 | - name: Submit coverage to Coveralls 34 | uses: coverallsapp/github-action@v2 35 | with: 36 | format: lcov 37 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2019-2020 Lutra Consulting 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in 13 | all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | THE SOFTWARE. 22 | -------------------------------------------------------------------------------- /.github/workflows/python_packages.yml: -------------------------------------------------------------------------------- 1 | name: Build and upload to PyPI 2 | 3 | on: [push] 4 | jobs: 5 | build_sdist: 6 | name: Build source distribution 7 | runs-on: ubuntu-latest 8 | steps: 9 | - uses: actions/checkout@v2 10 | 11 | - uses: actions/setup-python@v2 12 | name: Install Python 13 | with: 14 | python-version: '3.x' 15 | 16 | - name: Install deps 17 | run: | 18 | pip install --upgrade pip 19 | pip install setuptools twine wheel 20 | 21 | - name: Build sdist 22 | run: python setup.py sdist 23 | 24 | - uses: actions/upload-artifact@v4 25 | with: 26 | path: dist/*.tar.gz 27 | 28 | upload_pypi: 29 | needs: [build_sdist] 30 | runs-on: ubuntu-latest 31 | if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags') 32 | steps: 33 | - uses: actions/download-artifact@v4.1.7 34 | with: 35 | name: artifact 36 | path: dist 37 | 38 | - uses: pypa/gh-action-pypi-publish@release/v1 39 | with: 40 | user: __token__ 41 | password: ${{ secrets.PYPI_TOKEN }} 42 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2019 Lutra Consulting. All rights reserved. 2 | # Do not distribute without the express permission of the author. 3 | 4 | from setuptools import setup, find_packages 5 | 6 | setup( 7 | name="mergin-client", 8 | version="0.11.0", 9 | url="https://github.com/MerginMaps/python-api-client", 10 | license="MIT", 11 | author="Lutra Consulting Ltd.", 12 | author_email="info@merginmaps.com", 13 | description="Mergin Maps utils and client", 14 | long_description="Mergin Maps utils and client", 15 | packages=find_packages(), 16 | platforms="any", 17 | install_requires=[ 18 | "python-dateutil==2.8.2", 19 | "pygeodiff==2.0.4", 20 | "pytz==2022.1", 21 | "click==8.1.3", 22 | ], 23 | entry_points={ 24 | "console_scripts": ["mergin=mergin.cli:cli"], 25 | }, 26 | classifiers=[ 27 | "Development Status :: 5 - Production/Stable", 28 | "Intended Audience :: Developers", 29 | "License :: OSI Approved :: MIT License", 30 | "Operating System :: OS Independent", 31 | "Programming Language :: Python :: 3", 32 | ], 33 | package_data={"mergin": ["cert.pem"]}, 34 | ) 35 | -------------------------------------------------------------------------------- /mergin/common.py: -------------------------------------------------------------------------------- 1 | import os 2 | from enum import Enum 3 | 4 | CHUNK_SIZE = 100 * 1024 * 1024 5 | 6 | # there is an upper limit for chunk size on server, ideally should be requested from there once implemented 7 | UPLOAD_CHUNK_SIZE = 10 * 1024 * 1024 8 | 9 | # size of the log file part to send (if file is larger only this size from end will be sent) 10 | MAX_LOG_FILE_SIZE_TO_SEND = 5 * 1024 * 1024 11 | 12 | # default URL for submitting logs 13 | MERGIN_DEFAULT_LOGS_URL = "https://g4pfq226j0.execute-api.eu-west-1.amazonaws.com/mergin_client_log_submit" 14 | 15 | this_dir = os.path.dirname(os.path.realpath(__file__)) 16 | 17 | 18 | # Error code from the public API, add to the end of enum as we handle more eror 19 | class ErrorCode(Enum): 20 | ProjectsLimitHit = "ProjectsLimitHit" 21 | StorageLimitHit = "StorageLimitHit" 22 | 23 | 24 | class ClientError(Exception): 25 | def __init__(self, detail, url=None, server_code=None, server_response=None, http_error=None, http_method=None): 26 | self.detail = detail 27 | self.url = url 28 | self.http_error = http_error 29 | self.http_method = http_method 30 | 31 | self.server_code = server_code 32 | self.server_response = server_response 33 | 34 | self.extra = None 35 | 36 | def __str__(self): 37 | string_res = f"Detail: {self.detail}\n" 38 | if self.http_error: 39 | string_res += f"HTTP Error: {self.http_error}\n" 40 | if self.url: 41 | string_res += f"URL: {self.url}\n" 42 | if self.http_method: 43 | string_res += f"Method: {self.http_method}\n" 44 | if self.server_code: 45 | string_res += f"Error code: {self.server_code}\n" 46 | if self.extra: 47 | string_res += f"{self.extra}\n" 48 | return string_res 49 | 50 | 51 | class LoginError(Exception): 52 | pass 53 | 54 | 55 | class InvalidProject(Exception): 56 | pass 57 | 58 | 59 | try: 60 | import dateutil.parser 61 | from dateutil.tz import tzlocal 62 | except ImportError: 63 | # this is to import all dependencies shipped with package (e.g. to use in qgis-plugin) 64 | deps_dir = os.path.join(this_dir, "deps") 65 | if os.path.exists(deps_dir): 66 | import sys 67 | 68 | for f in os.listdir(os.path.join(deps_dir)): 69 | sys.path.append(os.path.join(deps_dir, f)) 70 | 71 | import dateutil.parser 72 | from dateutil.tz import tzlocal 73 | 74 | 75 | class WorkspaceRole(Enum): 76 | """ 77 | Workspace roles 78 | """ 79 | 80 | GUEST = "guest" 81 | READER = "reader" 82 | EDITOR = "editor" 83 | WRITER = "writer" 84 | ADMIN = "admin" 85 | OWNER = "owner" 86 | 87 | 88 | class ProjectRole(Enum): 89 | """ 90 | Project roles 91 | """ 92 | 93 | READER = "reader" 94 | EDITOR = "editor" 95 | WRITER = "writer" 96 | OWNER = "owner" 97 | -------------------------------------------------------------------------------- /mergin/editor.py: -------------------------------------------------------------------------------- 1 | from itertools import filterfalse 2 | from typing import Callable, Dict, List 3 | 4 | from .utils import is_mergin_config, is_qgis_file, is_versioned_file 5 | 6 | EDITOR_ROLE_NAME = "editor" 7 | 8 | """ 9 | Determines whether a given file change should be disallowed based on the file path. 10 | 11 | Returns: 12 | bool: True if the file change should be disallowed, False otherwise. 13 | """ 14 | _disallowed_changes: Callable[[dict], bool] = lambda change: is_qgis_file(change["path"]) 15 | 16 | 17 | def is_editor_enabled(mc, project_info: dict) -> bool: 18 | """ 19 | The function checks if the server supports editor access, and if the current user's project role matches the expected role name for editors. 20 | """ 21 | server_support = mc.has_editor_support() 22 | project_role = project_info.get("role") 23 | 24 | return server_support and project_role == EDITOR_ROLE_NAME 25 | 26 | 27 | def _apply_editor_filters(changes: Dict[str, List[dict]]) -> Dict[str, List[dict]]: 28 | """ 29 | Applies editor-specific filters to the changes dictionary, removing any changes to files that are not in the editor's list of allowed files. 30 | 31 | Args: 32 | changes (dict[str, list[dict]]): A dictionary containing the added, updated, and removed changes. 33 | 34 | Returns: 35 | dict[str, list[dict]]: The filtered changes dictionary. 36 | """ 37 | updated = changes.get("updated", []) 38 | 39 | changes["updated"] = list(filterfalse(_disallowed_changes, updated)) 40 | return changes 41 | 42 | 43 | def filter_changes(mc, project_info: dict, changes: Dict[str, List[dict]]) -> Dict[str, List[dict]]: 44 | """ 45 | Filters the given changes dictionary based on the editor's enabled state. 46 | 47 | If the editor is not enabled, the changes dictionary is returned as-is. Otherwise, the changes are passed through the `_apply_editor_filters` method to apply any configured filters. 48 | 49 | Args: 50 | changes (dict[str, list[dict]]): A dictionary mapping file paths to lists of change dictionaries. 51 | 52 | Returns: 53 | dict[str, list[dict]]: The filtered changes dictionary. 54 | """ 55 | if not is_editor_enabled(mc, project_info): 56 | return changes 57 | return _apply_editor_filters(changes) 58 | 59 | 60 | def prevent_conflicted_copy(path: str, mc, project_info: dict) -> bool: 61 | """ 62 | Decides whether a file path should be blocked from creating a conflicted copy. 63 | Note: This is used when the editor is active and attempting to modify files (e.g., .ggs) that are also updated on the server, preventing unnecessary conflict files creation. 64 | 65 | Args: 66 | path (str): The file path to check. 67 | mc: The Mergin client object. 68 | project_info (dict): Information about the Mergin project from server. 69 | 70 | Returns: 71 | bool: True if the file path should be prevented from ceating conflicted copy, False otherwise. 72 | """ 73 | return is_editor_enabled(mc, project_info) and any([is_qgis_file(path)]) 74 | -------------------------------------------------------------------------------- /mergin/test/test_push_logging.py: -------------------------------------------------------------------------------- 1 | from types import SimpleNamespace 2 | from pathlib import Path 3 | import logging 4 | import tempfile 5 | from unittest.mock import MagicMock 6 | import pytest 7 | 8 | from pygeodiff import GeoDiff 9 | from mergin.client_push import push_project_finalize, UploadJob 10 | from mergin.common import ClientError 11 | from mergin.merginproject import MerginProject 12 | from mergin.client import MerginClient 13 | 14 | 15 | @pytest.mark.parametrize("status_code", [502, 504]) 16 | def test_push_finalize_logs_on_5xx_real_diff(caplog, status_code, tmp_path): 17 | # test data 18 | data_dir = Path(__file__).resolve().parent / "test_data" 19 | base = data_dir / "base.gpkg" 20 | modified = data_dir / "inserted_1_A.gpkg" 21 | assert base.exists() and modified.exists() 22 | 23 | # real MerginProject in temp dir 24 | proj_dir = tmp_path / "proj" 25 | meta_dir = proj_dir / ".mergin" 26 | meta_dir.mkdir(parents=True) 27 | mp = MerginProject(str(proj_dir)) 28 | 29 | # route MP logs into pytest caplog 30 | logger = logging.getLogger("mergin_test") 31 | logger.setLevel(logging.DEBUG) 32 | logger.propagate = True 33 | caplog.set_level(logging.ERROR, logger="mergin_test") 34 | mp.log = logger 35 | 36 | # generate a real diff into .mergin/ 37 | diff_path = meta_dir / "base_to_inserted_1_A.diff" 38 | GeoDiff().create_changeset(str(base), str(modified), str(diff_path)) 39 | diff_rel = diff_path.name 40 | diff_size = diff_path.stat().st_size 41 | file_size = modified.stat().st_size 42 | 43 | # mock MerginClient: only patch post(); simulate 5xx on finish 44 | tx = "tx-1" 45 | 46 | def mc_post(url, *args, **kwargs): 47 | if url.endswith(f"/v1/project/push/finish/{tx}"): 48 | err = ClientError("Gateway error") 49 | setattr(err, "http_error", status_code) # emulate HTTP code on the exception 50 | raise err 51 | if url.endswith(f"/v1/project/push/cancel/{tx}"): 52 | return SimpleNamespace(msg="cancelled") 53 | return SimpleNamespace(msg="ok") 54 | 55 | mc = MagicMock(spec=MerginClient) 56 | mc.post.side_effect = mc_post 57 | 58 | tmp_dir = tempfile.TemporaryDirectory(prefix="python-api-client-test-") 59 | 60 | # build a real UploadJob that references the diff/file sizes 61 | job = UploadJob( 62 | project_path="u/p", 63 | changes={ 64 | "added": [], 65 | "updated": [ 66 | { 67 | "path": modified.name, 68 | "size": file_size, 69 | "diff": {"path": diff_rel, "size": diff_size}, 70 | "chunks": [1], 71 | } 72 | ], 73 | "removed": [], 74 | }, 75 | transaction_id=tx, 76 | mp=mp, 77 | mc=mc, 78 | tmp_dir=tmp_dir, 79 | ) 80 | 81 | job.total_size = 1234 82 | job.transferred_size = 1234 83 | job.upload_queue_items = [1] 84 | job.executor = SimpleNamespace(shutdown=lambda wait=True: None) 85 | job.futures = [SimpleNamespace(done=lambda: True, exception=lambda: None, running=lambda: False)] 86 | job.server_resp = {"version": "n/a"} 87 | 88 | with pytest.raises(ClientError): 89 | push_project_finalize(job) 90 | 91 | text = caplog.text 92 | assert f"Push failed with HTTP error {status_code}" in text 93 | assert "Upload details:" in text 94 | assert "Files:" in text 95 | assert modified.name in text 96 | assert f"size={file_size}" in text 97 | assert f"diff_size={diff_size}" in text 98 | assert ("changes=" in text) or ("changes=n/a" in text) 99 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [![PyPI version](https://badge.fury.io/py/mergin-client.svg)](https://badge.fury.io/py/mergin-client) 2 | [![Build and upload to PyPI](https://github.com/MerginMaps/python-api-client/actions/workflows/python_packages.yml/badge.svg)](https://github.com/MerginMaps/python-api-client/actions/workflows/python_packages.yml) 3 | 4 | [![Code Style](https://github.com/MerginMaps/python-api-client/actions/workflows/code_style.yml/badge.svg)](https://github.com/MerginMaps/python-api-client/actions/workflows/code_style.yml) 5 | [![Auto Tests/Package](https://github.com/MerginMaps/python-api-client/workflows/Auto%20Tests/badge.svg)](https://github.com/MerginMaps/python-api-client/actions?query=workflow%3A%22Auto+Tests%22) 6 | [![Coverage Status](https://img.shields.io/coveralls/MerginMaps/python-api-client.svg)](https://coveralls.io/github/MerginMaps/python-api-client) 7 | 8 | # Mergin Maps Python Client 9 | 10 | This repository contains a Python client module for access to [Mergin Maps](https://merginmaps.com/) service and a command-line tool for easy access to data stored in Mergin Maps. 11 | 12 |
Join our community chat
and ask questions!

13 | 14 | 15 | To install the module: 16 | ```bash 17 | pip3 install mergin-client 18 | ``` 19 | 20 | Note: Check also [Mergin Maps Cpp Client](https://github.com/MerginMaps/cpp-api-client) 21 | 22 | ## Using Python API 23 | 24 | To use Mergin Maps from Python, it is only needed to create `MerginClient` object and then use it: 25 | 26 | ```python 27 | import mergin 28 | 29 | client = mergin.MerginClient(login='john', password='topsecret') 30 | client.download_project('lutraconsulting/Basic survey', '/tmp/basic-survey') 31 | ``` 32 | 33 | If you have Mergin Maps plugin for QGIS installed and you want to use it from QGIS' Python console 34 | 35 | ```python 36 | import Mergin.mergin as mergin 37 | 38 | client = mergin.MerginClient(login='john', password='topsecret') 39 | ``` 40 | 41 | ## Command-line Tool 42 | 43 | When the module is installed, it comes with `mergin` command line tool. 44 | 45 | ``` 46 | $ mergin --help 47 | Usage: mergin [OPTIONS] COMMAND [ARGS]... 48 | 49 | Command line interface for the Mergin Maps client module. For user 50 | authentication on server there are two options: 51 | 52 | 1. authorization token environment variable (MERGIN_AUTH) is defined, or 53 | 2. username and password need to be given either as environment variables 54 | (MERGIN_USERNAME, MERGIN_PASSWORD), or as command options (--username, 55 | --password). 56 | 57 | Run `mergin --username login` to see how to set the token 58 | variable manually. 59 | 60 | Options: 61 | --url TEXT Mergin Maps server URL. Default is: 62 | https://app.merginmaps.com/ 63 | --auth-token TEXT Mergin authentication token string 64 | --username TEXT 65 | --password TEXT 66 | --help Show this message and exit. 67 | 68 | Commands: 69 | clone Clone project from server. 70 | create Create a new project on Mergin Maps server. 71 | download Download last version of Mergin Maps project 72 | download-file Download project file at specified version. 73 | list-projects List projects on the server 74 | list-files List files in a project 75 | login Login to the service and see how to set the token... 76 | pull Fetch changes from Mergin Maps repository 77 | push Upload local changes into Mergin Maps repository 78 | remove Remove project from server. 79 | rename Rename project in Mergin Maps repository. 80 | reset Reset local changes in project. 81 | share Fetch permissions to project 82 | share-add Add permissions to [users] to project 83 | share-remove Remove [users] permissions from project 84 | show-file-changeset Displays information about project changes. 85 | show-file-history Displays information about a single version of a... 86 | show-version Displays information about a single version of a... 87 | status Show all changes in project files - upstream and... 88 | ``` 89 | 90 | ### Examples 91 | 92 | For example, to download a project: 93 | 94 | ``` 95 | $ mergin --username john download john/project1 ~/mergin/project1 96 | ``` 97 | To download a specific version of a project: 98 | ``` 99 | $ mergin --username john download --version v42 john/project1 ~/mergin/project1 100 | ``` 101 | 102 | To download a sepecific version of a single file: 103 | 104 | 1. First you need to download the project: 105 | ``` 106 | mergin --username john download john/myproject 107 | ``` 108 | 109 | 2. Go to the project directory 110 | ``` 111 | cd myproject 112 | ``` 113 | 114 | 3. Download the version of a file you want: 115 | ``` 116 | mergin --username john download-file --version v273 myfile.gpkg /tmp/myfile-v273.gpkg 117 | ``` 118 | 119 | If you do not want to specify username on the command line and be asked for you password every time, 120 | it is possible to set env variables MERGIN_USERNAME and MERGIN_PASSWORD. 121 | 122 | When a project is downloaded, `mergin` commands can be run in the project's 123 | working directory: 124 | 125 | 1. get status of the project (check if there are any local/remote changes) 126 | ``` 127 | $ mergin status 128 | ``` 129 | 2. pull changes from Mergin Maps service 130 | ``` 131 | $ mergin pull 132 | ``` 133 | 3. push local changes to Mergin Maps service 134 | ``` 135 | $ mergin push 136 | ``` 137 | 138 | ### Using CLI with auth token 139 | 140 | If you plan to run `mergin` command multiple times and you wish to avoid logging in every time, 141 | you can use "login" command to get authorization token. 142 | It will ask for password and then output environment variable with auth token. The returned token 143 | is not permanent - it will expire after several hours. 144 | ```bash 145 | $ mergin --username john login 146 | Password: topsecret 147 | Login successful! 148 | To set the MERGIN_AUTH variable run in Linux: 149 | export MERGIN_AUTH="Bearer ......." 150 | In Windows: 151 | SET MERGIN_AUTH=Bearer ....... 152 | ``` 153 | When setting the variable in Windows you do not quote the value. 154 | 155 | When the MERGIN_AUTH env variable is set (or passed with `--auth-token` command line argument), 156 | it is possible to run other commands without specifying username/password. 157 | 158 | 159 | ## Development 160 | 161 | ### Installing deps 162 | 163 | Python 3.7+ required. Create `mergin/deps` folder where [geodiff](https://github.com/MerginMaps/geodiff) lib is supposed to be and install dependencies: 164 | ```bash 165 | rm -r mergin/deps 166 | mkdir mergin/deps 167 | pip install python-dateutil pytz 168 | pip install pygeodiff --target=mergin/deps 169 | ``` 170 | 171 | For using mergin client with its dependencies packaged locally run: 172 | ```bash 173 | pip install wheel 174 | python3 setup.py sdist bdist_wheel 175 | mkdir -p mergin/deps 176 | pip wheel -r mergin_client.egg-info/requires.txt -w mergin/deps 177 | unzip mergin/deps/pygeodiff-*.whl -d mergin/deps 178 | pip install --editable . 179 | ``` 180 | 181 | ### Tests 182 | For running test do: 183 | 184 | ```bash 185 | cd mergin 186 | export TEST_MERGIN_URL= # testing server 187 | export TEST_API_USERNAME= 188 | export TEST_API_PASSWORD= 189 | export TEST_API_USERNAME2= 190 | export TEST_API_PASSWORD2= 191 | # workspace name with controlled available storage space (e.g. 20MB), default value: testpluginstorage 192 | export TEST_STORAGE_WORKSPACE= 193 | pip install pytest pytest-cov coveralls 194 | pytest --cov-report html --cov=mergin mergin/test/ 195 | ``` 196 | -------------------------------------------------------------------------------- /examples/02_sync.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "eff75b76", 6 | "metadata": {}, 7 | "source": [ 8 | "# Mergin Maps Synchronisation\n", 9 | "\n", 10 | "Mergin Maps synchronisation operates using a push/pull mechanism for your project. \n", 11 | "\n", 12 | "- **Push**: Synchronise your local project changes to the Mergin Maps server\n", 13 | "- **Pull**: Updates from the server are synchronised to the local device\n", 14 | "\n", 15 | "## Example project\n", 16 | "\n", 17 | "Imagine you are preparing a project for tree surveyors in Vienna.\n", 18 | "\n", 19 | "The task for the surveyors is to collect data about existing trees in the city. They are focusing on the health of the trees. In this example, we will use the Mergin Maps Python API client to automatically synchronise data to the Mergin Maps server. We will import CSV data into a GeoPackage and synchronize it. This GeoPackage can then be used for further data collection in the field." 20 | ] 21 | }, 22 | { 23 | "cell_type": "markdown", 24 | "id": "8eb25fff", 25 | "metadata": {}, 26 | "source": [ 27 | "Let's install Mergin Maps client and necessary libraries for this example." 28 | ] 29 | }, 30 | { 31 | "cell_type": "code", 32 | "execution_count": null, 33 | "id": "33ac4583", 34 | "metadata": {}, 35 | "outputs": [], 36 | "source": [ 37 | "!pip install mergin-client" 38 | ] 39 | }, 40 | { 41 | "cell_type": "markdown", 42 | "id": "611a93c1", 43 | "metadata": {}, 44 | "source": [ 45 | "Fill the following variables with your Mergin Maps credentials (username / email and password)." 46 | ] 47 | }, 48 | { 49 | "cell_type": "code", 50 | "execution_count": null, 51 | "id": "1bd4f48d", 52 | "metadata": {}, 53 | "outputs": [], 54 | "source": [ 55 | "LOGIN=\"...\"\n", 56 | "PASSWORD=\"...\"" 57 | ] 58 | }, 59 | { 60 | "cell_type": "markdown", 61 | "id": "8a68900f", 62 | "metadata": {}, 63 | "source": [ 64 | "Let's login to your account to be able to use the `MerginClient` class methods to automate your workflows." 65 | ] 66 | }, 67 | { 68 | "cell_type": "code", 69 | "execution_count": null, 70 | "id": "c332f11f", 71 | "metadata": {}, 72 | "outputs": [], 73 | "source": [ 74 | "import mergin\n", 75 | "\n", 76 | "client = mergin.MerginClient(\n", 77 | " login=LOGIN,\n", 78 | " password=PASSWORD\n", 79 | ")" 80 | ] 81 | }, 82 | { 83 | "cell_type": "markdown", 84 | "id": "3b3b55cd", 85 | "metadata": {}, 86 | "source": [ 87 | "Now you can use the client to call the API. Let's try to clone the project available for this example (`lutraconsulting/Vienna trees example`) to your Mergin Maps project. You need to specify to which project our sample project will be cloned to (edit the `PROJECT` variable in the form `{WORKSPACE NAME}/{PROJECT NAME}` in Mergin Maps cloud)." 88 | ] 89 | }, 90 | { 91 | "cell_type": "code", 92 | "execution_count": null, 93 | "id": "70f17d60", 94 | "metadata": {}, 95 | "outputs": [], 96 | "source": [ 97 | "PROJECT=\".../...\"\n", 98 | "\n", 99 | "client.clone_project(\"lutraconsulting/Vienna trees example\", PROJECT)" 100 | ] 101 | }, 102 | { 103 | "cell_type": "markdown", 104 | "id": "ff9dd71b", 105 | "metadata": {}, 106 | "source": [ 107 | "Project contains GeoPackage `Ready to survey trees` where surveyors can collect trees health and `vienna_trees_gansehauffel.csv` file with all trees from Gänsehäufel in Vienna. Let's download project to your computer using `download_project` method. " 108 | ] 109 | }, 110 | { 111 | "cell_type": "code", 112 | "execution_count": null, 113 | "id": "08fc0642", 114 | "metadata": {}, 115 | "outputs": [], 116 | "source": [ 117 | "# download project to local folder.\n", 118 | "LOCAL_FOLDER=\"/tmp/project\"\n", 119 | "\n", 120 | "client.download_project(PROJECT, LOCAL_FOLDER)" 121 | ] 122 | }, 123 | { 124 | "cell_type": "markdown", 125 | "id": "400194dc", 126 | "metadata": {}, 127 | "source": [ 128 | "We can now add sample points from the `.csv` file to the GeoPackage. These points within the GeoPackage will then be available to surveyors in the field for editing the health column using the Mergin Maps mobile app." 129 | ] 130 | }, 131 | { 132 | "cell_type": "code", 133 | "execution_count": null, 134 | "id": "23469139", 135 | "metadata": {}, 136 | "outputs": [], 137 | "source": [ 138 | "# Install geopandas to export csv to geopackage\n", 139 | "!pip install geopandas" 140 | ] 141 | }, 142 | { 143 | "cell_type": "code", 144 | "execution_count": null, 145 | "id": "f3002ce3", 146 | "metadata": {}, 147 | "outputs": [], 148 | "source": [ 149 | "import pandas as pd\n", 150 | "import geopandas as gpd\n", 151 | "import os\n", 152 | "\n", 153 | "# Get the data from the CSV (use just sample of data)\n", 154 | "csv_file = os.path.join(LOCAL_FOLDER, \"vienna_trees_gansehauffel.csv\")\n", 155 | "csv_df = pd.read_csv(csv_file, nrows=20, dtype={\"health\": str})\n", 156 | "# Convert geometry in WKT format to GeoDataFrame\n", 157 | "gdf = gpd.GeoDataFrame(csv_df, geometry=gpd.GeoSeries.from_wkt(csv_df.geometry))\n", 158 | "print(gdf.head())\n", 159 | "# Save the GeoDataFrame to a Geopackage\n", 160 | "gdf.to_file(\n", 161 | " os.path.join(LOCAL_FOLDER, \"Ready_to_survey_trees.gpkg\"), \n", 162 | " layer=\"Ready_to_survey_trees\", driver=\"GPKG\",\n", 163 | ")\n" 164 | ] 165 | }, 166 | { 167 | "cell_type": "markdown", 168 | "id": "d440ee5d", 169 | "metadata": {}, 170 | "source": [ 171 | "You can now see changes in GeoPackage file using `project_status` method. " 172 | ] 173 | }, 174 | { 175 | "cell_type": "code", 176 | "execution_count": null, 177 | "id": "d1a67839", 178 | "metadata": {}, 179 | "outputs": [ 180 | { 181 | "name": "stdout", 182 | "output_type": "stream", 183 | "text": [ 184 | "[{'path': 'Ready_to_survey_trees.gpkg', 'checksum': '3ba7658d231fefe30d9410f41c75f37d1ba5e614', 'size': 98304, 'mtime': datetime.datetime(2025, 5, 27, 16, 24, 30, 122463, tzinfo=tzlocal()), 'origin_checksum': '19b3331abc515a955691401918804d8bcf397ee4', 'chunks': ['be489067-d078-4862-bae1-c1cb222f680a']}]\n" 185 | ] 186 | } 187 | ], 188 | "source": [ 189 | "_, push_changes, __ = client.project_status(LOCAL_FOLDER)\n", 190 | "print(push_changes.get(\"updated\"))" 191 | ] 192 | }, 193 | { 194 | "cell_type": "markdown", 195 | "id": "506cfa68", 196 | "metadata": {}, 197 | "source": [ 198 | "You can now use `push_project` method to push data to the server." 199 | ] 200 | }, 201 | { 202 | "cell_type": "code", 203 | "execution_count": null, 204 | "id": "1a167b17", 205 | "metadata": {}, 206 | "outputs": [], 207 | "source": [ 208 | "client.push_project(LOCAL_FOLDER)" 209 | ] 210 | }, 211 | { 212 | "cell_type": "markdown", 213 | "id": "caccc6da", 214 | "metadata": {}, 215 | "source": [ 216 | "To pull the latest version of the project, use `pull_project` method." 217 | ] 218 | }, 219 | { 220 | "cell_type": "code", 221 | "execution_count": null, 222 | "id": "90e5e64a", 223 | "metadata": {}, 224 | "outputs": [], 225 | "source": [ 226 | "client.pull_project(LOCAL_FOLDER)" 227 | ] 228 | }, 229 | { 230 | "cell_type": "markdown", 231 | "id": "e65bb7c9", 232 | "metadata": {}, 233 | "source": [ 234 | "Mobile app users are now enabled to perform updates to the imported tree data directly in the field.\n", 235 | "\n", 236 | "\"drawing\"\n", 237 | "\n", 238 | "Editing tree health with predefined values.\n", 239 | "\n", 240 | "\"drawing\"" 241 | ] 242 | } 243 | ], 244 | "metadata": { 245 | "kernelspec": { 246 | "display_name": "python-api-client", 247 | "language": "python", 248 | "name": "python3" 249 | }, 250 | "language_info": { 251 | "codemirror_mode": { 252 | "name": "ipython", 253 | "version": 3 254 | }, 255 | "file_extension": ".py", 256 | "mimetype": "text/x-python", 257 | "name": "python", 258 | "nbconvert_exporter": "python", 259 | "pygments_lexer": "ipython3", 260 | "version": "3.10.14" 261 | } 262 | }, 263 | "nbformat": 4, 264 | "nbformat_minor": 5 265 | } 266 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | 3 | Changelog is moved to [GitHub releases](https://github.com/MerginMaps/python-api-client/releases). 4 | 5 | ## 0.9.3 6 | 7 | - Added ```list-files``` CLI command for list of project files (#199 by @kaloyan13) 8 | - Upgrade pygeodiff to 2.0.4 (#217) 9 | - Do not use paginated endpoint for project version detail (#216) 10 | - Improve client error handling (#209) 11 | 12 | ## 0.9.2 13 | 14 | - Update rules when pushing with editor permission level (#208) 15 | 16 | ## 0.9.1 17 | 18 | - Support for "editor" permission level for server >= 2024.4.0. Editors have more limited functionality than "writer" permission: they are not allowed to modify the QGIS project file or change structure of tables (#202, #207) 19 | - Better handling of unexpected errors during the initial download of a project (#201) 20 | - Fixes to CI (#205) 21 | 22 | ## 0.9.0 23 | 24 | - Add `reset_local_changes()` API call and `reset` CLI command to make it possible to discard any changes in the local project directory (#107) 25 | - Add `rename_project()` API call and `rename` CLI command to rename a project in its workspace (#190) 26 | - Add `delete_project_now()` API call for immediate project removal (mainly meant for testing) 27 | - Project info can be also queried by project's ID in `MerginClient.project_info()` (#179) 28 | - Improve project metadata handling 29 | - Add metadata getters in `MerginProject` class: `project_full_name()`, `project_name()`, `workspace_name()`, `project_id()`, `workspace_id()`, `files()` 30 | - Add metadata setters in `MerginProject` class: `update_metadata()`, `write_metadata()` (should not be needed by module users) 31 | - Deprecate `MerginProject.metadata` property, replaced by the methods above 32 | - Store all information from server's project info to `.mergin/mergin.json` instead of just some bits. Keep backwards compatibility to read `mergin.json` if it uses old-style syntax, new pulls/pushes will write new-style syntax (#83, #151) 33 | - Fix awkward way of passing full project names to `create_project()`, `create_project_and_push()`, `clone_project()` and in CLI for `create` command (#180) 34 | - Fix CLI for `list-projects` command (#172) 35 | - Write unhandled Python errors from download/pull/push to the client log (#156) 36 | - Keep client log if download of a project fails for some reason (#155) 37 | 38 | 39 | ## 0.8.3 40 | 41 | - Clean up temporary files in .mergin folder (#47) 42 | - Improved hint how to set MERGIN_AUTH on Windows (#170) 43 | - Better error reporting of HTTP errors (#174) 44 | 45 | ## 0.8.2 46 | 47 | - Updated to pygeodiff 2.0.2 48 | 49 | ## 0.8.1 50 | 51 | - Use new endpoint to retrieve detailed information about project version (#167) 52 | 53 | ## 0.8.0 54 | 55 | - Added workspace API to list available workspaces, creating a new workspace and listing projects (#150, #152, #158) 56 | - Removed check for available storage when syncing to personal namespace (#159) 57 | - Added storing project IDs to project metadata files (#154) 58 | 59 | ## 0.7.4 60 | 61 | - Added set_tables_to_skip() to optionally configure which tables to ignore in geodiff 62 | - Updated to pygeodiff 2.0.1 63 | 64 | ## 0.7.3 65 | 66 | - New default public server URL (#136) 67 | - Fix issues with files and changesets downloading (#140) 68 | - Fix handling of conflict files (MerginMaps/qgis-mergin-plugin#382) 69 | - CLI: Fix mergin-py-client list project (#133) 70 | - CLI: Fix individual file download (#140) 71 | 72 | ## 0.7.2 73 | 74 | - Reporting can be run without end version specified (lutraconsulting/qgis-mergin-plugin#365) 75 | - Fixed reporting when some tables were missing (lutraconsulting/qgis-mergin-plugin#362) 76 | 77 | ## 0.7.1 78 | 79 | - Added has_writing_permissions() API call 80 | 81 | ## 0.7.0 82 | 83 | - Added support for reporting of changes based on project history 84 | - Fixed sync issues on Windows when schema changes (#117) 85 | - Better naming of conflict files (#62) 86 | - Unhandled exceptions during pull get written to the client log (#103) 87 | - Fixed download of earlier version of GeoPackage files (#119) 88 | 89 | ## 0.6.6 90 | 91 | - Add user_service() API call (#113) 92 | - Updated to pygeodiff 1.0.5 93 | 94 | ## 0.6.5 95 | 96 | - CLI: add, remove and list user permissions in a project (#98, #110) 97 | - Update to pygeodiff 1.0.4 98 | 99 | ## 0.6.4 100 | 101 | - Allow any namespace in create_project_and_push() (#104) 102 | - CLI: create project & push data using --from-dir (#105) 103 | - Update to pygeodiff 1.0.3 104 | 105 | ## 0.6.3 106 | 107 | - Update to pygeodiff 1.0.2 108 | 109 | ## 0.6.2 110 | 111 | - Fixed python error when pushing changes 112 | 113 | ## 0.6.1 114 | 115 | - Added APIs to download individual files at any version (download_file()) and diffs (get_file_diff()) (#93) 116 | - Robustness fixes (#30, #53, #96) 117 | - Always require geodiff to be available (#92, #63) 118 | 119 | ## 0.6.0 120 | 121 | - Moved to pygeodiff 1.0.0 - adding various new APIs 122 | 123 | ## 0.5.12 124 | 125 | - CLI: simplified authentication, with multiple options now (#76) 126 | - user can pass MERGIN_USERNAME and MERGIN_PASSWORD env variables to commands 127 | - user can pass --username (and optionally --password) command line arguments to commands 128 | - user can still use "mergin login" command to first get auth token and set MERGIN_AUTH_TOKEN or pass --auth-token command line argument 129 | - CLI: it is possible to create a project in a different namespace (#81) 130 | - Fixed removal of projects in CLI (#82) 131 | - Fixed possible error when removing project on Windows (#57) 132 | - Fixed issue when a file was deleted both locally and on the server (qgis-mergin-plugin#232) 133 | - Added optional global log (in addition to per-project logs) 134 | - Improved handling of auth token problems 135 | - Better error reporting 136 | 137 | ## 0.5.11 138 | 139 | - Update to geodiff 0.8.8 (fixed issues with unicode paths, better error reporting) 140 | 141 | ## 0.5.10 142 | 143 | - Added API (paginated_projects_list()) for paginated listing of projects (#77) 144 | - Fixed sync error that have happened when moving to version >= 10 (#79, fixes lutraconsulting/qgis-mergin-plugin#219) 145 | - Added more details to diagnostic logs to help finding errors 146 | 147 | ## 0.5.9 148 | 149 | - Added API to download older version of a project (#74) 150 | - Added API to query project details of an older version of a project 151 | 152 | ## 0.5.8 153 | 154 | - Added API (get_projects_by_names()) to request project info for specified projects 155 | 156 | ## 0.5.7 157 | 158 | - Fixed quota check when uploading to organisations (#70) 159 | - Added API to manage collaborators in a project 160 | 161 | ## 0.5.6 162 | 163 | - Added API for cloning of projects 164 | - Projects can be created on behalf of an organisation 165 | 166 | ## 0.5.5 167 | 168 | - Update to geodiff 0.8.6 (fixes non-ascii paths, quotes in paths, rebase with gpkg triggers) 169 | - Fix "transferred size and expected total size do not match" (qgis-mergin-plugin#142) 170 | 171 | ## 0.5.4 172 | 173 | - Disable upload of renames of files to prevent corrupt projects (#27) 174 | 175 | ## 0.5.3 176 | 177 | - Fix download/pull of subdirs with just empty file(s) (qgis-mergin-plugin#160) 178 | - Add version info to user agent + log versions in log file (qgis-mergin-plugin#150) 179 | - Raise exception when username is passed but no password 180 | - Automatic running of tests in continuous integration on GitHub (#64) 181 | 182 | ## 0.5.2 183 | 184 | - Release fix 185 | 186 | ## 0.5.1 187 | 188 | - Update to geodiff 0.8.5 (leaking file handles in hasChanges() call) 189 | - Better error reporting for CLI, detect expired access token 190 | - Project push: fail early if user does not have write permissions 191 | - Support more endpoints with project info + CLI integration 192 | - Raise ClientError when request fails when trying to log in 193 | 194 | ## 0.5.0 195 | 196 | - Update to geodiff 0.8.4 (fixes a rebase bug, adds logging, supports conflict files) 197 | - More detailed logging 198 | - Better handling of exceptions 199 | - CLI improvements and fixes 200 | - Fixed sync error with missing basefile 201 | - Changed version because the module is going to have also independent releases from QGIS plugin 202 | (they had synchronized version numbers until now) 203 | 204 | ## 2020.4.1 205 | 206 | - Fixed load error if pip is not available (#133) 207 | 208 | ## 2020.4.0 209 | 210 | - Asynchronous push/pull/download (#44) 211 | - Basic logging of sync oprations to .mergin/client-log.txt 212 | - Modified create_project() API - does not do project upload anymore 213 | 214 | ## 2020.3.1 215 | 216 | - Second fix of the issue with sync (#119) 217 | 218 | ## 2020.3 219 | 220 | - Fixed an issue with synchronization (#119) 221 | 222 | ## 2020.2 223 | 224 | - Added checkpointing for GeoPackages (#93) 225 | - More details in exceptions (#106) 226 | - Fix SSL issues on macOS (#70) 227 | - Handling of expired token (#108) 228 | - Fix issue with checksums 229 | 230 | ## 2020.1 231 | 232 | - Check for enough free space (#25) 233 | - Added user agent in requests (#31) 234 | - Fix problems with empty files 235 | 236 | ## 2019.6 237 | 238 | - show list of local changes 239 | 240 | ## 2019.5 241 | 242 | - Support for geodiff 243 | - Added server compatibility checks (#1) 244 | - Added tests (#16) 245 | 246 | ## 2019.4.1 247 | - Improved push/pull speed using parallel requests 248 | - Added possibility to create blank mergin project 249 | 250 | ## 2019.4 251 | 252 | - Added filters for listing projects (owner, shared, query) 253 | - Changed Basic auth to Bearer token-based auth 254 | - Improved CLI: added login, credentials in env variables, delete project 255 | - Download/upload files with multiple sequential requests (chunked transfer) 256 | - Fixed push with no changes 257 | - Blacklisting various temp files from sync 258 | 259 | ## 2019.3 260 | 261 | Initial release containing following actions for project: 262 | 263 | - list 264 | - create (from local dir) 265 | - download 266 | - sync 267 | 268 | with basic CLI icluded. 269 | -------------------------------------------------------------------------------- /mergin/utils.py: -------------------------------------------------------------------------------- 1 | import os 2 | import io 3 | import json 4 | import hashlib 5 | import re 6 | import sqlite3 7 | from datetime import datetime 8 | from pathlib import Path 9 | import tempfile 10 | from .common import ClientError 11 | 12 | 13 | def generate_checksum(file, chunk_size=4096): 14 | """ 15 | Generate checksum for file from chunks. 16 | 17 | :param file: file to calculate checksum 18 | :param chunk_size: size of chunk 19 | :return: sha1 checksum 20 | """ 21 | checksum = hashlib.sha1() 22 | with open(file, "rb") as f: 23 | while True: 24 | chunk = f.read(chunk_size) 25 | if not chunk: 26 | return checksum.hexdigest() 27 | checksum.update(chunk) 28 | 29 | 30 | def save_to_file(stream, path): 31 | """ 32 | Save readable object in file. 33 | :param stream: object implementing readable interface 34 | :param path: destination file path 35 | """ 36 | directory = os.path.abspath(os.path.dirname(path)) 37 | 38 | os.makedirs(directory, exist_ok=True) 39 | 40 | with open(path, "wb") as output: 41 | writer = io.BufferedWriter(output, buffer_size=32768) 42 | while True: 43 | part = stream.read(4096) 44 | if part: 45 | writer.write(part) 46 | else: 47 | writer.flush() 48 | break 49 | 50 | 51 | def move_file(src, dest): 52 | dest_dir = os.path.dirname(dest) 53 | os.makedirs(dest_dir, exist_ok=True) 54 | os.rename(src, dest) 55 | 56 | 57 | class DateTimeEncoder(json.JSONEncoder): 58 | def default(self, obj): 59 | if isinstance(obj, datetime): 60 | return obj.isoformat() 61 | 62 | return super().default(obj) 63 | 64 | 65 | def find(items, fn): 66 | for item in items: 67 | if fn(item): 68 | return item 69 | 70 | 71 | def int_version(version): 72 | """Convert v format of version to integer representation.""" 73 | return int(version.lstrip("v")) if re.match(r"v\d", version) else None 74 | 75 | 76 | def do_sqlite_checkpoint(path, log=None): 77 | """ 78 | Function to do checkpoint over the geopackage file which was not able to do diff file. 79 | 80 | :param path: file's absolute path on disk 81 | :type path: str 82 | :param log: optional reference to a logger 83 | :type log: logging.Logger 84 | :returns: new size and checksum of file after checkpoint 85 | :rtype: int, str 86 | """ 87 | new_size = None 88 | new_checksum = None 89 | if ".gpkg" in path and os.path.exists(f"{path}-wal"): 90 | if log: 91 | log.info("checkpoint - going to add it in " + path) 92 | conn = sqlite3.connect(path) 93 | cursor = conn.cursor() 94 | cursor.execute("PRAGMA wal_checkpoint=FULL") 95 | if log: 96 | log.info("checkpoint - return value: " + str(cursor.fetchone())) 97 | cursor.execute("VACUUM") 98 | conn.commit() 99 | conn.close() 100 | new_size = os.path.getsize(path) 101 | new_checksum = generate_checksum(path) 102 | if log: 103 | log.info("checkpoint - new size {} checksum {}".format(new_size, new_checksum)) 104 | 105 | return new_size, new_checksum 106 | 107 | 108 | def get_versions_with_file_changes(mc, project_path, file_path, version_from=None, version_to=None, file_history=None): 109 | """ 110 | Get the project version tags where the file was added, modified or deleted. 111 | 112 | Args: 113 | mc (MerginClient): MerginClient instance 114 | project_path (str): project full name (/) 115 | file_path (str): relative path of file to download in the project directory 116 | version_from (str): optional minimum version to fetch, for example "v3" 117 | version_to (str): optional maximum version to fetch 118 | file_history (dict): optional file history info, result of project_file_history_info(). 119 | 120 | Returns: 121 | list of version tags, for example ["v4", "v7", "v8"] 122 | """ 123 | if file_history is None: 124 | file_history = mc.project_file_history_info(project_path, file_path) 125 | all_version_numbers = sorted([int(k[1:]) for k in file_history["history"].keys()]) 126 | version_from = all_version_numbers[0] if version_from is None else int_version(version_from) 127 | version_to = all_version_numbers[-1] if version_to is None else int_version(version_to) 128 | if version_from is None or version_to is None: 129 | err = f"Wrong version parameters: {version_from}-{version_to} while getting diffs for {file_path}. " 130 | err += f"Version tags required in the form: 'v2', 'v11', etc." 131 | raise ClientError(err) 132 | if version_from >= version_to: 133 | err = f"Wrong version parameters: {version_from}-{version_to} while getting diffs for {file_path}. " 134 | err += f"version_from needs to be smaller than version_to." 135 | raise ClientError(err) 136 | if version_from not in all_version_numbers or version_to not in all_version_numbers: 137 | err = f"Wrong version parameters: {version_from}-{version_to} while getting diffs for {file_path}. " 138 | err += f"Available versions: {all_version_numbers}" 139 | raise ClientError(err) 140 | 141 | # Find versions to fetch between the 'from' and 'to' versions 142 | idx_from = idx_to = None 143 | for idx, version in enumerate(all_version_numbers): 144 | if version == version_from: 145 | idx_from = idx 146 | elif version == version_to: 147 | idx_to = idx 148 | break 149 | return [f"v{ver_nr}" for ver_nr in all_version_numbers[idx_from : idx_to + 1]] 150 | 151 | 152 | def unique_path_name(path): 153 | """ 154 | Generates an unique name for the given path. If the given path does 155 | not exist yet it will be returned unchanged, otherwise a sequntial 156 | number will be added to the path in format: 157 | - if path is a directory: "folder" -> "folder (1)" 158 | - if path is a file: "filename.txt" -> "filename (1).txt" 159 | 160 | :param path: path to file or directory 161 | :type path: str 162 | :returns: unique path 163 | :rtype: str 164 | """ 165 | unique_path = str(path) 166 | 167 | is_dir = os.path.isdir(path) 168 | head, tail = os.path.split(os.path.normpath(path)) 169 | ext = "".join(Path(tail).suffixes) 170 | file_name = tail.replace(ext, "") 171 | 172 | i = 0 173 | while os.path.exists(unique_path): 174 | i += 1 175 | 176 | if is_dir: 177 | unique_path = f"{path} ({i})" 178 | else: 179 | unique_path = os.path.join(head, file_name) + f" ({i}){ext}" 180 | 181 | return unique_path 182 | 183 | 184 | def conflicted_copy_file_name(path, user, version): 185 | """ 186 | Generates a file name for the conflict copy file in the following form 187 | (conflicted copy, v).gpkg. Example: 188 | * data (conflicted copy, martin v5).gpkg 189 | 190 | :param path: name of the file 191 | :type path: str 192 | :param user: name of the user 193 | :type user: str 194 | :param version: version of the Mergin Maps project 195 | :type version: str 196 | :returns: new file name 197 | :rtype: str 198 | """ 199 | if not path: 200 | return "" 201 | 202 | head, tail = os.path.split(os.path.normpath(path)) 203 | ext = "".join(Path(tail).suffixes) 204 | file_name = tail.replace(ext, "") 205 | # in case of QGIS project files we TemporaryDirectoryhave to add "~" (tilde) to suffix 206 | # to avoid having several QGIS project files inside Mergin Maps project. 207 | # See https://github.com/lutraconsulting/qgis-mergin-plugin/issues/382 208 | # for more details 209 | if is_qgis_file(path): 210 | ext += "~" 211 | return os.path.join(head, file_name) + f" (conflicted copy, {user} v{version}){ext}" 212 | 213 | 214 | def edit_conflict_file_name(path, user, version): 215 | """ 216 | Generates a file name for the edit conflict file in the following form 217 | (edit conflict, v).gpkg. Example: 218 | * data (edit conflict, martin v5).gpkg 219 | 220 | :param path: name of the file 221 | :type path: str 222 | :param user: name of the user 223 | :type user: str 224 | :param version: version of the Mergin Maps project 225 | :type version: str 226 | :returns: new file name 227 | :rtype: str 228 | """ 229 | if not path: 230 | return "" 231 | 232 | head, tail = os.path.split(os.path.normpath(path)) 233 | ext = "".join(Path(tail).suffixes) 234 | file_name = tail.replace(ext, "") 235 | return os.path.join(head, file_name) + f" (edit conflict, {user} v{version}).json" 236 | 237 | 238 | def is_version_acceptable(version, min_version): 239 | """ 240 | Checks whether given version is at least min_version or later (both given as strings). 241 | 242 | Versions are expected to be using syntax X.Y.Z 243 | 244 | Returns true if version >= min_version 245 | """ 246 | m = re.search("(\\d+)[.](\\d+)", min_version) 247 | min_major, min_minor = m.group(1), m.group(2) 248 | 249 | if len(version) == 0: 250 | return False # unknown version is assumed to be old 251 | 252 | m = re.search("(\\d+)[.](\\d+)", version) 253 | if m is None: 254 | return False 255 | 256 | major, minor = m.group(1), m.group(2) 257 | 258 | return major > min_major or (major == min_major and minor >= min_minor) 259 | 260 | 261 | def is_versioned_file(path: str) -> bool: 262 | diff_extensions = [".gpkg", ".sqlite"] 263 | f_extension = os.path.splitext(path)[1] 264 | return f_extension.lower() in diff_extensions 265 | 266 | 267 | def is_qgis_file(path: str) -> bool: 268 | """ 269 | Check if file is a QGIS project file. 270 | """ 271 | f_extension = os.path.splitext(path)[1] 272 | return f_extension.lower() in [".qgs", ".qgz"] 273 | 274 | 275 | def is_mergin_config(path: str) -> bool: 276 | """Check if the given path is for file mergin-config.json""" 277 | filename = os.path.basename(path).lower() 278 | return filename == "mergin-config.json" 279 | 280 | 281 | def bytes_to_human_size(bytes: int): 282 | """ 283 | Convert bytes to human readable size 284 | example : 285 | bytes_to_human_size(5600000) -> "5.3 MB" 286 | """ 287 | precision = 1 288 | if bytes < 1e-5: 289 | return "0.0 MB" 290 | elif bytes < 1024.0 * 1024.0: 291 | return f"{round( bytes / 1024.0, precision )} KB" 292 | elif bytes < 1024.0 * 1024.0 * 1024.0: 293 | return f"{round( bytes / 1024.0 / 1024.0, precision)} MB" 294 | elif bytes < 1024.0 * 1024.0 * 1024.0 * 1024.0: 295 | return f"{round( bytes / 1024.0 / 1024.0 / 1024.0, precision )} GB" 296 | else: 297 | return f"{round( bytes / 1024.0 / 1024.0 / 1024.0 / 1024.0, precision )} TB" 298 | 299 | 300 | def cleanup_tmp_dir(mp, tmp_dir: tempfile.TemporaryDirectory): 301 | """ 302 | Remove temporary from tempfile.TemporaryDirectory instance 303 | This is needed beacause ignore_clanup_errors is not accepted under < Python 3.10 304 | """ 305 | 306 | try: 307 | tmp_dir.cleanup() 308 | except PermissionError: 309 | mp.log.warning(f"Permission error during tmp dir cleanup: {tmp_dir.name}") 310 | except Exception as e: 311 | mp.log.error(f"Error during tmp dir cleanup: {tmp_dir.name}: {e}") 312 | -------------------------------------------------------------------------------- /examples/03_projects.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "1db9d8bc-6de0-47dc-90f3-400e395bc3e2", 6 | "metadata": {}, 7 | "source": [ 8 | "# Mergin Maps Projects Management" 9 | ] 10 | }, 11 | { 12 | "cell_type": "markdown", 13 | "id": "8c4330a7-a7ad-441d-9f2d-16d5e0dfc8f7", 14 | "metadata": {}, 15 | "source": [ 16 | "Mergin Maps API allows you to manage your projects in a simple and effective way. See the [API reference](https://merginmaps.com/docs/dev/integration/) for more details about methods used in this notebook.\n", 17 | "\n", 18 | "First let's install Mergin Maps client (if not installed yet)" 19 | ] 20 | }, 21 | { 22 | "cell_type": "code", 23 | "execution_count": null, 24 | "id": "fb42bae4-f313-4196-9299-2c8d2dacca11", 25 | "metadata": { 26 | "trusted": true 27 | }, 28 | "outputs": [], 29 | "source": [ 30 | "!pip install mergin-client" 31 | ] 32 | }, 33 | { 34 | "cell_type": "markdown", 35 | "id": "f7517ef0-7b3f-47f6-8140-c3076ac02215", 36 | "metadata": {}, 37 | "source": [ 38 | "Login to Mergin Maps using your an existing user" 39 | ] 40 | }, 41 | { 42 | "cell_type": "code", 43 | "execution_count": null, 44 | "id": "c380887c-271b-48b6-b435-ed8628dd0a81", 45 | "metadata": { 46 | "trusted": true 47 | }, 48 | "outputs": [], 49 | "source": [ 50 | "# Use here your login username and password\n", 51 | "LOGIN=\"...\"\n", 52 | "PASSWORD=\"...\"\n", 53 | "\n", 54 | "import mergin\n", 55 | "\n", 56 | "client = mergin.MerginClient(login=LOGIN, password=PASSWORD)" 57 | ] 58 | }, 59 | { 60 | "cell_type": "markdown", 61 | "id": "46bcada4-5dac-44fe-8c63-5f35932404c4", 62 | "metadata": {}, 63 | "source": [ 64 | "Now you can use the client to call the API. Let's try to clone the project available for this example (`lutraconsulting/Vienna trees example`) to your Mergin Maps projects. \n", 65 | "\n", 66 | "You need to specify a workspace and project where our sample project will be cloned to. We can split our template to two projects." 67 | ] 68 | }, 69 | { 70 | "cell_type": "code", 71 | "execution_count": null, 72 | "id": "1dfb7e3c-bf38-4867-b47e-b895c4672212", 73 | "metadata": { 74 | "trusted": true 75 | }, 76 | "outputs": [], 77 | "source": [ 78 | "WORKSPACE = \"...\"\n", 79 | "PROJECT = \"project\"\n", 80 | "\n", 81 | "client.clone_project(\n", 82 | " source_project_path=f\"lutraconsulting/Vienna trees example\", cloned_project_name=f\"{WORKSPACE}/{PROJECT}-team-a\"\n", 83 | ")\n", 84 | "client.clone_project(\n", 85 | " source_project_path=f\"lutraconsulting/Vienna trees example\", cloned_project_name=f\"{WORKSPACE}/{PROJECT}-team-b\"\n", 86 | ")" 87 | ] 88 | }, 89 | { 90 | "cell_type": "markdown", 91 | "id": "75b88a5e-efc3-4569-a15e-6f49c591180f", 92 | "metadata": {}, 93 | "source": [ 94 | "Download your newly created Mergin Maps project to prepare data for your teams (Team A and Team B)." 95 | ] 96 | }, 97 | { 98 | "cell_type": "code", 99 | "execution_count": 7, 100 | "id": "34f8279f-76b9-452b-b067-20923f0783b6", 101 | "metadata": { 102 | "trusted": true 103 | }, 104 | "outputs": [], 105 | "source": [ 106 | "# download project to local folder.\n", 107 | "LOCAL_FOLDER_TEAM_A=\"/tmp/project-team-a\"\n", 108 | "LOCAL_FOLDER_TEAM_B=\"/tmp/project-team-b\"\n", 109 | "client.download_project(project_path=f\"{WORKSPACE}/{PROJECT}-team-a\", directory=LOCAL_FOLDER_TEAM_A)\n", 110 | "client.download_project(project_path=f\"{WORKSPACE}/{PROJECT}-team-b\", directory=LOCAL_FOLDER_TEAM_B)" 111 | ] 112 | }, 113 | { 114 | "cell_type": "markdown", 115 | "id": "10ea77c1-98b7-4d22-ac2c-a2b051db400e", 116 | "metadata": {}, 117 | "source": [ 118 | "Let's add 20 trees to the GeoPackage for each team. See GeoPackages prepared for this example in the [./03_projects_assets](./03_projects_assets) folder." 119 | ] 120 | }, 121 | { 122 | "cell_type": "code", 123 | "execution_count": null, 124 | "id": "11c18fc4-3202-498f-a733-7bf76f069926", 125 | "metadata": { 126 | "trusted": true 127 | }, 128 | "outputs": [ 129 | { 130 | "data": { 131 | "text/plain": [ 132 | "'/tmp/project-team-b/Ready_to_survey_trees.gpkg'" 133 | ] 134 | }, 135 | "execution_count": 9, 136 | "metadata": {}, 137 | "output_type": "execute_result" 138 | } 139 | ], 140 | "source": [ 141 | "# copy GeoPackage for team A to .gpkg file in LOCAL_FOLDER_TEAM_A\n", 142 | "import shutil\n", 143 | "shutil.copyfile(\n", 144 | " f\"./03_projects_assets/Ready_to_survey_trees_team_A.gpkg\",\n", 145 | " f\"{LOCAL_FOLDER_TEAM_A}/Ready_to_survey_trees.gpkg\"\n", 146 | ")\n", 147 | "# copy GeoPackage for team B to .gpkg file in LOCAL_FOLDER_TEAM_B\n", 148 | "import shutil\n", 149 | "shutil.copyfile(\n", 150 | " f\"./03_projects_assets/Ready_to_survey_trees_team_B.gpkg\",\n", 151 | " f\"{LOCAL_FOLDER_TEAM_B}/Ready_to_survey_trees.gpkg\"\n", 152 | ")" 153 | ] 154 | }, 155 | { 156 | "cell_type": "markdown", 157 | "id": "4ab30e8b", 158 | "metadata": {}, 159 | "source": [ 160 | "If you open your QGIS with projects for teams, you can see that every team is having different points to check.\n", 161 | "\n", 162 | "\"Team \"Team" 163 | ] 164 | }, 165 | { 166 | "cell_type": "markdown", 167 | "id": "a53d4a66", 168 | "metadata": {}, 169 | "source": [ 170 | "Let's sync your changes to Mergin Maps." 171 | ] 172 | }, 173 | { 174 | "cell_type": "code", 175 | "execution_count": 11, 176 | "id": "ce7aafe2-4e9d-4990-89f4-3654d9e15987", 177 | "metadata": { 178 | "trusted": true 179 | }, 180 | "outputs": [], 181 | "source": [ 182 | "client.push_project(directory=f\"{LOCAL_FOLDER_TEAM_A}\")\n", 183 | "client.push_project(directory=f\"{LOCAL_FOLDER_TEAM_B}\")" 184 | ] 185 | }, 186 | { 187 | "cell_type": "markdown", 188 | "id": "796d3cd2", 189 | "metadata": {}, 190 | "source": [ 191 | "If you have prepared data for your teams, you can now add some surveyors to projects. We can use the guest user role for this, as it only grants access to a specific projects. Let's add `john.doeA@example.com` and `john.doeB@example.com` to the teams." 192 | ] 193 | }, 194 | { 195 | "cell_type": "code", 196 | "execution_count": 12, 197 | "id": "d25078b0", 198 | "metadata": {}, 199 | "outputs": [ 200 | { 201 | "name": "stdout", 202 | "output_type": "stream", 203 | "text": [ 204 | "WORKSPACE_ID: 60538\n" 205 | ] 206 | } 207 | ], 208 | "source": [ 209 | "from mergin.common import WorkspaceRole\n", 210 | "\n", 211 | "# First, let's get workspace ID\n", 212 | "WORKSPACE_ID = None\n", 213 | "for p in client.workspaces_list():\n", 214 | " if p['name'] == WORKSPACE:\n", 215 | " WORKSPACE_ID = p['id']\n", 216 | "\n", 217 | "print(f\"WORKSPACE_ID: {WORKSPACE_ID}\")\n", 218 | "\n", 219 | "user_team_a = client.create_user(\n", 220 | " email=\"john.doeA@example.com\",\n", 221 | " password=\"JohnDoe123\",\n", 222 | " workspace_id=WORKSPACE_ID,\n", 223 | " workspace_role=WorkspaceRole.GUEST\n", 224 | ")\n", 225 | "user_team_b = client.create_user(\n", 226 | " email=\"john.doeB@example.com\",\n", 227 | " password=\"JohnDoe123\",\n", 228 | " workspace_id=WORKSPACE_ID,\n", 229 | " workspace_role=WorkspaceRole.GUEST\n", 230 | ")" 231 | ] 232 | }, 233 | { 234 | "cell_type": "markdown", 235 | "id": "5ffce828", 236 | "metadata": {}, 237 | "source": [ 238 | "Created users do not have any permissions yet. Let's add users as collaborators to the projects. They will be able to start the survey with Mergin Maps mobile app." 239 | ] 240 | }, 241 | { 242 | "cell_type": "code", 243 | "execution_count": null, 244 | "id": "fd204b84", 245 | "metadata": {}, 246 | "outputs": [ 247 | { 248 | "data": { 249 | "text/plain": [ 250 | "{'email': 'john.doeB@example.com',\n", 251 | " 'id': 122393,\n", 252 | " 'project_role': 'editor',\n", 253 | " 'role': 'editor',\n", 254 | " 'username': 'john.doeb',\n", 255 | " 'workspace_role': 'guest'}" 256 | ] 257 | }, 258 | "execution_count": 15, 259 | "metadata": {}, 260 | "output_type": "execute_result" 261 | } 262 | ], 263 | "source": [ 264 | "from mergin.common import ProjectRole\n", 265 | "\n", 266 | "team_a_project = client.project_info(f\"{WORKSPACE}/{PROJECT}-team-a\")\n", 267 | "team_b_project = client.project_info(f\"{WORKSPACE}/{PROJECT}-team-b\")\n", 268 | "\n", 269 | "# Add users to the projects\n", 270 | "client.add_project_collaborator(\n", 271 | " project_id=team_a_project.get(\"id\"), user=\"john.doeA@example.com\", project_role=ProjectRole.EDITOR\n", 272 | ")\n", 273 | "client.add_project_collaborator(\n", 274 | " project_id=team_b_project.get(\"id\"), user=\"john.doeB@example.com\", project_role=ProjectRole.EDITOR\n", 275 | ")" 276 | ] 277 | }, 278 | { 279 | "cell_type": "markdown", 280 | "id": "51879308-c8aa-480e-9f8c-530f282f4ec6", 281 | "metadata": {}, 282 | "source": [ 283 | "Let's delete cloned projects to make cleanup after the example.\n", 284 | "\n", 285 | "NOTE: using `delete_project_now` will bypass the default value `DELETED_PROJECT_EXPIRATION`. See: https://merginmaps.com/docs/server/environment/#data-synchronisation-and-management" 286 | ] 287 | }, 288 | { 289 | "cell_type": "code", 290 | "execution_count": 16, 291 | "id": "62abc678-75fe-4f7e-9f71-8dddee05d81e", 292 | "metadata": { 293 | "trusted": true 294 | }, 295 | "outputs": [], 296 | "source": [ 297 | "client.delete_project_now(f\"{WORKSPACE}/{PROJECT}-team-a\")\n", 298 | "client.delete_project_now(f\"{WORKSPACE}/{PROJECT}-team-b\")" 299 | ] 300 | } 301 | ], 302 | "metadata": { 303 | "kernelspec": { 304 | "display_name": "python-api-client", 305 | "language": "python", 306 | "name": "python3" 307 | }, 308 | "language_info": { 309 | "codemirror_mode": { 310 | "name": "ipython", 311 | "version": 3 312 | }, 313 | "file_extension": ".py", 314 | "mimetype": "text/x-python", 315 | "name": "python", 316 | "nbconvert_exporter": "python", 317 | "pygments_lexer": "ipython3", 318 | "version": "3.10.14" 319 | } 320 | }, 321 | "nbformat": 4, 322 | "nbformat_minor": 5 323 | } 324 | -------------------------------------------------------------------------------- /mergin/report.py: -------------------------------------------------------------------------------- 1 | import csv 2 | import json 3 | import os 4 | from collections import defaultdict 5 | from datetime import datetime 6 | from itertools import groupby 7 | 8 | from . import ClientError 9 | from .merginproject import MerginProject, pygeodiff 10 | from .utils import int_version 11 | 12 | try: 13 | from qgis.core import ( 14 | QgsGeometry, 15 | QgsDistanceArea, 16 | QgsCoordinateReferenceSystem, 17 | QgsCoordinateTransformContext, 18 | QgsWkbTypes, 19 | ) 20 | 21 | has_qgis = True 22 | except ImportError: 23 | has_qgis = False 24 | 25 | 26 | # inspired by C++ implementation https://github.com/lutraconsulting/geodiff/blob/master/geodiff/src/drivers/sqliteutils.cpp 27 | # in geodiff lib (MIT licence) 28 | # ideally there should be pygeodiff api for it 29 | def parse_gpkgb_header_size(gpkg_wkb): 30 | """Parse header of geopackage wkb and return its size""" 31 | # some constants 32 | no_envelope_header_size = 8 33 | flag_byte_pos = 3 34 | envelope_size_mask = 14 35 | 36 | try: 37 | flag_byte = gpkg_wkb[flag_byte_pos] 38 | except IndexError: 39 | return -1 # probably some invalid input 40 | envelope_byte = (flag_byte & envelope_size_mask) >> 1 41 | envelope_size = 0 42 | 43 | if envelope_byte == 1: 44 | envelope_size = 32 45 | elif envelope_byte == 2: 46 | envelope_size = 48 47 | elif envelope_byte == 3: 48 | envelope_size = 48 49 | elif envelope_byte == 4: 50 | envelope_size = 64 51 | 52 | return no_envelope_header_size + envelope_size 53 | 54 | 55 | def qgs_geom_from_wkb(geom): 56 | if not has_qgis: 57 | raise NotImplementedError 58 | g = QgsGeometry() 59 | wkb_header_length = parse_gpkgb_header_size(geom) 60 | wkb_geom = geom[wkb_header_length:] 61 | g.fromWkb(wkb_geom) 62 | return g 63 | 64 | 65 | class ChangesetReportEntry: 66 | """Derivative of geodiff ChangesetEntry suitable for further processing/reporting""" 67 | 68 | def __init__(self, changeset_entry, geom_idx, geom, qgs_distance_area=None): 69 | self.table = changeset_entry.table.name 70 | self.geom_type = geom["type"] 71 | self.crs = "EPSG:" + geom["srs_id"] 72 | self.length = None 73 | self.area = None 74 | self.dim = 0 75 | 76 | if changeset_entry.operation == changeset_entry.OP_DELETE: 77 | self.operation = "delete" 78 | elif changeset_entry.operation == changeset_entry.OP_UPDATE: 79 | self.operation = "update" 80 | elif changeset_entry.operation == changeset_entry.OP_INSERT: 81 | self.operation = "insert" 82 | else: 83 | self.operation = "unknown" 84 | 85 | # only calculate geom properties when qgis api is available 86 | if not qgs_distance_area: 87 | return 88 | 89 | crs = QgsCoordinateReferenceSystem() 90 | crs.createFromString(self.crs) 91 | qgs_distance_area.setSourceCrs(crs, QgsCoordinateTransformContext()) 92 | 93 | if hasattr(changeset_entry, "old_values"): 94 | old_wkb = changeset_entry.old_values[geom_idx] 95 | if isinstance(old_wkb, pygeodiff.UndefinedValue): 96 | old_wkb = None 97 | else: 98 | old_wkb = None 99 | if hasattr(changeset_entry, "new_values"): 100 | new_wkb = changeset_entry.new_values[geom_idx] 101 | if isinstance(new_wkb, pygeodiff.UndefinedValue): 102 | new_wkb = None 103 | else: 104 | new_wkb = None 105 | 106 | # no geometry at all 107 | if old_wkb is None and new_wkb is None: 108 | return 109 | 110 | updated_qgs_geom = None 111 | if self.operation == "delete": 112 | qgs_geom = qgs_geom_from_wkb(old_wkb) 113 | elif self.operation == "update": 114 | qgs_geom = qgs_geom_from_wkb(old_wkb) 115 | # get new geom if it was updated, there can be updates also without change of geom 116 | updated_qgs_geom = qgs_geom_from_wkb(new_wkb) if new_wkb else qgs_geom 117 | elif self.operation == "insert": 118 | qgs_geom = qgs_geom_from_wkb(new_wkb) 119 | 120 | self.dim = QgsWkbTypes.wkbDimensions(qgs_geom.wkbType()) 121 | if self.dim == 1: 122 | self.length = qgs_distance_area.measureLength(qgs_geom) 123 | if updated_qgs_geom: 124 | self.length = qgs_distance_area.measureLength(updated_qgs_geom) - self.length 125 | elif self.dim == 2: 126 | self.length = qgs_distance_area.measurePerimeter(qgs_geom) 127 | self.area = qgs_distance_area.measureArea(qgs_geom) 128 | if updated_qgs_geom: 129 | self.length = qgs_distance_area.measurePerimeter(updated_qgs_geom) - self.length 130 | self.area = qgs_distance_area.measureArea(updated_qgs_geom) - self.area 131 | 132 | 133 | def changeset_report(changeset_reader, schema, mp): 134 | """Parse Geodiff changeset reader and create report from it. 135 | Aggregate individual entries based on common table, operation and geom type. 136 | If QGIS API is available, then lengths and areas of individual entries are summed. 137 | 138 | :Example: 139 | 140 | >>> geodiff.schema("sqlite", "", "/tmp/base.gpkg", "/tmp/base-schema.json") 141 | >>> with open("/tmp/base-schema.json", 'r') as sf: 142 | schema = json.load(sf).get("geodiff_schema") 143 | >>> cr = geodiff.read_changeset("/tmp/base.gpkg-diff") 144 | >>> changeset_report(cr, schema) 145 | [{"table": "Lines", "operation": "insert", "length": 1.234, "area": 0.0, "count": 3}] 146 | 147 | Args: 148 | changeset_reader (pygeodiff.ChangesetReader): changeset reader from geodiff changeset (diff file) 149 | schema: geopackage schema with list of tables (from full .gpkg file) 150 | Returns: 151 | list of dict of aggregated records 152 | """ 153 | entries = [] 154 | records = [] 155 | 156 | if has_qgis: 157 | distance_area = QgsDistanceArea() 158 | distance_area.setEllipsoid("WGS84") 159 | else: 160 | distance_area = None 161 | # let's iterate through reader and populate entries 162 | for entry in changeset_reader: 163 | schema_table = next((t for t in schema if t["table"] == entry.table.name), None) 164 | if schema_table: 165 | # get geometry index in both gpkg schema and diffs values 166 | geom_idx = next( 167 | (index for (index, col) in enumerate(schema_table["columns"]) if col["type"] == "geometry"), None 168 | ) 169 | if geom_idx is None: 170 | continue 171 | 172 | geom_col = schema_table["columns"][geom_idx]["geometry"] 173 | report_entry = ChangesetReportEntry(entry, geom_idx, geom_col, distance_area) 174 | entries.append(report_entry) 175 | else: 176 | mp.log.warning(f"Table {entry.table.name} is not present in the changeset.") 177 | 178 | # create a map of entries grouped by tables within single .gpkg file 179 | tables = defaultdict(list) 180 | for obj in entries: 181 | tables[obj.table].append(obj) 182 | 183 | # iterate through all tables and aggregate changes by operation type (e.g. insert) and geometry type (e.g point) 184 | # for example 3 point features inserted in 'Points' table would be single row with count 3 185 | for table, entries in tables.items(): 186 | items = groupby(entries, lambda i: (i.operation, i.geom_type)) 187 | for k, v in items: 188 | values = list(v) 189 | # sum lenghts and areas only if it makes sense (valid dimension) 190 | area = sum([entry.area for entry in values if entry.area]) if values[0].dim == 2 else None 191 | length = sum([entry.length for entry in values if entry.length]) if values[0].dim > 0 else None 192 | records.append({"table": table, "operation": k[0], "length": length, "area": area, "count": len(values)}) 193 | return records 194 | 195 | 196 | def create_report(mc, directory, since, to, out_file): 197 | """Creates report from geodiff changesets for a range of project versions in CSV format. 198 | 199 | Report is created for all .gpkg files and all tables within where updates were done using Geodiff lib. 200 | Changeset contains operation (insert/update/delete) and geometry properties like length/perimeter and area. 201 | Each row is an aggregate of the features modified by the same operation and of the same geometry type and contains 202 | these values: "file", "table", "author", "date", "time", "version", "operation", "length", "area", "count" 203 | 204 | No filtering and grouping is done here, this is left for third-party office software to use pivot table functionality. 205 | 206 | Args: 207 | mc (MerginClient): MerginClient instance. 208 | directory (str): local project directory (must already exist). 209 | since (str): starting project version tag, for example 'v3'. 210 | to (str): ending project version tag, for example 'v6'. If empty string is passed the latest version will be used. 211 | out_file (str): output file to save csv in 212 | 213 | Returns: 214 | List of warnings/issues for versions which could not be processed (e.g. broken history with missing diff) 215 | """ 216 | mp = MerginProject(directory) 217 | project = mp.project_full_name() 218 | mp.log.info(f"--- Creating changesets report for {project} from {since} to {to if to else 'latest'} versions ----") 219 | versions = mc.project_versions(project, since, to if to else None) 220 | versions_map = {v["name"]: v for v in versions} 221 | headers = ["file", "table", "author", "date", "time", "version", "operation", "length", "area", "count"] 222 | records = [] 223 | warnings = [] 224 | info = mc.project_info(project, since=since) 225 | num_since = int_version(since) 226 | num_to = int_version(to) if to else int_version(versions[-1]["name"]) 227 | # filter only .gpkg files 228 | files = [f for f in info["files"] if mp.is_versioned_file(f["path"])] 229 | for f in files: 230 | mp.log.debug(f"analyzing {f['path']} ...") 231 | try: 232 | if "history" not in f: 233 | mp.log.debug(f"no history field, skip") 234 | continue 235 | 236 | # get version list (keys) within range 237 | history_keys = list(filter(lambda v: (num_since <= int_version(v) <= num_to), f["history"].keys())) 238 | if not history_keys: 239 | mp.log.debug(f"no file history within range, skip") 240 | continue 241 | 242 | # download diffs 243 | mc.download_file_diffs(directory, f["path"], history_keys) 244 | 245 | # download full gpkg in "to" version to analyze its schema to determine which col is geometry 246 | full_gpkg = mp.fpath_cache(f["path"], version=to) 247 | if not os.path.exists(full_gpkg): 248 | mc.download_file(directory, f["path"], full_gpkg, to) 249 | 250 | # get gpkg schema 251 | schema_file = full_gpkg + "-schema.json" # geodiff writes schema into a file 252 | if not os.path.exists(schema_file): 253 | mp.geodiff.schema("sqlite", "", full_gpkg, schema_file) 254 | with open(schema_file, "r") as sf: 255 | schema = json.load(sf).get("geodiff_schema") 256 | 257 | # add records for every version (diff) and all tables within geopackage 258 | for version in history_keys: 259 | if "diff" not in f["history"][version]: 260 | if f["history"][version]["change"] == "updated": 261 | warnings.append(f"Missing diff: {f['path']} was overwritten in {version} - broken diff history") 262 | else: 263 | warnings.append(f"Missing diff: {f['path']} was {f['history'][version]['change']} in {version}") 264 | continue 265 | 266 | v_diff_file = mp.fpath_cache(f["history"][version]["diff"]["path"], version=version) 267 | version_data = versions_map[version] 268 | cr = mp.geodiff.read_changeset(v_diff_file) 269 | report = changeset_report(cr, schema, mp) 270 | # append version info to changeset info 271 | dt = datetime.fromisoformat(version_data["created"].rstrip("Z")) 272 | version_fields = { 273 | "file": f["path"], 274 | "author": version_data["author"], 275 | "date": dt.date().isoformat(), 276 | "time": dt.time().isoformat(), 277 | "version": version_data["name"], 278 | } 279 | for row in report: 280 | records.append({**row, **version_fields}) 281 | mp.log.debug(f"done") 282 | except (ClientError, pygeodiff.GeoDiffLibError) as e: 283 | mp.log.warning(f"Skipping from report {f['path']}, issue found: {str(e)}") 284 | raise ClientError("Reporting failed, please check log for details") 285 | 286 | # export report to csv file 287 | out_dir = os.path.dirname(out_file) 288 | os.makedirs(out_dir, exist_ok=True) 289 | with open(out_file, "w", newline="") as f_csv: 290 | writer = csv.DictWriter(f_csv, fieldnames=headers) 291 | writer.writeheader() 292 | writer.writerows(records) 293 | mp.log.info(f"--- Report saved to {out_file} ----") 294 | return warnings 295 | -------------------------------------------------------------------------------- /mergin/client_push.py: -------------------------------------------------------------------------------- 1 | """ 2 | To push projects asynchronously. Start push: (does not block) 3 | 4 | job = push_project_async(mergin_client, '/tmp/my_project') 5 | 6 | Then we need to wait until we are finished uploading - either by periodically 7 | calling push_project_is_running(job) that will just return True/False or by calling 8 | push_project_wait(job) that will block the current thread (not good for GUI). 9 | To finish the upload job, we have to call push_project_finalize(job). 10 | """ 11 | 12 | import json 13 | import hashlib 14 | import pprint 15 | import tempfile 16 | import concurrent.futures 17 | import os 18 | 19 | from .common import UPLOAD_CHUNK_SIZE, ClientError 20 | from .merginproject import MerginProject, pygeodiff 21 | from .editor import filter_changes 22 | from .utils import cleanup_tmp_dir 23 | 24 | 25 | class UploadJob: 26 | """Keeps all the important data about a pending upload job""" 27 | 28 | def __init__(self, project_path, changes, transaction_id, mp, mc, tmp_dir): 29 | self.project_path = project_path # full project name ("username/projectname") 30 | self.changes = changes # dictionary of local changes to the project 31 | self.transaction_id = transaction_id # ID of the transaction assigned by the server 32 | self.total_size = 0 # size of data to upload (in bytes) 33 | self.transferred_size = 0 # size of data already uploaded (in bytes) 34 | self.upload_queue_items = [] # list of items to upload in the background 35 | self.mp = mp # MerginProject instance 36 | self.mc = mc # MerginClient instance 37 | self.tmp_dir = tmp_dir # TemporaryDirectory instance for any temp file we need 38 | self.is_cancelled = False # whether upload has been cancelled 39 | self.executor = None # ThreadPoolExecutor that manages background upload tasks 40 | self.futures = [] # list of futures submitted to the executor 41 | self.server_resp = None # server response when transaction is finished 42 | 43 | def dump(self): 44 | print("--- JOB ---", self.total_size, "bytes") 45 | for item in self.upload_queue_items: 46 | print("- {} {} {}".format(item.file_path, item.chunk_index, item.size)) 47 | print("--- END ---") 48 | 49 | 50 | class UploadQueueItem: 51 | """A single chunk of data that needs to be uploaded""" 52 | 53 | def __init__(self, file_path, size, transaction_id, chunk_id, chunk_index): 54 | self.file_path = file_path # full path to the file 55 | self.size = size # size of the chunk in bytes 56 | self.chunk_id = chunk_id # ID of the chunk within transaction 57 | self.chunk_index = chunk_index # index (starting from zero) of the chunk within the file 58 | self.transaction_id = transaction_id # ID of the transaction 59 | 60 | def upload_blocking(self, mc, mp): 61 | with open(self.file_path, "rb") as file_handle: 62 | file_handle.seek(self.chunk_index * UPLOAD_CHUNK_SIZE) 63 | data = file_handle.read(UPLOAD_CHUNK_SIZE) 64 | 65 | checksum = hashlib.sha1() 66 | checksum.update(data) 67 | 68 | mp.log.debug(f"Uploading {self.file_path} part={self.chunk_index}") 69 | 70 | headers = {"Content-Type": "application/octet-stream"} 71 | resp = mc.post( 72 | "/v1/project/push/chunk/{}/{}".format(self.transaction_id, self.chunk_id), 73 | data, 74 | headers, 75 | ) 76 | resp_dict = json.load(resp) 77 | mp.log.debug(f"Upload finished: {self.file_path}") 78 | if not (resp_dict["size"] == len(data) and resp_dict["checksum"] == checksum.hexdigest()): 79 | try: 80 | mc.post("/v1/project/push/cancel/{}".format(self.transaction_id)) 81 | except ClientError: 82 | pass 83 | raise ClientError("Mismatch between uploaded file chunk {} and local one".format(self.chunk_id)) 84 | 85 | 86 | def push_project_async(mc, directory): 87 | """Starts push of a project and returns pending upload job""" 88 | 89 | mp = MerginProject(directory) 90 | if mp.has_unfinished_pull(): 91 | raise ClientError("Project is in unfinished pull state. Please resolve unfinished pull and try again.") 92 | 93 | project_path = mp.project_full_name() 94 | local_version = mp.version() 95 | 96 | mp.log.info("--- version: " + mc.user_agent_info()) 97 | mp.log.info(f"--- start push {project_path}") 98 | 99 | try: 100 | project_info = mc.project_info(project_path) 101 | except ClientError as err: 102 | mp.log.error("Error getting project info: " + str(err)) 103 | mp.log.info("--- push aborted") 104 | raise 105 | server_version = project_info["version"] if project_info["version"] else "v0" 106 | 107 | mp.log.info(f"got project info: local version {local_version} / server version {server_version}") 108 | 109 | username = mc.username() 110 | # permissions field contains information about update, delete and upload privileges of the user 111 | # on a specific project. This is more accurate information then "writernames" field, as it takes 112 | # into account namespace privileges. So we have to check only "permissions", namely "upload" one 113 | if not mc.has_writing_permissions(project_path): 114 | mp.log.error(f"--- push {project_path} - username {username} does not have write access") 115 | raise ClientError(f"You do not seem to have write access to the project (username '{username}')") 116 | 117 | if local_version != server_version: 118 | mp.log.error(f"--- push {project_path} - not up to date (local {local_version} vs server {server_version})") 119 | raise ClientError( 120 | "There is a new version of the project on the server. Please update your local copy." 121 | + f"\n\nLocal version: {local_version}\nServer version: {server_version}" 122 | ) 123 | 124 | changes = mp.get_push_changes() 125 | changes = filter_changes(mc, project_info, changes) 126 | mp.log.debug("push changes:\n" + pprint.pformat(changes)) 127 | 128 | tmp_dir = tempfile.TemporaryDirectory(prefix="python-api-client-") 129 | 130 | # If there are any versioned files (aka .gpkg) that are not updated through a diff, 131 | # we need to make a temporary copy somewhere to be sure that we are uploading full content. 132 | # That's because if there are pending transactions, checkpointing or switching from WAL mode 133 | # won't work, and we would end up with some changes left in -wal file which do not get 134 | # uploaded. The temporary copy using geodiff uses sqlite backup API and should copy everything. 135 | for f in changes["updated"]: 136 | if mp.is_versioned_file(f["path"]) and "diff" not in f: 137 | mp.copy_versioned_file_for_upload(f, tmp_dir.name) 138 | 139 | for f in changes["added"]: 140 | if mp.is_versioned_file(f["path"]): 141 | mp.copy_versioned_file_for_upload(f, tmp_dir.name) 142 | 143 | if not sum(len(v) for v in changes.values()): 144 | mp.log.info(f"--- push {project_path} - nothing to do") 145 | return 146 | 147 | # drop internal info from being sent to server 148 | for item in changes["updated"]: 149 | item.pop("origin_checksum", None) 150 | data = {"version": local_version, "changes": changes} 151 | 152 | try: 153 | resp = mc.post( 154 | f"/v1/project/push/{project_path}", 155 | data, 156 | {"Content-Type": "application/json"}, 157 | ) 158 | except ClientError as err: 159 | mp.log.error("Error starting transaction: " + str(err)) 160 | mp.log.info("--- push aborted") 161 | raise 162 | server_resp = json.load(resp) 163 | 164 | upload_files = data["changes"]["added"] + data["changes"]["updated"] 165 | 166 | transaction_id = server_resp["transaction"] if upload_files else None 167 | job = UploadJob(project_path, changes, transaction_id, mp, mc, tmp_dir) 168 | 169 | if not upload_files: 170 | mp.log.info("not uploading any files") 171 | job.server_resp = server_resp 172 | push_project_finalize(job) 173 | return None # all done - no pending job 174 | 175 | mp.log.info(f"got transaction ID {transaction_id}") 176 | 177 | upload_queue_items = [] 178 | total_size = 0 179 | # prepare file chunks for upload 180 | for file in upload_files: 181 | if "diff" in file: 182 | # versioned file - uploading diff 183 | file_location = mp.fpath_meta(file["diff"]["path"]) 184 | file_size = file["diff"]["size"] 185 | elif "upload_file" in file: 186 | # versioned file - uploading full (a temporary copy) 187 | file_location = file["upload_file"] 188 | file_size = file["size"] 189 | else: 190 | # non-versioned file 191 | file_location = mp.fpath(file["path"]) 192 | file_size = file["size"] 193 | 194 | for chunk_index, chunk_id in enumerate(file["chunks"]): 195 | size = min(UPLOAD_CHUNK_SIZE, file_size - chunk_index * UPLOAD_CHUNK_SIZE) 196 | upload_queue_items.append(UploadQueueItem(file_location, size, transaction_id, chunk_id, chunk_index)) 197 | 198 | total_size += file_size 199 | 200 | job.total_size = total_size 201 | job.upload_queue_items = upload_queue_items 202 | 203 | mp.log.info(f"will upload {len(upload_queue_items)} items with total size {total_size}") 204 | 205 | # start uploads in background 206 | job.executor = concurrent.futures.ThreadPoolExecutor(max_workers=4) 207 | for item in upload_queue_items: 208 | future = job.executor.submit(_do_upload, item, job) 209 | job.futures.append(future) 210 | 211 | return job 212 | 213 | 214 | def push_project_wait(job): 215 | """blocks until all upload tasks are finished""" 216 | 217 | concurrent.futures.wait(job.futures) 218 | 219 | 220 | def push_project_is_running(job): 221 | """ 222 | Returns true/false depending on whether we have some pending uploads 223 | 224 | It also forwards any exceptions from workers (e.g. some network errors). If an exception 225 | is raised, it is advised to call push_project_cancel() to abort the job. 226 | """ 227 | for future in job.futures: 228 | if future.done() and future.exception() is not None: 229 | job.mp.log.error("Error while pushing data: " + str(future.exception())) 230 | job.mp.log.info("--- push aborted") 231 | raise future.exception() 232 | if future.running(): 233 | return True 234 | return False 235 | 236 | 237 | def push_project_finalize(job): 238 | """ 239 | To be called when push in the background is finished and we need to do the finalization 240 | 241 | This should not be called from a worker thread (e.g. directly from a handler when push is complete). 242 | 243 | If any of the workers has thrown any exception, it will be re-raised (e.g. some network errors). 244 | That also means that the whole job has been aborted. 245 | """ 246 | 247 | with_upload_of_files = job.executor is not None 248 | 249 | if with_upload_of_files: 250 | job.executor.shutdown(wait=True) 251 | 252 | # make sure any exceptions from threads are not lost 253 | for future in job.futures: 254 | if future.exception() is not None: 255 | job.mp.log.error("Error while pushing data: " + str(future.exception())) 256 | job.mp.log.info("--- push aborted") 257 | raise future.exception() 258 | 259 | if job.transferred_size != job.total_size: 260 | error_msg = "Transferred size ({}) and expected total size ({}) do not match!".format( 261 | job.transferred_size, job.total_size 262 | ) 263 | job.mp.log.error("--- push finish failed! " + error_msg) 264 | cleanup_tmp_dir(job.mp, job.tmp_dir) # delete our temporary dir and all its content 265 | raise ClientError("Upload error: " + error_msg) 266 | 267 | if with_upload_of_files: 268 | try: 269 | job.mp.log.info(f"Finishing transaction {job.transaction_id}") 270 | resp = job.mc.post("/v1/project/push/finish/%s" % job.transaction_id) 271 | job.server_resp = json.load(resp) 272 | except ClientError as err: 273 | # Log additional metadata on server error 502 or 504 (extended logging only) 274 | http_code = getattr(err, "http_error", None) 275 | if http_code in (502, 504): 276 | job.mp.log.error( 277 | f"Push failed with HTTP error {http_code}. " 278 | f"Upload details: {len(job.upload_queue_items)} file chunks, total size {job.total_size} bytes." 279 | ) 280 | job.mp.log.error("Files:") 281 | for f in job.changes.get("added", []) + job.changes.get("updated", []): 282 | path = f.get("path", "") 283 | size = f.get("size", "?") 284 | if "diff" in f: 285 | diff_info = f.get("diff", {}) 286 | diff_size = diff_info.get("size", "?") 287 | # best-effort: number of geodiff changes (if available) 288 | changes_cnt = _geodiff_changes_count(job.mp, diff_info.get("path")) 289 | if changes_cnt is None: 290 | job.mp.log.error(f" - {path}, size={size}, diff_size={diff_size}, changes=n/a") 291 | else: 292 | job.mp.log.error(f" - {path}, size={size}, diff_size={diff_size}, changes={changes_cnt}") 293 | else: 294 | job.mp.log.error(f" - {path}, size={size}") 295 | 296 | # server returns various error messages with filename or something generic 297 | # it would be better if it returned list of failed files (and reasons) whenever possible 298 | job.mp.log.error("--- push finish failed! " + str(err)) 299 | 300 | # if push finish fails, the transaction is not killed, so we 301 | # need to cancel it so it does not block further uploads 302 | job.mp.log.info("canceling the pending transaction...") 303 | try: 304 | resp_cancel = job.mc.post("/v1/project/push/cancel/%s" % job.transaction_id) 305 | job.mp.log.info("cancel response: " + resp_cancel.msg) 306 | except ClientError as err2: 307 | job.mp.log.info("cancel response: " + str(err2)) 308 | cleanup_tmp_dir(job.mp, job.tmp_dir) # delete our temporary dir and all its content 309 | raise err 310 | 311 | job.mp.update_metadata(job.server_resp) 312 | try: 313 | job.mp.apply_push_changes(job.changes) 314 | except Exception as e: 315 | job.mp.log.error("Failed to apply push changes: " + str(e)) 316 | job.mp.log.info("--- push aborted") 317 | cleanup_tmp_dir(job.mp, job.tmp_dir) # delete our temporary dir and all its content 318 | raise ClientError("Failed to apply push changes: " + str(e)) 319 | 320 | cleanup_tmp_dir(job.mp, job.tmp_dir) # delete our temporary dir and all its content 321 | remove_diff_files(job) 322 | 323 | job.mp.log.info("--- push finished - new project version " + job.server_resp["version"]) 324 | 325 | 326 | def _geodiff_changes_count(mp: MerginProject, diff_rel_path: str): 327 | """ 328 | Best-effort: return number of changes in the .gpkg diff (int) or None. 329 | Never raises – diagnostics/logging must not fail. 330 | """ 331 | 332 | diff_abs = mp.fpath_meta(diff_rel_path) 333 | try: 334 | return pygeodiff.GeoDiff().changes_count(diff_abs) 335 | except ( 336 | pygeodiff.GeoDiffLibError, 337 | pygeodiff.GeoDiffLibConflictError, 338 | pygeodiff.GeoDiffLibUnsupportedChangeError, 339 | pygeodiff.GeoDiffLibVersionError, 340 | FileNotFoundError, 341 | ): 342 | return None 343 | 344 | 345 | def push_project_cancel(job): 346 | """ 347 | To be called (from main thread) to cancel a job that has uploads in progress. 348 | Returns once all background tasks have exited (may block for a bit of time). 349 | """ 350 | 351 | job.mp.log.info("user cancelled the push...") 352 | # set job as cancelled 353 | job.is_cancelled = True 354 | 355 | job.executor.shutdown(wait=True) 356 | try: 357 | resp_cancel = job.mc.post("/v1/project/push/cancel/%s" % job.transaction_id) 358 | job.server_resp = resp_cancel.msg 359 | except ClientError as err: 360 | job.mp.log.error("--- push cancelling failed! " + str(err)) 361 | cleanup_tmp_dir(job.mp, job.tmp_dir) 362 | raise err 363 | cleanup_tmp_dir(job.mp, job.tmp_dir) # delete our temporary dir and all its content 364 | job.mp.log.info("--- push cancel response: " + str(job.server_resp)) 365 | 366 | 367 | def _do_upload(item, job): 368 | """runs in worker thread""" 369 | if job.is_cancelled: 370 | return 371 | 372 | item.upload_blocking(job.mc, job.mp) 373 | job.transferred_size += item.size 374 | 375 | 376 | def remove_diff_files(job) -> None: 377 | """Looks for diff files in the job and removes them.""" 378 | 379 | for change in job.changes["updated"]: 380 | if "diff" in change.keys(): 381 | diff_file = job.mp.fpath_meta(change["diff"]["path"]) 382 | if os.path.exists(diff_file): 383 | os.remove(diff_file) 384 | -------------------------------------------------------------------------------- /examples/01_users.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": { 6 | "id": "ngTJ3YbHlb8s" 7 | }, 8 | "source": [ 9 | "# Mergin Maps Users Management" 10 | ] 11 | }, 12 | { 13 | "cell_type": "markdown", 14 | "metadata": { 15 | "id": "q0eLjSMzlwdx" 16 | }, 17 | "source": [ 18 | "The Mergin Maps client allows you to manage your workspace users, their roles, and project-specific permissions. Every user has a workspace role and a project role. A project role is defined for a specific project and overrides the workspace role for that project. For more details, see the [permissions documentation](https://dev.merginmaps.com/docs/manage/permissions/).\n", 19 | "\n", 20 | "See the [API reference](https://merginmaps.com/docs/dev/integration/) for more details about methods used in this notebook." 21 | ] 22 | }, 23 | { 24 | "cell_type": "markdown", 25 | "metadata": { 26 | "id": "IKmFEjG-mmL6" 27 | }, 28 | "source": [ 29 | "First let's install mergin maps client" 30 | ] 31 | }, 32 | { 33 | "cell_type": "code", 34 | "execution_count": null, 35 | "metadata": { 36 | "id": "I_vpP6NnmqV7" 37 | }, 38 | "outputs": [], 39 | "source": [ 40 | "!pip install mergin-client" 41 | ] 42 | }, 43 | { 44 | "cell_type": "markdown", 45 | "metadata": { 46 | "id": "u05lxbRQm2VF" 47 | }, 48 | "source": [ 49 | "Login to Mergin Maps using your workspace user with `Owner` permission." 50 | ] 51 | }, 52 | { 53 | "cell_type": "code", 54 | "execution_count": null, 55 | "metadata": { 56 | "colab": { 57 | "base_uri": "https://localhost:8080/" 58 | }, 59 | "executionInfo": { 60 | "elapsed": 1822, 61 | "status": "ok", 62 | "timestamp": 1748364457967, 63 | "user": { 64 | "displayName": "Fernando Ribeiro", 65 | "userId": "15488710231554262191" 66 | }, 67 | "user_tz": -60 68 | }, 69 | "id": "dWQorVqZnRNl", 70 | "outputId": "778487c5-b0b5-4a7f-a024-33122e49b6fb", 71 | "trusted": true 72 | }, 73 | "outputs": [], 74 | "source": [ 75 | "# Use here your login username and password\n", 76 | "LOGIN = \"...\"\n", 77 | "PASSWORD = \"...\"\n", 78 | "\n", 79 | "import mergin\n", 80 | "\n", 81 | "client = mergin.MerginClient(login=LOGIN, password=PASSWORD)" 82 | ] 83 | }, 84 | { 85 | "cell_type": "markdown", 86 | "metadata": { 87 | "id": "gFN3jXIjntwf" 88 | }, 89 | "source": [ 90 | "Let's create a workspace and project as base for our users management example.\n", 91 | "\n", 92 | "Set the `WORKSPACE` variable to your desired workspace name and the `PROJECT` variable to the name of the project created in your workspace." 93 | ] 94 | }, 95 | { 96 | "cell_type": "code", 97 | "execution_count": null, 98 | "metadata": { 99 | "executionInfo": { 100 | "elapsed": 745, 101 | "status": "ok", 102 | "timestamp": 1748364795430, 103 | "user": { 104 | "displayName": "Fernando Ribeiro", 105 | "userId": "15488710231554262191" 106 | }, 107 | "user_tz": -60 108 | }, 109 | "id": "27rA4VfgoJjy" 110 | }, 111 | "outputs": [], 112 | "source": [ 113 | "# Add here your existing workspace name and the new project name\n", 114 | "WORKSPACE = \"...\"\n", 115 | "PROJECT = \"...\"\n", 116 | "\n", 117 | "# Create new workspace\n", 118 | "# INFO: Only uncomment if you are able to create a new workspace. Mergin Maps free tier only allows for 1 workspace per user. In this case use your existing workspace.\n", 119 | "# client.create_workspace(WORKSPACE)\n", 120 | "\n", 121 | "# Create new project\n", 122 | "client.create_project(project_name=PROJECT, namespace=WORKSPACE)" 123 | ] 124 | }, 125 | { 126 | "cell_type": "markdown", 127 | "metadata": { 128 | "id": "SXimIDIDqb9J" 129 | }, 130 | "source": [ 131 | "Create some users on your Mergin Maps example workspace from the provided example file in `01_users_assets/users.csv` with random permissions." 132 | ] 133 | }, 134 | { 135 | "cell_type": "code", 136 | "execution_count": null, 137 | "metadata": { 138 | "trusted": true 139 | }, 140 | "outputs": [ 141 | { 142 | "name": "stdout", 143 | "output_type": "stream", 144 | "text": [ 145 | "WORKSPACE_ID: 15639\n" 146 | ] 147 | } 148 | ], 149 | "source": [ 150 | "# First, let's get workspace ID\n", 151 | "WORKSPACE_ID = None\n", 152 | "for p in client.workspaces_list():\n", 153 | " if p['name'] == WORKSPACE:\n", 154 | " WORKSPACE_ID = p['id']\n", 155 | "\n", 156 | "print(f\"WORKSPACE_ID: {WORKSPACE_ID}\")" 157 | ] 158 | }, 159 | { 160 | "cell_type": "markdown", 161 | "metadata": {}, 162 | "source": [ 163 | "We can now use the client to create multiple users at once in your workspace by importing them from a CSV file. In this case, we will use the example users with random choice of password and workspace role. Username is optional, Megin Maps will generate username from email address." 164 | ] 165 | }, 166 | { 167 | "cell_type": "code", 168 | "execution_count": null, 169 | "metadata": { 170 | "id": "Lp351dFYquVs" 171 | }, 172 | "outputs": [ 173 | { 174 | "name": "stdout", 175 | "output_type": "stream", 176 | "text": [ 177 | "User with email jdoe@example.com created, password: PpWVMDIpGB164\n", 178 | "User with email asmith@example.com created, password: yvhWPxvZkU230\n", 179 | "User with email bwilliams@example.com created, password: ofgPmxerDW473\n", 180 | "User with email cjohnson@example.com created, password: ktAhUKuOnu295\n", 181 | "User with email omartin@example.com created, password: hIWhvYoNNh661\n" 182 | ] 183 | } 184 | ], 185 | "source": [ 186 | "from os.path import join\n", 187 | "import csv\n", 188 | "import string\n", 189 | "import random\n", 190 | "\n", 191 | "from mergin.common import WorkspaceRole\n", 192 | "\n", 193 | "filename = \"01_users_assets/users.csv\"\n", 194 | "\n", 195 | "with open(filename, mode=\"r\", newline=\"\", encoding=\"utf-8\") as csvfile:\n", 196 | " reader = csv.reader(csvfile)\n", 197 | " header = next(reader) # Skip header\n", 198 | " for row in reader:\n", 199 | " username = row[0]\n", 200 | " email = row[1]\n", 201 | " # add new mergin maps user\n", 202 | " password = \"\".join(\n", 203 | " random.choices(string.ascii_uppercase + string.ascii_lowercase, k=10) + random.choices(string.digits, k=3)\n", 204 | " )\n", 205 | " client.create_user(\n", 206 | " email=email,\n", 207 | " password=password,\n", 208 | " workspace_id=WORKSPACE_ID,\n", 209 | " workspace_role=WorkspaceRole.READER,\n", 210 | " username=username,\n", 211 | " )\n", 212 | " print(f\"User with email {email} created, password: {password}\")" 213 | ] 214 | }, 215 | { 216 | "cell_type": "markdown", 217 | "metadata": {}, 218 | "source": [ 219 | "We can now list all users in workspace using `list_workspace_members` method." 220 | ] 221 | }, 222 | { 223 | "cell_type": "code", 224 | "execution_count": 15, 225 | "metadata": {}, 226 | "outputs": [ 227 | { 228 | "name": "stdout", 229 | "output_type": "stream", 230 | "text": [ 231 | "[{'email': 'asmith@example.com', 'id': 22597, 'username': 'asmith', 'workspace_role': 'reader'}, {'email': 'bwilliams@example.com', 'id': 22598, 'username': 'bwilliams', 'workspace_role': 'reader'}, {'email': 'cjohnson@example.com', 'id': 22599, 'username': 'cjohnson', 'workspace_role': 'reader'}, {'email': 'jdoe@example.com', 'id': 22596, 'username': 'jdoe', 'workspace_role': 'reader'}, {'email': 'marcel.kocisek+r22overviews@lutraconsulting.co.uk', 'id': 12800, 'username': 'r22overviews', 'workspace_role': 'owner'}, {'email': 'omartin@example.com', 'id': 22600, 'username': 'omartin', 'workspace_role': 'reader'}]\n" 232 | ] 233 | } 234 | ], 235 | "source": [ 236 | "members = client.list_workspace_members(WORKSPACE_ID)\n", 237 | "print(members)" 238 | ] 239 | }, 240 | { 241 | "cell_type": "markdown", 242 | "metadata": { 243 | "id": "OoCFI9u7uClQ" 244 | }, 245 | "source": [ 246 | "Let's change workspace permission level for a specific user to `EDITOR`." 247 | ] 248 | }, 249 | { 250 | "cell_type": "code", 251 | "execution_count": null, 252 | "metadata": { 253 | "id": "6QG8Smj2uI2l" 254 | }, 255 | "outputs": [ 256 | { 257 | "name": "stdout", 258 | "output_type": "stream", 259 | "text": [ 260 | "Changing role of user asmith to editor\n" 261 | ] 262 | }, 263 | { 264 | "data": { 265 | "text/plain": [ 266 | "{'email': 'asmith@example.com',\n", 267 | " 'id': 22597,\n", 268 | " 'projects_roles': [],\n", 269 | " 'username': 'asmith',\n", 270 | " 'workspace_role': 'editor'}" 271 | ] 272 | }, 273 | "execution_count": 18, 274 | "metadata": {}, 275 | "output_type": "execute_result" 276 | } 277 | ], 278 | "source": [ 279 | "new_editor = members[0]\n", 280 | "print(f\"Changing role of user {new_editor['username']} to editor\")\n", 281 | "client.update_workspace_member(\n", 282 | " user_id=new_editor.get(\"id\", 0), workspace_id=WORKSPACE_ID, workspace_role=WorkspaceRole.EDITOR\n", 283 | ")" 284 | ] 285 | }, 286 | { 287 | "cell_type": "markdown", 288 | "metadata": { 289 | "id": "PpSxZfRfujj7" 290 | }, 291 | "source": [ 292 | "The user is now an editor for the entire workspace and every project within it. If you want to change the user's role to 'writer' for a specific project, you need to add them as a project collaborator." 293 | ] 294 | }, 295 | { 296 | "cell_type": "code", 297 | "execution_count": null, 298 | "metadata": { 299 | "id": "5_7lhNhVuukI" 300 | }, 301 | "outputs": [ 302 | { 303 | "data": { 304 | "text/plain": [ 305 | "{'email': 'asmith@example.com',\n", 306 | " 'id': 22597,\n", 307 | " 'project_role': 'writer',\n", 308 | " 'role': 'writer',\n", 309 | " 'username': 'asmith',\n", 310 | " 'workspace_role': 'editor'}" 311 | ] 312 | }, 313 | "execution_count": 21, 314 | "metadata": {}, 315 | "output_type": "execute_result" 316 | } 317 | ], 318 | "source": [ 319 | "from mergin.common import ProjectRole\n", 320 | "\n", 321 | "# find project id of project used in this example\n", 322 | "PROJECT_ID = None\n", 323 | "for p in client.projects_list(namespace=WORKSPACE):\n", 324 | " if p[\"name\"] == PROJECT:\n", 325 | " PROJECT_ID = p[\"id\"]\n", 326 | "\n", 327 | "\n", 328 | "client.add_project_collaborator(\n", 329 | " user=new_editor.get(\"email\", \"\"), project_id=PROJECT_ID, project_role=ProjectRole.WRITER\n", 330 | ")" 331 | ] 332 | }, 333 | { 334 | "cell_type": "markdown", 335 | "metadata": {}, 336 | "source": [ 337 | "We can now see that the user has beed added to project with different role than the `workspace_role`. Project role is now `writer`." 338 | ] 339 | }, 340 | { 341 | "cell_type": "markdown", 342 | "metadata": {}, 343 | "source": [ 344 | "We can now upgrade user to full control over project." 345 | ] 346 | }, 347 | { 348 | "cell_type": "code", 349 | "execution_count": 22, 350 | "metadata": {}, 351 | "outputs": [ 352 | { 353 | "data": { 354 | "text/plain": [ 355 | "{'email': 'asmith@example.com',\n", 356 | " 'id': 22597,\n", 357 | " 'project_role': 'owner',\n", 358 | " 'role': 'owner',\n", 359 | " 'username': 'asmith',\n", 360 | " 'workspace_role': 'editor'}" 361 | ] 362 | }, 363 | "execution_count": 22, 364 | "metadata": {}, 365 | "output_type": "execute_result" 366 | } 367 | ], 368 | "source": [ 369 | "client.update_project_collaborator(\n", 370 | " user_id=new_editor.get(\"id\", 0), project_id=PROJECT_ID, project_role=ProjectRole.OWNER\n", 371 | ")" 372 | ] 373 | }, 374 | { 375 | "cell_type": "markdown", 376 | "metadata": {}, 377 | "source": [ 378 | "To simply remove that user from project and use his default role, you can use the `remove_project_collaborator` method." 379 | ] 380 | }, 381 | { 382 | "cell_type": "code", 383 | "execution_count": 23, 384 | "metadata": {}, 385 | "outputs": [], 386 | "source": [ 387 | "client.remove_project_collaborator(\n", 388 | " user_id=new_editor.get(\"id\", 0), project_id=PROJECT_ID\n", 389 | ")" 390 | ] 391 | }, 392 | { 393 | "cell_type": "markdown", 394 | "metadata": {}, 395 | "source": [ 396 | "Check which users have access to the project using `list_project_collaborators` method." 397 | ] 398 | }, 399 | { 400 | "cell_type": "code", 401 | "execution_count": 24, 402 | "metadata": {}, 403 | "outputs": [ 404 | { 405 | "data": { 406 | "text/plain": [ 407 | "[{'email': 'asmith@example.com',\n", 408 | " 'id': 22597,\n", 409 | " 'project_role': None,\n", 410 | " 'role': 'editor',\n", 411 | " 'username': 'asmith',\n", 412 | " 'workspace_role': 'editor'},\n", 413 | " {'email': 'bwilliams@example.com',\n", 414 | " 'id': 22598,\n", 415 | " 'project_role': None,\n", 416 | " 'role': 'reader',\n", 417 | " 'username': 'bwilliams',\n", 418 | " 'workspace_role': 'reader'},\n", 419 | " {'email': 'cjohnson@example.com',\n", 420 | " 'id': 22599,\n", 421 | " 'project_role': None,\n", 422 | " 'role': 'reader',\n", 423 | " 'username': 'cjohnson',\n", 424 | " 'workspace_role': 'reader'},\n", 425 | " {'email': 'jdoe@example.com',\n", 426 | " 'id': 22596,\n", 427 | " 'project_role': None,\n", 428 | " 'role': 'reader',\n", 429 | " 'username': 'jdoe',\n", 430 | " 'workspace_role': 'reader'},\n", 431 | " {'email': 'marcel.kocisek+r22overviews@lutraconsulting.co.uk',\n", 432 | " 'id': 12800,\n", 433 | " 'project_role': None,\n", 434 | " 'role': 'owner',\n", 435 | " 'username': 'r22overviews',\n", 436 | " 'workspace_role': 'owner'},\n", 437 | " {'email': 'omartin@example.com',\n", 438 | " 'id': 22600,\n", 439 | " 'project_role': None,\n", 440 | " 'role': 'reader',\n", 441 | " 'username': 'omartin',\n", 442 | " 'workspace_role': 'reader'}]" 443 | ] 444 | }, 445 | "execution_count": 24, 446 | "metadata": {}, 447 | "output_type": "execute_result" 448 | } 449 | ], 450 | "source": [ 451 | "client.list_project_collaborators(project_id=PROJECT_ID)" 452 | ] 453 | }, 454 | { 455 | "cell_type": "markdown", 456 | "metadata": {}, 457 | "source": [ 458 | "Remove a user from an workspace to completely remove them from every project." 459 | ] 460 | }, 461 | { 462 | "cell_type": "code", 463 | "execution_count": 25, 464 | "metadata": { 465 | "trusted": true 466 | }, 467 | "outputs": [], 468 | "source": [ 469 | "client.remove_workspace_member(user_id=new_editor.get(\"id\", 0), workspace_id=WORKSPACE_ID)" 470 | ] 471 | } 472 | ], 473 | "metadata": { 474 | "colab": { 475 | "authorship_tag": "ABX9TyPXXcFNdfLOsA7CkWfkKXfJ", 476 | "collapsed_sections": [ 477 | "x8IVfy9K5Z4l", 478 | "7VMIPQo2yTmQ" 479 | ], 480 | "provenance": [] 481 | }, 482 | "kernelspec": { 483 | "display_name": "python-api-client", 484 | "language": "python", 485 | "name": "python3" 486 | }, 487 | "language_info": { 488 | "codemirror_mode": { 489 | "name": "ipython", 490 | "version": 3 491 | }, 492 | "file_extension": ".py", 493 | "mimetype": "text/x-python", 494 | "name": "python", 495 | "nbconvert_exporter": "python", 496 | "pygments_lexer": "ipython3", 497 | "version": "3.10.14" 498 | } 499 | }, 500 | "nbformat": 4, 501 | "nbformat_minor": 4 502 | } 503 | -------------------------------------------------------------------------------- /mergin/cli.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | """ 4 | Command line interface for the Mergin Maps client module. When installed with pip, this script 5 | is installed as 'mergin' command line tool (defined in setup.py). If you have installed the module 6 | but the tool is not available, you may need to fix your PATH (e.g. add ~/.local/bin where 7 | pip puts these tools). 8 | """ 9 | 10 | from datetime import datetime, timezone, date 11 | import click 12 | import json 13 | import os 14 | import platform 15 | import sys 16 | import time 17 | import traceback 18 | 19 | from mergin import ( 20 | ClientError, 21 | InvalidProject, 22 | LoginError, 23 | MerginClient, 24 | MerginProject, 25 | __version__, 26 | ) 27 | from mergin.client_pull import ( 28 | download_project_async, 29 | download_project_cancel, 30 | download_file_async, 31 | download_file_finalize, 32 | download_project_finalize, 33 | download_project_is_running, 34 | ) 35 | from mergin.client_pull import pull_project_async, pull_project_is_running, pull_project_finalize, pull_project_cancel 36 | from mergin.client_push import push_project_async, push_project_is_running, push_project_finalize, push_project_cancel 37 | 38 | 39 | from pygeodiff import GeoDiff 40 | 41 | 42 | class OptionPasswordIfUser(click.Option): 43 | """Custom option class for getting a password only if the --username option was specified.""" 44 | 45 | def handle_parse_result(self, ctx, opts, args): 46 | self.has_username = "username" in opts 47 | return super(OptionPasswordIfUser, self).handle_parse_result(ctx, opts, args) 48 | 49 | def prompt_for_value(self, ctx): 50 | if self.has_username: 51 | return super(OptionPasswordIfUser, self).prompt_for_value(ctx) 52 | return None 53 | 54 | 55 | def get_changes_count(diff): 56 | attrs = ["added", "removed", "updated"] 57 | return sum([len(diff[attr]) for attr in attrs]) 58 | 59 | 60 | def pretty_diff(diff): 61 | added = diff["added"] 62 | removed = diff["removed"] 63 | updated = diff["updated"] 64 | if removed: 65 | click.secho("\n>>> Removed:", fg="cyan") 66 | click.secho("\n".join("- " + f["path"] for f in removed), fg="red") 67 | if added: 68 | click.secho("\n>>> Added:", fg="cyan") 69 | click.secho("\n".join("+ " + f["path"] for f in added), fg="green") 70 | if updated: 71 | click.secho("\n>>> Modified:", fg="cyan") 72 | click.secho("\n".join("M " + f["path"] for f in updated), fg="yellow") 73 | 74 | 75 | def pretty_summary(summary): 76 | for k, v in summary.items(): 77 | click.secho("Details " + k) 78 | click.secho( 79 | "".join( 80 | "layer name - " 81 | + d["table"] 82 | + ": inserted: " 83 | + str(d["insert"]) 84 | + ", modified: " 85 | + str(d["update"]) 86 | + ", deleted: " 87 | + str(d["delete"]) 88 | + "\n" 89 | for d in v["geodiff_summary"] 90 | if d["table"] != "gpkg_contents" 91 | ) 92 | ) 93 | 94 | 95 | def get_token(url, username, password): 96 | """Get authorization token for given user and password.""" 97 | mc = MerginClient(url) 98 | 99 | try: 100 | session = mc.login(username, password) 101 | except LoginError as e: 102 | click.secho("Unable to log in: " + str(e), fg="red") 103 | return None 104 | return session["token"] 105 | 106 | 107 | def get_client(url=None, auth_token=None, username=None, password=None): 108 | """Return Mergin Maps client.""" 109 | if auth_token is not None: 110 | try: 111 | mc = MerginClient(url, auth_token=auth_token) 112 | except ClientError as e: 113 | click.secho(str(e), fg="red") 114 | return None 115 | # Check if the token has expired or is just about to expire 116 | delta = mc._auth_session["expire"] - datetime.now(timezone.utc) 117 | if delta.total_seconds() > 5: 118 | return mc 119 | if username and password: 120 | auth_token = get_token(url, username, password) 121 | if auth_token is None: 122 | return None 123 | mc = MerginClient(url, auth_token=f"Bearer {auth_token}") 124 | else: 125 | click.secho( 126 | "Missing authorization data.\n" 127 | "Either set environment variables (MERGIN_USERNAME and MERGIN_PASSWORD) " 128 | "or specify --username / --password options.\n" 129 | "Note: if --username is specified but password is missing you will be prompted for password.", 130 | fg="red", 131 | ) 132 | return None 133 | return mc 134 | 135 | 136 | def _print_unhandled_exception(): 137 | """Outputs details of an unhandled exception that is being handled right now""" 138 | click.secho("Unhandled exception!", fg="red") 139 | for line in traceback.format_exception(*sys.exc_info()): 140 | click.echo(line) 141 | 142 | 143 | @click.group( 144 | epilog=f"Copyright (C) 2019-{date.today().year} Lutra Consulting\n\n(python-api-client v{__version__} / pygeodiff v{GeoDiff().version()})" 145 | ) 146 | @click.option( 147 | "--url", 148 | envvar="MERGIN_URL", 149 | default=MerginClient.default_url(), 150 | help=f"Mergin Maps server URL. Default is: {MerginClient.default_url()}", 151 | ) 152 | @click.option("--auth-token", envvar="MERGIN_AUTH", help="Mergin Maps authentication token string") 153 | @click.option("--username", envvar="MERGIN_USERNAME") 154 | @click.option("--password", cls=OptionPasswordIfUser, prompt=True, hide_input=True, envvar="MERGIN_PASSWORD") 155 | @click.pass_context 156 | def cli(ctx, url, auth_token, username, password): 157 | """ 158 | Command line interface for the Mergin Maps client module. 159 | For user authentication on server there are two options: 160 | 161 | 1. authorization token environment variable (MERGIN_AUTH) is defined, or 162 | 2. username and password need to be given either as environment variables (MERGIN_USERNAME, MERGIN_PASSWORD), 163 | or as command options (--username, --password). 164 | 165 | Run `mergin --username login` to see how to set the token variable manually. 166 | """ 167 | mc = get_client(url=url, auth_token=auth_token, username=username, password=password) 168 | ctx.obj = {"client": mc} 169 | 170 | 171 | @cli.command() 172 | @click.pass_context 173 | def login(ctx): 174 | """Login to the service and see how to set the token environment variable.""" 175 | mc = ctx.obj["client"] 176 | if mc is not None: 177 | click.secho("Login successful!", fg="green") 178 | token = mc._auth_session["token"] 179 | if platform.system() == "Windows": 180 | hint = f"To set the MERGIN_AUTH variable run:\nset MERGIN_AUTH={token}" 181 | else: 182 | hint = f'To set the MERGIN_AUTH variable run:\nexport MERGIN_AUTH="{token}"' 183 | click.secho(hint) 184 | 185 | 186 | @cli.command() 187 | @click.argument("project") 188 | @click.option("--public", is_flag=True, default=False, help="Public project, visible to everyone") 189 | @click.option( 190 | "--from-dir", 191 | default=None, 192 | help="Content of the directory will be uploaded to the newly created project. " 193 | "The directory will get assigned to the project.", 194 | ) 195 | @click.pass_context 196 | def create(ctx, project, public, from_dir): 197 | """Create a new project on Mergin Maps server. `project` needs to be a combination of namespace/project.""" 198 | mc = ctx.obj["client"] 199 | if mc is None: 200 | return 201 | try: 202 | if from_dir is None: 203 | mc.create_project(project, is_public=public) 204 | click.echo("Created project " + project) 205 | else: 206 | mc.create_project_and_push(project, from_dir, is_public=public) 207 | click.echo("Created project " + project + " and pushed content from directory " + from_dir) 208 | except ClientError as e: 209 | click.secho("Error: " + str(e), fg="red") 210 | return 211 | except Exception as e: 212 | _print_unhandled_exception() 213 | 214 | 215 | @cli.command() 216 | @click.argument("namespace") 217 | @click.option( 218 | "--name", 219 | help="Filter projects with name like name", 220 | ) 221 | @click.option( 222 | "--order_params", 223 | help="optional attributes for sorting the list. " 224 | "It should be a comma separated attribute names " 225 | "with _asc or _desc appended for sorting direction. " 226 | 'For example: "namespace_asc,disk_usage_desc". ' 227 | "Available attrs: namespace, name, created, updated, disk_usage, creator", 228 | ) 229 | @click.pass_context 230 | def list_projects(ctx, name, namespace, order_params): 231 | """List projects on the server.""" 232 | 233 | mc = ctx.obj["client"] 234 | if mc is None: 235 | return 236 | 237 | projects_list = mc.projects_list(name=name, namespace=namespace, order_params=order_params) 238 | 239 | click.echo("Fetched {} projects .".format(len(projects_list))) 240 | for project in projects_list: 241 | full_name = "{} / {}".format(project["namespace"], project["name"]) 242 | click.echo( 243 | " {:40}\t{:6.1f} MB\t{}".format(full_name, project["disk_usage"] / (1024 * 1024), project["version"]) 244 | ) 245 | 246 | 247 | @cli.command() 248 | @click.argument("project") 249 | @click.argument("directory", type=click.Path(), required=False) 250 | @click.option("--version", default=None, help="Version of project to download") 251 | @click.pass_context 252 | def download(ctx, project, directory, version): 253 | """Download last version of mergin project.""" 254 | mc = ctx.obj["client"] 255 | if mc is None: 256 | return 257 | directory = directory or os.path.basename(project) 258 | click.echo("Downloading into {}".format(directory)) 259 | try: 260 | job = download_project_async(mc, project, directory, version) 261 | with click.progressbar(length=job.total_size) as bar: 262 | last_transferred_size = 0 263 | while download_project_is_running(job): 264 | time.sleep(1 / 10) # 100ms 265 | new_transferred_size = job.transferred_size 266 | bar.update(new_transferred_size - last_transferred_size) # the update() needs increment only 267 | last_transferred_size = new_transferred_size 268 | download_project_finalize(job) 269 | click.echo("Done") 270 | except KeyboardInterrupt: 271 | click.secho("Cancelling...") 272 | download_project_cancel(job) 273 | except ClientError as e: 274 | click.secho("Error: " + str(e), fg="red") 275 | except Exception as e: 276 | _print_unhandled_exception() 277 | 278 | 279 | @cli.command() 280 | @click.argument("project") 281 | @click.argument("usernames", nargs=-1) 282 | @click.option("--permissions", help="permissions to be granted to project (reader, writer, owner)") 283 | @click.pass_context 284 | def share_add(ctx, project, usernames, permissions): 285 | """Add permissions to [users] to project.""" 286 | mc = ctx.obj["client"] 287 | if mc is None: 288 | return 289 | usernames = list(usernames) 290 | mc.add_user_permissions_to_project(project, usernames, permissions) 291 | 292 | 293 | @cli.command() 294 | @click.argument("project") 295 | @click.argument("usernames", nargs=-1) 296 | @click.pass_context 297 | def share_remove(ctx, project, usernames): 298 | """Remove [users] permissions from project.""" 299 | mc = ctx.obj["client"] 300 | if mc is None: 301 | return 302 | usernames = list(usernames) 303 | mc.remove_user_permissions_from_project(project, usernames) 304 | 305 | 306 | @cli.command() 307 | @click.argument("project") 308 | @click.pass_context 309 | def share(ctx, project): 310 | """Fetch permissions to project.""" 311 | mc = ctx.obj["client"] 312 | if mc is None: 313 | return 314 | access_list = mc.project_user_permissions(project) 315 | 316 | owners = access_list.get("owners", []) 317 | writers = access_list.get("writers", []) 318 | editors = access_list.get("editors", []) 319 | readers = access_list.get("readers", []) 320 | 321 | for username in owners: 322 | click.echo("{:20}\t{:20}".format(username, "owner")) 323 | for username in writers: 324 | if username not in owners: 325 | click.echo("{:20}\t{:20}".format(username, "writer")) 326 | for username in editors: 327 | if username not in writers: 328 | click.echo("{:20}\t{:20}".format(username, "editor")) 329 | for username in readers: 330 | if username not in editors: 331 | click.echo("{:20}\t{:20}".format(username, "reader")) 332 | 333 | 334 | @cli.command() 335 | @click.argument("filepath") 336 | @click.argument("output") 337 | @click.option("--version", help="Project version tag, for example 'v3'") 338 | @click.pass_context 339 | def download_file(ctx, filepath, output, version): 340 | """ 341 | Download project file at specified version. `project` needs to be a combination of namespace/project. 342 | If no version is given, the latest will be fetched. 343 | """ 344 | mc = ctx.obj["client"] 345 | if mc is None: 346 | return 347 | try: 348 | job = download_file_async(mc, os.getcwd(), filepath, output, version) 349 | with click.progressbar(length=job.total_size) as bar: 350 | last_transferred_size = 0 351 | while download_project_is_running(job): 352 | time.sleep(1 / 10) # 100ms 353 | new_transferred_size = job.transferred_size 354 | bar.update(new_transferred_size - last_transferred_size) # the update() needs increment only 355 | last_transferred_size = new_transferred_size 356 | download_file_finalize(job) 357 | click.echo("Done") 358 | except KeyboardInterrupt: 359 | click.secho("Cancelling...") 360 | download_project_cancel(job) 361 | except ClientError as e: 362 | click.secho("Error: " + str(e), fg="red") 363 | except Exception as e: 364 | _print_unhandled_exception() 365 | 366 | 367 | def num_version(name): 368 | return int(name.lstrip("v")) 369 | 370 | 371 | @cli.command() 372 | @click.pass_context 373 | def status(ctx): 374 | """Show all changes in project files - upstream and local.""" 375 | mc = ctx.obj["client"] 376 | if mc is None: 377 | return 378 | try: 379 | pull_changes, push_changes, push_changes_summary = mc.project_status(os.getcwd()) 380 | except InvalidProject as e: 381 | click.secho("Invalid project directory ({})".format(str(e)), fg="red") 382 | return 383 | except ClientError as e: 384 | click.secho("Error: " + str(e), fg="red") 385 | return 386 | except Exception as e: 387 | _print_unhandled_exception() 388 | return 389 | 390 | if mc.has_unfinished_pull(os.getcwd()): 391 | click.secho( 392 | "The previous pull has not finished completely: status " 393 | "of some files may be reported incorrectly. Use " 394 | "resolve_unfinished_pull command to try to fix that.", 395 | fg="yellow", 396 | ) 397 | 398 | click.secho("### Server changes:", fg="magenta") 399 | pretty_diff(pull_changes) 400 | click.secho("### Local changes:", fg="magenta") 401 | pretty_diff(push_changes) 402 | click.secho("### Local changes summary ###") 403 | pretty_summary(push_changes_summary) 404 | 405 | 406 | @cli.command() 407 | @click.pass_context 408 | def push(ctx): 409 | """Upload local changes into Mergin Maps repository.""" 410 | mc = ctx.obj["client"] 411 | if mc is None: 412 | return 413 | directory = os.getcwd() 414 | try: 415 | job = push_project_async(mc, directory) 416 | if job is not None: # if job is none, we don't upload any files, and the transaction is finished already 417 | with click.progressbar(length=job.total_size) as bar: 418 | last_transferred_size = 0 419 | while push_project_is_running(job): 420 | time.sleep(1 / 10) # 100ms 421 | new_transferred_size = job.transferred_size 422 | bar.update(new_transferred_size - last_transferred_size) # the update() needs increment only 423 | last_transferred_size = new_transferred_size 424 | push_project_finalize(job) 425 | click.echo("Done") 426 | except InvalidProject as e: 427 | click.secho("Invalid project directory ({})".format(str(e)), fg="red") 428 | except ClientError as e: 429 | click.secho("Error: " + str(e), fg="red") 430 | return 431 | except KeyboardInterrupt: 432 | click.secho("Cancelling...") 433 | push_project_cancel(job) 434 | except Exception as e: 435 | _print_unhandled_exception() 436 | 437 | 438 | @cli.command() 439 | @click.pass_context 440 | def pull(ctx): 441 | """Fetch changes from Mergin Maps repository.""" 442 | mc = ctx.obj["client"] 443 | if mc is None: 444 | return 445 | directory = os.getcwd() 446 | try: 447 | job = pull_project_async(mc, directory) 448 | if job is None: 449 | click.echo("Project is up to date") 450 | return 451 | with click.progressbar(length=job.total_size) as bar: 452 | last_transferred_size = 0 453 | while pull_project_is_running(job): 454 | time.sleep(1 / 10) # 100ms 455 | new_transferred_size = job.transferred_size 456 | bar.update(new_transferred_size - last_transferred_size) # the update() needs increment only 457 | last_transferred_size = new_transferred_size 458 | pull_project_finalize(job) 459 | click.echo("Done") 460 | except InvalidProject as e: 461 | click.secho("Invalid project directory ({})".format(str(e)), fg="red") 462 | except ClientError as e: 463 | click.secho("Error: " + str(e), fg="red") 464 | return 465 | except KeyboardInterrupt: 466 | click.secho("Cancelling...") 467 | pull_project_cancel(job) 468 | except Exception as e: 469 | _print_unhandled_exception() 470 | 471 | 472 | @cli.command() 473 | @click.argument("version") 474 | @click.pass_context 475 | def show_version(ctx, version): 476 | """Displays information about a single version of a project. `version` is 'v1', 'v2', etc.""" 477 | mc = ctx.obj["client"] 478 | if mc is None: 479 | return 480 | directory = os.getcwd() 481 | mp = MerginProject(directory) 482 | project_id = mp.project_id() 483 | # TODO: handle exception when version not found 484 | version_info_dict = mc.project_version_info(project_id, version) 485 | click.secho("Project: " + version_info_dict["namespace"] + "/" + version_info_dict["project_name"]) 486 | click.secho("Version: " + version_info_dict["name"] + " by " + version_info_dict["author"]) 487 | click.secho("Time: " + version_info_dict["created"]) 488 | pretty_diff(version_info_dict["changes"]) 489 | 490 | 491 | @cli.command() 492 | @click.argument("path") 493 | @click.pass_context 494 | def show_file_history(ctx, path): 495 | """Displays information about a single version of a project.""" 496 | mc = ctx.obj["client"] 497 | if mc is None: 498 | return 499 | directory = os.getcwd() 500 | mp = MerginProject(directory) 501 | project_path = mp.project_full_name() 502 | info_dict = mc.project_file_history_info(project_path, path) 503 | # TODO: handle exception if history not found 504 | history_dict = info_dict["history"] 505 | click.secho("File history: " + info_dict["path"]) 506 | click.secho("-----") 507 | for version, version_data in history_dict.items(): 508 | diff_info = "" 509 | if "diff" in version_data: 510 | diff_info = "diff ({} bytes)".format(version_data["diff"]["size"]) 511 | click.secho(" {:5} {:10} {}".format(version, version_data["change"], diff_info)) 512 | 513 | 514 | @cli.command() 515 | @click.argument("path") 516 | @click.argument("version") 517 | @click.pass_context 518 | def show_file_changeset(ctx, path, version): 519 | """Displays information about project changes.""" 520 | mc = ctx.obj["client"] 521 | if mc is None: 522 | return 523 | directory = os.getcwd() 524 | mp = MerginProject(directory) 525 | project_path = mp.project_full_name() 526 | info_dict = mc.project_file_changeset_info(project_path, path, version) 527 | # TODO: handle exception if Diff not found 528 | click.secho(json.dumps(info_dict, indent=2)) 529 | 530 | 531 | @cli.command() 532 | @click.argument("source_project_path", required=True) 533 | @click.argument("cloned_project_name", required=True) 534 | @click.argument("cloned_project_namespace", required=False) 535 | @click.pass_context 536 | def clone(ctx, source_project_path, cloned_project_name, cloned_project_namespace=None): 537 | """Clone project from server.""" 538 | mc = ctx.obj["client"] 539 | if mc is None: 540 | return 541 | try: 542 | if cloned_project_namespace: 543 | click.secho( 544 | "The usage of `cloned_project_namespace` parameter in `mergin clone` is deprecated." 545 | "Specify `cloned_project_name` as full name (/) instead.", 546 | fg="yellow", 547 | ) 548 | if cloned_project_namespace is None and "/" not in cloned_project_name: 549 | click.secho( 550 | "The use of only project name as `cloned_project_name` in `clone_project()` is deprecated." 551 | "The `cloned_project_name` should be full name (/).", 552 | fg="yellow", 553 | ) 554 | if cloned_project_namespace and "/" not in cloned_project_name: 555 | cloned_project_name = f"{cloned_project_namespace}/{cloned_project_name}" 556 | mc.clone_project(source_project_path, cloned_project_name) 557 | click.echo("Done") 558 | except ClientError as e: 559 | click.secho("Error: " + str(e), fg="red") 560 | except Exception as e: 561 | _print_unhandled_exception() 562 | 563 | 564 | @cli.command() 565 | @click.argument("project", required=True) 566 | @click.pass_context 567 | def remove(ctx, project): 568 | """Remove project from server.""" 569 | mc = ctx.obj["client"] 570 | if mc is None: 571 | return 572 | if "/" in project: 573 | try: 574 | namespace, project = project.split("/") 575 | assert namespace, "No namespace given" 576 | assert project, "No project name given" 577 | except (ValueError, AssertionError) as e: 578 | click.secho(f"Incorrect namespace/project format: {e}", fg="red") 579 | return 580 | else: 581 | # namespace not specified, use current user namespace 582 | namespace = mc.username() 583 | try: 584 | mc.delete_project(f"{namespace}/{project}") 585 | click.echo("Remote project removed") 586 | except ClientError as e: 587 | click.secho("Error: " + str(e), fg="red") 588 | except Exception as e: 589 | _print_unhandled_exception() 590 | 591 | 592 | @cli.command() 593 | @click.pass_context 594 | def resolve_unfinished_pull(ctx): 595 | """Try to resolve unfinished pull.""" 596 | mc = ctx.obj["client"] 597 | if mc is None: 598 | return 599 | 600 | try: 601 | mc.resolve_unfinished_pull(os.getcwd()) 602 | click.echo("Unfinished pull successfully resolved") 603 | except InvalidProject as e: 604 | click.secho("Invalid project directory ({})".format(str(e)), fg="red") 605 | except ClientError as e: 606 | click.secho("Error: " + str(e), fg="red") 607 | except Exception as e: 608 | _print_unhandled_exception() 609 | 610 | 611 | @cli.command() 612 | @click.argument("project_path") 613 | @click.argument("new_project_name") 614 | @click.pass_context 615 | def rename(ctx, project_path: str, new_project_name: str): 616 | """Rename project in Mergin Maps repository.""" 617 | mc = ctx.obj["client"] 618 | if mc is None: 619 | return 620 | 621 | if "/" not in project_path: 622 | click.secho(f"Specify `project_path` as full name (/) instead.", fg="red") 623 | return 624 | 625 | if "/" in new_project_name: 626 | old_workspace, old_project_name = project_path.split("/") 627 | new_workspace, new_project_name = new_project_name.split("/") 628 | 629 | if old_workspace != new_workspace: 630 | click.secho( 631 | "`new_project_name` should not contain namespace, project can only be rename within their namespace.\nTo move project to another workspace use web dashboard.", 632 | fg="red", 633 | ) 634 | return 635 | 636 | try: 637 | mc.rename_project(project_path, new_project_name) 638 | click.echo("Project renamed") 639 | except ClientError as e: 640 | click.secho("Error: " + str(e), fg="red") 641 | except Exception as e: 642 | _print_unhandled_exception() 643 | 644 | 645 | @cli.command() 646 | @click.pass_context 647 | def reset(ctx): 648 | """Reset local changes in project.""" 649 | directory = os.getcwd() 650 | mc: MerginClient = ctx.obj["client"] 651 | if mc is None: 652 | return 653 | try: 654 | mc.reset_local_changes(directory) 655 | except InvalidProject as e: 656 | click.secho("Invalid project directory ({})".format(str(e)), fg="red") 657 | except ClientError as e: 658 | click.secho("Error: " + str(e), fg="red") 659 | except Exception as e: 660 | _print_unhandled_exception() 661 | 662 | 663 | @cli.command() 664 | @click.argument("project") 665 | @click.option("--json", is_flag=True, default=False, help="Output in JSON format") 666 | @click.pass_context 667 | def list_files(ctx, project, json): 668 | """List files in a project.""" 669 | 670 | mc = ctx.obj["client"] 671 | if mc is None: 672 | return 673 | 674 | project_info = mc.project_info(project) 675 | project_files = project_info["files"] 676 | 677 | if json: 678 | print(project_files) 679 | else: 680 | click.echo("Fetched {} files .".format(len(project_files))) 681 | for file in project_files: 682 | click.echo(" {:40}\t{:6.1f} MB".format(file["path"], file["size"] / (1024 * 1024))) 683 | 684 | 685 | if __name__ == "__main__": 686 | cli() 687 | -------------------------------------------------------------------------------- /mergin/client_pull.py: -------------------------------------------------------------------------------- 1 | """ 2 | To download projects asynchronously. Start download: (does not block) 3 | 4 | job = download_project_async(mergin_client, 'user/project', '/tmp/my_project) 5 | 6 | Then we need to wait until we are finished downloading - either by periodically 7 | calling download_project_is_running(job) that will just return True/False or by calling 8 | download_project_wait(job) that will block the current thread (not good for GUI). 9 | To finish the download job, we have to call download_project_finalize(job). 10 | """ 11 | 12 | import copy 13 | import math 14 | import os 15 | import pprint 16 | import shutil 17 | import tempfile 18 | import typing 19 | import traceback 20 | 21 | import concurrent.futures 22 | 23 | from .common import CHUNK_SIZE, ClientError 24 | from .merginproject import MerginProject 25 | from .utils import cleanup_tmp_dir, save_to_file 26 | 27 | 28 | # status = download_project_async(...) 29 | # 30 | # for completely async approach: 31 | # - a method called (in worker thread(!)) when new data are received -- to update progress bar 32 | # - a method called (in worker thread(!)) when download is complete -- and we just need to do the final steps (in main thread) 33 | # - the methods in worker threads could send queued signals to some QObject instances owned by main thread to do updating/finalization 34 | # 35 | # polling approach: 36 | # - caller will caller a method every X ms to check the status 37 | # - once status says download is finished, the caller would call a function to do finalization 38 | 39 | 40 | class DownloadJob: 41 | """ 42 | Keeps all the important data about a pending download job. 43 | Used for downloading whole projects but also single files. 44 | """ 45 | 46 | def __init__( 47 | self, 48 | project_path, 49 | total_size, 50 | version, 51 | update_tasks, 52 | download_queue_items, 53 | tmp_dir: tempfile.TemporaryDirectory, 54 | mp, 55 | project_info, 56 | ): 57 | self.project_path = project_path 58 | self.total_size = total_size # size of data to download (in bytes) 59 | self.transferred_size = 0 60 | self.version = version 61 | self.update_tasks = update_tasks 62 | self.download_queue_items = download_queue_items 63 | self.tmp_dir = tmp_dir 64 | self.mp = mp # MerginProject instance 65 | self.is_cancelled = False 66 | self.project_info = project_info # parsed JSON with project info returned from the server 67 | self.failure_log_file = None # log file, copied from the project directory if download fails 68 | 69 | def dump(self): 70 | print("--- JOB ---", self.total_size, "bytes") 71 | for task in self.update_tasks: 72 | print("- {} ... {}".format(task.file_path, len(task.download_queue_items))) 73 | print("--") 74 | for item in self.download_queue_items: 75 | print("- {} {} {} {}".format(item.file_path, item.version, item.part_index, item.size)) 76 | print("--- END ---") 77 | 78 | 79 | def _download_items(file, directory, diff_only=False): 80 | """Returns an array of download queue items""" 81 | 82 | file_dir = os.path.dirname(os.path.normpath(os.path.join(directory, file["path"]))) 83 | basename = os.path.basename(file["diff"]["path"]) if diff_only else os.path.basename(file["path"]) 84 | file_size = file["diff"]["size"] if diff_only else file["size"] 85 | chunks = math.ceil(file_size / CHUNK_SIZE) 86 | 87 | items = [] 88 | for part_index in range(chunks): 89 | download_file_path = os.path.join(file_dir, basename + ".{}".format(part_index)) 90 | size = min(CHUNK_SIZE, file_size - part_index * CHUNK_SIZE) 91 | items.append(DownloadQueueItem(file["path"], size, file["version"], diff_only, part_index, download_file_path)) 92 | 93 | return items 94 | 95 | 96 | def _do_download(item, mc, mp, project_path, job): 97 | """runs in worker thread""" 98 | if job.is_cancelled: 99 | return 100 | 101 | # TODO: make download_blocking / save_to_file cancellable so that we can cancel as soon as possible 102 | 103 | item.download_blocking(mc, mp, project_path) 104 | job.transferred_size += item.size 105 | 106 | 107 | def _cleanup_failed_download(mergin_project: MerginProject = None): 108 | """ 109 | If a download job fails, there will be the newly created directory left behind with some 110 | temporary files in it. We want to remove it because a new download would fail because 111 | the directory already exists. 112 | 113 | Returns path to the client log file or None if log file does not exist. 114 | """ 115 | # First try to get the Mergin Maps project logger and remove its handlers to allow the log file deletion 116 | if mergin_project is not None: 117 | mergin_project.remove_logging_handler() 118 | 119 | # keep log file as it might contain useful debug info 120 | log_file = os.path.join(mergin_project.dir, ".mergin", "client-log.txt") 121 | dest_path = None 122 | 123 | if os.path.exists(log_file): 124 | tmp_file = tempfile.NamedTemporaryFile(prefix="mergin-", suffix=".txt", delete=False) 125 | tmp_file.close() 126 | dest_path = tmp_file.name 127 | shutil.copyfile(log_file, dest_path) 128 | 129 | return dest_path 130 | 131 | 132 | def download_project_async(mc, project_path, directory, project_version=None): 133 | """ 134 | Starts project download in background and returns handle to the pending project download. 135 | Using that object it is possible to watch progress or cancel the ongoing work. 136 | """ 137 | 138 | if "/" not in project_path: 139 | raise ClientError("Project name needs to be fully qualified, e.g. /") 140 | if os.path.exists(directory): 141 | raise ClientError("Project directory already exists") 142 | os.makedirs(directory) 143 | mp = MerginProject(directory) 144 | 145 | mp.log.info("--- version: " + mc.user_agent_info()) 146 | mp.log.info(f"--- start download {project_path}") 147 | 148 | tmp_dir = tempfile.TemporaryDirectory(prefix="python-api-client-") 149 | 150 | try: 151 | # check whether we download the latest version or not 152 | latest_proj_info = mc.project_info(project_path) 153 | if project_version: 154 | project_info = mc.project_info(project_path, version=project_version) 155 | else: 156 | project_info = latest_proj_info 157 | 158 | except ClientError: 159 | _cleanup_failed_download(mp) 160 | raise 161 | 162 | version = project_info["version"] if project_info["version"] else "v0" 163 | 164 | mp.log.info(f"got project info. version {version}") 165 | 166 | # prepare download 167 | update_tasks = [] # stuff to do at the end of download 168 | for file in project_info["files"]: 169 | file["version"] = version 170 | items = _download_items(file, tmp_dir.name) 171 | is_latest_version = project_version == latest_proj_info["version"] 172 | update_tasks.append(UpdateTask(file["path"], items, latest_version=is_latest_version)) 173 | 174 | # make a single list of items to download 175 | total_size = 0 176 | download_list = [] 177 | for task in update_tasks: 178 | download_list.extend(task.download_queue_items) 179 | for item in task.download_queue_items: 180 | total_size += item.size 181 | 182 | mp.log.info(f"will download {len(update_tasks)} files in {len(download_list)} chunks, total size {total_size}") 183 | 184 | job = DownloadJob(project_path, total_size, version, update_tasks, download_list, tmp_dir, mp, project_info) 185 | 186 | # start download 187 | job.executor = concurrent.futures.ThreadPoolExecutor(max_workers=4) 188 | job.futures = [] 189 | for item in download_list: 190 | future = job.executor.submit(_do_download, item, mc, mp, project_path, job) 191 | job.futures.append(future) 192 | 193 | return job 194 | 195 | 196 | def download_project_wait(job): 197 | """blocks until all download tasks are finished""" 198 | 199 | concurrent.futures.wait(job.futures) 200 | 201 | 202 | def download_project_is_running(job): 203 | """ 204 | Returns true/false depending on whether we have some pending downloads. 205 | 206 | It also forwards any exceptions from workers (e.g. some network errors). If an exception 207 | is raised, it is advised to call download_project_cancel() to abort the job. 208 | """ 209 | for future in job.futures: 210 | if future.done() and future.exception() is not None: 211 | exc = future.exception() 212 | traceback_lines = traceback.format_exception(type(exc), exc, exc.__traceback__) 213 | job.mp.log.error("Error while downloading project: " + "".join(traceback_lines)) 214 | job.mp.log.info("--- download aborted") 215 | job.failure_log_file = _cleanup_failed_download(job.mp) 216 | raise future.exception() 217 | if future.running(): 218 | return True 219 | return False 220 | 221 | 222 | def download_project_finalize(job): 223 | """ 224 | To be called when download in the background is finished and we need to do the finalization (merge chunks etc.) 225 | 226 | This should not be called from a worker thread (e.g. directly from a handler when download is complete). 227 | 228 | If any of the workers has thrown any exception, it will be re-raised (e.g. some network errors). 229 | That also means that the whole job has been aborted. 230 | """ 231 | 232 | job.executor.shutdown(wait=True) 233 | 234 | # make sure any exceptions from threads are not lost 235 | for future in job.futures: 236 | if future.exception() is not None: 237 | exc = future.exception() 238 | traceback_lines = traceback.format_exception(type(exc), exc, exc.__traceback__) 239 | job.mp.log.error("Error while downloading project: " + "".join(traceback_lines)) 240 | job.mp.log.info("--- download aborted") 241 | job.failure_log_file = _cleanup_failed_download(job.mp) 242 | raise future.exception() 243 | 244 | job.mp.log.info("--- download finished") 245 | 246 | for task in job.update_tasks: 247 | # right now only copy tasks... 248 | task.apply(job.mp.dir, job.mp) 249 | 250 | # final update of project metadata 251 | job.mp.update_metadata(job.project_info) 252 | 253 | cleanup_tmp_dir(job.mp, job.tmp_dir) 254 | 255 | 256 | def download_project_cancel(job): 257 | """ 258 | To be called (from main thread) to cancel a job that has downloads in progress. 259 | Returns once all background tasks have exited (may block for a bit of time). 260 | """ 261 | job.mp.log.info("user cancelled downloading...") 262 | # set job as cancelled 263 | job.is_cancelled = True 264 | job.executor.shutdown(wait=True) 265 | job.mp.log.info("--- download cancelled") 266 | cleanup_tmp_dir(job.mp, job.tmp_dir) 267 | 268 | 269 | class UpdateTask: 270 | """ 271 | Entry for each file that will be updated. 272 | At the end of a successful download of new data, all the tasks are executed. 273 | """ 274 | 275 | # TODO: methods other than COPY 276 | def __init__(self, file_path, download_queue_items, destination_file=None, latest_version=True): 277 | self.file_path = file_path 278 | self.destination_file = destination_file 279 | self.download_queue_items = download_queue_items 280 | self.latest_version = latest_version 281 | 282 | def apply(self, directory, mp): 283 | """assemble downloaded chunks into a single file""" 284 | 285 | if self.destination_file is None: 286 | basename = os.path.basename(self.file_path) 287 | file_dir = os.path.dirname(os.path.normpath(os.path.join(directory, self.file_path))) 288 | dest_file_path = os.path.join(file_dir, basename) 289 | else: 290 | file_dir = os.path.dirname(os.path.normpath(self.destination_file)) 291 | dest_file_path = self.destination_file 292 | os.makedirs(file_dir, exist_ok=True) 293 | 294 | # ignore check if we download not-latest version of gpkg file (possibly reconstructed on server on demand) 295 | check_size = self.latest_version or not mp.is_versioned_file(self.file_path) 296 | # merge chunks together (and delete them afterwards) 297 | file_to_merge = FileToMerge(dest_file_path, self.download_queue_items, check_size) 298 | file_to_merge.merge() 299 | 300 | # Make a copy of the file to meta dir only if there is no user-specified path for the file. 301 | # destination_file is None for full project download and takes a meaningful value for a single file download. 302 | if mp.is_versioned_file(self.file_path) and self.destination_file is None: 303 | mp.geodiff.make_copy_sqlite(mp.fpath(self.file_path), mp.fpath_meta(self.file_path)) 304 | 305 | 306 | class DownloadQueueItem: 307 | """a piece of data from a project that should be downloaded - it can be either a chunk or it can be a diff""" 308 | 309 | def __init__(self, file_path, size, version, diff_only, part_index, download_file_path): 310 | self.file_path = file_path # relative path to the file within project 311 | self.size = size # size of the item in bytes 312 | self.version = version # version of the file ("v123") 313 | self.diff_only = diff_only # whether downloading diff or full version 314 | self.part_index = part_index # index of the chunk 315 | self.download_file_path = download_file_path # full path to a temporary file which will receive the content 316 | 317 | def __repr__(self): 318 | return "".format( 319 | self.file_path, self.version, self.diff_only, self.part_index, self.size, self.download_file_path 320 | ) 321 | 322 | def download_blocking(self, mc, mp, project_path): 323 | """Starts download and only returns once the file has been fully downloaded and saved""" 324 | 325 | mp.log.debug( 326 | f"Downloading {self.file_path} version={self.version} diff={self.diff_only} part={self.part_index}" 327 | ) 328 | start = self.part_index * (1 + CHUNK_SIZE) 329 | resp = mc.get( 330 | "/v1/project/raw/{}".format(project_path), 331 | data={"file": self.file_path, "version": self.version, "diff": self.diff_only}, 332 | headers={"Range": "bytes={}-{}".format(start, start + CHUNK_SIZE)}, 333 | ) 334 | if resp.status in [200, 206]: 335 | mp.log.debug(f"Download finished: {self.file_path}") 336 | save_to_file(resp, self.download_file_path) 337 | else: 338 | mp.log.error(f"Download failed: {self.file_path}") 339 | raise ClientError("Failed to download part {} of file {}".format(self.part_index, self.file_path)) 340 | 341 | 342 | class PullJob: 343 | def __init__( 344 | self, 345 | project_path, 346 | pull_changes, 347 | total_size, 348 | version, 349 | files_to_merge, 350 | download_queue_items, 351 | tmp_dir, 352 | mp, 353 | project_info, 354 | basefiles_to_patch, 355 | mc, 356 | ): 357 | self.project_path = project_path 358 | self.pull_changes = ( 359 | pull_changes # dictionary with changes (dict[str, list[dict]] - keys: "added", "updated", ...) 360 | ) 361 | self.total_size = total_size # size of data to download (in bytes) 362 | self.transferred_size = 0 363 | self.version = version 364 | self.files_to_merge = files_to_merge # list of FileToMerge instances 365 | self.download_queue_items = download_queue_items 366 | self.tmp_dir = tmp_dir # TemporaryDirectory instance where we store downloaded files 367 | self.mp = mp # MerginProject instance 368 | self.is_cancelled = False 369 | self.project_info = project_info # parsed JSON with project info returned from the server 370 | self.basefiles_to_patch = ( 371 | basefiles_to_patch # list of tuples (relative path within project, list of diff files in temp dir to apply) 372 | ) 373 | self.mc = mc 374 | 375 | def dump(self): 376 | print("--- JOB ---", self.total_size, "bytes") 377 | for file_to_merge in self.files_to_merge: 378 | print("- {} ... download items={}".format(file_to_merge.dest_file, len(file_to_merge.downloaded_items))) 379 | print("--") 380 | for basefile, diffs in self.basefiles_to_patch: 381 | print("patch basefile {} with {} diffs".format(basefile, len(diffs))) 382 | print("--") 383 | for item in self.download_queue_items: 384 | print("- {} {} {} {}".format(item.file_path, item.version, item.part_index, item.size)) 385 | print("--- END ---") 386 | 387 | 388 | def pull_project_async(mc, directory): 389 | """ 390 | Starts project pull in background and returns handle to the pending job. 391 | Using that object it is possible to watch progress or cancel the ongoing work. 392 | """ 393 | 394 | mp = MerginProject(directory) 395 | if mp.has_unfinished_pull(): 396 | try: 397 | mp.resolve_unfinished_pull(mc.username()) 398 | except ClientError as err: 399 | mp.log.error("Error resolving unfinished pull: " + str(err)) 400 | mp.log.info("--- pull aborted") 401 | raise 402 | 403 | project_path = mp.project_full_name() 404 | local_version = mp.version() 405 | 406 | mp.log.info("--- version: " + mc.user_agent_info()) 407 | mp.log.info(f"--- start pull {project_path}") 408 | 409 | try: 410 | server_info = mc.project_info(project_path, since=local_version) 411 | except ClientError as err: 412 | mp.log.error("Error getting project info: " + str(err)) 413 | mp.log.info("--- pull aborted") 414 | raise 415 | server_version = server_info["version"] 416 | 417 | mp.log.info(f"got project info: local version {local_version} / server version {server_version}") 418 | 419 | if local_version == server_version: 420 | mp.log.info("--- pull - nothing to do (already at server version)") 421 | return # Project is up to date 422 | 423 | # we either download a versioned file using diffs (strongly preferred), 424 | # but if we don't have history with diffs (e.g. uploaded without diffs) 425 | # then we just download the whole file 426 | _pulling_file_with_diffs = lambda f: "diffs" in f and len(f["diffs"]) != 0 427 | 428 | tmp_dir = tempfile.TemporaryDirectory(prefix="mm-pull-") 429 | pull_changes = mp.get_pull_changes(server_info["files"]) 430 | mp.log.debug("pull changes:\n" + pprint.pformat(pull_changes)) 431 | fetch_files = [] 432 | for f in pull_changes["added"]: 433 | f["version"] = server_version 434 | fetch_files.append(f) 435 | # extend fetch files download list with various version of diff files (if needed) 436 | for f in pull_changes["updated"]: 437 | if _pulling_file_with_diffs(f): 438 | for diff in f["diffs"]: 439 | diff_file = copy.deepcopy(f) 440 | for k, v in f["history"].items(): 441 | if "diff" not in v: 442 | continue 443 | if diff == v["diff"]["path"]: 444 | diff_file["version"] = k 445 | diff_file["diff"] = v["diff"] 446 | fetch_files.append(diff_file) 447 | else: 448 | f["version"] = server_version 449 | fetch_files.append(f) 450 | 451 | files_to_merge = [] # list of FileToMerge instances 452 | 453 | for file in fetch_files: 454 | diff_only = _pulling_file_with_diffs(file) 455 | items = _download_items(file, tmp_dir.name, diff_only) 456 | 457 | # figure out destination path for the file 458 | file_dir = os.path.dirname(os.path.normpath(os.path.join(tmp_dir.name, file["path"]))) 459 | basename = os.path.basename(file["diff"]["path"]) if diff_only else os.path.basename(file["path"]) 460 | dest_file_path = os.path.join(file_dir, basename) 461 | os.makedirs(file_dir, exist_ok=True) 462 | files_to_merge.append(FileToMerge(dest_file_path, items)) 463 | 464 | # make sure we can update geodiff reference files (aka. basefiles) with diffs or 465 | # download their full versions so we have them up-to-date for applying changes 466 | basefiles_to_patch = [] # list of tuples (relative path within project, list of diff files in temp dir to apply) 467 | for file in pull_changes["updated"]: 468 | if not _pulling_file_with_diffs(file): 469 | continue # this is only for diffable files (e.g. geopackages) 470 | 471 | basefile = mp.fpath_meta(file["path"]) 472 | if not os.path.exists(basefile): 473 | # The basefile does not exist for some reason. This should not happen normally (maybe user removed the file 474 | # or we removed it within previous pull because we failed to apply patch the older version for some reason). 475 | # But it's not a problem - we will download the newest version and we're sorted. 476 | file_path = file["path"] 477 | mp.log.info(f"missing base file for {file_path} -> going to download it (version {server_version})") 478 | file["version"] = server_version 479 | items = _download_items(file, tmp_dir.name, diff_only=False) 480 | dest_file_path = mp.fpath(file["path"], tmp_dir.name) 481 | # dest_file_path = os.path.join(os.path.dirname(os.path.normpath(os.path.join(temp_dir, file['path']))), os.path.basename(file['path'])) 482 | files_to_merge.append(FileToMerge(dest_file_path, items)) 483 | continue 484 | 485 | basefiles_to_patch.append((file["path"], file["diffs"])) 486 | 487 | # make a single list of items to download 488 | total_size = 0 489 | download_list = [] 490 | for file_to_merge in files_to_merge: 491 | download_list.extend(file_to_merge.downloaded_items) 492 | for item in file_to_merge.downloaded_items: 493 | total_size += item.size 494 | 495 | mp.log.info(f"will download {len(download_list)} chunks, total size {total_size}") 496 | 497 | job = PullJob( 498 | project_path, 499 | pull_changes, 500 | total_size, 501 | server_version, 502 | files_to_merge, 503 | download_list, 504 | tmp_dir, 505 | mp, 506 | server_info, 507 | basefiles_to_patch, 508 | mc, 509 | ) 510 | 511 | # start download 512 | job.executor = concurrent.futures.ThreadPoolExecutor(max_workers=4) 513 | job.futures = [] 514 | for item in download_list: 515 | future = job.executor.submit(_do_download, item, mc, mp, project_path, job) 516 | job.futures.append(future) 517 | 518 | return job 519 | 520 | 521 | def pull_project_wait(job): 522 | """blocks until all download tasks are finished""" 523 | 524 | concurrent.futures.wait(job.futures) 525 | 526 | 527 | def pull_project_is_running(job): 528 | """ 529 | Returns true/false depending on whether we have some pending downloads 530 | 531 | It also forwards any exceptions from workers (e.g. some network errors). If an exception 532 | is raised, it is advised to call pull_project_cancel() to abort the job. 533 | """ 534 | for future in job.futures: 535 | if future.done() and future.exception() is not None: 536 | job.mp.log.error("Error while pulling data: " + str(future.exception())) 537 | job.mp.log.info("--- pull aborted") 538 | raise future.exception() 539 | if future.running(): 540 | return True 541 | return False 542 | 543 | 544 | def pull_project_cancel(job): 545 | """ 546 | To be called (from main thread) to cancel a job that has downloads in progress. 547 | Returns once all background tasks have exited (may block for a bit of time). 548 | """ 549 | job.mp.log.info("user cancelled the pull...") 550 | # set job as cancelled 551 | job.is_cancelled = True 552 | job.executor.shutdown(wait=True) 553 | job.mp.log.info("--- pull cancelled") 554 | cleanup_tmp_dir(job.mp, job.tmp_dir) # delete our temporary dir and all its content 555 | 556 | 557 | class FileToMerge: 558 | """ 559 | Keeps information about how to create a file (path specified by dest_file) from a couple 560 | of downloaded items (chunks) - each item is DownloadQueueItem object which has path 561 | to the temporary file containing its data. Calling merge() will create the destination file 562 | and remove the temporary files of the chunks 563 | """ 564 | 565 | def __init__(self, dest_file, downloaded_items, size_check=True): 566 | self.dest_file = dest_file # full path to the destination file to be created 567 | self.downloaded_items = downloaded_items # list of pieces of the destination file to be merged 568 | self.size_check = size_check # whether we want to do merged file size check 569 | 570 | def merge(self): 571 | with open(self.dest_file, "wb") as final: 572 | for item in self.downloaded_items: 573 | with open(item.download_file_path, "rb") as chunk: 574 | shutil.copyfileobj(chunk, final) 575 | os.remove(item.download_file_path) 576 | 577 | if not self.size_check: 578 | return 579 | expected_size = sum(item.size for item in self.downloaded_items) 580 | if os.path.getsize(self.dest_file) != expected_size: 581 | os.remove(self.dest_file) 582 | raise ClientError("Download of file {} failed. Please try it again.".format(self.dest_file)) 583 | 584 | 585 | def pull_project_finalize(job: PullJob): 586 | """ 587 | To be called when pull in the background is finished and we need to do the finalization (merge chunks etc.) 588 | 589 | This should not be called from a worker thread (e.g. directly from a handler when download is complete) 590 | 591 | If any of the workers has thrown any exception, it will be re-raised (e.g. some network errors). 592 | That also means that the whole job has been aborted. 593 | """ 594 | 595 | job.executor.shutdown(wait=True) 596 | 597 | # make sure any exceptions from threads are not lost 598 | for future in job.futures: 599 | if future.exception() is not None: 600 | job.mp.log.error("Error while pulling data: " + str(future.exception())) 601 | job.mp.log.info("--- pull aborted") 602 | raise future.exception() 603 | 604 | job.mp.log.info("finalizing pull") 605 | 606 | # merge downloaded chunks 607 | try: 608 | for file_to_merge in job.files_to_merge: 609 | file_to_merge.merge() 610 | except ClientError as err: 611 | job.mp.log.error("Error merging chunks of downloaded file: " + str(err)) 612 | job.mp.log.info("--- pull aborted") 613 | raise 614 | 615 | # make sure we can update geodiff reference files (aka. basefiles) with diffs or 616 | # download their full versions so we have them up-to-date for applying changes 617 | for file_path, file_diffs in job.basefiles_to_patch: 618 | basefile = job.mp.fpath_meta(file_path) 619 | server_file = job.mp.fpath(file_path, job.tmp_dir.name) 620 | 621 | shutil.copy(basefile, server_file) 622 | diffs = [job.mp.fpath(f, job.tmp_dir.name) for f in file_diffs] 623 | patch_error = job.mp.apply_diffs(server_file, diffs) 624 | if patch_error: 625 | # that's weird that we are unable to apply diffs to the basefile! 626 | # because it should be possible to apply them cleanly since the server 627 | # was also able to apply those diffs. It could be that someone modified 628 | # the basefile and we ended up in this inconsistent state. 629 | # let's remove the basefile and let the user retry - we should download clean version again 630 | job.mp.log.error(f"Error patching basefile {basefile}") 631 | job.mp.log.error("Diffs we were applying: " + str(diffs)) 632 | job.mp.log.error("Removing basefile because it would be corrupted anyway...") 633 | job.mp.log.info("--- pull aborted") 634 | os.remove(basefile) 635 | raise ClientError("Cannot patch basefile {}! Please try syncing again.".format(basefile)) 636 | 637 | try: 638 | conflicts = job.mp.apply_pull_changes(job.pull_changes, job.tmp_dir.name, job.project_info, job.mc) 639 | except Exception as e: 640 | job.mp.log.error("Failed to apply pull changes: " + str(e)) 641 | job.mp.log.info("--- pull aborted") 642 | cleanup_tmp_dir(job.mp, job.tmp_dir) # delete our temporary dir and all its content 643 | raise ClientError("Failed to apply pull changes: " + str(e)) 644 | 645 | job.mp.update_metadata(job.project_info) 646 | 647 | if job.mp.has_unfinished_pull(): 648 | job.mp.log.info("--- failed to complete pull -- project left in the unfinished pull state") 649 | else: 650 | job.mp.log.info("--- pull finished -- at version " + job.mp.version()) 651 | 652 | cleanup_tmp_dir(job.mp, job.tmp_dir) # delete our temporary dir and all its content 653 | return conflicts 654 | 655 | 656 | def download_file_async(mc, project_dir, file_path, output_file, version): 657 | """ 658 | Starts background download project file at specified version. 659 | Returns handle to the pending download. 660 | """ 661 | return download_files_async(mc, project_dir, [file_path], [output_file], version) 662 | 663 | 664 | def download_file_finalize(job): 665 | """ 666 | To be called when download_file_async is finished 667 | """ 668 | download_files_finalize(job) 669 | 670 | 671 | def download_diffs_async(mc, project_directory, file_path, versions): 672 | """ 673 | Starts background download project file diffs for specified versions. 674 | Returns handle to the pending download. 675 | 676 | Args: 677 | mc (MerginClient): MerginClient instance. 678 | project_directory (str): local project directory. 679 | file_path (str): file path relative to Mergin Maps project root. 680 | versions (list): list of versions to download diffs for, e.g. ['v1', 'v2']. 681 | 682 | Returns: 683 | PullJob/None: a handle for the pending download. 684 | """ 685 | mp = MerginProject(project_directory) 686 | project_path = mp.project_full_name() 687 | file_history = mc.project_file_history_info(project_path, file_path) 688 | mp.log.info(f"--- version: {mc.user_agent_info()}") 689 | mp.log.info(f"--- start download diffs for {file_path} of {project_path}, versions: {[v for v in versions]}") 690 | 691 | try: 692 | server_info = mc.project_info(project_path) 693 | if file_history is None: 694 | file_history = mc.project_file_history_info(project_path, file_path) 695 | except ClientError as err: 696 | mp.log.error("Error getting project info: " + str(err)) 697 | mp.log.info("--- downloading diffs aborted") 698 | raise 699 | 700 | fetch_files = [] 701 | 702 | for version in versions: 703 | if version not in file_history["history"]: 704 | continue # skip if this file was not modified at this version 705 | version_data = file_history["history"][version] 706 | if "diff" not in version_data: 707 | continue # skip if there is no diff in history 708 | diff_data = copy.deepcopy(version_data) 709 | diff_data["version"] = version 710 | diff_data["diff"] = version_data["diff"] 711 | fetch_files.append(diff_data) 712 | 713 | files_to_merge = [] # list of FileToMerge instances 714 | download_list = [] # list of all items to be downloaded 715 | total_size = 0 716 | for file in fetch_files: 717 | items = _download_items(file, mp.cache_dir, diff_only=True) 718 | dest_file_path = mp.fpath_cache(file["diff"]["path"], version=file["version"]) 719 | if os.path.exists(dest_file_path): 720 | continue 721 | files_to_merge.append(FileToMerge(dest_file_path, items)) 722 | download_list.extend(items) 723 | for item in items: 724 | total_size += item.size 725 | 726 | mp.log.info(f"will download {len(download_list)} chunks, total size {total_size}") 727 | 728 | job = PullJob( 729 | project_path, 730 | None, 731 | total_size, 732 | None, 733 | files_to_merge, 734 | download_list, 735 | mp.cache_dir, 736 | mp, 737 | server_info, 738 | {}, 739 | mc, 740 | ) 741 | 742 | # start download 743 | job.executor = concurrent.futures.ThreadPoolExecutor(max_workers=4) 744 | job.futures = [] 745 | for item in download_list: 746 | future = job.executor.submit(_do_download, item, mc, mp, project_path, job) 747 | job.futures.append(future) 748 | 749 | return job 750 | 751 | 752 | def download_diffs_finalize(job): 753 | """To be called after download_diffs_async 754 | 755 | Returns: 756 | diffs: list of downloaded diffs (their actual locations on disk) 757 | """ 758 | 759 | job.executor.shutdown(wait=True) 760 | 761 | # make sure any exceptions from threads are not lost 762 | for future in job.futures: 763 | if future.exception() is not None: 764 | job.mp.log.error("Error while pulling data: " + str(future.exception())) 765 | job.mp.log.info("--- diffs download aborted") 766 | raise future.exception() 767 | 768 | job.mp.log.info("finalizing diffs pull") 769 | diffs = [] 770 | 771 | # merge downloaded chunks 772 | try: 773 | for file_to_merge in job.files_to_merge: 774 | file_to_merge.merge() 775 | diffs.append(file_to_merge.dest_file) 776 | except ClientError as err: 777 | job.mp.log.error("Error merging chunks of downloaded file: " + str(err)) 778 | job.mp.log.info("--- diffs pull aborted") 779 | raise 780 | 781 | job.mp.log.info("--- diffs pull finished") 782 | return diffs 783 | 784 | 785 | def download_files_async( 786 | mc, project_dir: str, file_paths: typing.List[str], output_paths: typing.List[str], version: str 787 | ): 788 | """ 789 | Starts background download project files at specified version. 790 | Returns handle to the pending download. 791 | """ 792 | mp = MerginProject(project_dir) 793 | project_path = mp.project_full_name() 794 | ver_info = f"at version {version}" if version is not None else "at latest version" 795 | mp.log.info(f"Getting [{', '.join(file_paths)}] {ver_info}") 796 | latest_proj_info = mc.project_info(project_path) 797 | if version: 798 | project_info = mc.project_info(project_path, version=version) 799 | else: 800 | project_info = latest_proj_info 801 | mp.log.info(f"Got project info. version {project_info['version']}") 802 | 803 | # set temporary directory for download 804 | tmp_dir = tempfile.mkdtemp(prefix="python-api-client-") 805 | 806 | if output_paths is None: 807 | output_paths = [] 808 | for file in file_paths: 809 | output_paths.append(mp.fpath(file)) 810 | 811 | if len(output_paths) != len(file_paths): 812 | warn = "Output file paths are not of the same length as file paths. Cannot store required files." 813 | mp.log.warning(warn) 814 | shutil.rmtree(tmp_dir) 815 | raise ClientError(warn) 816 | 817 | download_list = [] 818 | update_tasks = [] 819 | total_size = 0 820 | # None can not be used to indicate latest version of the file, so 821 | # it is necessary to pass actual version. 822 | if version is None: 823 | version = latest_proj_info["version"] 824 | for file in project_info["files"]: 825 | if file["path"] in file_paths: 826 | index = file_paths.index(file["path"]) 827 | file["version"] = version 828 | items = _download_items(file, tmp_dir) 829 | is_latest_version = version == latest_proj_info["version"] 830 | task = UpdateTask(file["path"], items, output_paths[index], latest_version=is_latest_version) 831 | download_list.extend(task.download_queue_items) 832 | for item in task.download_queue_items: 833 | total_size += item.size 834 | update_tasks.append(task) 835 | 836 | missing_files = [] 837 | files_to_download = [] 838 | project_file_paths = [file["path"] for file in project_info["files"]] 839 | for file in file_paths: 840 | if file not in project_file_paths: 841 | missing_files.append(file) 842 | else: 843 | files_to_download.append(file) 844 | 845 | if not download_list or missing_files: 846 | warn = f"No [{', '.join(missing_files)}] exists at version {version}" 847 | mp.log.warning(warn) 848 | shutil.rmtree(tmp_dir) 849 | raise ClientError(warn) 850 | 851 | mp.log.info( 852 | f"will download files [{', '.join(files_to_download)}] in {len(download_list)} chunks, total size {total_size}" 853 | ) 854 | job = DownloadJob(project_path, total_size, version, update_tasks, download_list, tmp_dir, mp, project_info) 855 | job.executor = concurrent.futures.ThreadPoolExecutor(max_workers=4) 856 | job.futures = [] 857 | for item in download_list: 858 | future = job.executor.submit(_do_download, item, mc, mp, project_path, job) 859 | job.futures.append(future) 860 | 861 | return job 862 | 863 | 864 | def download_files_finalize(job): 865 | """ 866 | To be called when download_file_async is finished 867 | """ 868 | job.executor.shutdown(wait=True) 869 | 870 | # make sure any exceptions from threads are not lost 871 | for future in job.futures: 872 | if future.exception() is not None: 873 | raise future.exception() 874 | 875 | job.mp.log.info("--- download finished") 876 | 877 | for task in job.update_tasks: 878 | task.apply(job.tmp_dir, job.mp) 879 | 880 | # Remove temporary download directory 881 | if job.tmp_dir is not None and os.path.exists(job.tmp_dir): 882 | shutil.rmtree(job.tmp_dir) 883 | -------------------------------------------------------------------------------- /mergin/merginproject.py: -------------------------------------------------------------------------------- 1 | import json 2 | import logging 3 | import math 4 | import os 5 | import re 6 | import shutil 7 | import uuid 8 | import tempfile 9 | from datetime import datetime 10 | from dateutil.tz import tzlocal 11 | 12 | from .editor import prevent_conflicted_copy 13 | 14 | from .common import UPLOAD_CHUNK_SIZE, InvalidProject, ClientError 15 | from .utils import ( 16 | generate_checksum, 17 | is_versioned_file, 18 | int_version, 19 | do_sqlite_checkpoint, 20 | unique_path_name, 21 | conflicted_copy_file_name, 22 | edit_conflict_file_name, 23 | ) 24 | 25 | 26 | this_dir = os.path.dirname(os.path.realpath(__file__)) 27 | 28 | 29 | # Try to import pygeodiff from "deps" sub-directory (which should be present e.g. when 30 | # used within QGIS plugin), if that's not available then try to import it from standard 31 | # python paths. 32 | try: 33 | from .deps import pygeodiff 34 | except (ImportError, ModuleNotFoundError): 35 | import pygeodiff 36 | 37 | 38 | class MerginProject: 39 | """Base class for Mergin Maps local projects. 40 | 41 | Linked to existing local directory, with project metadata (mergin.json) and backups located in .mergin directory. 42 | """ 43 | 44 | def __init__(self, directory): 45 | self.dir = os.path.abspath(directory) 46 | if not os.path.exists(self.dir): 47 | raise InvalidProject("Project directory does not exist") 48 | 49 | self.meta_dir = os.path.join(self.dir, ".mergin") 50 | if not os.path.exists(self.meta_dir): 51 | os.mkdir(self.meta_dir) 52 | 53 | # location for files from unfinished pull 54 | self.unfinished_pull_dir = os.path.join(self.meta_dir, "unfinished_pull") 55 | 56 | self.cache_dir = os.path.join(self.meta_dir, ".cache") 57 | if not os.path.exists(self.cache_dir): 58 | os.mkdir(self.cache_dir) 59 | 60 | # metadata from JSON are lazy loaded 61 | self._metadata = None 62 | self.is_old_metadata = False 63 | 64 | self.setup_logging(directory) 65 | 66 | # make sure we can load correct pygeodiff 67 | try: 68 | self.geodiff = pygeodiff.GeoDiff() 69 | except pygeodiff.geodifflib.GeoDiffLibVersionError: 70 | # this is a fatal error, we can't live without geodiff 71 | self.log.error("Unable to load geodiff! (lib version error)") 72 | raise ClientError("Unable to load geodiff library!") 73 | 74 | # redirect any geodiff output to our log file 75 | def _logger_callback(level, text_bytes): 76 | text = text_bytes.decode() # convert bytes to str 77 | if level == pygeodiff.GeoDiff.LevelError: 78 | self.log.error("GEODIFF: " + text) 79 | elif level == pygeodiff.GeoDiff.LevelWarning: 80 | self.log.warning("GEODIFF: " + text) 81 | else: 82 | self.log.info("GEODIFF: " + text) 83 | 84 | self.geodiff.set_logger_callback(_logger_callback) 85 | self.geodiff.set_maximum_logger_level(pygeodiff.GeoDiff.LevelDebug) 86 | 87 | def setup_logging(self, logger_name): 88 | """Setup logging into project directory's .mergin/client-log.txt file.""" 89 | self.log = logging.getLogger("mergin.project." + str(logger_name)) 90 | self.log.setLevel(logging.DEBUG) # log everything (it would otherwise log just warnings+errors) 91 | if not self.log.handlers: 92 | # we only need to set the handler once 93 | # (otherwise we would get things logged multiple times as loggers are cached) 94 | log_handler = logging.FileHandler(os.path.join(self.meta_dir, "client-log.txt"), encoding="utf-8") 95 | log_handler.setFormatter(logging.Formatter("%(asctime)s %(message)s")) 96 | self.log.addHandler(log_handler) 97 | 98 | def remove_logging_handler(self): 99 | """Check if there is a logger handler defined for the project and remove it. There should be only one.""" 100 | if len(self.log.handlers) > 0: 101 | handler = self.log.handlers[0] 102 | self.log.removeHandler(handler) 103 | handler.flush() 104 | handler.close() 105 | 106 | def fpath(self, file, other_dir=None): 107 | """ 108 | Helper function to get absolute path of project file. Defaults to project dir but 109 | alternative dir get be provided (mostly meta or temp). Also making sure that parent dirs to file exist. 110 | 111 | :param file: relative file path in project (posix) 112 | :type file: str 113 | :param other_dir: alternative base directory for file, defaults to None 114 | :type other_dir: str 115 | :returns: file's absolute path 116 | :rtype: str 117 | """ 118 | root = other_dir or self.dir 119 | abs_path = os.path.abspath(os.path.join(root, file)) 120 | f_dir = os.path.dirname(abs_path) 121 | os.makedirs(f_dir, exist_ok=True) 122 | return abs_path 123 | 124 | def fpath_meta(self, file): 125 | """Helper function to get absolute path of file in meta dir.""" 126 | return self.fpath(file, self.meta_dir) 127 | 128 | def fpath_unfinished_pull(self, file): 129 | """Helper function to get absolute path of file in unfinished_pull dir.""" 130 | return self.fpath(file, self.unfinished_pull_dir) 131 | 132 | def fpath_cache(self, file, version=None): 133 | """Helper function to get absolute path of file in cache dir. 134 | It can be either in root cache directory (.mergin/.cache/) or in some version's subfolder 135 | """ 136 | if version: 137 | return self.fpath(file, os.path.join(self.cache_dir, version)) 138 | return self.fpath(file, self.cache_dir) 139 | 140 | def project_full_name(self) -> str: 141 | """Returns fully qualified project name: /""" 142 | self._read_metadata() 143 | if self.is_old_metadata: 144 | return self._metadata["name"] 145 | else: 146 | return f"{self._metadata['namespace']}/{self._metadata['name']}" 147 | 148 | def project_name(self) -> str: 149 | """Returns only project name, without its workspace name""" 150 | full_name = self.project_full_name() 151 | slash_index = full_name.index("/") 152 | return full_name[slash_index + 1 :] 153 | 154 | def workspace_name(self) -> str: 155 | """Returns name of the workspace where the project belongs""" 156 | full_name = self.project_full_name() 157 | slash_index = full_name.index("/") 158 | return full_name[:slash_index] 159 | 160 | def project_id(self) -> str: 161 | """Returns ID of the project (UUID using 8-4-4-4-12 formatting without braces) 162 | 163 | Raises ClientError if project id is not present in the project metadata. This should 164 | only happen with projects downloaded with old client, before February 2023, 165 | see https://github.com/MerginMaps/python-api-client/pull/154 166 | """ 167 | self._read_metadata() 168 | 169 | # "id" or "project_id" may not exist in projects downloaded with old client version 170 | if self.is_old_metadata: 171 | if "project_id" not in self._metadata: 172 | raise ClientError( 173 | "The project directory has been created with an old version of the Mergin Maps client. " 174 | "Please delete the project directory and re-download the project." 175 | ) 176 | return self._metadata["project_id"] 177 | else: 178 | return self._metadata["id"] 179 | 180 | def workspace_id(self) -> int: 181 | """Returns ID of the workspace where the project belongs""" 182 | # unfortunately we currently do not have information about workspace ID 183 | # in project's metadata... 184 | self._read_metadata() 185 | 186 | # "workspace_id" does not exist in projects downloaded with old client version 187 | if self.is_old_metadata: 188 | raise ClientError( 189 | "The project directory has been created with an old version of the Mergin Maps client. " 190 | "Please delete the project directory and re-download the project." 191 | ) 192 | return self._metadata["workspace_id"] 193 | 194 | def version(self) -> str: 195 | """Returns project version (e.g. "v123")""" 196 | self._read_metadata() 197 | return self._metadata["version"] 198 | 199 | def files(self) -> list: 200 | """Returns project's list of files (each file being a dictionary)""" 201 | self._read_metadata() 202 | return self._metadata["files"] 203 | 204 | @property 205 | def metadata(self) -> dict: 206 | """Gets raw access to metadata. Kept only for backwards compatibility and will be removed.""" 207 | # as we will change what is written in mergin.json, we do not want 208 | # client code to use this getter, and rather use project_full_name(), version() etc. 209 | from warnings import warn 210 | 211 | warn("MerginProject.metadata getter should not be used anymore", DeprecationWarning) 212 | self._read_metadata() 213 | return self._metadata 214 | 215 | def _read_metadata(self) -> None: 216 | """Loads the project's metadata from JSON""" 217 | if self._metadata is not None: 218 | return 219 | if not os.path.exists(self.fpath_meta("mergin.json")): 220 | raise InvalidProject("Project metadata has not been created yet") 221 | with open(self.fpath_meta("mergin.json"), "r") as file: 222 | self._metadata = json.load(file) 223 | 224 | self.is_old_metadata = "/" in self._metadata["name"] 225 | 226 | def update_metadata(self, data: dict): 227 | """Writes project metadata and updates cached metadata.""" 228 | self._metadata = data 229 | MerginProject.write_metadata(self.dir, data) 230 | 231 | @staticmethod 232 | def write_metadata(project_directory: str, data: dict): 233 | """Writes project metadata to /.mergin/mergin.json 234 | 235 | In most cases it is better to call update_metadata() as that will also 236 | update in-memory cache of metadata in MerginProject - this static method is 237 | useful for cases when this is the first time metadata are being written 238 | (and therefore creating MerginProject would fail). 239 | """ 240 | meta_dir = os.path.join(project_directory, ".mergin") 241 | os.makedirs(meta_dir, exist_ok=True) 242 | metadata_json_file = os.path.abspath(os.path.join(meta_dir, "mergin.json")) 243 | with open(metadata_json_file, "w") as file: 244 | json.dump(data, file, indent=2) 245 | 246 | def is_versioned_file(self, file): 247 | """Check if file is compatible with geodiff lib and hence suitable for versioning. 248 | 249 | :param file: file path 250 | :type file: str 251 | :returns: if file is compatible with geodiff lib 252 | :rtype: bool 253 | """ 254 | return is_versioned_file(file) 255 | 256 | def is_gpkg_open(self, path): 257 | """ 258 | Check whether geopackage file is open (and wal file exists) 259 | 260 | :param path: absolute path of file on disk 261 | :type path: str 262 | :returns: whether file is open 263 | :rtype: bool 264 | """ 265 | f_extension = os.path.splitext(path)[1] 266 | if f_extension != ".gpkg": 267 | return False 268 | if os.path.exists(f"{path}-wal"): 269 | return True 270 | return False 271 | 272 | def ignore_file(self, file): 273 | """ 274 | Helper function for blacklisting certain types of files. 275 | 276 | :param file: file path in project 277 | :type file: str 278 | :returns: whether file should be ignored 279 | :rtype: bool 280 | """ 281 | ignore_ext = re.compile(r"({})$".format("|".join(re.escape(x) for x in ["-shm", "-wal", "~", "pyc", "swap"]))) 282 | ignore_files = [".DS_Store", ".directory"] 283 | name, ext = os.path.splitext(file) 284 | if ext and ignore_ext.search(ext): 285 | return True 286 | if file in ignore_files: 287 | return True 288 | return False 289 | 290 | def inspect_files(self): 291 | """ 292 | Inspect files in project directory and return metadata. 293 | 294 | :returns: metadata for files in project directory in server required format 295 | :rtype: list[dict] 296 | """ 297 | files_meta = [] 298 | for root, dirs, files in os.walk(self.dir, topdown=True): 299 | dirs[:] = [d for d in dirs if d not in [".mergin"]] 300 | for file in files: 301 | if self.ignore_file(file): 302 | continue 303 | 304 | abs_path = os.path.abspath(os.path.join(root, file)) 305 | rel_path = os.path.relpath(abs_path, start=self.dir) 306 | proj_path = "/".join(rel_path.split(os.path.sep)) # we need posix path 307 | files_meta.append( 308 | { 309 | "path": proj_path, 310 | "checksum": generate_checksum(abs_path), 311 | "size": os.path.getsize(abs_path), 312 | "mtime": datetime.fromtimestamp(os.path.getmtime(abs_path), tzlocal()), 313 | } 314 | ) 315 | return files_meta 316 | 317 | def compare_file_sets(self, origin, current): 318 | """ 319 | Helper function to calculate difference between two sets of files metadata using file names and checksums. 320 | 321 | :Example: 322 | 323 | >>> origin = [{'checksum': '08b0e8caddafe74bf5c11a45f65cedf974210fed', 'path': 'base.gpkg', 'size': 2793, 'mtime': '2019-08-26T11:08:34.051221+02:00'}] 324 | >>> current = [{'checksum': 'c9a4fd2afd513a97aba19d450396a4c9df8b2ba4', 'path': 'test.qgs', 'size': 31980, 'mtime': '2019-08-26T11:09:30.051221+02:00'}] 325 | >>> self.compare_file_sets(origin, current) 326 | {"added": [{'checksum': 'c9a4fd2afd513a97aba19d450396a4c9df8b2ba4', 'path': 'test.qgs', 'size': 31980, 'mtime': '2019-08-26T11:09:30.051221+02:00'}], "removed": [[{'checksum': '08b0e8caddafe74bf5c11a45f65cedf974210fed', 'path': 'base.gpkg', 'size': 2793, 'mtime': '2019-08-26T11:08:34.051221+02:00'}]], "renamed": [], "updated": []} 327 | 328 | :param origin: origin set of files metadata 329 | :type origin: list[dict] 330 | :param current: current set of files metadata to be compared against origin 331 | :type current: list[dict] 332 | :returns: changes between two sets with change type 333 | :rtype: dict[str, list[dict]]' 334 | """ 335 | origin_map = {f["path"]: f for f in origin} 336 | current_map = {f["path"]: f for f in current} 337 | removed = [f for f in origin if f["path"] not in current_map] 338 | added = [f for f in current if f["path"] not in origin_map] 339 | updated = [] 340 | for f in current: 341 | path = f["path"] 342 | if path not in origin_map: 343 | continue 344 | # with open WAL files we don't know yet, better to mark file as updated 345 | if not self.is_gpkg_open(self.fpath(path)) and f["checksum"] == origin_map[path]["checksum"]: 346 | continue 347 | f["origin_checksum"] = origin_map[path]["checksum"] 348 | updated.append(f) 349 | 350 | return {"renamed": [], "added": added, "removed": removed, "updated": updated} 351 | 352 | def get_pull_changes(self, server_files): 353 | """ 354 | Calculate changes needed to be pulled from server. 355 | 356 | Calculate diffs between local files metadata and server's ones. Because simple metadata like file size or 357 | checksum are not enough to determine geodiff files changes, evaluate also their history (provided by server). 358 | For small files ask for full versions of geodiff files, otherwise determine list of diffs needed to update file. 359 | 360 | .. seealso:: self.compare_file_sets 361 | 362 | :param server_files: list of server files' metadata with mandatory 'history' field (see also self.inspect_files(), 363 | self.project_info(project_path, since=v1)) 364 | :type server_files: list[dict] 365 | :returns: changes metadata for files to be pulled from server 366 | :rtype: dict 367 | """ 368 | 369 | # first let's have a look at the added/updated/removed files 370 | changes = self.compare_file_sets(self.files(), server_files) 371 | 372 | # then let's inspect our versioned files (geopackages) if there are any relevant changes 373 | not_updated = [] 374 | for file in changes["updated"]: 375 | if not self.is_versioned_file(file["path"]): 376 | continue 377 | 378 | diffs = [] 379 | diffs_size = 0 380 | is_updated = False 381 | 382 | # get sorted list of the history (they may not be sorted or using lexical sorting - "v10", "v11", "v5", "v6", ...) 383 | history_list = [] 384 | for version_str, version_info in file["history"].items(): 385 | history_list.append((int_version(version_str), version_info)) 386 | history_list = sorted(history_list, key=lambda item: item[0]) # sort tuples based on version numbers 387 | 388 | # need to track geodiff file history to see if there were any changes 389 | for version, version_info in history_list: 390 | if version <= int_version(self.version()): 391 | continue # ignore history of no interest 392 | is_updated = True 393 | if "diff" in version_info: 394 | diffs.append(version_info["diff"]["path"]) 395 | diffs_size += version_info["diff"]["size"] 396 | else: 397 | diffs = [] 398 | break # we found force update in history, does not make sense to download diffs 399 | 400 | if is_updated: 401 | file["diffs"] = diffs 402 | else: 403 | not_updated.append(file) 404 | 405 | changes["updated"] = [f for f in changes["updated"] if f not in not_updated] 406 | return changes 407 | 408 | def get_push_changes(self): 409 | """ 410 | Calculate changes needed to be pushed to server. 411 | 412 | Calculate diffs between local files metadata and actual files in project directory. Because simple metadata like 413 | file size or checksum are not enough to determine geodiff files changes, geodiff tool is used to determine change 414 | of file content and update corresponding metadata. 415 | 416 | .. seealso:: self.compare_file_sets 417 | 418 | :returns: changes metadata for files to be pushed to server 419 | :rtype: dict 420 | """ 421 | changes = self.compare_file_sets(self.files(), self.inspect_files()) 422 | # do checkpoint to push changes from wal file to gpkg 423 | for file in changes["added"] + changes["updated"]: 424 | size, checksum = do_sqlite_checkpoint(self.fpath(file["path"]), self.log) 425 | if size and checksum: 426 | file["size"] = size 427 | file["checksum"] = checksum 428 | file["chunks"] = [str(uuid.uuid4()) for i in range(math.ceil(file["size"] / UPLOAD_CHUNK_SIZE))] 429 | 430 | # need to check for real changes in geodiff files using geodiff tool (comparing checksum is not enough) 431 | not_updated = [] 432 | for file in changes["updated"]: 433 | path = file["path"] 434 | if not self.is_versioned_file(path): 435 | continue 436 | 437 | # we use geodiff to check if we can push only diff files 438 | current_file = self.fpath(path) 439 | origin_file = self.fpath_meta(path) 440 | diff_id = str(uuid.uuid4()) 441 | diff_name = path + "-diff-" + diff_id 442 | diff_file = self.fpath_meta(diff_name) 443 | try: 444 | self.geodiff.create_changeset(origin_file, current_file, diff_file) 445 | if self.geodiff.has_changes(diff_file): 446 | diff_size = os.path.getsize(diff_file) 447 | file["checksum"] = file["origin_checksum"] # need to match basefile on server 448 | file["chunks"] = [str(uuid.uuid4()) for i in range(math.ceil(diff_size / UPLOAD_CHUNK_SIZE))] 449 | file["mtime"] = datetime.fromtimestamp(os.path.getmtime(current_file), tzlocal()) 450 | file["diff"] = { 451 | "path": diff_name, 452 | "checksum": generate_checksum(diff_file), 453 | "size": diff_size, 454 | "mtime": datetime.fromtimestamp(os.path.getmtime(diff_file), tzlocal()), 455 | } 456 | else: 457 | if os.path.exists(diff_file): 458 | os.remove(diff_file) 459 | not_updated.append(file) 460 | except (pygeodiff.GeoDiffLibError, pygeodiff.GeoDiffLibConflictError) as e: 461 | self.log.warning("failed to create changeset for " + path) 462 | # probably the database schema has been modified if geodiff cannot create changeset. 463 | # we will need to do full upload of the file 464 | pass 465 | 466 | changes["updated"] = [f for f in changes["updated"] if f not in not_updated] 467 | return changes 468 | 469 | def copy_versioned_file_for_upload(self, f, tmp_dir): 470 | """ 471 | Make a temporary copy of the versioned file using geodiff, to make sure that we have full 472 | content in a single file (nothing left in WAL journal) 473 | """ 474 | path = f["path"] 475 | self.log.info("Making a temporary copy (full upload): " + path) 476 | tmp_file = os.path.join(tmp_dir, path) 477 | os.makedirs(os.path.dirname(tmp_file), exist_ok=True) 478 | self.geodiff.make_copy_sqlite(self.fpath(path), tmp_file) 479 | f["size"] = os.path.getsize(tmp_file) 480 | f["checksum"] = generate_checksum(tmp_file) 481 | f["chunks"] = [str(uuid.uuid4()) for i in range(math.ceil(f["size"] / UPLOAD_CHUNK_SIZE))] 482 | f["upload_file"] = tmp_file 483 | return tmp_file 484 | 485 | def get_list_of_push_changes(self, push_changes): 486 | changes = {} 487 | for idx, file in enumerate(push_changes["updated"]): 488 | if "diff" in file: 489 | changeset_path = file["diff"]["path"] 490 | changeset = self.fpath_meta(changeset_path) 491 | result_file = self.fpath("change_list" + str(idx), self.meta_dir) 492 | try: 493 | self.geodiff.list_changes_summary(changeset, result_file) 494 | with open(result_file, "r") as f: 495 | change = f.read() 496 | changes[file["path"]] = json.loads(change) 497 | os.remove(result_file) 498 | except (pygeodiff.GeoDiffLibError, pygeodiff.GeoDiffLibConflictError): 499 | pass 500 | return changes 501 | 502 | def apply_pull_changes(self, changes, temp_dir, server_project, mc): 503 | """ 504 | Apply changes pulled from server. 505 | 506 | Update project files according to file changes. Apply changes to geodiff basefiles as well 507 | so they are up to date with server. In case of conflicts create backups from locally modified versions. 508 | 509 | .. seealso:: self.get_pull_changes 510 | 511 | :param changes: metadata for pulled files 512 | :type changes: dict[str, list[dict]] 513 | :param temp_dir: directory with downloaded files from server 514 | :type temp_dir: str 515 | :param user_name: name of the user that is pulling the changes 516 | :type user_name: str 517 | :param server_project: project metadata from the server 518 | :type server_project: dict 519 | :param mc: mergin client 520 | :type mc: mergin.client.MerginClient 521 | :returns: list of files with conflicts 522 | :rtype: list[str] 523 | """ 524 | conflicts = [] 525 | local_changes = self.get_push_changes() 526 | modified_local_paths = [f["path"] for f in local_changes.get("added", []) + local_changes.get("updated", [])] 527 | 528 | local_files_map = {} 529 | for f in self.inspect_files(): 530 | local_files_map.update({f["path"]: f}) 531 | 532 | for k, v in changes.items(): 533 | for item in v: 534 | path = item["path"] 535 | src = self.fpath(path, temp_dir) 536 | dest = self.fpath(path) 537 | basefile = self.fpath_meta(path) 538 | 539 | # special care is needed for geodiff files 540 | # 'src' here is server version of file and 'dest' is locally modified 541 | if self.is_versioned_file(path) and k == "updated": 542 | if path in modified_local_paths: 543 | conflict = self.update_with_rebase(path, src, dest, basefile, temp_dir, mc.username()) 544 | if conflict: 545 | conflicts.append(conflict) 546 | else: 547 | # The local file is not modified -> no rebase needed. 548 | # We just apply the diff between our copy and server to both the local copy and its basefile 549 | self.update_without_rebase(path, src, dest, basefile, temp_dir) 550 | else: 551 | # creating conflicted copy if both server and local changes are present on the files 552 | if ( 553 | path in modified_local_paths 554 | and item["checksum"] != local_files_map[path]["checksum"] 555 | and not prevent_conflicted_copy(path, mc, server_project) 556 | ): 557 | conflict = self.create_conflicted_copy(path, mc.username()) 558 | conflicts.append(conflict) 559 | 560 | if k == "removed": 561 | if os.path.exists(dest): 562 | os.remove(dest) 563 | else: 564 | # the file could be deleted via web interface AND also manually locally -> just log it 565 | self.log.warning(f"File to be removed locally doesn't exist: {dest}") 566 | if self.is_versioned_file(path): 567 | os.remove(basefile) 568 | else: 569 | if self.is_versioned_file(path): 570 | self.geodiff.make_copy_sqlite(src, dest) 571 | self.geodiff.make_copy_sqlite(src, basefile) 572 | else: 573 | shutil.copy(src, dest) 574 | 575 | return conflicts 576 | 577 | def update_with_rebase(self, path, src, dest, basefile, temp_dir, user_name): 578 | """ 579 | Update a versioned file with rebase. 580 | 581 | Try to peform automatic rebase, create conflict file if failed.pyt 582 | 583 | .. seealso:: self.update_without_rebase 584 | 585 | :param path: path to geodiff file 586 | :type path: str 587 | :param src: path to the server version of the file 588 | :type src: str 589 | :param dest: path to the local version of the file 590 | :type dest: str 591 | :param basefile: path to a file in meta dir 592 | :type basefile: str 593 | :param temp_dir: directory with downloaded files from server 594 | :type temp_dir: str 595 | :returns: path to conflict file if rebase fails, empty string on success 596 | :rtype: str 597 | """ 598 | self.log.info("updating file with rebase: " + path) 599 | 600 | server_diff = self.fpath(f"{path}-server_diff", temp_dir) # diff between server file and local basefile 601 | local_diff = self.fpath(f"{path}-local_diff", temp_dir) 602 | 603 | # temporary backup of file pulled from server for recovery 604 | f_server_backup = self.fpath(f"{path}-server_backup", temp_dir) 605 | self.geodiff.make_copy_sqlite(src, f_server_backup) 606 | 607 | # create temp backup (ideally with geodiff) of locally modified file if needed later 608 | f_conflict_file = self.fpath(f"{path}-local_backup", temp_dir) 609 | 610 | try: 611 | self.geodiff.create_changeset(basefile, dest, local_diff) 612 | self.geodiff.make_copy_sqlite(basefile, f_conflict_file) 613 | self.geodiff.apply_changeset(f_conflict_file, local_diff) 614 | except (pygeodiff.GeoDiffLibError, pygeodiff.GeoDiffLibConflictError): 615 | self.log.info("backup of local file with geodiff failed - need to do hard copy") 616 | self.geodiff.make_copy_sqlite(dest, f_conflict_file) 617 | 618 | # in case there will be any conflicting operations found during rebase, 619 | # they will be stored in a JSON file - if there are no conflicts, the file 620 | # won't even be created 621 | rebase_conflicts = unique_path_name( 622 | edit_conflict_file_name(self.fpath(path), user_name, int_version(self.version())) 623 | ) 624 | 625 | # try to do rebase magic 626 | try: 627 | self.geodiff.create_changeset(basefile, src, server_diff) 628 | self.geodiff.rebase(basefile, src, dest, rebase_conflicts) 629 | # make sure basefile is in the same state as remote server file (for calc of push changes) 630 | self.geodiff.apply_changeset(basefile, server_diff) 631 | self.log.info("rebase successful!") 632 | except (pygeodiff.GeoDiffLibError, pygeodiff.GeoDiffLibConflictError) as err: 633 | self.log.warning("rebase failed! going to create conflict file") 634 | try: 635 | # it would not be possible to commit local changes, they need to end up in new conflict file 636 | self.geodiff.make_copy_sqlite(f_conflict_file, dest) 637 | conflict = self.create_conflicted_copy(path, user_name) 638 | # original file synced with server 639 | self.geodiff.make_copy_sqlite(f_server_backup, basefile) 640 | self.geodiff.make_copy_sqlite(f_server_backup, dest) 641 | return conflict 642 | except pygeodiff.GeoDiffLibError as err: 643 | self.log.warning("creation of conflicted copy failed! going to create an unfinished pull") 644 | f_server_unfinished = self.fpath_unfinished_pull(path) 645 | self.geodiff.make_copy_sqlite(f_server_backup, f_server_unfinished) 646 | 647 | return "" 648 | 649 | def update_without_rebase(self, path, src, dest, basefile, temp_dir): 650 | """ 651 | Update a versioned file without rebase. 652 | 653 | Apply the diff between local copy and server to both the local 654 | copy and its basefile. 655 | 656 | .. seealso:: self.update_with_rebase 657 | 658 | :param path: path to geodiff file 659 | :type path: str 660 | :param src: path to the server version of the file 661 | :type src: str 662 | :param dest: path to the local version of the file 663 | :type dest: str 664 | :param basefile: path to a file in meta dir 665 | :type basefile: str 666 | :param temp_dir: directory with downloaded files from server 667 | :type temp_dir: str 668 | """ 669 | self.log.info("updating file without rebase: " + path) 670 | try: 671 | server_diff = self.fpath(f"{path}-server_diff", temp_dir) # diff between server file and local basefile 672 | # TODO: it could happen that basefile does not exist. 673 | # It was either never created (e.g. when pushing without geodiff) 674 | # or it was deleted by mistake(?) by the user. We should detect that 675 | # when starting pull and download it as well 676 | self.geodiff.create_changeset(basefile, src, server_diff) 677 | self.geodiff.apply_changeset(dest, server_diff) 678 | self.geodiff.apply_changeset(basefile, server_diff) 679 | self.log.info("update successful") 680 | except (pygeodiff.GeoDiffLibError, pygeodiff.GeoDiffLibConflictError): 681 | self.log.warning("update failed! going to copy file") 682 | # something bad happened and we have failed to patch our local files - this should not happen if there 683 | # wasn't a schema change or something similar that geodiff can't handle. 684 | self.geodiff.make_copy_sqlite(src, dest) 685 | self.geodiff.make_copy_sqlite(src, basefile) 686 | 687 | def apply_push_changes(self, changes): 688 | """ 689 | For geodiff files update basefiles according to changes pushed to server. 690 | 691 | :param changes: metadata for pulled files 692 | :type changes: dict[str, list[dict]] 693 | """ 694 | for k, v in changes.items(): 695 | for item in v: 696 | path = item["path"] 697 | if not self.is_versioned_file(path): 698 | continue 699 | 700 | basefile = self.fpath_meta(path) 701 | if k == "removed": 702 | os.remove(basefile) 703 | elif k == "added": 704 | self.geodiff.make_copy_sqlite(self.fpath(path), basefile) 705 | elif k == "updated": 706 | # in case for geopackage cannot be created diff (e.g. forced update with committed changes from wal file) 707 | if "diff" not in item: 708 | self.log.info("updating basefile (copy) for: " + path) 709 | self.geodiff.make_copy_sqlite(self.fpath(path), basefile) 710 | else: 711 | self.log.info("updating basefile (diff) for: " + path) 712 | # better to apply diff to previous basefile to avoid issues with geodiff tmp files 713 | changeset = self.fpath_meta(item["diff"]["path"]) 714 | patch_error = self.apply_diffs(basefile, [changeset]) 715 | if patch_error: 716 | # in case of local sync issues it is safier to remove basefile, next time it will be downloaded from server 717 | self.log.warning("removing basefile (because of apply diff error) for: " + path) 718 | os.remove(basefile) 719 | else: 720 | pass 721 | 722 | def create_conflicted_copy(self, file, user_name): 723 | """ 724 | Create conflicted copy file next to its origin. 725 | 726 | :param file: path of file in project 727 | :type file: str 728 | :returns: path to conflicted copy 729 | :rtype: str 730 | """ 731 | src = self.fpath(file) 732 | if not os.path.exists(src): 733 | return 734 | 735 | backup_path = unique_path_name( 736 | conflicted_copy_file_name(self.fpath(file), user_name, int_version(self.version())) 737 | ) 738 | 739 | if self.is_versioned_file(file): 740 | self.geodiff.make_copy_sqlite(src, backup_path) 741 | else: 742 | shutil.copy(src, backup_path) 743 | return backup_path 744 | 745 | def apply_diffs(self, basefile, diffs): 746 | """ 747 | Helper function to update content of geodiff file using list of diffs. 748 | Input file will be overwritten (make sure to create backup if needed). 749 | 750 | :param basefile: abs path to file to be updated 751 | :type basefile: str 752 | :param diffs: list of abs paths to geodiff changeset files 753 | :type diffs: list[str] 754 | :returns: error message if diffs were not successfully applied or None 755 | :rtype: str 756 | """ 757 | error = None 758 | if not self.is_versioned_file(basefile): 759 | return error 760 | 761 | for index, diff in enumerate(diffs): 762 | try: 763 | self.geodiff.apply_changeset(basefile, diff) 764 | except (pygeodiff.GeoDiffLibError, pygeodiff.GeoDiffLibConflictError) as e: 765 | self.log.warning("failed to apply changeset " + diff + " to " + basefile) 766 | error = str(e) 767 | break 768 | return error 769 | 770 | def has_unfinished_pull(self): 771 | """Check if there is an unfinished pull for this project. 772 | 773 | Unfinished pull means that a previous pull_project() call has 774 | failed in the final stage due to some files being in read-only 775 | mode. When a project has unfinished pull, it has to be resolved 776 | before allowing further pulls or pushes. 777 | 778 | .. seealso:: self.resolve_unfinished_pull 779 | 780 | :returns: whether there is an unfinished pull 781 | :rtype: bool 782 | """ 783 | return os.path.exists(self.unfinished_pull_dir) 784 | 785 | def resolve_unfinished_pull(self, user_name): 786 | """ 787 | Try to resolve unfinished pull. 788 | 789 | Unfinished pull means that a previous pull_project() call has 790 | failed in the final stage due to some files being in read-only 791 | mode. When a project has unfinished pull, it has to be resolved 792 | before allowing further pulls or pushes. 793 | 794 | Resolving unfinihed pull means creation of the conflicted copy 795 | and replacement of the original file by the new version from the 796 | server. 797 | 798 | .. seealso:: self.has_unfinished_pull 799 | 800 | :param user_name: name of the user 801 | :type user_name: str 802 | :returns: files where conflicts were found 803 | :rtype: list[str] 804 | """ 805 | conflicts = [] 806 | 807 | if not self.has_unfinished_pull(): 808 | self.log.warning("no unfinished pulls found") 809 | return 810 | 811 | self.log.info("resolving unfinished pull") 812 | 813 | temp_dir = tempfile.mkdtemp(prefix="python-api-client-") 814 | 815 | for root, dirs, files in os.walk(self.unfinished_pull_dir): 816 | for file_name in files: 817 | src = os.path.join(root, file_name) 818 | dest = self.fpath(file_name) 819 | basefile = self.fpath_meta(file_name) 820 | 821 | self.log.info("trying to resolve unfinished pull for: " + file_name) 822 | 823 | # 'src' here is a server version of the file from unfinished 824 | # pull and 'dest' is a local version of the same file. 825 | # to resolve unfinished pull we create a conflicted copy and 826 | # replace local file with the changes from the server. 827 | try: 828 | # conflicted copy 829 | conflict = self.create_conflicted_copy(dest, user_name) 830 | conflicts.append(conflict) 831 | # original file synced with server 832 | self.geodiff.make_copy_sqlite(src, basefile) 833 | self.geodiff.make_copy_sqlite(src, dest) 834 | except pygeodiff.GeoDiffLibError as err: 835 | self.log.error("unable to apply changes from previous unfinished pull!") 836 | raise ClientError("Unable to resolve unfinished pull!") 837 | 838 | shutil.rmtree(self.unfinished_pull_dir) 839 | self.log.info("unfinished pull resolved successfuly!") 840 | return conflicts 841 | 842 | def set_tables_to_skip(self, tables): 843 | """ 844 | Set list of tables to exclude from geodiff operations. Once defined, these 845 | tables will be excluded from the following operations: create changeset, 846 | apply changeset, rebase, get database schema, dump database contents, copy 847 | database between different drivers. 848 | 849 | If empty list is passed, list will be reset. 850 | 851 | :param tables: list of table names to ignore 852 | :type tables: list[str] 853 | """ 854 | self.geodiff.set_tables_to_skip(tables) 855 | --------------------------------------------------------------------------------