25 |
26 | This project follows the `all-contributors `_ specification.
27 |
28 | Contributions of any kind are welcome!
29 |
--------------------------------------------------------------------------------
/tests/test_pytest_gee/serialized_test_image_regression_with_region.yml:
--------------------------------------------------------------------------------
1 | result: '0'
2 | values:
3 | '0':
4 | functionInvocationValue:
5 | arguments:
6 | geometry:
7 | functionInvocationValue:
8 | arguments:
9 | distance:
10 | constantValue: 100
11 | geometry:
12 | functionInvocationValue:
13 | arguments:
14 | coordinates:
15 | constantValue:
16 | - 12.453585
17 | - 41.903115
18 | functionName: GeometryConstructors.Point
19 | functionName: Geometry.buffer
20 | input:
21 | functionInvocationValue:
22 | arguments:
23 | bandNames:
24 | constantValue:
25 | - SR_B5
26 | - SR_B4
27 | input:
28 | functionInvocationValue:
29 | arguments:
30 | id:
31 | constantValue: LANDSAT/LC08/C02/T1_L2/LC08_191031_20240607
32 | functionName: Image.load
33 | functionName: Image.normalizedDifference
34 | scale:
35 | constantValue: 30
36 | functionName: Image.clipToBoundsAndScale
37 |
--------------------------------------------------------------------------------
/tests/test_pytest_gee/serialized_test_image_regression_with_overlay.yml:
--------------------------------------------------------------------------------
1 | result: '0'
2 | values:
3 | '0':
4 | functionInvocationValue:
5 | arguments:
6 | geometry:
7 | functionInvocationValue:
8 | arguments:
9 | distance:
10 | constantValue: 20000
11 | geometry:
12 | functionInvocationValue:
13 | arguments:
14 | geometry:
15 | functionInvocationValue:
16 | arguments:
17 | feature:
18 | valueReference: '1'
19 | functionName: Image.geometry
20 | functionName: Geometry.centroid
21 | functionName: Geometry.buffer
22 | input:
23 | valueReference: '1'
24 | scale:
25 | constantValue: 100
26 | functionName: Image.clipToBoundsAndScale
27 | '1':
28 | functionInvocationValue:
29 | arguments:
30 | bandSelectors:
31 | constantValue:
32 | - SR_B4
33 | - SR_B3
34 | - SR_B2
35 | input:
36 | functionInvocationValue:
37 | arguments:
38 | id:
39 | constantValue: LANDSAT/LC08/C02/T1_L2/LC08_191031_20240607
40 | functionName: Image.load
41 | functionName: Image.select
42 |
--------------------------------------------------------------------------------
/.pre-commit-config.yaml:
--------------------------------------------------------------------------------
1 | default_install_hook_types: [pre-commit, commit-msg]
2 |
3 | repos:
4 | - repo: "https://github.com/commitizen-tools/commitizen"
5 | rev: "v2.18.0"
6 | hooks:
7 | - id: commitizen
8 | stages: [commit-msg]
9 |
10 | - repo: "https://github.com/kynan/nbstripout"
11 | rev: "0.5.0"
12 | hooks:
13 | - id: nbstripout
14 | stages: [pre-commit]
15 |
16 | - repo: "https://github.com/pycontribs/mirrors-prettier"
17 | rev: "v3.4.2"
18 | hooks:
19 | - id: prettier
20 | stages: [pre-commit]
21 | exclude: tests\/test_.+\.
22 |
23 | - repo: https://github.com/charliermarsh/ruff-pre-commit
24 | rev: "v0.7.0"
25 | hooks:
26 | - id: ruff
27 | stages: [pre-commit]
28 | - id: ruff-format
29 | stages: [pre-commit]
30 |
31 | - repo: https://github.com/sphinx-contrib/sphinx-lint
32 | rev: "v1.0.0"
33 | hooks:
34 | - id: sphinx-lint
35 | stages: [pre-commit]
36 |
37 | - repo: https://github.com/codespell-project/codespell
38 | rev: v2.3.0
39 | hooks:
40 | - id: codespell
41 | stages: [pre-commit]
42 | additional_dependencies:
43 | - tomli
44 |
45 | # Prevent committing inline conflict markers
46 | - repo: https://github.com/pre-commit/pre-commit-hooks
47 | rev: v6.0.0
48 | hooks:
49 | - id: check-merge-conflict
50 | stages: [pre-commit]
51 | args: [--assume-in-merge]
52 |
--------------------------------------------------------------------------------
/tests/check_warnings.py:
--------------------------------------------------------------------------------
1 | """Check the warnings from doc builds."""
2 |
3 | import sys
4 | from pathlib import Path
5 |
6 |
7 | def check_warnings(file: Path) -> int:
8 | """Check the list of warnings produced by the CI tests.
9 |
10 | Raises errors if there are unexpected ones and/or if some are missing.
11 |
12 | Args:
13 | file: the path to the generated warning.txt file from
14 | the CI build
15 |
16 | Returns:
17 | 0 if the warnings are all there
18 | 1 if some warning are not registered or unexpected
19 | """
20 | # print some log
21 | print("\n=== Sphinx Warnings test ===\n")
22 |
23 | # find the file where all the known warnings are stored
24 | warning_file = Path(__file__).parent / "data" / "warning_list.txt"
25 |
26 | test_warnings = file.read_text().strip().split("\n")
27 | ref_warnings = warning_file.read_text().strip().split("\n")
28 |
29 | print(
30 | f'Checking build warnings in file: "{file}" and comparing to expected '
31 | f'warnings defined in "{warning_file}"\n\n'
32 | )
33 |
34 | # find all the missing warnings
35 | missing_warnings = []
36 | for wa in ref_warnings:
37 | index = [i for i, twa in enumerate(test_warnings) if wa in twa]
38 | if len(index) == 0:
39 | missing_warnings += [wa]
40 | print(f"Warning was not raised: {wa}")
41 | else:
42 | test_warnings.pop(index[0])
43 |
44 | # the remaining one are unexpected
45 | for twa in test_warnings:
46 | print(f"Unexpected warning: {twa}")
47 |
48 | # delete the tmp warnings file
49 | file.unlink()
50 |
51 | return len(missing_warnings) != 0 or len(test_warnings) != 0
52 |
53 |
54 | if __name__ == "__main__":
55 | # cast the file to path and resolve to an absolute one
56 | file = Path.cwd() / "warnings.txt"
57 |
58 | # execute the test
59 | sys.exit(check_warnings(file))
60 |
--------------------------------------------------------------------------------
/.github/workflows/pypackage_check.yaml:
--------------------------------------------------------------------------------
1 | name: template update check
2 |
3 | on:
4 | workflow_dispatch:
5 |
6 | env:
7 | PIP_ROOT_USER_ACTION: ignore
8 |
9 | jobs:
10 | check_version:
11 | runs-on: ubuntu-latest
12 | steps:
13 | - uses: actions/checkout@v4
14 | - uses: actions/setup-python@v5
15 | with:
16 | python-version: "3.10"
17 | - name: install dependencies
18 | run: pip install requests
19 | - name: get latest pypackage release
20 | id: get_latest_release
21 | run: |
22 | RELEASE=$(curl -s https://api.github.com/repos/12rambau/pypackage/releases | jq -r '.[0].tag_name')
23 | echo "latest=$RELEASE" >> $GITHUB_OUTPUT
24 | echo "latest release: $RELEASE"
25 | - name: get current pypackage version
26 | id: get_current_version
27 | run: |
28 | RELEASE=$(yq -r "._commit" .copier-answers.yml)
29 | echo "current=$RELEASE" >> $GITHUB_OUTPUT
30 | echo "current release: $RELEASE"
31 | - name: open issue
32 | if: steps.get_current_version.outputs.current != steps.get_latest_release.outputs.latest
33 | uses: rishabhgupta/git-action-issue@v2
34 | with:
35 | token: ${{ secrets.GITHUB_TOKEN }}
36 | title: "Update template to ${{ steps.get_latest_release.outputs.latest }}"
37 | body: |
38 | The package is based on the ${{ steps.get_current_version.outputs.current }} version of [@12rambau/pypackage](https://github.com/12rambau/pypackage).
39 |
40 | The latest version of the template is ${{ steps.get_latest_release.outputs.latest }}.
41 |
42 | Please consider updating the template to the latest version to include all the latest developments.
43 |
44 | Run the following code in your project directory to update the template:
45 |
46 | ```
47 | copier update --trust --defaults --vcs-ref ${{ steps.get_latest_release.outputs.latest }}
48 | ```
49 |
50 | > **Note**
51 | > You may need to reinstall ``copier`` and ``jinja2-time`` if they are not available in your environment.
52 |
53 | After solving the merging issues you can push back the changes to your main branch.
54 |
--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
1 | # Byte-compiled / optimized / DLL files
2 | __pycache__/
3 | *.py[cod]
4 | *$py.class
5 |
6 | # C extensions
7 | *.so
8 |
9 | # Distribution / packaging
10 | .Python
11 | build/
12 | develop-eggs/
13 | dist/
14 | downloads/
15 | eggs/
16 | .eggs/
17 | lib/
18 | lib64/
19 | parts/
20 | sdist/
21 | var/
22 | wheels/
23 | pip-wheel-metadata/
24 | share/python-wheels/
25 | *.egg-info/
26 | .installed.cfg
27 | *.egg
28 | MANIFEST
29 |
30 | # PyInstaller
31 | # Usually these files are written by a python script from a template
32 | # before PyInstaller builds the exe, so as to inject date/other infos into it.
33 | *.manifest
34 | *.spec
35 |
36 | # Installer logs
37 | pip-log.txt
38 | pip-delete-this-directory.txt
39 |
40 | # Unit test / coverage reports
41 | htmlcov/
42 | .tox/
43 | .nox/
44 | .coverage
45 | .coverage.*
46 | .cache
47 | nosetests.xml
48 | coverage.xml
49 | *.cover
50 | *.py,cover
51 | .hypothesis/
52 | .pytest_cache/
53 | .ruff_cache/
54 |
55 | # Translations
56 | *.mo
57 | *.pot
58 |
59 | # Django stuff:
60 | *.log
61 | local_settings.py
62 | db.sqlite3
63 | db.sqlite3-journal
64 |
65 | # Flask stuff:
66 | instance/
67 | .webassets-cache
68 |
69 | # Scrapy stuff:
70 | .scrapy
71 |
72 | # Sphinx documentation
73 | docs/_build/
74 | docs/api/
75 |
76 | # PyBuilder
77 | target/
78 |
79 | # Jupyter Notebook
80 | .ipynb_checkpoints
81 |
82 | # IPython
83 | profile_default/
84 | ipython_config.py
85 |
86 | # pyenv
87 | .python-version
88 |
89 | # pipenv
90 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
91 | # However, in case of collaboration, if having platform-specific dependencies or dependencies
92 | # having no cross-platform support, pipenv may install dependencies that don't work, or not
93 | # install all needed dependencies.
94 | #Pipfile.lock
95 |
96 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow
97 | __pypackages__/
98 |
99 | # Celery stuff
100 | celerybeat-schedule
101 | celerybeat.pid
102 |
103 | # SageMath parsed files
104 | *.sage.py
105 |
106 | # Environments
107 | .env
108 | .venv
109 | env/
110 | venv/
111 | ENV/
112 | env.bak/
113 | venv.bak/
114 |
115 | # Spyder project settings
116 | .spyderproject
117 | .spyproject
118 |
119 | # Rope project settings
120 | .ropeproject
121 |
122 | # mkdocs documentation
123 | /site
124 |
125 | # mypy
126 | .mypy_cache/
127 | .dmypy.json
128 | dmypy.json
129 |
130 | # Pyre type checker
131 | .pyre/
132 |
133 | # system IDE
134 | .vscode/
135 |
136 | # image tmp file
137 | *Zone.Identifier
138 |
139 | # debugging notebooks
140 | test.ipynb
141 |
--------------------------------------------------------------------------------
/docs/_static/custom-icon.js:
--------------------------------------------------------------------------------
1 | /*******************************************************************************
2 | * Set a custom icon for pypi as it's not available in the fa built-in brands
3 | */
4 | FontAwesome.library.add(
5 | (faListOldStyle = {
6 | prefix: "fa-custom",
7 | iconName: "conda",
8 | icon: [
9 | 24, // viewBox width
10 | 24, // viewBox height
11 | [], // ligature
12 | "e001", // unicode codepoint - private use area
13 | "M12.045.033a12.181 12.182 0 00-1.361.078 17.512 17.513 0 011.813 1.433l.48.438-.465.45a15.047 15.048 0 00-1.126 1.205l-.178.215a8.527 8.527 0 01.86-.05 8.154 8.155 0 11-4.286 15.149 15.764 15.765 0 01-1.841.106h-.86a21.847 21.848 0 00.264 2.866 11.966 11.967 0 106.7-21.89zM8.17.678a12.181 12.182 0 00-2.624 1.275 15.506 15.507 0 011.813.43A18.551 18.552 0 018.17.678zM9.423.75a16.237 16.238 0 00-.995 1.998 16.15 16.152 0 011.605.66 6.98 6.98 0 01.43-.509c.234-.286.472-.559.716-.817A15.047 15.048 0 009.423.75zM4.68 2.949a14.969 14.97 0 000 2.336c.587-.065 1.196-.1 1.812-.107a16.617 16.617 0 01.48-1.748 16.48 16.481 0 00-2.292-.481zM3.62 3.5A11.938 11.938 0 001.762 5.88a17.004 17.004 0 011.877-.444A17.39 17.391 0 013.62 3.5zm4.406.287c-.143.437-.265.888-.38 1.347a8.255 8.255 0 011.67-.803c-.423-.2-.845-.38-1.29-.544zM6.3 6.216a14.051 14.052 0 00-1.555.108c.064.523.157 1.038.272 1.554a8.39 8.391 0 011.283-1.662zm-2.55.137a15.313 15.313 0 00-2.602.716h-.078v.079a17.104 17.105 0 001.267 2.544l.043.071.072-.049a16.309 16.31 0 011.734-1.083l.057-.035V8.54a16.867 16.868 0 01-.408-2.094v-.092zM.644 8.095l-.063.2A11.844 11.845 0 000 11.655v.209l.143-.152a17.706 17.707 0 011.584-1.447l.057-.043-.043-.064a16.18 16.18 0 01-1.025-1.87zm3.77 1.253l-.18.1c-.465.273-.93.573-1.375.889l-.065.05.05.064c.309.437.645.867.996 1.276l.137.165v-.208a8.176 8.176 0 01.364-2.15zM2.2 10.853l-.072.05a16.574 16.574 0 00-1.813 1.734l-.058.058.066.057a15.449 15.45 0 001.991 1.483l.072.05.043-.08a16.738 16.74 0 011.053-1.64v-.05l-.043-.05a16.99 16.99 0 01-1.19-1.54zm1.855 2.071l-.121.172a15.363 15.363 0 00-.917 1.433l-.043.072.071.043a16.61 16.61 0 001.562.766l.193.086-.086-.193a8.04 8.04 0 01-.66-2.172zm-3.976.48v.2a11.758 11.759 0 00.946 3.326l.078.186.072-.194a16.215 16.216 0 01.845-2l.057-.063-.064-.043a17.197 17.198 0 01-1.776-1.284zm2.543 1.805l-.035.08a15.764 15.765 0 00-.983 2.479v.08h.086a16.15 16.152 0 002.688.5l.072.007v-.086a17.562 17.563 0 01.164-2.056v-.065H4.55a16.266 16.266 0 01-1.849-.896zm2.544 1.169v.114a17.254 17.255 0 00-.151 1.828v.078h.931c.287 0 .624.014.946 0h.209l-.166-.129a8.011 8.011 0 01-1.64-1.834zm-3.29 2.1l.115.172a11.988 11.988 0 002.502 2.737l.157.129v-.201a22.578 22.58 0 01-.2-2.336v-.071h-.072a16.23 16.23 0 01-2.3-.387z", // svg path (https://simpleicons.org/icons/anaconda.svg)
14 | ],
15 | }),
16 | );
17 |
--------------------------------------------------------------------------------
/.github/workflows/unit.yaml:
--------------------------------------------------------------------------------
1 | name: Unit tests
2 |
3 | on:
4 | workflow_dispatch:
5 | workflow_call:
6 | push:
7 | branches:
8 | - main
9 | pull_request:
10 |
11 | env:
12 | EARTHENGINE_SERVICE_ACCOUNT: ${{ secrets.EARTHENGINE_SERVICE_ACCOUNT }}
13 | EARTHENGINE_PROJECT: ${{ secrets.EARTHENGINE_PROJECT }}
14 | FORCE_COLOR: 1
15 | PIP_ROOT_USER_ACTION: ignore
16 |
17 | jobs:
18 | lint:
19 | runs-on: ubuntu-latest
20 | steps:
21 | - uses: actions/checkout@v4
22 | - uses: actions/setup-python@v5
23 | with:
24 | python-version: "3.11"
25 | - uses: pre-commit/action@v3.0.0
26 |
27 | mypy:
28 | runs-on: ubuntu-latest
29 | steps:
30 | - uses: actions/checkout@v4
31 | - uses: actions/setup-python@v5
32 | with:
33 | python-version: "3.11"
34 | - name: Install nox
35 | run: pip install nox[uv]
36 | - name: run mypy checks
37 | run: nox -s mypy
38 |
39 | docs:
40 | needs: [lint, mypy]
41 | runs-on: ubuntu-latest
42 | steps:
43 | - uses: actions/checkout@v4
44 | - uses: actions/setup-python@v5
45 | with:
46 | python-version: "3.11"
47 | - name: Install nox
48 | run: pip install nox[uv]
49 | - name: build static docs
50 | run: nox -s docs
51 |
52 | build:
53 | needs: [lint, mypy]
54 | strategy:
55 | fail-fast: false
56 | matrix:
57 | os: [ubuntu-latest]
58 | python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"]
59 | include:
60 | - os: macos-latest # macos test
61 | python-version: "3.13"
62 | - os: windows-latest # windows test
63 | python-version: "3.13"
64 | runs-on: ${{ matrix.os }}
65 | steps:
66 | - uses: actions/checkout@v4
67 | - name: Set up Python ${{ matrix.python-version }}
68 | uses: actions/setup-python@v5
69 | with:
70 | python-version: ${{ matrix.python-version }}
71 | - name: Install nox
72 | run: pip install nox[uv]
73 | - name: test with pytest
74 | run: nox -s ci-test
75 | - name: assess dead fixtures
76 | if: ${{ matrix.python-version == '3.11' }}
77 | shell: bash
78 | run: nox -s dead-fixtures
79 | - uses: actions/upload-artifact@v4
80 | if: ${{ matrix.python-version == '3.11' }}
81 | with:
82 | name: coverage
83 | path: coverage.xml
84 |
85 | coverage:
86 | needs: [build]
87 | runs-on: ubuntu-latest
88 | steps:
89 | - uses: actions/checkout@v4
90 | - uses: actions/download-artifact@v4
91 | with:
92 | name: coverage
93 | - name: codecov
94 | uses: codecov/codecov-action@v4
95 | with:
96 | token: ${{ secrets.CODECOV_TOKEN }}
97 | verbose: true
98 | fail_ci_if_error: true
99 |
--------------------------------------------------------------------------------
/noxfile.py:
--------------------------------------------------------------------------------
1 | """All the process that can be run using nox.
2 |
3 | The nox run are build in isolated environment that will be stored in .nox. to force the venv update, remove the .nox/xxx folder.
4 | """
5 |
6 | import datetime
7 | import fileinput
8 |
9 | import nox
10 |
11 | nox.options.sessions = ["lint", "test", "docs", "mypy"]
12 |
13 |
14 | @nox.session(reuse_venv=True, venv_backend="uv")
15 | def lint(session):
16 | """Apply the pre-commits."""
17 | session.install("pre-commit")
18 | session.run("pre-commit", "run", "--all-files", *session.posargs)
19 |
20 |
21 | @nox.session(reuse_venv=True)
22 | def test(session):
23 | """Run the selected tests and report coverage in html."""
24 | session.install(".[test]")
25 | test_files = session.posargs or ["tests"]
26 | session.run("pytest", "--cov", "--cov-report=html", *test_files)
27 |
28 |
29 | @nox.session(reuse_venv=True, name="ci-test", venv_backend="uv")
30 | def ci_test(session):
31 | """Run all the test and report coverage in xml."""
32 | session.install(".[test]")
33 | session.run("pytest", "--cov", "--cov-report=xml")
34 |
35 |
36 | @nox.session(reuse_venv=True, name="dead-fixtures", venv_backend="uv")
37 | def dead_fixtures(session):
38 | """Check for dead fixtures within the tests."""
39 | session.install(".[test]")
40 | session.run("pytest", "--dead-fixtures")
41 |
42 |
43 | @nox.session(reuse_venv=True, venv_backend="uv")
44 | def docs(session):
45 | """Build the documentation."""
46 | build = session.posargs.pop() if session.posargs else "html"
47 | session.install(".[doc]")
48 | dst, warn = f"docs/_build/{build}", "warnings.txt"
49 | session.run("sphinx-build", "-v", "-b", build, "docs", dst, "-w", warn)
50 | session.run("python", "tests/check_warnings.py")
51 |
52 |
53 | @nox.session(name="mypy", reuse_venv=True, venv_backend="uv")
54 | def mypy(session):
55 | """Run a mypy check of the lib."""
56 | session.install("mypy", "types-requests", "types-Deprecated", "types-PyYAML")
57 | test_files = session.posargs or ["pytest_gee"]
58 | session.run("mypy", *test_files)
59 |
60 |
61 | @nox.session(reuse_venv=True, venv_backend="uv")
62 | def stubgen(session):
63 | """Generate stub files for the lib but requires human attention before merge."""
64 | session.install("mypy")
65 | package = session.posargs or ["pytest_gee"]
66 | session.run("stubgen", "-p", package[0], "-o", "stubs", "--include-private")
67 |
68 |
69 | @nox.session(name="release-date", reuse_venv=True, venv_backend="uv")
70 | def release_date(session):
71 | """Update the release date of the citation file."""
72 | current_date = datetime.datetime.now().strftime("%Y-%m-%d")
73 |
74 | with fileinput.FileInput("CITATION.cff", inplace=True) as file:
75 | for line in file:
76 | if line.startswith("date-released:"):
77 | print(f'date-released: "{current_date}"')
78 | else:
79 | print(line, end="")
80 |
--------------------------------------------------------------------------------
/docs/conf.py:
--------------------------------------------------------------------------------
1 | """Configuration file for the Sphinx documentation builder.
2 |
3 | This file only contains a selection of the most common options. For a full
4 | list see the documentation:
5 | https://www.sphinx-doc.org/en/master/usage/configuration.html
6 | """
7 |
8 | # -- Path setup ----------------------------------------------------------------
9 | from datetime import datetime
10 |
11 | # -- Project information -------------------------------------------------------
12 | project = "pytest-gee"
13 | author = "Pierrick Rambaud"
14 | copyright = f"2020-{datetime.now().year}, {author}"
15 | release = "0.8.1"
16 |
17 | # -- General configuration -----------------------------------------------------
18 | extensions = [
19 | "sphinx_copybutton",
20 | "sphinx.ext.napoleon",
21 | "sphinx.ext.viewcode",
22 | "sphinx.ext.intersphinx",
23 | "sphinx_design",
24 | "autoapi.extension",
25 | ]
26 | exclude_patterns = ["**.ipynb_checkpoints"]
27 | templates_path = ["_template"]
28 |
29 | # -- Options for HTML output ---------------------------------------------------
30 | html_theme = "pydata_sphinx_theme"
31 | html_static_path = ["_static"]
32 | html_logo = "_static/long-logo.png"
33 | html_favicon = "_static/logo.png"
34 | html_sidebars = {"content/*": []}
35 | html_theme_options = {
36 | "use_edit_page_button": True,
37 | "footer_end": ["theme-version", "pypackage-credit"],
38 | "icon_links": [
39 | {
40 | "name": "GitHub",
41 | "url": "https://github.com/gee-community/pytest-gee",
42 | "icon": "fa-brands fa-github",
43 | },
44 | {
45 | "name": "Pypi",
46 | "url": "https://pypi.org/project/pytest-gee/",
47 | "icon": "fa-brands fa-python",
48 | },
49 | {
50 | "name": "Conda",
51 | "url": "https://anaconda.org/conda-forge/pytest-gee",
52 | "icon": "fa-custom fa-conda",
53 | "type": "fontawesome",
54 | },
55 | ],
56 | }
57 | html_context = {
58 | "github_user": "gee-community",
59 | "github_repo": "pytest-gee",
60 | "github_version": "main",
61 | "doc_path": "docs",
62 | }
63 | html_css_files = ["custom.css"]
64 |
65 | # -- Options for autosummary/autodoc output ------------------------------------
66 | autodoc_typehints = "description"
67 | autoapi_dirs = ["../pytest_gee"]
68 | autoapi_python_class_content = "init"
69 | autoapi_member_order = "groupwise"
70 |
71 | # -- Options for intersphinx output --------------------------------------------
72 | # fmt: off
73 | intersphinx_mapping = {
74 | "python": ("https://docs.python.org/3", None),
75 | "pytest_regressions": ("https://pytest-regressions.readthedocs.io/en/latest/", None),
76 | "pytest": ("https://docs.pytest.org/en/stable/", None),
77 | "ee": ("https://developers.google.com/earth-engine/apidocs", "https://raw.githubusercontent.com/gee-community/sphinx-inventory/refs/heads/main/inventory/earthengine-api.inv"),
78 | }
79 | # fmt: on
80 |
--------------------------------------------------------------------------------
/README.rst:
--------------------------------------------------------------------------------
1 |
2 | pytest-gee
3 | ==========
4 |
5 | .. |license| image:: https://img.shields.io/badge/License-MIT-yellow.svg?logo=opensourceinitiative&logoColor=white
6 | :target: LICENSE
7 | :alt: License: MIT
8 |
9 | .. |commit| image:: https://img.shields.io/badge/Conventional%20Commits-1.0.0-yellow.svg?logo=git&logoColor=white
10 | :target: https://conventionalcommits.org
11 | :alt: conventional commit
12 |
13 | .. |ruff| image:: https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/ruff/main/assets/badge/v2.json
14 | :target: https://github.com/astral-sh/ruff
15 | :alt: ruff badge
16 |
17 | .. |prettier| image:: https://img.shields.io/badge/code_style-prettier-ff69b4.svg?logo=prettier&logoColor=white
18 | :target: https://github.com/prettier/prettier
19 | :alt: prettier badge
20 |
21 | .. |pre-commmit| image:: https://img.shields.io/badge/pre--commit-active-yellow?logo=pre-commit&logoColor=white
22 | :target: https://pre-commit.com/
23 | :alt: pre-commit
24 |
25 | .. |pypi| image:: https://img.shields.io/pypi/v/pytest-gee?color=blue&logo=pypi&logoColor=white
26 | :target: https://pypi.org/project/pytest-gee/
27 | :alt: PyPI version
28 |
29 | .. |conda| image:: https://img.shields.io/conda/vn/conda-forge/pytest-gee?logo=anaconda&logoColor=white&color=blue
30 | :target: https://anaconda.org/conda-forge/pytest-gee
31 | :alt: conda-forge version badge
32 |
33 | .. |build| image:: https://img.shields.io/github/actions/workflow/status/gee-community/pytest-gee/unit.yaml?logo=github&logoColor=white
34 | :target: https://github.com/gee-community/pytest-gee/actions/workflows/unit.yaml
35 | :alt: build
36 |
37 | .. |coverage| image:: https://img.shields.io/codecov/c/github/gee-community/pytest-gee?logo=codecov&logoColor=white
38 | :target: https://codecov.io/gh/gee-community/pytest-gee
39 | :alt: Test Coverage
40 |
41 | .. |docs| image:: https://img.shields.io/readthedocs/pytest-gee?logo=readthedocs&logoColor=white
42 | :target: https://pytest-gee.readthedocs.io/en/latest/
43 | :alt: Documentation Status
44 |
45 | |license| |commit| |ruff| |prettier| |pre-commmit| |pypi| |conda| |build| |coverage| |docs|
46 |
47 | Overview
48 | --------
49 |
50 | .. image:: https://raw.githubusercontent.com/gee-community/pytest-gee/main/docs/_static/logo.svg
51 | :width: 20%
52 | :align: right
53 |
54 | ``pytest-gee`` provides some fixtures that make it easy to generate independent tests that require Earth Engine asset filesystem.
55 | It also able to help maintaining tests that generate Earth Engine server side data.
56 |
57 | This plugin uses a data directory (courtesy of ``pytest-datadir``) to store expected data files,
58 | which are stored and used as baseline for future test runs.
59 | You can also define your own data directory directly as described in the ``pytest_regression`` documentation.
60 |
61 | Credits
62 | -------
63 |
64 | This package was created with `Copier `__ and the `@12rambau/pypackage `__ 0.1.16 project template.
65 |
--------------------------------------------------------------------------------
/pyproject.toml:
--------------------------------------------------------------------------------
1 | [build-system]
2 | requires = ["hatchling"]
3 | build-backend = "hatchling.build"
4 |
5 | [project]
6 | name = "pytest-gee"
7 | version = "0.8.1"
8 | description = "The Python plugin for your GEE based packages."
9 | keywords = [
10 | "skeleton",
11 | "Python"
12 | ]
13 | classifiers = [
14 | "Development Status :: 3 - Alpha",
15 | "Intended Audience :: Developers",
16 | "License :: OSI Approved :: MIT License",
17 | "Programming Language :: Python :: 3.9",
18 | "Programming Language :: Python :: 3.10",
19 | "Programming Language :: Python :: 3.11",
20 | "Programming Language :: Python :: 3.12",
21 | "Programming Language :: Python :: 3.13",
22 | ]
23 | requires-python = ">=3.9"
24 | dependencies = [
25 | "deprecated>=1.2.14",
26 | "earthengine-api>=0.1.397", # new ee.data.createFolder method
27 | "pytest",
28 | "pytest-regressions>=2.7.0", # get the fullpath parameter in the Imageregression
29 | "geopandas",
30 | "pillow",
31 | ]
32 |
33 | [[project.authors]]
34 | name = "Pierrick Rambaud"
35 | email = "pierrick.rambaud49@gmail.com"
36 |
37 | [project.license]
38 | text = "MIT"
39 |
40 | [project.readme]
41 | file = "README.rst"
42 | content-type = "text/x-rst"
43 |
44 | [project.urls]
45 | Homepage = "https://github.com/gee-community/pytest-gee"
46 |
47 | [project.entry-points."pytest11"]
48 | gee = "pytest_gee.plugin"
49 |
50 | [project.optional-dependencies]
51 | test = [
52 | "pytest",
53 | "pytest-cov",
54 | "pytest-deadfixtures"
55 | ]
56 | doc = [
57 | "sphinx>=6.2.1,<8",
58 | "pydata-sphinx-theme",
59 | "sphinx-copybutton",
60 | "sphinx-design",
61 | "sphinx-autoapi",
62 | "sphinxemoji",
63 | ]
64 |
65 | [tool.hatch.build.targets.wheel]
66 | only-include = ["pytest_gee"]
67 |
68 | [tool.hatch.envs.default]
69 | dependencies = [
70 | "pre-commit",
71 | "commitizen",
72 | "nox[uv]"
73 | ]
74 | post-install-commands = ["pre-commit install"]
75 |
76 | [tool.commitizen]
77 | tag_format = "v$major.$minor.$patch$prerelease"
78 | update_changelog_on_bump = false
79 | version = "0.8.1"
80 | version_files = [
81 | "pyproject.toml:version",
82 | "pytest_gee/__init__.py:__version__",
83 | "docs/conf.py:release",
84 | ]
85 |
86 | [tool.pytest.ini_options]
87 | testpaths = "tests"
88 |
89 | [tool.ruff]
90 | line-length = 100
91 | fix = true
92 |
93 | [tool.ruff.lint]
94 | select = ["E", "F", "W", "I", "D", "RUF"]
95 | ignore = [
96 | "E501", # line too long | Black take care of it
97 | "D212", # Multi-line docstring | We use D213
98 | "D101", # Missing docstring in public class | We use D106
99 | ]
100 |
101 | [tool.ruff.lint.flake8-quotes]
102 | docstring-quotes = "double"
103 |
104 | [tool.ruff.lint.pydocstyle]
105 | convention = "google"
106 |
107 | [tool.coverage.run]
108 | source = ["pytest_gee"]
109 |
110 | [tool.mypy]
111 | scripts_are_modules = true
112 | ignore_missing_imports = true
113 | install_types = true
114 | non_interactive = true
115 | warn_redundant_casts = true
116 |
117 | [tool.codespell]
118 | ignore-words-list = "nd"
119 |
--------------------------------------------------------------------------------
/tests/test_pytest_gee/test_feature_collection_regression_no_index.yml:
--------------------------------------------------------------------------------
1 | features:
2 | - geometry:
3 | coordinates:
4 | - - - 0.00030000000000000003
5 | - 0.00030000000000000003
6 | - - 0.0004
7 | - -0.0001
8 | - - 0.0002
9 | - -0.0004
10 | - - -0.0002
11 | - -0.0004
12 | - - -0.0004
13 | - -0.0001
14 | - - -0.0004
15 | - 0.00030000000000000003
16 | - - 0.0
17 | - 0.0004
18 | - - 0.00030000000000000003
19 | - 0.00030000000000000003
20 | type: Polygon
21 | id: '0'
22 | properties: {}
23 | type: Feature
24 | - geometry:
25 | coordinates:
26 | - - - 0.0004
27 | - 0.00030000000000000003
28 | - - 0.0005
29 | - -0.0001
30 | - - 0.0002
31 | - -0.0005
32 | - - -0.0002
33 | - -0.0005
34 | - - -0.0005
35 | - -0.0001
36 | - - -0.0004
37 | - 0.00030000000000000003
38 | - - 0.0
39 | - 0.0005
40 | - - 0.0004
41 | - 0.00030000000000000003
42 | type: Polygon
43 | id: '1'
44 | properties: {}
45 | type: Feature
46 | - geometry:
47 | coordinates:
48 | - - - 0.0005
49 | - 0.0004
50 | - - 0.0006000000000000001
51 | - -0.0001
52 | - - 0.00030000000000000003
53 | - -0.0006000000000000001
54 | - - -0.00030000000000000003
55 | - -0.0006000000000000001
56 | - - -0.0006000000000000001
57 | - -0.0001
58 | - - -0.0005
59 | - 0.0004
60 | - - 0.0
61 | - 0.0006000000000000001
62 | - - 0.0005
63 | - 0.0004
64 | type: Polygon
65 | id: '2'
66 | properties: {}
67 | type: Feature
68 | - geometry:
69 | coordinates:
70 | - - - 0.0006000000000000001
71 | - 0.0005
72 | - - 0.0007
73 | - -0.0001
74 | - - 0.00030000000000000003
75 | - -0.0006000000000000001
76 | - - -0.00030000000000000003
77 | - -0.0007
78 | - - -0.0007
79 | - -0.0002
80 | - - -0.0006000000000000001
81 | - 0.0004
82 | - - 0.0
83 | - 0.0007
84 | - - 0.0006000000000000001
85 | - 0.0005
86 | type: Polygon
87 | id: '3'
88 | properties: {}
89 | type: Feature
90 | - geometry:
91 | coordinates:
92 | - - - 0.0006000000000000001
93 | - 0.0005
94 | - - 0.0008
95 | - -0.0002
96 | - - 0.0004
97 | - -0.0007
98 | - - -0.00030000000000000003
99 | - -0.0007
100 | - - -0.0008
101 | - -0.0002
102 | - - -0.0006000000000000001
103 | - 0.0005
104 | - - 0.0
105 | - 0.0008
106 | - - 0.0006000000000000001
107 | - 0.0005
108 | type: Polygon
109 | id: '4'
110 | properties: {}
111 | type: Feature
112 | - geometry:
113 | coordinates:
114 | - - - 0.0007
115 | - 0.0006000000000000001
116 | - - 0.0009000000000000001
117 | - -0.0002
118 | - - 0.0004
119 | - -0.0008
120 | - - -0.0004
121 | - -0.0008
122 | - - -0.0009000000000000001
123 | - -0.0002
124 | - - -0.0007
125 | - 0.0006000000000000001
126 | - - 0.0
127 | - 0.0009000000000000001
128 | - - 0.0007
129 | - 0.0006000000000000001
130 | type: Polygon
131 | id: '5'
132 | properties: {}
133 | type: Feature
134 | type: FeatureCollection
135 |
--------------------------------------------------------------------------------
/pytest_gee/list_regression.py:
--------------------------------------------------------------------------------
1 | """Implementation of the ``list_regression`` fixture."""
2 |
3 | import os
4 | from contextlib import suppress
5 | from typing import Optional
6 |
7 | import ee
8 | from pytest import fail
9 | from pytest_regressions.data_regression import DataRegressionFixture
10 |
11 | from .utils import build_fullpath, check_serialized, round_data
12 |
13 |
14 | class ListFixture(DataRegressionFixture):
15 | """Fixture for regression testing of :py:class:`ee.List`."""
16 |
17 | def check(
18 | self,
19 | data_list: ee.List,
20 | basename: Optional[str] = None,
21 | fullpath: Optional[os.PathLike] = None,
22 | prescision: int = 6,
23 | ):
24 | """Check the given list against a previously recorded version, or generate a new file.
25 |
26 | Parameters:
27 | data_list: The list to check.
28 | basename: The basename of the file to test/record. If not given the name of the test is used.
29 | fullpath: complete path to use as a reference file. This option will ignore ``datadir`` fixture when reading *expected* files but will still use it to write *obtained* files. Useful if a reference file is located in the session data dir for example.
30 | precision: The number of decimal places to round to when comparing floats.
31 | """
32 | # build the different filename to be consistent between our 3 checks
33 | data_name = build_fullpath(
34 | datadir=self.original_datadir,
35 | request=self.request,
36 | extension=".yml",
37 | basename=basename,
38 | fullpath=fullpath,
39 | with_test_class_names=self.with_test_class_names,
40 | )
41 | serialized_name = data_name.with_stem(f"serialized_{data_name.stem}").with_suffix(".yml")
42 |
43 | # check the previously registered serialized call from GEE. If it matches the current call,
44 | # we don't need to check the data
45 | with suppress(AssertionError, fail.Exception):
46 | check_serialized(
47 | object=data_list,
48 | path=serialized_name,
49 | datadir=self.datadir,
50 | original_datadir=self.original_datadir,
51 | request=self.request,
52 | with_test_class_names=self.with_test_class_names,
53 | )
54 | return
55 |
56 | # delete the previously created file if wasn't successful
57 | serialized_name.unlink(missing_ok=True)
58 |
59 | # if it needs to be checked, we need to round the float values to the same precision as the
60 | # reference file
61 | data = round_data(data_list.getInfo(), prescision)
62 | try:
63 | super().check(data, fullpath=data_name)
64 |
65 | # IF we are here it means the data has been modified so we edit the API call accordingly
66 | # to make sure next run will not be forced to call the API for a response.
67 | with suppress(AssertionError, fail.Exception):
68 | check_serialized(
69 | object=data_list,
70 | path=serialized_name,
71 | datadir=self.datadir,
72 | original_datadir=self.original_datadir,
73 | request=self.request,
74 | with_test_class_names=self.with_test_class_names,
75 | force_regen=True,
76 | )
77 |
78 | except (AssertionError, fail.Exception) as e:
79 | raise e
80 |
--------------------------------------------------------------------------------
/pytest_gee/dictionary_regression.py:
--------------------------------------------------------------------------------
1 | """Implementation of the ``dictionary_regression`` fixture."""
2 |
3 | import os
4 | from contextlib import suppress
5 | from typing import Optional
6 |
7 | import ee
8 | from pytest import fail
9 | from pytest_regressions.data_regression import DataRegressionFixture
10 |
11 | from .utils import build_fullpath, check_serialized, round_data
12 |
13 |
14 | class DictionaryFixture(DataRegressionFixture):
15 | """Fixture for regression testing of :py:class:`ee.Dictionary`."""
16 |
17 | def check(
18 | self,
19 | data_dict: ee.Dictionary,
20 | basename: Optional[str] = None,
21 | fullpath: Optional[os.PathLike] = None,
22 | prescision: int = 6,
23 | ):
24 | """Check the given list against a previously recorded version, or generate a new file.
25 |
26 | Parameters:
27 | data_dict: The dictionary to check.
28 | basename: The basename of the file to test/record. If not given the name of the test is used.
29 | fullpath: complete path to use as a reference file. This option will ignore ``datadir`` fixture when reading *expected* files but will still use it to write *obtained* files. Useful if a reference file is located in the session data dir for example.
30 | precision: The number of decimal places to round to when comparing floats.
31 | """
32 | # build the different filename to be consistent between our 3 checks
33 | data_name = build_fullpath(
34 | datadir=self.original_datadir,
35 | request=self.request,
36 | extension=".yml",
37 | basename=basename,
38 | fullpath=fullpath,
39 | with_test_class_names=self.with_test_class_names,
40 | )
41 | serialized_name = data_name.with_stem(f"serialized_{data_name.stem}").with_suffix(".yml")
42 |
43 | # check the previously registered serialized call from GEE. If it matches the current call,
44 | # we don't need to check the data
45 | with suppress(AssertionError, fail.Exception):
46 | check_serialized(
47 | object=ee.Dictionary(data_dict),
48 | path=serialized_name,
49 | datadir=self.datadir,
50 | original_datadir=self.original_datadir,
51 | request=self.request,
52 | with_test_class_names=self.with_test_class_names,
53 | )
54 | return
55 |
56 | # delete the previously created file if wasn't successful
57 | serialized_name.unlink(missing_ok=True)
58 |
59 | # if it needs to be checked, we need to round the float values to the same precision as the
60 | # reference file
61 | data = round_data(data_dict.getInfo(), prescision)
62 | try:
63 | super().check(data, fullpath=data_name)
64 |
65 | # IF we are here it means the data has been modified so we edit the API call accordingly
66 | # to make sure next run will not be forced to call the API for a response.
67 | with suppress(AssertionError, fail.Exception):
68 | check_serialized(
69 | object=data_dict,
70 | path=serialized_name,
71 | datadir=self.datadir,
72 | original_datadir=self.original_datadir,
73 | request=self.request,
74 | with_test_class_names=self.with_test_class_names,
75 | force_regen=True,
76 | )
77 |
78 | except (AssertionError, fail.Exception) as e:
79 | raise e
80 |
--------------------------------------------------------------------------------
/tests/test_pytest_gee/serialized_test_feature_collection_regression_no_index.yml:
--------------------------------------------------------------------------------
1 | result: '0'
2 | values:
3 | '0':
4 | functionInvocationValue:
5 | arguments:
6 | baseAlgorithm:
7 | functionDefinitionValue:
8 | argumentNames:
9 | - _MAPPING_VAR_0_0
10 | body: '1'
11 | collection:
12 | functionInvocationValue:
13 | arguments:
14 | features:
15 | functionInvocationValue:
16 | arguments:
17 | baseAlgorithm:
18 | functionDefinitionValue:
19 | argumentNames:
20 | - _MAPPING_VAR_0_0
21 | body: '2'
22 | dropNulls:
23 | constantValue: false
24 | list:
25 | functionInvocationValue:
26 | arguments:
27 | baseAlgorithm:
28 | functionDefinitionValue:
29 | argumentNames:
30 | - _MAPPING_VAR_0_0
31 | body: '3'
32 | dropNulls:
33 | constantValue: false
34 | list:
35 | functionInvocationValue:
36 | arguments:
37 | end:
38 | constantValue: 100
39 | start:
40 | constantValue: 50
41 | step:
42 | constantValue: 10
43 | functionName: List.sequence
44 | functionName: List.map
45 | functionName: List.map
46 | functionName: Collection
47 | functionName: Collection.map
48 | '1':
49 | functionInvocationValue:
50 | arguments:
51 | input:
52 | argumentReference: _MAPPING_VAR_0_0
53 | propertySelectors:
54 | functionInvocationValue:
55 | arguments:
56 | element:
57 | constantValue: system:index
58 | list:
59 | functionInvocationValue:
60 | arguments:
61 | element:
62 | argumentReference: _MAPPING_VAR_0_0
63 | functionName: Element.propertyNames
64 | functionName: List.remove
65 | functionName: Feature.select
66 | '2':
67 | functionInvocationValue:
68 | arguments:
69 | geometry:
70 | argumentReference: _MAPPING_VAR_0_0
71 | functionName: Feature
72 | '3':
73 | functionInvocationValue:
74 | arguments:
75 | distance:
76 | argumentReference: _MAPPING_VAR_0_0
77 | geometry:
78 | functionInvocationValue:
79 | arguments:
80 | coordinates:
81 | constantValue:
82 | - 0
83 | - 0
84 | functionName: GeometryConstructors.Point
85 | maxError:
86 | functionInvocationValue:
87 | arguments:
88 | value:
89 | functionInvocationValue:
90 | arguments:
91 | left:
92 | argumentReference: _MAPPING_VAR_0_0
93 | right:
94 | constantValue: 5
95 | functionName: Number.divide
96 | functionName: ErrorMargin
97 | functionName: Geometry.buffer
98 |
--------------------------------------------------------------------------------
/CONTRIBUTING.rst:
--------------------------------------------------------------------------------
1 | Contribute
2 | ==========
3 |
4 | Thank you for your help improving **pytest-gee**!
5 |
6 | **pytest-gee** uses `nox `__ to automate several development-related tasks.
7 | Currently, the project uses four automation processes (called sessions) in ``noxfile.py``:
8 |
9 | - ``mypy``: to perform a mypy check on the lib;
10 | - ``test``: to run the test with pytest;
11 | - ``docs``: to build the documentation in the ``build`` folder;
12 | - ``lint``: to run the pre-commits in an isolated environment
13 |
14 | Every nox session is run in its own virtual environment, and the dependencies are installed automatically.
15 |
16 | To run a specific nox automation process, use the following command:
17 |
18 | .. code-block:: console
19 |
20 | nox -s
21 |
22 | For example: ``nox -s test`` or ``nox -s docs``.
23 |
24 | Workflow for contributing changes
25 | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
26 |
27 | We follow a typical GitHub workflow of:
28 |
29 | - Create a personal fork of this repo
30 | - Create a branch
31 | - Open a pull request
32 | - Fix findings of various linters and checks
33 | - Work through code review
34 |
35 | See the following sections for more details.
36 |
37 | Clone the repository
38 | ^^^^^^^^^^^^^^^^^^^^
39 |
40 | First off, you'll need your own copy of **pytest-gee** codebase. You can clone it for local development like so:
41 |
42 | Fork the repository so you have your own copy on GitHub. See the `GitHub forking guide for more information `__.
43 |
44 | Then, clone the repository locally so that you have a local copy to work on:
45 |
46 | .. code-block:: console
47 |
48 | git clone https://github.com//pytest-gee
49 | cd pytest-gee
50 |
51 | Then install the development version of the extension:
52 |
53 | .. code-block:: console
54 |
55 | pip install -e .[dev]
56 |
57 | This will install the **pytest-gee** library, together with two additional tools:
58 | - `pre-commit `__ for automatically enforcing code standards and quality checks before commits.
59 | - `nox `__, for automating common development tasks.
60 |
61 | Lastly, activate the pre-commit hooks by running:
62 |
63 | .. code-block:: console
64 |
65 | pre-commit install
66 |
67 | This will install the necessary dependencies to run pre-commit every time you make a commit with Git.
68 |
69 | Contribute to the codebase
70 | ^^^^^^^^^^^^^^^^^^^^^^^^^^
71 |
72 | Any larger updates to the codebase should include tests and documentation. The tests are located in the ``tests`` folder, and the documentation is located in the ``docs`` folder.
73 |
74 | To run the tests locally, use the following command:
75 |
76 | .. code-block:: console
77 |
78 | nox -s test
79 |
80 | See :ref:`below ` for more information on how to update the documentation.
81 |
82 | .. _contributing-docs:
83 |
84 | Contribute to the docs
85 | ^^^^^^^^^^^^^^^^^^^^^^
86 |
87 | The documentation is built using `Sphinx `__ and deployed to `Read the Docs `__.
88 |
89 | To build the documentation locally, use the following command:
90 |
91 | .. code-block:: console
92 |
93 | nox -s docs
94 |
95 | For each pull request, the documentation is built and deployed to make it easier to review the changes in the PR. To access the docs build from a PR, click on the "Read the Docs" preview in the CI/CD jobs.
96 |
97 | Release new version
98 | ^^^^^^^^^^^^^^^^^^^
99 |
100 | To release a new version, start by pushing a new bump from the local directory:
101 |
102 | .. code-block::
103 |
104 | cz bump
105 |
106 | The commitizen-tool will detect the semantic version name based on the existing commits messages.
107 |
108 | Then push to Github. In Github design a new release using the same tag name nad the ``release.yaml`` job will send it to pipy.
109 |
--------------------------------------------------------------------------------
/pytest_gee/feature_collection_regression.py:
--------------------------------------------------------------------------------
1 | """Implementation of the ``feature_collection_regression`` fixture."""
2 |
3 | import os
4 | from contextlib import suppress
5 | from typing import Optional
6 |
7 | import ee
8 | import geopandas as gpd
9 | from pytest import fail
10 | from pytest_regressions.data_regression import DataRegressionFixture
11 |
12 | from .utils import build_fullpath, check_serialized, round_data
13 |
14 |
15 | class FeatureCollectionFixture(DataRegressionFixture):
16 | """Fixture for regression testing of :py:class:`ee.FeatureCollection`."""
17 |
18 | def check(
19 | self,
20 | data_fc: ee.FeatureCollection,
21 | basename: Optional[str] = None,
22 | fullpath: Optional[os.PathLike] = None,
23 | prescision: int = 6,
24 | drop_index=False,
25 | ):
26 | """Check the given list against a previously recorded version, or generate a new file.
27 |
28 | Parameters:
29 | data_fc: The feature collection to check.
30 | basename: The basename of the file to test/record. If not given the name of the test is used.
31 | fullpath: complete path to use as a reference file. This option will ignore ``datadir`` fixture when reading *expected* files but will still use it to write *obtained* files. Useful if a reference file is located in the session data dir for example.
32 | precision: The number of decimal places to round to when comparing floats.
33 | drop_index: If True, the ``system:index`` property will be removed from the feature collection before checking.
34 | """
35 | if drop_index is True:
36 | data_fc = data_fc.map(lambda f: f.select(f.propertyNames().remove("system:index")))
37 |
38 | # build the different filename to be consistent between our 3 checks
39 | data_name = build_fullpath(
40 | datadir=self.original_datadir,
41 | request=self.request,
42 | extension=".yml",
43 | basename=basename,
44 | fullpath=fullpath,
45 | with_test_class_names=self.with_test_class_names,
46 | )
47 | serialized_name = data_name.with_stem(f"serialized_{data_name.stem}").with_suffix(".yml")
48 |
49 | # check the previously registered serialized call from GEE. If it matches the current call,
50 | # we don't need to check the data
51 | with suppress(AssertionError, fail.Exception):
52 | check_serialized(
53 | object=data_fc,
54 | path=serialized_name,
55 | datadir=self.datadir,
56 | original_datadir=self.original_datadir,
57 | request=self.request,
58 | with_test_class_names=self.with_test_class_names,
59 | )
60 | return
61 |
62 | # delete the previously created file if wasn't successful
63 | serialized_name.unlink(missing_ok=True)
64 |
65 | # round the geometry using geopandas to make sre with use the specific number of decimal places
66 | gdf = gpd.GeoDataFrame.from_features(data_fc.getInfo())
67 | gdf.geometry = gdf.set_precision(grid_size=10 ** (-prescision)).remove_repeated_points()
68 |
69 | # round any float value before serving the data to the check function
70 | data = gdf.to_geo_dict()
71 | data = round_data(data, prescision)
72 |
73 | # if it needs to be checked, we need to round the float values to the same precision as the
74 | # reference file
75 | try:
76 | super().check(data, fullpath=data_name)
77 |
78 | # IF we are here it means the data has been modified so we edit the API call accordingly
79 | # to make sure next run will not be forced to call the API for a response.
80 | with suppress(AssertionError, fail.Exception):
81 | check_serialized(
82 | object=data_fc,
83 | path=serialized_name,
84 | datadir=self.datadir,
85 | original_datadir=self.original_datadir,
86 | request=self.request,
87 | with_test_class_names=self.with_test_class_names,
88 | force_regen=True,
89 | )
90 |
91 | except (AssertionError, fail.Exception) as e:
92 | raise e
93 |
--------------------------------------------------------------------------------
/pytest_gee/plugin.py:
--------------------------------------------------------------------------------
1 | """A pytest plugin to build a GEE environment for a test session."""
2 |
3 | from __future__ import annotations
4 |
5 | import uuid
6 | from pathlib import Path
7 |
8 | import pytest
9 | from ee._state import get_state
10 |
11 | from . import utils
12 | from .dictionary_regression import DictionaryFixture
13 | from .feature_collection_regression import FeatureCollectionFixture
14 | from .image_regression import ImageFixture
15 | from .list_regression import ListFixture
16 |
17 |
18 | @pytest.fixture(scope="session")
19 | def gee_hash():
20 | """Generate a unique hash for the test session."""
21 | return uuid.uuid4().hex
22 |
23 |
24 | @pytest.fixture(scope="session")
25 | def gee_folder_root():
26 | """Link to the root folder of the connected account."""
27 | project_id = get_state().cloud_api_user_project
28 | return Path(f"projects/{project_id}/assets")
29 |
30 |
31 | @pytest.fixture(scope="session")
32 | def gee_folder_structure():
33 | """The structure of the generated test folder."""
34 | return {}
35 |
36 |
37 | @pytest.fixture(scope="session")
38 | def gee_test_folder(gee_hash, gee_folder_root, gee_folder_structure):
39 | """Create a test folder for the duration of the test session."""
40 | folder = utils.init_tree(gee_folder_structure, gee_hash, gee_folder_root)
41 |
42 | yield folder
43 |
44 | utils.delete_assets(folder, False)
45 |
46 |
47 | @pytest.fixture
48 | def ee_list_regression(
49 | datadir: Path, original_datadir: Path, request: pytest.FixtureRequest
50 | ) -> ListFixture:
51 | """Fixture to test :py:class:`ee.List` objects.
52 |
53 | Args:
54 | datadir: The directory where the data files are stored.
55 | original_datadir: The original data directory.
56 | request: The pytest request object.
57 |
58 | Returns:
59 | The ListFixture object.
60 |
61 | Example:
62 | .. code-block:: python
63 |
64 | def test_list_regression(list_regression):
65 | data = ee.List([1, 2, 3])
66 | list_regression.check(data)
67 | """
68 | return ListFixture(datadir, original_datadir, request)
69 |
70 |
71 | @pytest.fixture
72 | def ee_feature_collection_regression(
73 | datadir: Path, original_datadir: Path, request: pytest.FixtureRequest
74 | ) -> FeatureCollectionFixture:
75 | """Fixture to test :py:class:`ee.FeatureCollection` objects.
76 |
77 | Args:
78 | datadir: The directory where the data files are stored.
79 | original_datadir: The original data directory.
80 | request: The pytest request object.
81 |
82 | Returns:
83 | The FeatureCollectionFixture object.
84 |
85 | Example:
86 | .. code-block:: python
87 |
88 | def test_feature_collection_regression(feature_collection_regression):
89 | data = ee.FeatureCollection("FAO/GAUL/2015/level0").filter(ee.Filter.eq("ADM0_NAME", "Holy See"))
90 | feature_collection_regression.check(data)
91 | """
92 | return FeatureCollectionFixture(datadir, original_datadir, request)
93 |
94 |
95 | @pytest.fixture
96 | def ee_dictionary_regression(
97 | datadir: Path, original_datadir: Path, request: pytest.FixtureRequest
98 | ) -> DictionaryFixture:
99 | """Fixture to test `ee.Dictionary` objects.
100 |
101 | Args:
102 | datadir: The directory where the data files are stored.
103 | original_datadir: The original data directory.
104 | request: The pytest request object.
105 |
106 | Returns:
107 | The DictionaryFixture object.
108 |
109 | Example:
110 | .. code-block:: python
111 |
112 | def test_dictionary_regression(dictionary_regression):
113 | data = ee.Dictionary({"a": 1, "b": 2})
114 | dictionary_regression.check(data)
115 | """
116 | return DictionaryFixture(datadir, original_datadir, request)
117 |
118 |
119 | @pytest.fixture
120 | def ee_image_regression(
121 | datadir: Path, original_datadir: Path, request: pytest.FixtureRequest
122 | ) -> ImageFixture:
123 | """Fixture to test :py:class:`ee.Image` objects.
124 |
125 | Args:
126 | datadir: The directory where the data files are stored.
127 | original_datadir: The original data directory.
128 | request: The pytest request object.
129 |
130 | Returns:
131 | The ImageFixture object.
132 |
133 | Example:
134 | .. code-block:: python
135 |
136 | def test_image_regression(image_regression):
137 | data = ee.Image("LANDSAT/LC08/C02/T1_L2/LC08_191031_20210514")
138 | image_regression.check(data, scale=1000)
139 | """
140 | return ImageFixture(datadir, original_datadir, request)
141 |
--------------------------------------------------------------------------------
/docs/content/filesystem.rst:
--------------------------------------------------------------------------------
1 | GEE Filesystem
2 | ==============
3 |
4 | Generate a test file tree in GEE
5 | --------------------------------
6 |
7 | Using the ``pytest_gee`` plugin, you can easily generate a test file tree in GEE that will be used to run your tests.
8 | This tree will start in a folder named with the ``gee_hash`` fixture and will be deleted at the end of the test session.
9 |
10 | By using this method you will ensure that the folder you are using for your test is unique and that it will not interfere with other tests (e.g. parallel tests).
11 |
12 | .. code-block:: python
13 |
14 | # test_something.py
15 |
16 | def test_something(gee_hash, gee_folder_root, gee_test_folder):
17 | # this folder is existing within your GEE account and will be deleted at the end of the test session
18 | print(gee_folder_root)
19 |
20 | .. warning::
21 |
22 | To avoid piling up fake folder in your GEE account, make sure to let the test reach the end of the session.
23 | It means that you should **never** cancel a session with ``ctrl+c`` or by killing the process.
24 |
25 |
26 | Customize the test folder tree
27 | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
28 |
29 | By default the test folder tree is empty and will be deleted at the end of the test session.
30 | You can decide to populate it with some assets that will be used in your tests.
31 |
32 | To do so customize the ``gee_folder_structure`` fixture in your ``conftest.py`` file.
33 | This fixture is a ``dict`` that will be used to create the folder tree in GEE.
34 | First you can create containers assets (namely folders or image collections) to store your assets.
35 | These container are simply marked as keys in the dict and specify their types after a "::" symbol as shown in the following example.
36 | assets need to be ``ee.Image`` or ``ee.FeatureCollection`` objects and remaining small as the creation operation is taken care of by the plugin.
37 | Specifically for ``ee.Image`` objects, please use the ``clipToBoundsAndScale`` method to make sure the asset has a geometry and a scale.
38 |
39 | .. code-block:: python
40 |
41 | # conftest.py
42 |
43 | import pytest
44 |
45 | @pytest.fixture(scope="session")
46 | def gee_folder_structure():
47 | """Override the default test folder structure."""
48 | point = ee.Geometry.Point([0, 0])
49 | return {
50 | "folder::Folder": {
51 | "image": ee.Image(1).clipToBoundsAndScale(point.buffer(100), scale=30),
52 | "fc": ee.FeatureCollection(point),
53 | },
54 | "image_collection::ImageCollection": {
55 | "image1": ee.Image(1).clipToBoundsAndScale(point.buffer(100), scale=30),
56 | "image2": ee.Image(1).clipToBoundsAndScale(point.buffer(100), scale=30),
57 | }
58 | }
59 |
60 | Which will render in your GEE account as:
61 |
62 | .. code-block::
63 |
64 | 8d98a5be574041a6a54d6def9d915c67/
65 | └── folder/
66 | ├── fc (FeatureCollection)
67 | └── image (Image)
68 | └── image_collection/ (ImageCollection)
69 | ├── image1 (Image)
70 | └── image2 (Image)
71 |
72 | Customize the root folder
73 | ^^^^^^^^^^^^^^^^^^^^^^^^^
74 |
75 | By default the test folder will be created at the root of the user account. There are situation where one could prefer to store it in a specific folder.
76 |
77 | To do so customize the ``gee_folder_root`` fixture in your ``conftest.py`` file, simply return the asset id of the folder you want to use as root.
78 |
79 | .. code-block:: python
80 |
81 | # conftest.py
82 |
83 | import pytest
84 |
85 | @pytest.fixture(scope="session")
86 | def gee_folder_root():
87 | """Override the default test folder root."""
88 | return "project/username/assets/my_root_folder"
89 |
90 | .. note::
91 |
92 | This is compulsory if you use a service account to connect to GEE as the service account has no associated root folder.
93 |
94 | Create assets
95 | -------------
96 |
97 | Most of tests pipelines are checking different python versions in parallel which can create multiple issues from a GEE perspective:
98 |
99 | - The assets names need to be unique
100 | - The tasks names also need to be unique
101 |
102 | To avoid this issue, the plugin is shipped with a session wise unique hex fixture ``gee_hash`` that can be used to suffix or prefix your assets and tasks names.
103 | To make sure the asset exist when you run your tests, you can use the ``pytest_gee.wait`` method to wait until the asset is effectively generated.
104 |
105 | .. code-block:: python
106 |
107 | # test.py
108 |
109 | import pytest
110 | import pytest_gee
111 |
112 |
113 | def test_create_asset(gee_hash):
114 | # create an asset name
115 | asset_name = f"asset_{gee_hash}"
116 |
117 | # export the an object to this asset
118 | task = ee.batch.Export.image.toAsset(
119 | image=ee.Image(1),
120 | description=asset_name,
121 | assetId=asset_name,
122 | scale=1,
123 | maxPixels=1e9,
124 | )
125 | task.start()
126 |
127 | # wait for the asset to be created
128 | pytest_gee.wait(task)
129 |
130 | # Do something with the asset name
131 |
--------------------------------------------------------------------------------
/pytest_gee/__init__.py:
--------------------------------------------------------------------------------
1 | """The init file of the package."""
2 |
3 | from __future__ import annotations
4 |
5 | import json
6 | import os
7 | import re
8 | from pathlib import Path
9 | from typing import Union
10 |
11 | import ee
12 | import httplib2
13 | from deprecated.sphinx import deprecated
14 | from ee.cli.utils import wait_for_task
15 |
16 | __version__ = "0.8.1"
17 | __author__ = "Pierrick Rambaud"
18 | __email__ = "pierrick.rambaud49@gmail.com"
19 |
20 |
21 | def init_ee_from_token():
22 | r"""Initialize earth engine according using a token.
23 |
24 | THe environment used to run the tests need to have a EARTHENGINE_TOKEN variable.
25 | The content of this variable must be the copy of a personal credential file that you can find on your local computer if you already run the earth engine command line tool. See the usage question for a github action example.
26 |
27 | - Windows: ``C:\Users\USERNAME\\.config\\earthengine\\credentials``
28 | - Linux: ``/home/USERNAME/.config/earthengine/credentials``
29 | - MacOS: ``/Users/USERNAME/.config/earthengine/credentials``
30 |
31 | Note:
32 | As all init method of pytest-gee, this method will fallback to a regular ``ee.Initialize()`` if the environment variable is not found e.g. on your local computer.
33 | """
34 | if "EARTHENGINE_TOKEN" in os.environ:
35 | # read the ee_token from the environment variable
36 | ee_token = os.environ["EARTHENGINE_TOKEN"]
37 |
38 | # small workaround to remove the quotes around the token
39 | # related to a very specific issue with readthedocs interface
40 | # https://github.com/readthedocs/readthedocs.org/issues/10553
41 | pattern = re.compile(r"^'[^']*'$")
42 | ee_token = ee_token[1:-1] if pattern.match(ee_token) else ee_token
43 |
44 | # write the token to the appropriate folder
45 | credential_folder_path = Path.home() / ".config" / "earthengine"
46 | credential_folder_path.mkdir(parents=True, exist_ok=True)
47 | credential_file_path = credential_folder_path / "credentials"
48 | credential_file_path.write_text(ee_token)
49 |
50 | project_id = os.environ.get("EARTHENGINE_PROJECT")
51 | if project_id is None:
52 | raise ValueError(
53 | "The project name cannot be detected."
54 | "Please set the EARTHENGINE_PROJECT environment variable."
55 | )
56 |
57 | # if the user is in local development the authentication should
58 | # already be available
59 | ee.Initialize(project=project_id, http_transport=httplib2.Http())
60 |
61 |
62 | def init_ee_from_service_account():
63 | """Initialize earth engine according using a service account.
64 |
65 | The environment used to run the tests need to have a EARTHENGINE_SERVICE_ACCOUNT variable.
66 | The content of this variable must be the copy of a personal credential file that you can generate from the google cloud console.
67 |
68 | Note:
69 | As all init method of ``pytest-gee``, this method will fallback to a regular ``ee.Initialize`` using the ``EARTHENGINE_PROJECT`` environment variable.
70 | """
71 | if "EARTHENGINE_SERVICE_ACCOUNT" in os.environ:
72 | # extract the environment variables data
73 | private_key = os.environ["EARTHENGINE_SERVICE_ACCOUNT"]
74 |
75 | # small workaround to remove the quotes around the token
76 | # related to a very specific issue with readthedocs interface
77 | # https://github.com/readthedocs/readthedocs.org/issues/10553
78 | pattern = re.compile(r"^'[^']*'$")
79 | private_key = private_key[1:-1] if pattern.match(private_key) else private_key
80 |
81 | # connect to GEE using a a ServiceAccountCredential object based on the
82 | # private key data
83 | ee_user = json.loads(private_key)["client_email"]
84 | credentials = ee.ServiceAccountCredentials(ee_user, key_data=private_key)
85 | ee.Initialize(
86 | credentials=credentials, project=credentials.project_id, http_transport=httplib2.Http()
87 | )
88 |
89 | elif "EARTHENGINE_PROJECT" in os.environ:
90 | # if the user is in local development the authentication should already be available
91 | # we simply need to use the provided project name
92 | ee.Initialize(project=os.environ["EARTHENGINE_PROJECT"], http_transport=httplib2.Http())
93 |
94 | else:
95 | msg = "EARTHENGINE_SERVICE_ACCOUNT or EARTHENGINE_PROJECT environment variable is missing"
96 | raise ValueError(msg)
97 |
98 |
99 | @deprecated(version="0.3.5", reason="Use the vanilla GEE ``wait_for_task`` function instead.")
100 | def wait(task: Union[ee.batch.Task, str], timeout: int = 5 * 60) -> str:
101 | """Wait until the selected process is finished or we reached timeout value.
102 |
103 | Args:
104 | task: name of the running task or the Task object itself.
105 | timeout: timeout in seconds. if set to 0 the parameter is ignored. default to 5 minutes.
106 |
107 | Returns:
108 | the final state of the task
109 | """
110 | # just expose the utils function
111 | # this is compulsory as wait is also needed in the utils module
112 | task_id = task.id if isinstance(task, ee.batch.Task) else task
113 | return wait_for_task(task_id, timeout, log_progress=False)
114 |
--------------------------------------------------------------------------------
/tests/test_pytest_gee.py:
--------------------------------------------------------------------------------
1 | """Test the pytest_gee package."""
2 |
3 | import ee
4 |
5 | import pytest_gee
6 |
7 | landsat_image = "LANDSAT/LC08/C02/T1_L2/LC08_191031_20240607"
8 | "landsat image from 2024-06-07 on top of Rome"
9 |
10 |
11 | def test_hash_fixture(gee_hash):
12 | """Test the hash fixture."""
13 | assert isinstance(gee_hash, str)
14 | assert len(gee_hash) == 32
15 |
16 |
17 | def test_gee_init():
18 | """Test the init_ee_from_token function."""
19 | assert ee.Number(1).getInfo() == 1
20 |
21 |
22 | def test_structure(gee_folder_structure):
23 | """Test the structure fixture."""
24 | assert isinstance(gee_folder_structure, dict)
25 | assert "folder::Folder" in gee_folder_structure
26 | assert "image" in gee_folder_structure["folder::Folder"]
27 | assert "fc" in gee_folder_structure["folder::Folder"]
28 | assert "ic::ImageCollection" in gee_folder_structure
29 | assert "image1" in gee_folder_structure["ic::ImageCollection"]
30 | assert "image2" in gee_folder_structure["ic::ImageCollection"]
31 |
32 |
33 | def test_init_tree(gee_folder_root, gee_test_folder):
34 | """Test the init_tree function."""
35 | # search all the assets contained in the test_folder
36 | asset_list = pytest_gee.utils.get_assets(gee_folder_root)
37 | asset_list = [i["name"] for i in asset_list]
38 |
39 | # identify specific files and folders
40 | folder = gee_test_folder / "folder"
41 | image = folder / "image"
42 | feature_collection = folder / "fc"
43 |
44 | # check that they exist
45 | assert str(gee_test_folder) in asset_list
46 | assert str(folder) in asset_list
47 | assert str(image) in asset_list
48 | assert str(feature_collection) in asset_list
49 |
50 |
51 | def test_list_regression(ee_list_regression):
52 | """Test the ee_list_regression fixture."""
53 | data = ee.List([1, 2, 3])
54 | ee_list_regression.check(data)
55 |
56 |
57 | def test_list_regression_prescision(ee_list_regression):
58 | """Test the ee_list_regression fixture with a different precision."""
59 | data = ee.List([1.123456789, 2.123456789, 3.123456789])
60 | ee_list_regression.check(data, prescision=3)
61 |
62 |
63 | def test_feature_collection_regression(ee_feature_collection_regression):
64 | """Test the ee_feature_collection_regression fixture."""
65 | fc = ee.FeatureCollection("FAO/GAUL/2015/level0").filter(ee.Filter.eq("ADM0_NAME", "Holy See"))
66 | ee_feature_collection_regression.check(fc)
67 |
68 |
69 | def test_feature_collection_regression_prescision(ee_feature_collection_regression):
70 | """Test the ee_feature_collection_regression fixture."""
71 | fc = ee.FeatureCollection("FAO/GAUL/2015/level0").filter(ee.Filter.eq("ADM0_NAME", "Holy See"))
72 | ee_feature_collection_regression.check(fc, prescision=4)
73 |
74 |
75 | def test_feature_collection_regression_no_index(ee_feature_collection_regression):
76 | """Test the ee_feature_collection_regression fixture."""
77 | point = ee.Geometry.Point([0, 0])
78 | size = ee.List.sequence(50, 100, 10)
79 | geometries = size.map(lambda s: point.buffer(s, ee.Number(s).divide(5)))
80 | fc = ee.FeatureCollection(geometries.map(lambda g: ee.Feature(ee.Geometry(g))))
81 | ee_feature_collection_regression.check(fc, drop_index=True, prescision=4)
82 |
83 |
84 | def test_dictionary_regression(ee_dictionary_regression):
85 | """Test the ee_dictionary_regression fixture."""
86 | data = ee.Dictionary({"a": 1, "b": 2})
87 | ee_dictionary_regression.check(data)
88 |
89 |
90 | def test_dictionary_regression_prescision(ee_dictionary_regression):
91 | """Test the ee_dictionary_regression fixture with a different precision."""
92 | data = ee.Dictionary({"a": 1.123456789, "b": 2.123456789})
93 | ee_dictionary_regression.check(data, prescision=3)
94 |
95 |
96 | def test_image_regression_3_bands(ee_image_regression):
97 | """Test the image_regression fixture."""
98 | image = ee.Image(landsat_image).select(["SR_B4", "SR_B3", "SR_B2"])
99 | ee_image_regression.check(image, scale=1000)
100 |
101 |
102 | def test_image_regression_1_band(ee_image_regression):
103 | """Test the image_regression fixture."""
104 | image = ee.Image(landsat_image).normalizedDifference(["SR_B5", "SR_B4"])
105 | ee_image_regression.check(image, scale=1000)
106 |
107 |
108 | def test_image_regression_with_viz(ee_image_regression):
109 | """Test the image_regression fixture."""
110 | image = ee.Image(landsat_image).normalizedDifference(["SR_B5", "SR_B4"])
111 | # use magma palette and stretched to 2 sigma
112 | palette = ["#000004", "#2C105C", "#711F81", "#B63679", "#EE605E", "#FDAE78", "#FCFDBF"]
113 | viz = {"bands": ["nd"], "min": 0.0122, "max": 1.237, "palette": palette}
114 | ee_image_regression.check(image, scale=1000, viz_params=viz)
115 |
116 |
117 | def test_image_regression_with_region(ee_image_regression):
118 | """Test the image_regression fixture."""
119 | image = ee.Image(landsat_image).normalizedDifference(["SR_B5", "SR_B4"])
120 | vatican = ee.Geometry.Point([12.453585, 41.903115]).buffer(100)
121 | ee_image_regression.check(image, scale=30, region=vatican)
122 |
123 |
124 | def test_image_regression_with_overlay(ee_image_regression):
125 | """Test the image_regression fixture with overlay param."""
126 | image = ee.Image(landsat_image).select(["SR_B4", "SR_B3", "SR_B2"])
127 | centroid = image.geometry().centroid()
128 | overlay = ee.FeatureCollection(
129 | [ee.Feature(centroid, {"style": {"color": "red", "pointShape": "plus", "pointSize": 10}})]
130 | )
131 | ee_image_regression.check(image, scale=100, region=centroid.buffer(20000), overlay=overlay)
132 |
--------------------------------------------------------------------------------
/CODE_OF_CONDUCT.rst:
--------------------------------------------------------------------------------
1 | Contributor Covenant Code of Conduct
2 | ====================================
3 |
4 | Our Pledge
5 | ----------
6 |
7 | We as members, contributors, and leaders pledge to make participation in our
8 | community a harassment-free experience for everyone, regardless of age, body
9 | size, visible or invisible disability, ethnicity, sex characteristics, gender
10 | identity and expression, level of experience, education, socioeconomic status,
11 | nationality, personal appearance, race, religion, or sexual identity
12 | and orientation.
13 |
14 | We pledge to act and interact in ways that contribute to an open, welcoming,
15 | diverse, inclusive, and healthy community.
16 |
17 | Our Standards
18 | -------------
19 |
20 | Examples of behavior that contributes to a positive environment for our
21 | community include:
22 |
23 | * Demonstrating empathy and kindness toward other people
24 | * Being respectful of differing opinions, viewpoints, and experiences
25 | * Giving and gracefully accepting constructive feedback
26 | * Accepting responsibility and apologizing to those affected by our mistakes,
27 | and learning from the experience
28 | * Focusing on what is best not just for us as individuals, but for the
29 | overall community
30 |
31 | Examples of unacceptable behavior include:
32 |
33 | * The use of sexualized language or imagery, and sexual attention or
34 | advances of any kind
35 | * Trolling, insulting or derogatory comments, and personal or political attacks
36 | * Public or private harassment
37 | * Publishing others' private information, such as a physical or email
38 | address, without their explicit permission
39 | * Other conduct which could reasonably be considered inappropriate in a
40 | professional setting
41 |
42 | Enforcement Responsibilities
43 | ----------------------------
44 |
45 | Community leaders are responsible for clarifying and enforcing our standards of
46 | acceptable behavior and will take appropriate and fair corrective action in
47 | response to any behavior that they deem inappropriate, threatening, offensive,
48 | or harmful.
49 |
50 | Community leaders have the right and responsibility to remove, edit, or reject
51 | comments, commits, code, wiki edits, issues, and other contributions that are
52 | not aligned to this Code of Conduct, and will communicate reasons for moderation
53 | decisions when appropriate.
54 |
55 | Scope
56 | -----
57 |
58 | This Code of Conduct applies within all community spaces, and also applies when
59 | an individual is officially representing the community in public spaces.
60 | Examples of representing our community include using an official e-mail address,
61 | posting via an official social media account, or acting as an appointed
62 | representative at an online or offline event.
63 |
64 | Enforcement
65 | -----------
66 |
67 | Instances of abusive, harassing, or otherwise unacceptable behavior may be
68 | reported to the FAO team responsible for enforcement at
69 | pierrick.rambaud49@gmail.com.
70 | All complaints will be reviewed and investigated promptly and fairly.
71 |
72 | All community leaders are obligated to respect the privacy and security of the
73 | reporter of any incident.
74 |
75 | Enforcement Guidelines
76 | ----------------------
77 |
78 | Community leaders will follow these Community Impact Guidelines in determining
79 | the consequences for any action they deem in violation of this Code of Conduct:
80 |
81 | Correction
82 | ^^^^^^^^^^
83 |
84 | **Community Impact**: Use of inappropriate language or other behavior deemed
85 | unprofessional or unwelcome in the community.
86 |
87 | **Consequence**: A private, written warning from community leaders, providing
88 | clarity around the nature of the violation and an explanation of why the
89 | behavior was inappropriate. A public apology may be requested.
90 |
91 | Warning
92 | ^^^^^^^
93 |
94 | **Community Impact**: A violation through a single incident or series
95 | of actions.
96 |
97 | **Consequence**: A warning with consequences for continued behavior. No
98 | interaction with the people involved, including unsolicited interaction with
99 | those enforcing the Code of Conduct, for a specified period of time. This
100 | includes avoiding interactions in community spaces as well as external channels
101 | like social media. Violating these terms may lead to a temporary or
102 | permanent ban.
103 |
104 | Temporary Ban
105 | ^^^^^^^^^^^^^
106 |
107 | **Community Impact**: A serious violation of community standards, including
108 | sustained inappropriate behavior.
109 |
110 | **Consequence**: A temporary ban from any sort of interaction or public
111 | communication with the community for a specified period of time. No public or
112 | private interaction with the people involved, including unsolicited interaction
113 | with those enforcing the Code of Conduct, is allowed during this period.
114 | Violating these terms may lead to a permanent ban.
115 |
116 | Permanent Ban
117 | ^^^^^^^^^^^^^
118 |
119 | **Community Impact**: Demonstrating a pattern of violation of community
120 | standards, including sustained inappropriate behavior, harassment of an
121 | individual, or aggression toward or disparagement of classes of individuals.
122 |
123 | **Consequence**: A permanent ban from any sort of public interaction within
124 | the community.
125 |
126 | Attribution
127 | -----------
128 |
129 | This Code of Conduct is adapted from the [Contributor Covenant][homepage],
130 | version 2.0, available at
131 | https://www.contributor-covenant.org/version/2/0/code_of_conduct.html.
132 |
133 | Community Impact Guidelines were inspired by [Mozilla's code of conduct
134 | enforcement ladder](https://github.com/mozilla/diversity).
135 |
136 | [homepage]: https://www.contributor-covenant.org
137 |
138 | For answers to common questions about this code of conduct, see the FAQ at
139 | https://www.contributor-covenant.org/faq. Translations are available at
140 | https://www.contributor-covenant.org/translations.
141 |
--------------------------------------------------------------------------------
/pytest_gee/image_regression.py:
--------------------------------------------------------------------------------
1 | """implementation of the ``image_regression`` fixture."""
2 |
3 | import os
4 | from contextlib import suppress
5 | from typing import Optional, Union
6 |
7 | import ee
8 | import requests
9 | from pytest import fail
10 | from pytest_regressions.image_regression import ImageRegressionFixture
11 |
12 | from .utils import build_fullpath, check_serialized
13 |
14 |
15 | class ImageFixture(ImageRegressionFixture):
16 | """Fixture for regression testing of :py:class:`ee.Image`."""
17 |
18 | def check(
19 | self,
20 | data_image: ee.Image,
21 | diff_threshold: float = 0.1,
22 | expect_equal: bool = True,
23 | basename: Optional[str] = None,
24 | fullpath: Optional[os.PathLike] = None,
25 | scale: Optional[int] = 30,
26 | viz_params: Optional[dict] = None,
27 | region: Optional[Union[ee.FeatureCollection, ee.Feature, ee.Geometry]] = None,
28 | overlay: Optional[ee.FeatureCollection] = None,
29 | ):
30 | """Check the given image against a previously recorded version, or generate a new file.
31 |
32 | This method will create a thumbnail version of the requested image. It is made to allow a human user to check the result of the
33 | Computation. The thumbnail will be computed on the fly using earthengine. This mean that the test must be reasonable in size and scale.
34 | We will perform no feasibility checks and your computation might crash if you are too greedy.
35 | The input image will be either a single band image (displayed using black&white colormap) or a 3 band image (displayed using as fake RGB bands).
36 | If the ``viz_params`` parameter is omitted then it will detect the available ands, and use default viz params.
37 |
38 | Parameters:
39 | data_image: The image to check. The image needs to be clipped to a geometry or have an existing footprint.
40 | diff_threshold: The threshold for the difference between the expected and obtained images.
41 | expect_equal: If ``True`` the images are expected to be equal, otherwise they are expected to be different.
42 | basename: The basename of the file to test/record. If not given the name of the test is used.
43 | fullpath: complete path to use as a reference file. This option will ignore ``datadir`` fixture when reading *expected* files but will still use it to write *obtained* files. Useful if a reference file is located in the session data dir for example.
44 | scale: The scale to use for the thumbnail.
45 | viz_params: The visualization parameters to use for the thumbnail. If not given, the min and max values of the image will be used.
46 | region: The region to use for clipping the image. If not given, the image's region will be used.
47 | overlay: A FeatureCollection to draw on top of the image. The style will be taken from each Feature's "style" property.
48 | """
49 | # rescale the original image
50 | region = data_image if region is None else region
51 | geometry = region if isinstance(region, ee.Geometry) else region.geometry()
52 | data_image = data_image.clipToBoundsAndScale(geometry, scale=scale)
53 |
54 | # build the different filename to be consistent between our 3 checks
55 | data_name = build_fullpath(
56 | datadir=self.original_datadir,
57 | request=self.request,
58 | extension=".png",
59 | basename=basename,
60 | fullpath=fullpath,
61 | with_test_class_names=self.with_test_class_names,
62 | )
63 | serialized_name = data_name.with_stem(f"serialized_{data_name.stem}").with_suffix(".yml")
64 |
65 | # check the previously registered serialized call from GEE. If it matches the current call,
66 | # we don't need to check the data
67 | with suppress(AssertionError, fail.Exception):
68 | check_serialized(
69 | object=data_image,
70 | path=serialized_name,
71 | datadir=self.datadir,
72 | original_datadir=self.original_datadir,
73 | request=self.request,
74 | with_test_class_names=self.with_test_class_names,
75 | )
76 | return
77 |
78 | # delete the previously created file if wasn't successful
79 | serialized_name.unlink(missing_ok=True)
80 |
81 | # extract min and max for visualization
82 | minMax = data_image.reduceRegion(ee.Reducer.minMax(), geometry, scale)
83 |
84 | # create visualization parameters based on the computed minMax values
85 | if viz_params is None:
86 | nbBands = ee.Algorithms.If(data_image.bandNames().size().gte(3), 3, 1)
87 | bands = data_image.bandNames().slice(0, ee.Number(nbBands))
88 | min = bands.map(lambda b: minMax.get(ee.String(b).cat("_min")))
89 | max = bands.map(lambda b: minMax.get(ee.String(b).cat("_max")))
90 | viz_params = ee.Dictionary({"bands": bands, "min": min, "max": max}).getInfo()
91 |
92 | rgb = data_image.visualize(**viz_params)
93 | if overlay:
94 | rgb = rgb.blend(overlay.style(styleProperty="style"))
95 |
96 | # get the thumbnail image
97 | thumb_url = rgb.getThumbURL()
98 | byte_data = requests.get(thumb_url).content
99 |
100 | # if it needs to be checked, we need to round the float values to the same precision as the
101 | # reference file
102 | try:
103 | super().check(byte_data, diff_threshold, expect_equal, fullpath=data_name)
104 |
105 | # IF we are here it means the data has been modified so we edit the API call accordingly
106 | # to make sure next run will not be forced to call the API for a response.
107 | with suppress(AssertionError, fail.Exception):
108 | check_serialized(
109 | object=data_image,
110 | path=serialized_name,
111 | datadir=self.datadir,
112 | original_datadir=self.original_datadir,
113 | request=self.request,
114 | with_test_class_names=self.with_test_class_names,
115 | force_regen=True,
116 | )
117 |
118 | except (AssertionError, fail.Exception) as e:
119 | raise e
120 |
--------------------------------------------------------------------------------
/docs/content/installation.rst:
--------------------------------------------------------------------------------
1 | Installation
2 | ============
3 |
4 | Getting started
5 | ---------------
6 |
7 | Use pip/conda to install **pytest-gee** in your environment:
8 |
9 | .. tab-set::
10 |
11 | .. tab-item:: pip
12 |
13 | .. code-block:: console
14 |
15 | pip install pytest-gee
16 |
17 | .. tab-item:: conda
18 |
19 | .. code-block:: console
20 |
21 | conda install -c conda-forge pytest-gee
22 |
23 | It will then be automatically detected by ``pytest`` when you run your test suit.
24 |
25 | Connect To Google Earth Engine
26 | ------------------------------
27 |
28 | The main purpose of this plugin is to facilitate the connection to Google Earth Engine API in both CI/CD tests and local tests.
29 | To do so, the lib will provide a number of connection methods that will hopefully cover your favorite way of connecting to GEE.
30 |
31 | .. note::
32 |
33 | If you would like us to code an extra connection method please `open an issue `__ on the github repo and never forget that contribution are very welcome!
34 |
35 | .. note::
36 |
37 | All the methods presented in this section will fallback to a regular ``ee.Initialize()`` if the environment parameter are not found.
38 | This means that you can use this plugin in your local environment without having to change anything as long as the ``ee`` module is installed and that you already run once ``ee.Authenticate()``.
39 |
40 | .. danger::
41 |
42 | Never forget that this method can potentially expose your personal credential to GEE so take some safety precautions before starting:
43 |
44 | - make sure the CI/CD platform support private variable (that are not exposed in the build logs)
45 | - make sure to review PR from new users before starting the build to make sure nobody steal your credentials
46 | - make sure the account you are using will have access to all the assets you need to run your tests
47 | - create small tests that will run quickly to make sure you don't overload your own GEE account with concurrent tasks
48 |
49 | Private Token
50 | ^^^^^^^^^^^^^
51 |
52 | The first method is to use a private token. This is the easiest way to connect to GEE in a CI/CD environment.
53 |
54 | First authenticate to GEE API in your local computer using ``ee.Authenticate()``.
55 |
56 | Then copy the ``credentials`` file content. This file is located in a different folder depending on the platform you use:
57 |
58 | - Windows: ``C:\Users\USERNAME\\.config\\earthengine\\credentials``
59 | - Linux: ``/home/USERNAME/.config/earthengine/credentials``
60 | - MacOS: ``/Users/USERNAME/.config/earthengine/credentials``
61 |
62 | Paste this content in your CI/CD environment in a ``EARTHENGINE_TOKEN`` variable.
63 |
64 | Here is a github action example:
65 |
66 | .. image:: ../_static/github_env_var.png
67 | :alt: Github action environment variable setup
68 | :align: center
69 |
70 | #. First go to the :guilabel:`settings`` of your Github repository
71 | #. Then to :guilabel:`secretes and variables` -> :guilabel:`Actions`
72 | #. In this page, set a :guilabel:`new repository secret` with the name ``EARTHENGINE_TOKEN`` and paste the content of your ``credentials`` file in the value field.
73 |
74 | Since earthengine-api v0.1.370, it's not possible to use EE without providing a GCS project bucket. Save this value in a `EARTHENGINE_PROJECT` variable, it will be used in the method.
75 |
76 | To make the variable available in your CI environment, you will need to add the following line in your action `.yaml` file:
77 |
78 | .. code-block:: yaml
79 |
80 | # .github/action.yaml
81 |
82 | env:
83 | EARTHENGINE_TOKEN: ${{ secrets.EARTHENGINE_TOKEN }}
84 | EARTHENGINE_PROJECT: ${{ secrets.EARTHENGINE_PROJECT }}
85 |
86 | # The rest of your tests configuration
87 |
88 | When working in your local environment export a ``EARTHENGINE_PROJECT`` variable as well:
89 |
90 | .. code-block:: console
91 |
92 | export EARTHENGINE_PROJECT=ee-community
93 |
94 | Finally you need to configure the ``pytest`` execution environment itself. Add the following line in your ``conftest.py`` file:
95 |
96 | .. code-block:: python
97 |
98 | # conftest.py
99 |
100 | import pytest_gee
101 |
102 |
103 | def pytest_configure():
104 | pytest_gee.init_ee_from_token()
105 |
106 | You are now ready to make API calls within your tests!
107 |
108 | Service account
109 | ^^^^^^^^^^^^^^^
110 |
111 | .. warning::
112 |
113 | This documentation assumes that you already have a Google cloud service account and that you have generated an API key for it. If not, please refer to Google own `documentation `__ to proceed.
114 |
115 | Paste this content of the `private-key.json` in your CI/CD environment in a ``EARTHENGINE_SERVICE_ACCOUNT`` variable.
116 |
117 | Here is a github action example:
118 |
119 | .. image:: ../_static/github_env_var.png
120 | :alt: Github action environment variable setup
121 | :align: center
122 |
123 | #. First go to the :guilabel:`settings` of your Github repository
124 | #. Then to :guilabel:`secretes and variables` -> :guilabel:`Actions`
125 | #. In this page, set a :guilabel:`new repository secret` with the name ``EARTHENGINE_SERVICE_ACCOUNT`` and paste the content of your ``credentials`` file in the value field.
126 |
127 | Currently when the earthengine-api is Initialized using a service account, the name of the associated cloud project is not detectable. It will prevent the initialization of the test folder generated from `pytest-gee`. To avoid this issue the method rely also on a ``EARTHENGINE_PROJECT`` env variable where you can set the name of your project.
128 |
129 | To make the variable available in your CI environment, you will need to add the following line in your action `.yaml` file:
130 |
131 | .. code-block:: yaml
132 |
133 | # .github/action.yaml
134 |
135 | env:
136 | EARTHENGINE_SERVICE_ACCOUNT: ${{ secrets.EARTHENGINE_SERVICE_ACCOUNT }}
137 | EARTHENGINE_PROJECT: ${{ secrets.EARTHENGINE_PROJECT }}
138 |
139 | # The rest of your tests configuration
140 |
141 | When working in your local environment export a ``EARTHENGINE_PROJECT`` variable as well:
142 |
143 | .. code-block:: console
144 |
145 | export EARTHENGINE_PROJECT=ee-community
146 |
147 | Finally you need to configure the ``pytest`` execution environment itself. Add the following line in your ``conftest.py`` file:
148 |
149 | .. code-block:: python
150 |
151 | # conftest.py
152 |
153 | import pytest_gee
154 |
155 |
156 | def pytest_configure():
157 | pytest_gee.init_ee_from_service_account()
158 |
159 | You are now ready to make API calls within your tests!
160 |
--------------------------------------------------------------------------------
/docs/content/regression.rst:
--------------------------------------------------------------------------------
1 | Regression fixtures
2 | ===================
3 |
4 | **pytest-gee** fixtures that will make it easy to generate test data for your test suits using Earth
5 | Engine server side objects as sources. It is based on the ``pytest-regressions`` plugin and add GEE
6 | data management backend. We highly recommend users to read their `documentation `__
7 | to understand how to use some parameter of the fixtures like the saving directory and the management
8 | of parametrized tests.
9 |
10 | .. note::
11 |
12 | The fixtures provided by this plugin are based on the ``pytest-regressions`` plugin and are fully
13 | compatible with it. This means that you can use the ``pytest-regressions`` fixtures and the ``pytest-gee``
14 | fixtures in the same test suit.
15 |
16 | ee_list_regression
17 | ------------------
18 |
19 | Suppose the output of our test is a :py:class:`ee.List` object containing informmation. We could
20 | test the result as follow:
21 |
22 | .. code-block:: python
23 |
24 | def test_list():
25 | list = ee.List([ee.Number(0.11111111), ee.String("test")])
26 | assert list.getInfo()[0] == 0.11111111
27 | assert list.getInfo()[1] == "test"
28 |
29 | But this presents a number of problems:
30 |
31 | - Gets old quickly.
32 | - Error-prone.
33 | - If a check fails, we don’t know what else might be wrong with the obtained data.
34 | - Does not scale for large data.
35 |
36 | Instead, we can use the :py:func:`ee_list_regression ` fixture to save
37 | the expected output and compare it with the obtained output in a humanly readable format:
38 |
39 | .. code-block:: python
40 |
41 | def test_list(ee_list_regression):
42 | list = ee.List([ee.Number(0.11111111), ee.String("test")])
43 | ee_list_regression.check(list)
44 |
45 | That will be saved in a yaml file:
46 |
47 | .. code-block:: yaml
48 |
49 | # //.yaml
50 | - 0.11111111
51 | - test
52 |
53 | ee_dictionary_regression
54 | ------------------------
55 |
56 | Suppose the output of our test is a :py:class:`ee.Dictionary` object containing informmation. We could
57 | test the result as follow:
58 |
59 | .. code-block:: python
60 |
61 | def test_dict():
62 | dict = ee.Dictionary({"key1": ee.Number(0.11111111), "key2": ee.String("test")})
63 | assert dict.getInfo()["key1"] == 0.11111111
64 | assert dict.getInfo()["key2"] == "test"
65 |
66 | But this presents a number of problems:
67 |
68 | - Gets old quickly.
69 | - Error-prone.
70 | - If a check fails, we don’t know what else might be wrong with the obtained data.
71 | - Does not scale for large data.
72 |
73 | Instead, we can use the :py:func:`ee_dictionary_regression ` fixture
74 | to save the expected output and compare it with the obtained output in a humanly readable format:
75 |
76 | .. code-block:: python
77 |
78 | def test_dict(ee_dictionary_regression):
79 | dict = ee.Dictionary({"key1": ee.Number(0.11111111), "key2": ee.String("test")})
80 | ee_dictionary_regression.check(dict)
81 |
82 | That will be saved in a yaml file:
83 |
84 | .. code-block:: yaml
85 |
86 | # //.yaml
87 | key1: 0.11111111
88 | key2: test
89 |
90 |
91 | ee_feature_collection_regression
92 | --------------------------------
93 |
94 | Suppose the output of our test is a :py:class:`ee.FeatureCollection` object containing informmation.
95 | We could test the result as follow:
96 |
97 | .. code-block:: python
98 |
99 | def test_fc():
100 | fc = ee.FeatureCollection([ee.Feature(ee.Geometry.Point([0, 0]), {"key1": 0.11111111, "key2": "test"})])
101 | assert fc.getInfo()["features"][0]["properties"]["key1"] == 0.11111111
102 | assert fc.getInfo()["features"][0]["properties"]["key2"] == "test"
103 | assert fc.getInfo()["features"][0]["geometry"]["type"] == "Point"
104 |
105 | Instead we can use the :py:func:`ee_feature_collection_regression `
106 | fixture to save the expected output and compare it with the obtained output in a humanly readable format
107 | compatible with the ``geo_interface`` standard:
108 |
109 | .. code-block:: python
110 |
111 | def test_fc(ee_feature_collection_regression):
112 | fc = ee.FeatureCollection([ee.Feature(ee.Geometry.Point([0, 0]), {"key1": 0.11111111, "key2": "test"})])
113 | ee_feature_collection_regression.check(fc)
114 |
115 | That will be saved in a yaml file:
116 |
117 | .. code-block:: yaml
118 |
119 | # //.yaml
120 | type: FeatureCollection
121 | features:
122 | - type: Feature
123 | geometry:
124 | type: Point
125 | coordinates:
126 | - 0
127 | - 0
128 | properties:
129 | key1: 0.11111111
130 | key2: test
131 |
132 | ee_image_regression
133 | -------------------
134 |
135 | Suppose the output of our test is a :py:class:`ee.Image` object containing informmation. We could test the result as follow:
136 |
137 | .. code-block:: python
138 |
139 | import ee
140 | import pytest
141 |
142 | def test_image():
143 | image = ee.Image("LANDSAT/LC08/C02/T1_L2/LC08_191031_20240607")
144 | image = image.normalizedDifference(["SR_B5", "SR_B4"])
145 | assert image.bandNames().size().getInfo() == 1
146 |
147 | Instead we can use the :py:func:`ee_image_regression ` fixture to save the expected output and compare
148 | it with the obtained output as a processed `.png` image:
149 |
150 | .. code-block:: python
151 |
152 | import ee
153 | import pytest
154 |
155 | def test_image(ee_image_regression):
156 | image = ee.Image("LANDSAT/LC08/C02/T1_L2/LC08_191031_20240607")
157 | image = image.normalizedDifference(["SR_B5", "SR_B4"])
158 | ee_image_regression.check(image, scale=1000)
159 |
160 | That will be saved in a png file:
161 |
162 | .. image:: ../_static/ee_image_regression.png
163 | :alt: ee.Image regression
164 |
165 | :py:func:`ee_image_regression ` comes with more extra options than the other regressions as you need to give information
166 | about the scale, region and bands you want to see on the final output. First the image must be **small** as the
167 | image is generated via the :py:meth:`ee.Image.getThumbURL` method hat has a maximum limit of pixels.
168 | By default the fixture will autodetect the number of bands and display either a black&white single band or a
169 | RGB representation of the 3 first bands. You can customize the output by passing the ``viz_params`` argument to the
170 | :py:func:`check() ` method. The ``viz_params`` argument is a dictionary that
171 | is the same as the one used in the :py:meth:`ee.Image.getThumbURL` method. For example to display the image in
172 | magma colormap with the value stretched to 2 standard deviation (instead of the default min-max) you can do:
173 |
174 | .. code-block:: python
175 |
176 | import ee
177 | import pytest
178 |
179 | def test_image(ee_image_regression):
180 | image = ee.Image("LANDSAT/LC08/C02/T1_L2/LC08_191031_20240607")
181 | image = image.normalizedDifference(["SR_B5", "SR_B4"])
182 | palette = ["#000004", "#2C105C", "#711F81", "#B63679", "#EE605E", "#FDAE78", "#FCFDBF"]
183 | viz = {"bands": ["nd"], "min": 0.0122, "max": 1.237, "palette": palette}
184 | ee_image_regression.check(image, viz_params=viz, scale=1000)
185 |
186 | .. image:: ../_static/ee_image_regression_viz.png
187 | :alt: ee.Image regression with custom viz_params
188 |
--------------------------------------------------------------------------------
/docs/_static/logo.svg:
--------------------------------------------------------------------------------
1 |
2 |
3 |
18 |
--------------------------------------------------------------------------------
/pytest_gee/utils.py:
--------------------------------------------------------------------------------
1 | """functions used to build the API that we don't want to expose to end users.
2 |
3 | .. danger::
4 |
5 | This module is for internal use only and should not be used directly.
6 | """
7 |
8 | from __future__ import annotations
9 |
10 | import json
11 | import os
12 | import re
13 | from functools import partial
14 | from pathlib import Path, PurePosixPath
15 | from typing import List, Optional, Union
16 | from warnings import warn
17 |
18 | import ee
19 | import pytest
20 | import yaml
21 | from deprecated.sphinx import deprecated
22 | from ee.cli.utils import wait_for_task
23 | from pytest_regressions.common import check_text_files, perform_regression_check
24 | from pytest_regressions.data_regression import RegressionYamlDumper
25 |
26 | TASK_FINISHED_STATES: tuple[str, str, str] = (
27 | ee.batch.Task.State.COMPLETED,
28 | ee.batch.Task.State.FAILED,
29 | ee.batch.Task.State.CANCELLED,
30 | )
31 |
32 |
33 | @deprecated(version="0.3.5", reason="Use the vanilla GEE ``wait_for_task`` function instead.")
34 | def wait(task: Union[ee.batch.Task, str], timeout: int = 10 * 60) -> str:
35 | """Wait until the selected process is finished or we reached timeout value.
36 |
37 | Args:
38 | task: name of the running task or the Task object itself.
39 | timeout: timeout in seconds. if set to 0 the parameter is ignored. default to 1 minutes.
40 |
41 | Returns:
42 | the final state of the task
43 | """
44 | task_id = task.id if isinstance(task, ee.batch.Task) else task
45 | return wait_for_task(task_id, timeout, log_progress=False)
46 |
47 |
48 | def get_task(task_descripsion: str) -> Optional[ee.batch.Task]:
49 | """Search for the described task in the user Task list return None if nothing is found.
50 |
51 | Args:
52 | task_descripsion: the task description
53 |
54 | Returns:
55 | return the found task else None
56 | """
57 | task = None
58 | for t in ee.batch.Task.list():
59 | if t.config["description"] == task_descripsion:
60 | task = t
61 | break
62 |
63 | return task
64 |
65 |
66 | def get_assets(folder: Union[str, Path]) -> List[dict]:
67 | """Get all the assets from the parameter folder. every nested asset will be displayed.
68 |
69 | Args:
70 | folder: the initial GEE folder
71 |
72 | Returns:
73 | the asset list. each asset is a dict with 3 keys: 'type', 'name' and 'id'
74 | """
75 | # set the folder and init the list
76 | asset_list: list = []
77 | folder = folder if isinstance(folder, str) else folder.as_posix()
78 |
79 | # recursive function to get all the assets
80 | def _recursive_get(folder, asset_list):
81 | for asset in ee.data.listAssets({"parent": folder})["assets"]:
82 | asset_list.append(asset)
83 | if asset["type"] in ["FOLDER", "IMAGE_COLLECTION"]:
84 | asset_list = _recursive_get(asset["name"], asset_list)
85 | return asset_list
86 |
87 | return _recursive_get(folder, asset_list)
88 |
89 |
90 | def export_asset(
91 | object: ee.ComputedObject, asset_id: Union[str, Path], description: str
92 | ) -> PurePosixPath:
93 | """Export assets to the GEE platform, only working for very simple objects.
94 |
95 | Args:
96 | object: the object to export
97 | asset_id: the name of the asset to create
98 | description: the description of the task
99 |
100 | Returns:
101 | the path of the created asset
102 | """
103 | # convert the asset_id to a string note that GEE only supports unix style separator
104 | asset_id = asset_id if isinstance(asset_id, str) else asset_id.as_posix()
105 |
106 | if isinstance(object, ee.FeatureCollection):
107 | task = ee.batch.Export.table.toAsset(
108 | collection=object,
109 | description=description,
110 | assetId=asset_id,
111 | )
112 | elif isinstance(object, ee.Image):
113 | task = ee.batch.Export.image.toAsset(
114 | region=object.geometry(),
115 | image=object,
116 | description=description,
117 | assetId=asset_id,
118 | )
119 | else:
120 | raise ValueError("Only ee.Image and ee.FeatureCollection are supported")
121 |
122 | # launch the task and wait for the end of exportation
123 | task.start()
124 | wait_for_task(task.id, 10 * 60, False)
125 |
126 | return PurePosixPath(asset_id)
127 |
128 |
129 | def _create_container(asset_request: str) -> str:
130 | """Create a container for the asset request depending on the requested type.
131 |
132 | Args:
133 | asset_request: the asset request specifying the type of asset to create. Convention is ::
134 |
135 | Returns:
136 | the asset_id of the container
137 | """
138 | # deprecation management for older version of the lib
139 | parts = asset_request.split("::")
140 | if len(parts) == 1:
141 | parts.append("Folder")
142 | warn(f"Asset {asset_request} is not specifying asset Type, it will be created as a FOLDER.")
143 |
144 | # extract the asset_id and the asset_type from the different parts
145 | # if more than 2 splits are identified they will be ignored
146 | asset_id, asset_type = parts[:2]
147 |
148 | # create the container
149 | if asset_type in ["Folder", "ImageCollection"]:
150 | ee.data.createAsset({"type": asset_type}, asset_id)
151 | else:
152 | raise ValueError(f"Asset type {asset_type} is not supported.")
153 |
154 | return asset_id
155 |
156 |
157 | def init_tree(structure: dict, prefix: str, root: Union[str, PurePosixPath]) -> PurePosixPath:
158 | """Create an EarthEngine folder tree from a dictionary.
159 |
160 | The input ditionary should described the structure of the folder you want to create.
161 | The keys are the folder names and the values are the subfolders.
162 | Once you reach an ``ee.FeatureCollection`` and/or an ``ee.Image`` set it in the dictionary and the function will export the object.
163 |
164 | Args:
165 | structure: the structure of the folder to create
166 | prefix: the prefix to use on every item (folder, tasks, asset_id, etc.)
167 | root: the root folder of the test where to create the test folder.
168 |
169 | Returns:
170 | the path of the created folder
171 |
172 | Examples:
173 | >>> structure = {
174 | ... "folder_1": {
175 | ... "image": ee.image(1),
176 | ... "fc": ee.FeatureCollection(ee.Geometry.Point([0, 0])),
177 | ... },
178 | ... }
179 | ... init_tree(structure, "toto")
180 | """
181 |
182 | # recursive function to create the folder tree
183 | def _recursive_create(structure, prefix, folder):
184 | for name, content in structure.items():
185 | asset_id = PurePosixPath(folder) / name
186 | description = f"{prefix}_{name}"
187 | if isinstance(content, dict):
188 | asset_id = _create_container(str(asset_id))
189 | _recursive_create(content, prefix, asset_id)
190 | else:
191 | export_asset(content, asset_id, description)
192 |
193 | # create the root folder
194 | root = PurePosixPath(root) if isinstance(root, str) else root
195 | root_folder = f"{root.as_posix()}/{prefix}"
196 | ee.data.createFolder(root_folder)
197 |
198 | # start the recursive function
199 | _recursive_create(structure, prefix, root_folder)
200 |
201 | return PurePosixPath(root_folder)
202 |
203 |
204 | def delete_assets(asset_id: Union[str, Path], dry_run: bool = True) -> list:
205 | """Delete the selected asset and all its content.
206 |
207 | This method will delete all the files and folders existing in an asset folder.
208 | By default a dry run will be launched and if you are satisfyed with the displayed names, change the ``dry_run`` variable to ``False``.
209 | No other warnng will be displayed.
210 |
211 | .. warning::
212 |
213 | If this method is used on the root directory you will loose all your data, it's highly recommended to use a dry run first and carefully review the destroyed files.
214 |
215 | Args:
216 | asset_id: the Id of the asset or a folder
217 | dry_run: whether or not a dry run should be launched. dry run will only display the files name without deleting them.
218 |
219 | Returns:
220 | a list of all the files deleted or to be deleted
221 | """
222 | # convert the asset_id to a string
223 | asset_id = asset_id if isinstance(asset_id, str) else asset_id.as_posix()
224 |
225 | # define a delete function to change the behaviour of the method depending of the mode
226 | # in dry mode, the function only store the assets to be destroyed as a dictionary.
227 | # in non dry mode, the function store the asset names in a dictionary AND delete them.
228 | output = []
229 |
230 | def delete(id: str):
231 | output.append(id)
232 | dry_run is True or ee.data.deleteAsset(id)
233 |
234 | # identify the type of asset
235 | asset_info = ee.data.getAsset(asset_id)
236 |
237 | if asset_info["type"] in ["FOLDER", "IMAGE_COLLECTION"]:
238 | # get all the assets
239 | asset_list = get_assets(folder=asset_id)
240 |
241 | # split the files by nesting levels
242 | # we will need to delete the more nested files first
243 | assets_ordered: dict = {}
244 | for asset in asset_list:
245 | lvl = len(asset["id"].split("/"))
246 | assets_ordered.setdefault(lvl, [])
247 | assets_ordered[lvl].append(asset)
248 |
249 | # delete all items starting from the more nested ones
250 | assets_ordered = dict(sorted(assets_ordered.items(), reverse=True))
251 | print(assets_ordered)
252 | for lvl in assets_ordered:
253 | for i in assets_ordered[lvl]:
254 | delete(i["name"])
255 |
256 | # delete the initial folder/asset
257 | delete(asset_id)
258 |
259 | return output
260 |
261 |
262 | def round_data(data: Union[list, dict], prescision: int = 6) -> Union[list, dict]:
263 | """Recusrsively Round the values of a list to the given prescision."""
264 | # change the generator depending on the collection type
265 | generator = enumerate(data) if isinstance(data, list) else data.items()
266 | for k, v in generator:
267 | if isinstance(v, (list, dict)):
268 | data[k] = round_data(v, prescision)
269 | elif isinstance(v, float):
270 | data[k] = round(v, prescision)
271 | else:
272 | data[k] = v
273 | return data
274 |
275 |
276 | def build_fullpath(
277 | datadir: Path,
278 | request: pytest.FixtureRequest,
279 | extension: str,
280 | basename: Optional[str] = None,
281 | fullpath: Optional["os.PathLike[str]"] = None,
282 | with_test_class_names: bool = False,
283 | ) -> Path:
284 | """Generate a fullpath from parameters of the test.
285 |
286 | Args:
287 | datadir: Fixture embed_data.
288 | request: Pytest request object.
289 | extension: Extension of files compared by this check.
290 | basename: basename of the file to test/record. If not given the name of the test is used. Use either `basename` or `fullpath`.
291 | fullpath: complete path to use as a reference file. This option will ignore ``datadir`` fixture when reading *expected* files but will still use it to write *obtained* files. Useful if a reference file is located in the session data dir for example.
292 | with_test_class_names: if true it will use the test class name (if any) to compose the basename.
293 | """
294 | assert not (basename and fullpath), "pass either basename or fullpath, but not both"
295 |
296 | __tracebackhide__ = True
297 |
298 | with_test_class_names = with_test_class_names or request.config.getoption(
299 | "with_test_class_names"
300 | )
301 |
302 | if basename is None:
303 | if (request.node.cls is not None) and (with_test_class_names):
304 | basename = re.sub(r"[\W]", "_", request.node.cls.__name__) + "_"
305 | else:
306 | basename = ""
307 | basename += re.sub(r"[\W]", "_", request.node.name)
308 |
309 | if fullpath:
310 | filename = Path(fullpath)
311 | else:
312 | filename = (datadir / basename).with_suffix(extension)
313 |
314 | return filename
315 |
316 |
317 | def check_serialized(
318 | object: ee.ComputedObject,
319 | path: Path,
320 | datadir: Path,
321 | original_datadir: Path,
322 | request: pytest.FixtureRequest,
323 | force_regen: bool = False,
324 | with_test_class_names: bool = False,
325 | ):
326 | """Check if the serialized GEE object is the same as the saved one.
327 |
328 | Args:
329 | object: the earthnegine object to check
330 | path: the full path to the file to check against.
331 | datadir: Fixture embed_data.
332 | original_datadir: Fixture embed_data.
333 | request: Pytest request object.
334 | force_regen: if True, the file will be regenerated even if it exists.
335 | with_test_class_names: if true it will use the test class name (if any) to compose the basename.
336 |
337 | Raise:
338 | AssertionError if the serialized object is different from the saved one.
339 | """
340 | # serialize the object# extract the data from the computed object
341 | data_dict = json.loads(object.serialize())
342 |
343 | # delete the file upstream if force_regen is set
344 | if force_regen is True:
345 | path.unlink(missing_ok=True)
346 |
347 | def dump(filename: Path) -> None:
348 | """Dump dict contents to the given filename."""
349 | dumped_str = yaml.dump_all(
350 | [data_dict],
351 | Dumper=RegressionYamlDumper,
352 | default_flow_style=False,
353 | allow_unicode=True,
354 | indent=2,
355 | encoding="utf-8",
356 | )
357 | filename.write_bytes(dumped_str)
358 |
359 | # check the previously registered serialized call from GEE. If it matches the current call,
360 | # we don't need to check the data
361 | perform_regression_check(
362 | datadir=datadir,
363 | original_datadir=original_datadir,
364 | request=request,
365 | check_fn=partial(check_text_files, encoding="UTF-8"),
366 | dump_fn=dump,
367 | extension=".yml",
368 | fullpath=path,
369 | with_test_class_names=with_test_class_names,
370 | )
371 |
--------------------------------------------------------------------------------
