├── .github
├── FUNDING.yml
└── workflows
│ ├── python-package.yml
│ └── readme-publish.yml
├── .gitignore
├── CNAME
├── LICENSE
├── README.md
├── docs
├── CNAME
├── changelog.md
├── index.md
├── installation.md
├── license.md
├── projects
│ ├── access.md
│ ├── app2script.md
│ ├── cancel_tasks.md
│ ├── copy.md
│ ├── delete.md
│ ├── delete_metadata.md
│ ├── move.md
│ ├── projects.md
│ ├── quota.md
│ ├── report.md
│ ├── search.md
│ ├── size.md
│ └── task_status.md
└── stylesheets
│ ├── codehilite.css
│ └── dark_theme.css
├── geeadd
├── __init__.py
├── acl_changer.py
├── app2script.py
├── batch_copy.py
├── batch_delete.py
├── batch_mover.py
├── ee_del_meta.py
├── ee_projects.py
├── ee_report.py
└── geeadd.py
├── mkdocs.yml
├── overrides
├── assets
│ ├── javascripts
│ │ ├── custom.a678ee80.min.js
│ │ ├── custom.a678ee80.min.js.map
│ │ └── iconsearch_index.json
│ └── stylesheets
│ │ ├── custom.f7ec4df2.min.css
│ │ └── custom.f7ec4df2.min.css.map
└── main.html
├── requirements.txt
└── setup.py
/.github/FUNDING.yml:
--------------------------------------------------------------------------------
1 | github: samapriya
2 | custom:
3 | - buymeacoffee.com/samapriya
4 |
--------------------------------------------------------------------------------
/.github/workflows/python-package.yml:
--------------------------------------------------------------------------------
1 | # This workflow will install Python dependencies, run tests and lint with a variety of Python versions
2 | # For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
3 |
4 | name: CI geeadd
5 |
6 | on:
7 | push:
8 | branches: [master]
9 | pull_request:
10 | branches: [master]
11 |
12 | jobs:
13 | build:
14 | runs-on: ${{ matrix.os }}
15 | strategy:
16 | matrix:
17 | os: [macos-latest, ubuntu-latest, windows-latest]
18 | python-version: ["3.9", "3.10"]
19 | steps:
20 | - name: Checkout repository
21 | continue-on-error: true
22 | uses: actions/checkout@v3
23 | with:
24 | fetch-depth: 0
25 | - uses: actions/setup-python@v4
26 | with:
27 | python-version: ${{ matrix.python-version }}
28 | - name: Create whl files
29 | continue-on-error: true
30 | run: |
31 | pip install setuptools
32 | pip install wheel
33 | python setup.py sdist bdist_wheel
34 | cd dist
35 | - name: List & install wheel files(ubuntu)
36 | continue-on-error: true
37 | if: matrix.os == 'ubuntu-latest'
38 | run: |
39 | for file in dist/*.whl; do
40 | echo "Installing $file..."
41 | pip install "$file"
42 | done
43 | - name: List & install wheel files(mac)
44 | continue-on-error: true
45 | if: matrix.os == 'macos-latest'
46 | run: |
47 | for file in dist/*.whl; do
48 | echo "Installing $file..."
49 | pip install "$file"
50 | done
51 | - name: List & install wheel files(windows)
52 | continue-on-error: true
53 | if: matrix.os == 'windows-latest'
54 | run: |
55 | foreach ($file in Get-ChildItem -Path dist -Filter *.whl) {
56 | Write-Host "Installing $file..."
57 | pip.exe install "$file"
58 | }
59 | - name: test pkg
60 | run: |
61 | geeadd -h
62 |
--------------------------------------------------------------------------------
/.github/workflows/readme-publish.yml:
--------------------------------------------------------------------------------
1 | name: Publish docs via GitHub
2 | on:
3 | workflow_dispatch:
4 | push:
5 | branches:
6 | - master
7 |
8 | jobs:
9 | deploy:
10 | runs-on: ubuntu-latest
11 | steps:
12 | - name: Checkout repository
13 | uses: actions/checkout@v3
14 | with:
15 | fetch-depth: 0
16 | - uses: actions/setup-python@v4
17 | with:
18 | python-version: "3.10"
19 | - name: Install Python dependencies
20 | run: |
21 | pip install \
22 | "wheel" \
23 | "lxml" \
24 | "mkdocs-material" \
25 | "cairosvg>=2.5" \
26 | "mkdocs-git-committers-plugin-2>=1.1.1" \
27 | "mkdocs-git-revision-date-localized-plugin>=1.0" \
28 | "mkdocs-minify-plugin>=0.3" \
29 | "mkdocs-rss-plugin>=1.2" \
30 | "mkdocs-redirects>=1.0" \
31 | "pillow<10"
32 | - name: Deploy documentation
33 | env:
34 | GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
35 | run: |
36 | mkdocs gh-deploy --force
37 | mkdocs --version
38 |
--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
1 | # Byte-compiled / optimized / DLL files
2 | __pycache__/
3 | *.py[cod]
4 | *$py.class
5 |
6 | # C extensions
7 | *.so
8 |
9 |
10 | # Distribution / packaging
11 | .Python
12 | build/
13 | develop-eggs/
14 | dist/
15 | downloads/
16 | eggs/
17 | .eggs/
18 | lib/
19 | lib64/
20 | parts/
21 | sdist/
22 | var/
23 | wheels/
24 | share/python-wheels/
25 | *.egg-info/
26 | .installed.cfg
27 | *.egg
28 | MANIFEST
29 |
30 | # PyInstaller
31 | # Usually these files are written by a python script from a template
32 | # before PyInstaller builds the exe, so as to inject date/other infos into it.
33 | *.manifest
34 | *.spec
35 |
36 | # Installer logs
37 | pip-log.txt
38 | pip-delete-this-directory.txt
39 |
40 | # Unit test / coverage reports
41 | htmlcov/
42 | .tox/
43 | .nox/
44 | .coverage
45 | .coverage.*
46 | .cache
47 | nosetests.xml
48 | coverage.xml
49 | *.cover
50 | *.py,cover
51 | .hypothesis/
52 | .pytest_cache/
53 | cover/
54 |
55 | # Translations
56 | *.mo
57 | *.pot
58 |
59 | # Django stuff:
60 | *.log
61 | local_settings.py
62 | db.sqlite3
63 | db.sqlite3-journal
64 |
65 | # Flask stuff:
66 | instance/
67 | .webassets-cache
68 |
69 | # Scrapy stuff:
70 | .scrapy
71 |
72 | # Sphinx documentation
73 | docs/_build/
74 |
75 | # PyBuilder
76 | .pybuilder/
77 | target/
78 |
79 | # Jupyter Notebook
80 | .ipynb_checkpoints
81 |
82 | # IPython
83 | profile_default/
84 | ipython_config.py
85 |
86 | # pyenv
87 | # For a library or package, you might want to ignore these files since the code is
88 | # intended to run in multiple environments; otherwise, check them in:
89 | # .python-version
90 |
91 | # pipenv
92 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
93 | # However, in case of collaboration, if having platform-specific dependencies or dependencies
94 | # having no cross-platform support, pipenv may install dependencies that don't work, or not
95 | # install all needed dependencies.
96 | #Pipfile.lock
97 |
98 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow
99 | __pypackages__/
100 |
101 | # Celery stuff
102 | celerybeat-schedule
103 | celerybeat.pid
104 |
105 | # SageMath parsed files
106 | *.sage.py
107 |
108 | # Environments
109 | .env
110 | .venv
111 | env/
112 | venv/
113 | ENV/
114 | env.bak/
115 | venv.bak/
116 |
117 | # Spyder project settings
118 | .spyderproject
119 | .spyproject
120 |
121 | # Rope project settings
122 | .ropeproject
123 |
124 | # mkdocs documentation
125 | /site
126 |
127 | # mypy
128 | .mypy_cache/
129 | .dmypy.json
130 | dmypy.json
131 |
132 | # Pyre type checker
133 | .pyre/
134 |
135 | # pytype static type analyzer
136 | .pytype/
137 |
138 | # Cython debug symbols
139 | cython_debug/
--------------------------------------------------------------------------------
/CNAME:
--------------------------------------------------------------------------------
1 | geeadd.geetools.xyz
2 |
--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
1 | Apache License
2 | Version 2.0, January 2004
3 | http://www.apache.org/licenses/
4 |
5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
6 |
7 | 1. Definitions.
8 |
9 | "License" shall mean the terms and conditions for use, reproduction,
10 | and distribution as defined by Sections 1 through 9 of this document.
11 |
12 | "Licensor" shall mean the copyright owner or entity authorized by
13 | the copyright owner that is granting the License.
14 |
15 | "Legal Entity" shall mean the union of the acting entity and all
16 | other entities that control, are controlled by, or are under common
17 | control with that entity. For the purposes of this definition,
18 | "control" means (i) the power, direct or indirect, to cause the
19 | direction or management of such entity, whether by contract or
20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the
21 | outstanding shares, or (iii) beneficial ownership of such entity.
22 |
23 | "You" (or "Your") shall mean an individual or Legal Entity
24 | exercising permissions granted by this License.
25 |
26 | "Source" form shall mean the preferred form for making modifications,
27 | including but not limited to software source code, documentation
28 | source, and configuration files.
29 |
30 | "Object" form shall mean any form resulting from mechanical
31 | transformation or translation of a Source form, including but
32 | not limited to compiled object code, generated documentation,
33 | and conversions to other media types.
34 |
35 | "Work" shall mean the work of authorship, whether in Source or
36 | Object form, made available under the License, as indicated by a
37 | copyright notice that is included in or attached to the work
38 | (an example is provided in the Appendix below).
39 |
40 | "Derivative Works" shall mean any work, whether in Source or Object
41 | form, that is based on (or derived from) the Work and for which the
42 | editorial revisions, annotations, elaborations, or other modifications
43 | represent, as a whole, an original work of authorship. For the purposes
44 | of this License, Derivative Works shall not include works that remain
45 | separable from, or merely link (or bind by name) to the interfaces of,
46 | the Work and Derivative Works thereof.
47 |
48 | "Contribution" shall mean any work of authorship, including
49 | the original version of the Work and any modifications or additions
50 | to that Work or Derivative Works thereof, that is intentionally
51 | submitted to Licensor for inclusion in the Work by the copyright owner
52 | or by an individual or Legal Entity authorized to submit on behalf of
53 | the copyright owner. For the purposes of this definition, "submitted"
54 | means any form of electronic, verbal, or written communication sent
55 | to the Licensor or its representatives, including but not limited to
56 | communication on electronic mailing lists, source code control systems,
57 | and issue tracking systems that are managed by, or on behalf of, the
58 | Licensor for the purpose of discussing and improving the Work, but
59 | excluding communication that is conspicuously marked or otherwise
60 | designated in writing by the copyright owner as "Not a Contribution."
61 |
62 | "Contributor" shall mean Licensor and any individual or Legal Entity
63 | on behalf of whom a Contribution has been received by Licensor and
64 | subsequently incorporated within the Work.
65 |
66 | 2. Grant of Copyright License. Subject to the terms and conditions of
67 | this License, each Contributor hereby grants to You a perpetual,
68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable
69 | copyright license to reproduce, prepare Derivative Works of,
70 | publicly display, publicly perform, sublicense, and distribute the
71 | Work and such Derivative Works in Source or Object form.
72 |
73 | 3. Grant of Patent License. Subject to the terms and conditions of
74 | this License, each Contributor hereby grants to You a perpetual,
75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable
76 | (except as stated in this section) patent license to make, have made,
77 | use, offer to sell, sell, import, and otherwise transfer the Work,
78 | where such license applies only to those patent claims licensable
79 | by such Contributor that are necessarily infringed by their
80 | Contribution(s) alone or by combination of their Contribution(s)
81 | with the Work to which such Contribution(s) was submitted. If You
82 | institute patent litigation against any entity (including a
83 | cross-claim or counterclaim in a lawsuit) alleging that the Work
84 | or a Contribution incorporated within the Work constitutes direct
85 | or contributory patent infringement, then any patent licenses
86 | granted to You under this License for that Work shall terminate
87 | as of the date such litigation is filed.
88 |
89 | 4. Redistribution. You may reproduce and distribute copies of the
90 | Work or Derivative Works thereof in any medium, with or without
91 | modifications, and in Source or Object form, provided that You
92 | meet the following conditions:
93 |
94 | (a) You must give any other recipients of the Work or
95 | Derivative Works a copy of this License; and
96 |
97 | (b) You must cause any modified files to carry prominent notices
98 | stating that You changed the files; and
99 |
100 | (c) You must retain, in the Source form of any Derivative Works
101 | that You distribute, all copyright, patent, trademark, and
102 | attribution notices from the Source form of the Work,
103 | excluding those notices that do not pertain to any part of
104 | the Derivative Works; and
105 |
106 | (d) If the Work includes a "NOTICE" text file as part of its
107 | distribution, then any Derivative Works that You distribute must
108 | include a readable copy of the attribution notices contained
109 | within such NOTICE file, excluding those notices that do not
110 | pertain to any part of the Derivative Works, in at least one
111 | of the following places: within a NOTICE text file distributed
112 | as part of the Derivative Works; within the Source form or
113 | documentation, if provided along with the Derivative Works; or,
114 | within a display generated by the Derivative Works, if and
115 | wherever such third-party notices normally appear. The contents
116 | of the NOTICE file are for informational purposes only and
117 | do not modify the License. You may add Your own attribution
118 | notices within Derivative Works that You distribute, alongside
119 | or as an addendum to the NOTICE text from the Work, provided
120 | that such additional attribution notices cannot be construed
121 | as modifying the License.
122 |
123 | You may add Your own copyright statement to Your modifications and
124 | may provide additional or different license terms and conditions
125 | for use, reproduction, or distribution of Your modifications, or
126 | for any such Derivative Works as a whole, provided Your use,
127 | reproduction, and distribution of the Work otherwise complies with
128 | the conditions stated in this License.
129 |
130 | 5. Submission of Contributions. Unless You explicitly state otherwise,
131 | any Contribution intentionally submitted for inclusion in the Work
132 | by You to the Licensor shall be under the terms and conditions of
133 | this License, without any additional terms or conditions.
134 | Notwithstanding the above, nothing herein shall supersede or modify
135 | the terms of any separate license agreement you may have executed
136 | with Licensor regarding such Contributions.
137 |
138 | 6. Trademarks. This License does not grant permission to use the trade
139 | names, trademarks, service marks, or product names of the Licensor,
140 | except as required for reasonable and customary use in describing the
141 | origin of the Work and reproducing the content of the NOTICE file.
142 |
143 | 7. Disclaimer of Warranty. Unless required by applicable law or
144 | agreed to in writing, Licensor provides the Work (and each
145 | Contributor provides its Contributions) on an "AS IS" BASIS,
146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
147 | implied, including, without limitation, any warranties or conditions
148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
149 | PARTICULAR PURPOSE. You are solely responsible for determining the
150 | appropriateness of using or redistributing the Work and assume any
151 | risks associated with Your exercise of permissions under this License.
152 |
153 | 8. Limitation of Liability. In no event and under no legal theory,
154 | whether in tort (including negligence), contract, or otherwise,
155 | unless required by applicable law (such as deliberate and grossly
156 | negligent acts) or agreed to in writing, shall any Contributor be
157 | liable to You for damages, including any direct, indirect, special,
158 | incidental, or consequential damages of any character arising as a
159 | result of this License or out of the use or inability to use the
160 | Work (including but not limited to damages for loss of goodwill,
161 | work stoppage, computer failure or malfunction, or any and all
162 | other commercial damages or losses), even if such Contributor
163 | has been advised of the possibility of such damages.
164 |
165 | 9. Accepting Warranty or Additional Liability. While redistributing
166 | the Work or Derivative Works thereof, You may choose to offer,
167 | and charge a fee for, acceptance of support, warranty, indemnity,
168 | or other liability obligations and/or rights consistent with this
169 | License. However, in accepting such obligations, You may act only
170 | on Your own behalf and on Your sole responsibility, not on behalf
171 | of any other Contributor, and only if You agree to indemnify,
172 | defend, and hold each Contributor harmless for any liability
173 | incurred by, or claims asserted against, such Contributor by reason
174 | of your accepting any such warranty or additional liability.
175 |
176 | END OF TERMS AND CONDITIONS
177 |
178 | APPENDIX: How to apply the Apache License to your work.
179 |
180 | To apply the Apache License to your work, attach the following
181 | boilerplate notice, with the fields enclosed by brackets "{}"
182 | replaced with your own identifying information. (Don't include
183 | the brackets!) The text should be enclosed in the appropriate
184 | comment syntax for the file format. We also recommend that a
185 | file or class name and description of purpose be included on the
186 | same "printed page" as the copyright notice for easier
187 | identification within third-party archives.
188 |
189 | Copyright {2017} {Samapriya Roy}
190 |
191 | Licensed under the Apache License, Version 2.0 (the "License");
192 | you may not use this file except in compliance with the License.
193 | You may obtain a copy of the License at
194 |
195 | http://www.apache.org/licenses/LICENSE-2.0
196 |
197 | Unless required by applicable law or agreed to in writing, software
198 | distributed under the License is distributed on an "AS IS" BASIS,
199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
200 | See the License for the specific language governing permissions and
201 | limitations under the License.
202 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | # [Google Earth Engine Batch Asset Manager with Addons](https://samapriya.github.io/gee_asset_manager_addon/)
2 |
3 | [](https://www.linkedin.com/in/samapriya/)
4 | [](https://medium.com/@samapriyaroy)
5 | [](https://twitter.com/intent/follow?screen_name=samapriyaroy)
6 | [](https://mapstodon.space/@samapriya)
7 | [](https://hitsofcode.com/github/samapriya/gee_asset_manager_addon?branch=master)
8 | 
9 | [](https://pepy.tech/project/geeadd)
10 | [](https://opensource.org/licenses/Apache-2.0)
11 | [](https://doi.org/10.5281/zenodo.11482497)
12 | 
13 | [](https://www.buymeacoffee.com/samapriya)
14 | [](https://github.com/sponsors/samapriya)
15 |
16 | **geeadd** (which stands for **Google Earth Engine** batch asset manager with **addons** ) provides a command-line tool for managing Google Earth Engine assets in batch. It allows you to perform a variety of operations on your assets that add to the existing earthengine command line tool. This includes tools that allows a user to estimate their quota usage, asset sizes, set permissions on assets, move and copy and many others. There are also some functions that are build to provided extended functionality like convert any earth engine app script to the source code and search tool which allows you to search both the GEE official data catalog and the community catalog.
17 |
18 | This project has been a lot of work starting a few years ago and current improvements and updates can be seen reflected in v1.0.0 and higher up.
19 |
20 | 
21 |
22 | Like, share and support the Github project. And you can now cite it too
23 |
24 |
25 |
26 | ```
27 | Samapriya Roy, & Biplov Bhandari. (2024). samapriya/gee_asset_manager_addon: GEE Asset Manager with Addons (1.2.0).
28 | Zenodo. https://doi.org/10.5281/zenodo.11482497
29 | ```
30 |
31 |
32 |
33 | Find the [readme docs here](https://geeadd.geetools.xyz)
34 |
--------------------------------------------------------------------------------
/docs/CNAME:
--------------------------------------------------------------------------------
1 | geeadd.geetools.xyz
2 |
--------------------------------------------------------------------------------
/docs/changelog.md:
--------------------------------------------------------------------------------
1 | # Changelog
2 |
3 | #### v1.2.0
4 | - removed deprecated pkg_resources module
5 | - added better version checker
6 | - upgraded app2script module to handle conversions
7 | - added a projects tool to fetch projects with earthengine api enabled
8 | - upgraded quota tool to handle root assets for earthengine-api >=v0.1.400
9 |
10 | #### v1.1.0
11 | - added recursive handling for folder object count
12 | - fixed [Issue 18](https://github.com/samapriya/gee_asset_manager_addon/issues/18) to handle featureview operations
13 | - fixed issue with copy, move, asset size estimation and acl
14 | - Added extra fields for geeadd search tool
15 |
16 | #### v1.0.1
17 | - Updated pending/ready tasks as they are called in tasklist
18 | - Fixed task cancellation options
19 |
20 | ### v1.0.0
21 | - Gives better parsing of tasking info
22 | - Search by task ID now
23 | - Now outputs EECU and path for tasks that have those fields
24 | - Fixed size estimation for image vs image collection
25 | - Enhanced user docs and readme
26 | - added output from delete operation
27 | - Added some function descriptions & general improvements
28 |
29 | ### v0.6.0
30 | - Updated to use API v1 with some updates to avoid breaking changes
31 | - Updated some core tools like size update and quota updation
32 | - The geeadd access tool is now user type agnostic and you can simply pass if the user is service account, group or email
33 | - Better handling of acl delete function
34 | - Added some function descriptions & general improvements
35 | - Reduced client initialization steps
36 |
37 | ### v0.5.6
38 | - fixed ee_report tool to allow for report exports for all EE asset types
39 | - updated task search and task by state search
40 | - general cleanup and improvements
41 |
42 | ### v0.5.5
43 | - added folder migration tools
44 | - improved recursive folder search & copy, move and permissions tools
45 | - major improvements to copy and move tools for better migration of nested Folders
46 | - Minor improvements and cleanup
47 |
48 | ### v0.5.4
49 | - Updated search tool to use updated endpoint
50 | - Search tool no longer downloads zip or parses CSV
51 | - Minor improvements
52 |
53 | ### v0.5.2
54 | - Updated copy tool to allow for non mirrored copy
55 | - Updated task and task cancel tools to account for states Pending and Cancelling
56 |
57 | ### v0.5.1
58 | - Updated quota tool to handle GCP projects inside GEE
59 | - Updated Folder size reporting
60 |
61 | ### v0.5.0
62 | - Updated to use earthengine-api>= 0.1.222
63 | - Copy and move tool improvements to facilitate cloud alpha support.
64 | - Updated task check tool to account for operations based handling.
65 | -
66 | ### v0.4.9
67 | - Fixed [issue 11](https://github.com/samapriya/gee_asset_manager_addon/issues/11).
68 | - Updated to recent API calls based on Issue and general improvements
69 | - Added auto version check from pypi.
70 |
71 | ### v0.4.7
72 | - Fixed issue with delete tool and shell call.
73 | - Fixed issue with copy and move function for single collections
74 |
75 | ### v0.4.6
76 | - Now inclues asset_url and thumbnail_url for search.
77 | - Formatting and general improvements.
78 |
79 | ### v0.4.5
80 | - Now inclues license in sdist
81 | - Fixed issue with app2script tool and string and text parsing.
82 | - Added readme and version tools.
83 | - Added readme docs and deployed environment.
84 |
85 | ### v0.4.4
86 | - Removed git dependency and used urllib instead based on [feedback](https://github.com/samapriya/gee_asset_manager_addon/issues/10)
87 | - Created conda forge release based on [Issue 10](https://github.com/samapriya/gee_asset_manager_addon/issues/10)
88 |
89 | ### v0.4.2
90 | - Fixed relative import issue for earthengine.
91 | - Fixed image collection move tool to parse ee object type correctly as image_collection.
92 |
93 | ### v0.4.1
94 | - Made enhancement [Issue 9](https://github.com/samapriya/gee_asset_manager_addon/issues/9).
95 | - Search tool now return earth engine asset snippet and start and end dates as JSON object.
96 | - Removed pretty table dependency.
97 |
98 | ### v0.4.0
99 | - Improved quota tools to get all quota and asset counts.
100 | - Added a search tool to search GEE catalog using keywords.
101 | - Improved parsing for app to script tool.
102 | - Detailed asset root for all root folders and recursively
103 | - Cancel tasks now allows you to choose, running, ready or specific task ids.
104 | - Assets copy and move now allows you to copy entire folders, collectiona and assets recursively
105 | - Updated assets access tool
106 | - Delete metadata allows you to delete metadata for existing collection.
107 | - Overall general improvements and optimization.
108 |
109 | ### v0.3.3
110 | - General improvements
111 | - Added tool to get underlying code from earthengine app
112 |
113 | ### v0.3.1
114 | - Updated list and asset size functions
115 | - Updated function to generate earthengine asset report
116 | - General optimization and improvements to distribution
117 | - Better error handling
118 |
119 | ### v0.3.0
120 | - Removed upload function
121 | - Upload handles by [geeup](https://github.com/samapriya/geeup)
122 | - General optimization and improvements to distribution
123 | - Better error handling
124 |
125 | ### v0.2.8
126 | - Uses poster for streaming upload more stable with memory issues and large files
127 | - Poster dependency limits use to Py 2.7 will fix in the new version
128 |
129 | ### v0.2.6
130 | - Major improvement to move, batch copy, and task reporting
131 | - Major improvements to access tool to allow users read/write permission to entire Folder/collection.
132 |
133 | ### v0.2.5
134 | - Handles bandnames during upload thanks to Lukasz for original upload code
135 | - Removed manifest option, that can be handled by seperate tool (ppipe)
136 |
137 | ### v0.2.3
138 | - Removing the initialization loop error
139 |
140 | ### v0.2.2
141 | - Added improvement to earthengine authorization
142 |
143 | ### v0.2.1
144 | - Added capability to handle PlanetScope 4Band Surface Reflectance Metadata Type
145 | - General Improvements
146 |
147 | ### v0.2.0
148 | - Tool improvements and enhancements
149 |
150 | ### v0.1.9
151 | - New tool EE_Report was added
152 |
153 | ### v0.1.8
154 | - Fixed issues with install
155 | - Dependencies now part of setup.py
156 | - Updated Parser and general improvements
157 |
--------------------------------------------------------------------------------
/docs/index.md:
--------------------------------------------------------------------------------
1 | # Google Earth Engine Batch Asset Manager with Addons
2 |
3 | [](https://www.linkedin.com/in/samapriya/)
4 | [](https://medium.com/@samapriyaroy)
5 | [](https://twitter.com/intent/follow?screen_name=samapriyaroy)
6 | [](https://mapstodon.space/@samapriya)
7 | [](https://hitsofcode.com/github/samapriya/gee_asset_manager_addon?branch=master)
8 | 
9 | [](https://pepy.tech/project/geeadd)
10 | [](https://opensource.org/licenses/Apache-2.0)
11 | [](https://doi.org/10.5281/zenodo.11482497)
12 | 
13 | [](https://www.buymeacoffee.com/samapriya)
14 | [](https://github.com/sponsors/samapriya)
15 |
16 | **geeadd** (which stands for **Google Earth Engine** batch asset manager with **addons** ) provides a command-line tool for managing Google Earth Engine assets in batch. It allows you to perform a variety of operations on your assets that add to the existing earthengine command line tool. This includes tools that allows a user to estimate their quota usage, asset sizes, set permissions on assets, move and copy and many others. There are also some functions that are build to provided extended functionality like convert any earth engine app script to the source code and search tool which allows you to search both the GEE official data catalog and the community catalog.
17 |
18 | This project has been a lot of work starting a few years ago and current improvements and updates can be seen reflected in v1.0.0 and higher up.
19 |
20 | 
21 |
22 | Like, share and support the Github project. And you can now cite it too
23 |
24 |
25 |
26 | ```
27 | Samapriya Roy, & Biplov Bhandari. (2024). samapriya/gee_asset_manager_addon: GEE Asset Manager with Addons (1.2.0).
28 | Zenodo. https://doi.org/10.5281/zenodo.11482497
29 | ```
30 |
31 |
32 |
--------------------------------------------------------------------------------
/docs/installation.md:
--------------------------------------------------------------------------------
1 | # Prerequisites and Installation
2 | We assume Earth Engine Python API is installed and EE authorised as desribed [here](https://developers.google.com/earth-engine/python_install). From v0.3.4 onwards geeadd will only run on Python 3. Also with the new changes to the Earth Engine API library, the tool was completely modified to work with earthengine-api v0.1.127 and higher. Authenticate your earth engine client by using the following in your command line or terminal setup.
3 |
4 |
5 |
6 | ```
7 | earthengine authenticate
8 | ```
9 |
10 |
11 |
12 | Quick installation
13 |
14 | ```
15 | pip install geeadd
16 |
17 | pip install geeadd --user
18 | ```
19 |
20 |
21 | To get always fresh install using GitHub (**This could be a staging version and will include a pop up on top to remind you of that**)
22 |
23 |
24 |
25 | ```
26 | pip install git+https://github.com/samapriya/gee_asset_manager_addon.git
27 | ```
28 |
29 |
30 |
31 | The advantage of having it installed is being able to execute geeadd as any command line tool. I recommend installation within virtual environment.
32 |
--------------------------------------------------------------------------------
/docs/license.md:
--------------------------------------------------------------------------------
1 |
2 |
3 | ```
4 | Apache License
5 | Version 2.0, January 2004
6 | http://www.apache.org/licenses/
7 | ```
8 |
9 |
10 |
11 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
12 |
13 | 1. Definitions.
14 |
15 | "License" shall mean the terms and conditions for use, reproduction,
16 | and distribution as defined by Sections 1 through 9 of this document.
17 |
18 | "Licensor" shall mean the copyright owner or entity authorized by
19 | the copyright owner that is granting the License.
20 |
21 | "Legal Entity" shall mean the union of the acting entity and all
22 | other entities that control, are controlled by, or are under common
23 | control with that entity. For the purposes of this definition,
24 | "control" means (i) the power, direct or indirect, to cause the
25 | direction or management of such entity, whether by contract or
26 | otherwise, or (ii) ownership of fifty percent (50%) or more of the
27 | outstanding shares, or (iii) beneficial ownership of such entity.
28 |
29 | "You" (or "Your") shall mean an individual or Legal Entity
30 | exercising permissions granted by this License.
31 |
32 | "Source" form shall mean the preferred form for making modifications,
33 | including but not limited to software source code, documentation
34 | source, and configuration files.
35 |
36 | "Object" form shall mean any form resulting from mechanical
37 | transformation or translation of a Source form, including but
38 | not limited to compiled object code, generated documentation,
39 | and conversions to other media types.
40 |
41 | "Work" shall mean the work of authorship, whether in Source or
42 | Object form, made available under the License, as indicated by a
43 | copyright notice that is included in or attached to the work
44 | (an example is provided in the Appendix below).
45 |
46 | "Derivative Works" shall mean any work, whether in Source or Object
47 | form, that is based on (or derived from) the Work and for which the
48 | editorial revisions, annotations, elaborations, or other modifications
49 | represent, as a whole, an original work of authorship. For the purposes
50 | of this License, Derivative Works shall not include works that remain
51 | separable from, or merely link (or bind by name) to the interfaces of,
52 | the Work and Derivative Works thereof.
53 |
54 | "Contribution" shall mean any work of authorship, including
55 | the original version of the Work and any modifications or additions
56 | to that Work or Derivative Works thereof, that is intentionally
57 | submitted to Licensor for inclusion in the Work by the copyright owner
58 | or by an individual or Legal Entity authorized to submit on behalf of
59 | the copyright owner. For the purposes of this definition, "submitted"
60 | means any form of electronic, verbal, or written communication sent
61 | to the Licensor or its representatives, including but not limited to
62 | communication on electronic mailing lists, source code control systems,
63 | and issue tracking systems that are managed by, or on behalf of, the
64 | Licensor for the purpose of discussing and improving the Work, but
65 | excluding communication that is conspicuously marked or otherwise
66 | designated in writing by the copyright owner as "Not a Contribution."
67 |
68 | "Contributor" shall mean Licensor and any individual or Legal Entity
69 | on behalf of whom a Contribution has been received by Licensor and
70 | subsequently incorporated within the Work.
71 |
72 | 2. Grant of Copyright License. Subject to the terms and conditions of
73 | this License, each Contributor hereby grants to You a perpetual,
74 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable
75 | copyright license to reproduce, prepare Derivative Works of,
76 | publicly display, publicly perform, sublicense, and distribute the
77 | Work and such Derivative Works in Source or Object form.
78 |
79 | 3. Grant of Patent License. Subject to the terms and conditions of
80 | this License, each Contributor hereby grants to You a perpetual,
81 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable
82 | (except as stated in this section) patent license to make, have made,
83 | use, offer to sell, sell, import, and otherwise transfer the Work,
84 | where such license applies only to those patent claims licensable
85 | by such Contributor that are necessarily infringed by their
86 | Contribution(s) alone or by combination of their Contribution(s)
87 | with the Work to which such Contribution(s) was submitted. If You
88 | institute patent litigation against any entity (including a
89 | cross-claim or counterclaim in a lawsuit) alleging that the Work
90 | or a Contribution incorporated within the Work constitutes direct
91 | or contributory patent infringement, then any patent licenses
92 | granted to You under this License for that Work shall terminate
93 | as of the date such litigation is filed.
94 |
95 | 4. Redistribution. You may reproduce and distribute copies of the
96 | Work or Derivative Works thereof in any medium, with or without
97 | modifications, and in Source or Object form, provided that You
98 | meet the following conditions:
99 |
100 | (a) You must give any other recipients of the Work or
101 | Derivative Works a copy of this License; and
102 |
103 | (b) You must cause any modified files to carry prominent notices
104 | stating that You changed the files; and
105 |
106 | (c) You must retain, in the Source form of any Derivative Works
107 | that You distribute, all copyright, patent, trademark, and
108 | attribution notices from the Source form of the Work,
109 | excluding those notices that do not pertain to any part of
110 | the Derivative Works; and
111 |
112 | (d) If the Work includes a "NOTICE" text file as part of its
113 | distribution, then any Derivative Works that You distribute must
114 | include a readable copy of the attribution notices contained
115 | within such NOTICE file, excluding those notices that do not
116 | pertain to any part of the Derivative Works, in at least one
117 | of the following places: within a NOTICE text file distributed
118 | as part of the Derivative Works; within the Source form or
119 | documentation, if provided along with the Derivative Works; or,
120 | within a display generated by the Derivative Works, if and
121 | wherever such third-party notices normally appear. The contents
122 | of the NOTICE file are for informational purposes only and
123 | do not modify the License. You may add Your own attribution
124 | notices within Derivative Works that You distribute, alongside
125 | or as an addendum to the NOTICE text from the Work, provided
126 | that such additional attribution notices cannot be construed
127 | as modifying the License.
128 |
129 | You may add Your own copyright statement to Your modifications and
130 | may provide additional or different license terms and conditions
131 | for use, reproduction, or distribution of Your modifications, or
132 | for any such Derivative Works as a whole, provided Your use,
133 | reproduction, and distribution of the Work otherwise complies with
134 | the conditions stated in this License.
135 |
136 | 5. Submission of Contributions. Unless You explicitly state otherwise,
137 | any Contribution intentionally submitted for inclusion in the Work
138 | by You to the Licensor shall be under the terms and conditions of
139 | this License, without any additional terms or conditions.
140 | Notwithstanding the above, nothing herein shall supersede or modify
141 | the terms of any separate license agreement you may have executed
142 | with Licensor regarding such Contributions.
143 |
144 | 6. Trademarks. This License does not grant permission to use the trade
145 | names, trademarks, service marks, or product names of the Licensor,
146 | except as required for reasonable and customary use in describing the
147 | origin of the Work and reproducing the content of the NOTICE file.
148 |
149 | 7. Disclaimer of Warranty. Unless required by applicable law or
150 | agreed to in writing, Licensor provides the Work (and each
151 | Contributor provides its Contributions) on an "AS IS" BASIS,
152 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
153 | implied, including, without limitation, any warranties or conditions
154 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
155 | PARTICULAR PURPOSE. You are solely responsible for determining the
156 | appropriateness of using or redistributing the Work and assume any
157 | risks associated with Your exercise of permissions under this License.
158 |
159 | 8. Limitation of Liability. In no event and under no legal theory,
160 | whether in tort (including negligence), contract, or otherwise,
161 | unless required by applicable law (such as deliberate and grossly
162 | negligent acts) or agreed to in writing, shall any Contributor be
163 | liable to You for damages, including any direct, indirect, special,
164 | incidental, or consequential damages of any character arising as a
165 | result of this License or out of the use or inability to use the
166 | Work (including but not limited to damages for loss of goodwill,
167 | work stoppage, computer failure or malfunction, or any and all
168 | other commercial damages or losses), even if such Contributor
169 | has been advised of the possibility of such damages.
170 |
171 | 9. Accepting Warranty or Additional Liability. While redistributing
172 | the Work or Derivative Works thereof, You may choose to offer,
173 | and charge a fee for, acceptance of support, warranty, indemnity,
174 | or other liability obligations and/or rights consistent with this
175 | License. However, in accepting such obligations, You may act only
176 | on Your own behalf and on Your sole responsibility, not on behalf
177 | of any other Contributor, and only if You agree to indemnify,
178 | defend, and hold each Contributor harmless for any liability
179 | incurred by, or claims asserted against, such Contributor by reason
180 | of your accepting any such warranty or additional liability.
181 |
182 | END OF TERMS AND CONDITIONS
183 |
184 | APPENDIX: How to apply the Apache License to your work.
185 |
186 | To apply the Apache License to your work, attach the following
187 | boilerplate notice, with the fields enclosed by brackets "{}"
188 | replaced with your own identifying information. (Don't include
189 | the brackets!) The text should be enclosed in the appropriate
190 | comment syntax for the file format. We also recommend that a
191 | file or class name and description of purpose be included on the
192 | same "printed page" as the copyright notice for easier
193 | identification within third-party archives.
194 |
195 | Copyright {2019} {Samapriya Roy}
196 |
197 | Licensed under the Apache License, Version 2.0 (the "License");
198 | you may not use this file except in compliance with the License.
199 | You may obtain a copy of the License at
200 |
201 | http://www.apache.org/licenses/LICENSE-2.0
202 |
203 | Unless required by applicable law or agreed to in writing, software
204 | distributed under the License is distributed on an "AS IS" BASIS,
205 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
206 | See the License for the specific language governing permissions and
207 | limitations under the License.
208 |
--------------------------------------------------------------------------------
/docs/projects/access.md:
--------------------------------------------------------------------------------
1 | # Change asset permissions
2 |
3 | The Access tool in geeadd empowers users to efficiently manage asset access permissions within Google Earth Engine (GEE). This tool streamlines the process of setting access properties for folders, collections, or images, and it does so recursively. This means you can apply access configurations to multiple assets simultaneously, saving you valuable time and effort.
4 |
5 | #### Key Features
6 |
7 | - **Recursive Access Configuration**: The Access tool enables you to apply access permissions recursively. This functionality allows you to set access properties for multiple assets at once, ensuring efficient and consistent management of your Earth Engine assets.
8 |
9 | - **Simplified User Identification**: Starting from version 1.0.0 and onwards, the Access tool eliminates the need for manual email parsing. You can now directly provide an individual user email, a Google group, or a Google service account without specifying the type. This enhancement streamlines the process and makes it more user-friendly.
10 |
11 | #### Usage
12 | Using the Access tool is straightforward. Simply call the function and provide the necessary arguments.
13 |
14 | 
15 |
16 | ```
17 | > geeadd access -h
18 | usage: geeadd access [-h] --asset ASSET --user USER --role ROLE
19 |
20 | options:
21 | -h, --help show this help message and exit
22 |
23 | Required named arguments.:
24 | --asset ASSET This is the path to the earth engine asset whose permission you are changing folder/collection/image
25 | --user USER Can be user email or serviceAccount like account@gserviceaccount.com or groups like group@googlegroups.com or try using "allUsers" to make it public
26 | --role ROLE Choose between reader, writer or delete
27 | ```
28 |
--------------------------------------------------------------------------------
/docs/projects/app2script.md:
--------------------------------------------------------------------------------
1 | # GEE App to Script tool
2 |
3 | The App to Script tool in geeadd offers a convenient way to extract the underlying Earth Engine code from any public Earth Engine application. This tool provides two options: it can either print out the code or export it into a JavaScript file for easy integration into the Google Earth Engine code editor.
4 |
5 | #### Key Features
6 |
7 | - **Code Extraction**: The tool seamlessly retrieves the Earth Engine code that powers any public Earth Engine application, providing users with a clear understanding of the underlying code structure.
8 |
9 | - **JavaScript Export**: Users have the option to export the code directly into a JavaScript file. This file can be opened in any text editor, allowing for easy modification and integration into the Google Earth Engine code editor.
10 |
11 | #### Usage
12 |
13 | Using the App to Script tool is straightforward. Simply provide the URL of the public Earth Engine application you want to extract the code from.
14 |
15 | ```bash
16 | # Print out the Earth Engine code
17 | geeadd app2script --url "https://gena.users.earthengine.app/view/urban-lights"
18 | ```
19 |
20 | ```bash
21 | # Export the code to a JavaScript file
22 | geeadd app2script --url "https://gena.users.earthengine.app/view/urban-lights" --outfile "Full path to javascript.js"
23 | ```
24 |
25 | - `--url`: The URL of the public Earth Engine application.
26 |
27 | - `--outfile`: (Optional) The full path to the JavaScript file where you want to export the code.
28 |
29 | 
30 |
31 | #### Example
32 |
33 | The following example demonstrates how to use the App to Script tool to extract and export Earth Engine code:
34 |
35 | ```bash
36 | # Print out the Earth Engine code
37 | geeadd app2script --url "https://gena.users.earthengine.app/view/urban-lights"
38 | ```
39 |
40 |
41 |
42 |
--------------------------------------------------------------------------------
/docs/projects/cancel_tasks.md:
--------------------------------------------------------------------------------
1 | # Cancel Tasks tool
2 |
3 | The Cancel Tasks tool in geeadd is a powerful utility designed to streamline task management within Google Earth Engine (GEE). This tool provides users with precise control over task cancellation, allowing for the cancellation of various task types based on specific criteria. Whether you need to cancel all tasks, terminate running tasks, clear pending tasks, or selectively cancel a single task using its unique task ID, the Cancel Tasks tool has you covered.
4 |
5 | #### Key Features
6 |
7 | - **Efficient Task Cancellation**: The Cancel Tasks tool simplifies the process of terminating GEE tasks, ensuring efficient resource management.
8 |
9 | - **Cancel All Tasks**: Users can opt to cancel all active tasks, providing a quick and comprehensive way to clear ongoing processes.
10 |
11 | - **Terminate Running Tasks**: For situations requiring the immediate cessation of running tasks, this tool enables the cancellation of running tasks specifically.
12 |
13 | - **Clear Pending Tasks**: Pending tasks can be removed in one fell swoop, ensuring that resources are not tied up unnecessarily.
14 |
15 | - **Selective Task Cancellation**: Users have the flexibility to target a single task for cancellation by providing its unique task ID.
16 |
17 | #### Usage
18 |
19 | Utilizing the Cancel Tasks tool is straightforward, and it provides fine-grained control over task cancellation.
20 |
21 | ```bash
22 | geeadd cancel -h
23 | usage: geeadd cancel [-h] --tasks TASKS
24 |
25 | optional arguments:
26 | -h, --help show this help message and exit
27 |
28 | Required named arguments.:
29 | --tasks TASKS You can provide tasks as running or pending or all or even a
30 | single task id
31 | ```
32 |
33 | - `state`: (Optional) Specifies the state of tasks to cancel (e.g., 'RUNNING' for running tasks, 'READY' for pending tasks).
34 |
35 | - `task_id`: (Optional) The unique identifier of the task you want to cancel.
36 |
37 | #### Example
38 |
39 | Here's an example demonstrating how to use the Cancel Tasks tool to efficiently manage Earth Engine tasks:
40 |
41 | 
42 |
--------------------------------------------------------------------------------
/docs/projects/copy.md:
--------------------------------------------------------------------------------
1 | # Copy Assets tool
2 |
3 | The Copy Assets tool in geeadd offers a versatile solution for copying Earth Engine assets efficiently. With its recursive capabilities, this tool empowers users to duplicate entire folders, collections, images, or tables seamlessly. Additionally, it allows users with read access to assets from other users to copy assets from their repositories, enhancing collaboration and asset management.
4 |
5 | #### Key Features
6 |
7 | - **Comprehensive Asset Duplication**: The Copy Assets tool enables users to perform recursive copying of Earth Engine assets, ensuring that entire hierarchies of assets can be effortlessly duplicated.
8 |
9 | - **User-Friendly Interface**: The tool offers a straightforward command-line interface, making it accessible to users of all experience levels.
10 |
11 | - **Copy Assets from Other Users**: Users with read access to assets from other users can utilize this tool to copy assets from their repositories, facilitating collaborative projects and data sharing.
12 |
13 | #### Usage
14 |
15 | Using the Copy Assets tool is simple and intuitive, allowing users to specify the source and destination paths for asset duplication.
16 |
17 | ```bash
18 | geeadd copy --initial "existing_asset_path" --final "new_asset_path"
19 | ```
20 |
21 | - `--initial`: The existing path of the assets you want to copy.
22 |
23 | - `--final`: The new path where you want to duplicate the assets.
24 |
25 | #### Example
26 |
27 | Here's an example demonstrating how to use the Copy Assets tool to duplicate Earth Engine assets:
28 |
29 | ```bash
30 | geeadd copy --initial "users/your_username/your_collection" --final "users/your_username/copied_collection"
31 | ```
32 |
33 | 
34 |
35 | The Copy Assets tool in geeadd simplifies asset duplication and promotes efficient asset management within Google Earth Engine.
36 |
--------------------------------------------------------------------------------
/docs/projects/delete.md:
--------------------------------------------------------------------------------
1 | # Delete Assets tool
2 |
3 | The Delete Assets tool in GEEadd is a powerful utility designed to facilitate asset management within Google Earth Engine (GEE). This tool empowers users to perform recursive deletions of Earth Engine assets, including folders, collections, images, and their child assets. However, it is important to exercise caution while using this tool, as it permanently removes assets and their associated data.
4 |
5 | #### Key Features
6 |
7 | - **Comprehensive Asset Deletion**: The Delete Assets tool allows users to perform recursive deletions of assets, ensuring that entire hierarchies of assets can be removed with a single command.
8 |
9 | - **Use with Caution**: Due to the recursive nature of this tool, it will delete not only the specified asset but also all its child assets, including images, collections, and folders. Therefore, it is essential to use this tool with caution to avoid unintentional data loss.
10 |
11 | #### Usage
12 |
13 | Using the Delete Assets tool is straightforward, requiring only the specification of the target Earth Engine asset for deletion.
14 |
15 | ```bash
16 | geeadd delete --id "asset_path_to_delete"
17 | ```
18 |
19 | - `--id`: The full path to the asset you want to delete. This tool will recursively remove all child assets, including images, collections, and folders associated with the specified asset.
20 |
21 | #### Example
22 |
23 | Here's an example demonstrating how to use the Delete Assets tool to remove an Earth Engine asset and all its child assets:
24 |
25 | ```bash
26 | geeadd delete --id "users/your_username/your_collection"
27 | ```
28 |
29 | 
30 |
--------------------------------------------------------------------------------
/docs/projects/delete_metadata.md:
--------------------------------------------------------------------------------
1 | # Delete metadata tool
2 |
3 | The Asset Metadata Delete tool in geeadd provides users with a valuable capability to delete specific properties from metadata associated with Earth Engine assets. This tool is particularly useful when you need to reset or remove a property value from the metadata of an ingested collection, image, or table.
4 |
5 | #### Key Features
6 |
7 | - **Selective Metadata Property Deletion**: The Asset Metadata Delete tool allows users to selectively delete a specific property from the metadata associated with an Earth Engine asset.
8 |
9 | #### Usage
10 |
11 | Using the Asset Metadata Delete tool is straightforward, requiring the specification of the target Earth Engine asset and the property to be deleted from its metadata.
12 |
13 | ```bash
14 | geeadd delete_metadata --asset "asset_path_here" --property "metadata_property_to_delete"
15 | ```
16 |
17 | - `--asset`: The path to the Earth Engine asset from which you want to remove a specific metadata property.
18 |
19 | - `--property`: The name of the metadata property that you want to delete.
20 |
21 | #### Example
22 |
23 | Here's an example illustrating how to use the Asset Metadata Delete tool to remove a specific property from the metadata of an Earth Engine asset:
24 |
25 | ```bash
26 | geeadd delete_metadata --asset "users/your_username/your_collection" --property "description"
27 | ```
28 |
29 | 
30 |
--------------------------------------------------------------------------------
/docs/projects/move.md:
--------------------------------------------------------------------------------
1 | # Move Assets tool
2 |
3 | The Asset Move tool in geeadd is a versatile utility designed to simplify asset management within Google Earth Engine (GEE). This tool empowers users to perform recursive moves of entire Earth Engine assets, including folders, collections, images, and tables, from one location to another effortlessly.
4 |
5 | #### Key Features
6 |
7 | - **Effortless Asset Relocation**: The Asset Move tool streamlines the process of moving Earth Engine assets, ensuring that entire hierarchies of assets can be relocated with ease.
8 |
9 | - **User-Friendly Interface**: The tool offers an intuitive command-line interface, making it accessible to users of all experience levels.
10 |
11 | #### Usage
12 |
13 | Using the Asset Move tool is simple and straightforward, allowing users to specify the source and destination paths for asset relocation.
14 |
15 | ```bash
16 | geeadd move --initial "existing_asset_path" --final "new_asset_path"
17 | ```
18 |
19 | - `--initial`: The existing path of the assets you want to move.
20 |
21 | - `--final`: The new path where you want to relocate the assets.
22 |
23 | #### Example
24 |
25 | Here's an example illustrating how to use the Asset Move tool to efficiently manage Earth Engine assets:
26 |
27 | ```bash
28 | geeadd move --initial "users/your_username/your_collection" --final "users/your_username/new_collection"
29 | ```
30 |
31 | 
32 |
33 | The Asset Move tool in geeadd simplifies asset relocation, making it an invaluable asset for users seeking to organize and manage their Earth Engine assets efficiently.
34 |
--------------------------------------------------------------------------------
/docs/projects/projects.md:
--------------------------------------------------------------------------------
1 | # EE API Enabled Projects tool
2 |
3 | The projects tool is a relatively new add and allows the owners of cloud projects to understand which cloud projects have earth engine api enabled. The tool only works on cloud projects you own because it needs to query all permissions to check for this. These are also the projects you should be able to add to your earth engine code editor.
4 |
5 | #### Prerequisites
6 | * This tool assumes you have gcloud installed and authenticated. It does check for these two things as it tries to run the tool.
7 | * This assumes that the user has enabled the earth engine api on these projects since this is what the tool is checking for.
8 |
9 | #### Features
10 | - **Automatically checks gcloud installation and config** this is handy because it allows the user to know if they check their base configuration
11 | - **Prints Project Name and Number** this is useful as a list to know which projects to set if needed for your python projects as well or to find out which projects to use in code editor.
12 |
13 | #### Sample run
14 |
15 | ```
16 | geeadd projects
17 |
18 | ```
19 |
20 | 
21 |
22 | #### Sample output
23 |
24 | ```
25 | gcloud is installed and authenticated
26 |
27 | Checking Earth Engine permissions for all projects...
28 |
29 | Project Name: abc Project Number: 77544
30 | Project Name: def Project Number: 433
31 | Project Name: ghi Project Number: 107921
32 | Project Name: ijk Project Number: 225
33 |
34 | ```
35 |
--------------------------------------------------------------------------------
/docs/projects/quota.md:
--------------------------------------------------------------------------------
1 | # Quota tool
2 |
3 | The Quota tool is a fundamental component of the geeadd toolbox, providing essential functionality to monitor and manage your Google Earth Engine (GEE) resources. It enables you to retrieve crucial information about your current GEE quota, including both used and remaining resources. This information is aggregated across all your legacy folders and user root folders.
4 |
5 | #### Features
6 |
7 | - **Total Quota Overview**: The Quota tool provides a comprehensive overview of your GEE quota, including both the resources you've already consumed and the resources still available.
8 |
9 | - **Legacy and User Root Folders**: It automatically calculates and aggregates quota information from all your legacy folders and user root folders, ensuring you have a complete picture of your resource usage.
10 |
11 | 
12 |
13 | - **Compatibility with Google Cloud Projects**: The Quota tool has been enhanced to work seamlessly with Google Earth Engine in conjunction with Google Cloud Projects. You can now specify a project path as an argument to retrieve quota information specific to that project.
14 |
15 | The Quota tool in geeadd simplifies resource tracking and management, making it an indispensable tool for Earth Engine users.
16 |
--------------------------------------------------------------------------------
/docs/projects/report.md:
--------------------------------------------------------------------------------
1 | # Assets Report tool
2 |
3 | The Assets Report tool in geeadd is a robust utility designed to provide users with comprehensive insights into their Google Earth Engine (GEE) assets. This tool performs a recursive analysis of all your assets, including images, image collections, and tables, generating a detailed report that contains a wealth of information. The report includes essential fields such as asset type, path, number of assets, size in megabytes (MB), units, owner, readers, and writers for each asset. Please note that generating a detailed report may take some time due to the thorough analysis involved.
4 |
5 | #### Key Features
6 |
7 | - **Comprehensive Asset Reporting**: The Assets Report tool offers an in-depth analysis of all your GEE assets, enabling you to gain a comprehensive understanding of your asset inventory.
8 |
9 | - **Detailed Information**: The generated report includes crucial asset details, such as type, path, quantity, size, ownership, and access permissions, making it a valuable resource for asset management and monitoring.
10 |
11 | #### Usage
12 |
13 | Using the Assets Report tool is straightforward, requiring only the specification of the location where you want to save the generated report in CSV format.
14 |
15 | ```bash
16 | geeadd ee_report --outfile "report_file_location.csv"
17 | ```
18 |
19 | - `--outfile`: The full path to the location where you want to save the report in CSV format.
20 |
21 | #### Example
22 |
23 | Here's an example demonstrating how to use the Assets Report tool to generate a comprehensive report of your GEE assets:
24 |
25 | ```bash
26 | geeadd ee_report --outfile "C:\johndoe\report.csv"
27 | ```
28 |
29 | 
30 |
31 | The Assets Report Tool in geeadd empowers users with a wealth of asset-related insights, facilitating effective asset management, auditing, and monitoring within the Google Earth Engine environment.
32 |
--------------------------------------------------------------------------------
/docs/projects/search.md:
--------------------------------------------------------------------------------
1 | # Search Tool
2 |
3 | The Search tool, empowers users to explore the extensive [Google Earth Engine data catalog](https://developers.google.com/earth-engine/datasets/catalog) as well as the [Awesome GEE Community Catalog](https://gee-community-catalog.org/) effortlessly. It enables users to search for images that match specific keywords, searching based on names, IDs, tags, and more. The search results are conveniently presented in JSON format, providing valuable information such as Earth Engine asset types, start and end dates, and additional details.
4 |
5 | #### Key Features
6 |
7 | - **Efficient Data Catalog Search**: The Search tool allows users to efficiently explore the Google Earth Engine data catalog using custom keywords and search criteria. It simplifies the process of finding relevant Earth Engine assets.
8 |
9 | - **Comprehensive Search Results**: The tool reports search results in JSON format, presenting essential information about the matched assets, including asset types, start and end dates, and other relevant details.
10 |
11 | - **Community Dataset Support**: Starting from GEEadd version 0.5.4, the Search tool includes the capability to search within the community datasets catalog. This feature expands the search scope to include community-contributed datasets, further enriching the available data resources.
12 |
13 | #### Usage
14 |
15 | Using the Search tool is straightforward. You can specify keywords and, if desired, the source of the data catalog.
16 |
17 | ```bash
18 | # Search for images matching the keyword "fire" in the Earth Engine data catalog
19 | geeadd search --keywords "fire"
20 | ```
21 |
22 | ```bash
23 | # Search for images in the community-contributed datasets catalog
24 | geeadd search --keywords "ph" --source community
25 | ```
26 |
27 | - `--keywords`: The keywords or search terms you want to use to find matching Earth Engine assets.
28 |
29 | - `--source`: (Optional) The source of the data catalog you want to search in (default is Earth Engine data catalog).
30 |
31 | ## Example
32 |
33 | Here's an example demonstrating how to use the Search tool to find Earth Engine assets:
34 |
35 | ```bash
36 | # Search for images matching the keyword "fire" in the Earth Engine data catalog
37 | geeadd search --keywords "fire"
38 | ```
39 |
40 | 
41 |
42 | The Search tool in GEEadd simplifies the process of discovering relevant Earth Engine assets, providing users with a powerful tool for accessing geospatial data within the Earth Engine ecosystem. Additionally, the tool's support for community-contributed datasets enhances its utility, making it an invaluable resource for Earth Engine users.
43 |
44 | 
45 |
--------------------------------------------------------------------------------
/docs/projects/size.md:
--------------------------------------------------------------------------------
1 | # Asset Size tool
2 |
3 | The Asset Size tool in geeadd provides users with a convenient means to query the size of Earth Engine assets, including images, image collections, tables, and folders. This tool offers valuable insights by displaying the number of assets and their total size, presented in easily comprehensible formats such as kilobytes (KB), megabytes (MB), gigabytes (GB), or terabytes (TB), depending on the size.
4 |
5 | #### Key Features
6 |
7 | - **Asset Size Query**: The Asset Size tool enables users to query the size of Earth Engine assets swiftly and accurately.
8 |
9 | - **Informative Output**: The tool delivers clear and informative output, providing users with the number of assets and their cumulative size in human-readable formats.
10 |
11 | #### Usage
12 |
13 | Using the Asset Size tool is straightforward, requiring only the specification of the target Earth Engine asset.
14 |
15 | ```bash
16 | geeadd assetsize --asset "your_asset_path_here"
17 | ```
18 |
19 | - `--asset`: The Earth Engine asset for which you want to obtain size properties.
20 |
21 | #### Example
22 |
23 | Here's an example illustrating how to use the Asset Size tool to determine the size of an Earth Engine asset:
24 |
25 | ```bash
26 | geeadd assetsize --asset "users/your_username/your_collection"
27 | ```
28 |
29 | 
30 |
--------------------------------------------------------------------------------
/docs/projects/task_status.md:
--------------------------------------------------------------------------------
1 | # Tasks tool
2 |
3 | The Tasks tool in geeadd provides a streamlined approach to monitor and manage tasks within Google Earth Engine (GEE). It offers comprehensive insights into the status of ongoing tasks, including those that are running, cancelled, pending, and failed. Additionally, the tool allows for detailed task-specific information retrieval by providing either the task state or a specific task ID.
4 |
5 | #### Key Features
6 |
7 | - **Task Status Overview**: The Tasks tool offers a quick summary of tasks currently in progress, cancelled, pending, and those that have encountered failures. This enables users to effectively track the progress and status of their Earth Engine tasks.
8 |
9 | - **Detailed Task Information**: Users have the ability to retrieve detailed information about a specific task by providing either the task state or a unique task ID. This information includes task descriptions, URIs, and the resources (EECUs) utilized by the task.
10 |
11 | #### Usage
12 |
13 | Using the Tasks tool is straightforward. You can either get an overview of all tasks or retrieve specific information about a particular task.
14 |
15 | 
16 |
17 | ```
18 | > geeadd tasks -h
19 | usage: geeadd tasks [-h] [--state STATE] [--id ID]
20 |
21 | options:
22 | -h, --help show this help message and exit
23 |
24 | Optional named arguments:
25 | --state STATE Query by state type COMPLETED|PENDING|RUNNING|FAILED
26 | --id ID Query by task id
27 | ```
28 |
--------------------------------------------------------------------------------
/docs/stylesheets/codehilite.css:
--------------------------------------------------------------------------------
1 | /*
2 | /////////////////
3 | // Inline Code //
4 | /////////////////
5 | */
6 |
7 | .md-typeset code {
8 | background-color: #424242;
9 | color: #F5F5F5;
10 | margin: 0;
11 | padding: 0.07353em 0.29412em;
12 | box-shadow: none;
13 | }
14 |
15 | /*
16 | /////////////////
17 | // Code Blocks //
18 | /////////////////
19 | */
20 |
21 | /*
22 | line number
23 | */
24 | .linenos {
25 | color: #F5F5F5 !important;
26 | background-color: #313131 !important;
27 | }
28 |
29 | /*
30 | code block background
31 | */
32 | .codehilite {
33 | background-color: #424242 !important;
34 | }
35 |
36 | /*
37 | scroll bar size
38 | */
39 |
40 | .md-typeset .codehilite::-webkit-scrollbar {
41 | height: 1rem !important;
42 | }
43 |
44 | /*
45 | actual syntax highlighting
46 | */
47 | .codehilite pre { color: #FAFAFA !important; background-color: transparent !important; }
48 | .codehilite .hll { background-color: #272822 !important; }
49 | .codehilite .c { color: #75715e !important } /* Comment */
50 | .codehilite .err { color: #960050 !important; background-color: #1e0010 !important } /* Error */
51 | .codehilite .k { color: #66d9ef !important } /* Keyword */
52 | .codehilite .l { color: #ae81ff !important } /* Literal */
53 | .codehilite .n { color: #f8f8f2 !important } /* Name */
54 | .codehilite .o { color: #f92672 !important } /* Operator */
55 | .codehilite .p { color: #f8f8f2 !important } /* Punctuation */
56 | .codehilite .cm { color: #75715e !important } /* Comment.Multiline */
57 | .codehilite .cp { color: #75715e !important } /* Comment.Preproc */
58 | .codehilite .c1 { color: #75715e !important } /* Comment.Single */
59 | .codehilite .cs { color: #75715e !important } /* Comment.Special */
60 | .codehilite .ge { font-style: italic !important } /* Generic.Emph */
61 | .codehilite .gs { font-weight: bold !important } /* Generic.Strong */
62 | .codehilite .kc { color: #66d9ef !important } /* Keyword.Constant */
63 | .codehilite .kd { color: #66d9ef !important } /* Keyword.Declaration */
64 | .codehilite .kn { color: #f92672 !important } /* Keyword.Namespace */
65 | .codehilite .kp { color: #66d9ef !important } /* Keyword.Pseudo */
66 | .codehilite .kr { color: #66d9ef !important } /* Keyword.Reserved */
67 | .codehilite .kt { color: #66d9ef !important } /* Keyword.Type */
68 | .codehilite .ld { color: #e6db74 !important } /* Literal.Date */
69 | .codehilite .m { color: #ae81ff !important } /* Literal.Number */
70 | .codehilite .s { color: #e6db74 !important } /* Literal.String */
71 | .codehilite .na { color: #a6e22e !important } /* Name.Attribute */
72 | .codehilite .nb { color: #f8f8f2 !important } /* Name.Builtin */
73 | .codehilite .nc { color: #a6e22e !important } /* Name.Class */
74 | .codehilite .no { color: #66d9ef !important } /* Name.Constant */
75 | .codehilite .nd { color: #a6e22e !important } /* Name.Decorator */
76 | .codehilite .ni { color: #f8f8f2 !important } /* Name.Entity */
77 | .codehilite .ne { color: #a6e22e !important } /* Name.Exception */
78 | .codehilite .nf { color: #a6e22e !important } /* Name.Function */
79 | .codehilite .nl { color: #f8f8f2 !important } /* Name.Label */
80 | .codehilite .nn { color: #f8f8f2 !important } /* Name.Namespace */
81 | .codehilite .nx { color: #a6e22e !important } /* Name.Other */
82 | .codehilite .py { color: #f8f8f2 !important } /* Name.Property */
83 | .codehilite .nt { color: #f92672 !important } /* Name.Tag */
84 | .codehilite .nv { color: #f8f8f2 !important } /* Name.Variable */
85 | .codehilite .ow { color: #f92672 !important } /* Operator.Word */
86 | .codehilite .w { color: #f8f8f2 !important } /* Text.Whitespace */
87 | .codehilite .mf { color: #ae81ff !important } /* Literal.Number.Float */
88 | .codehilite .mh { color: #ae81ff !important } /* Literal.Number.Hex */
89 | .codehilite .mi { color: #ae81ff !important } /* Literal.Number.Integer */
90 | .codehilite .mo { color: #ae81ff !important } /* Literal.Number.Oct */
91 | .codehilite .sb { color: #e6db74 !important } /* Literal.String.Backtick */
92 | .codehilite .sc { color: #e6db74 !important } /* Literal.String.Char */
93 | .codehilite .sd { color: #e6db74 !important } /* Literal.String.Doc */
94 | .codehilite .s2 { color: #e6db74 !important } /* Literal.String.Double */
95 | .codehilite .se { color: #ae81ff !important } /* Literal.String.Escape */
96 | .codehilite .sh { color: #e6db74 !important } /* Literal.String.Heredoc */
97 | .codehilite .si { color: #e6db74 !important } /* Literal.String.Interpol */
98 | .codehilite .sx { color: #e6db74 !important } /* Literal.String.Other */
99 | .codehilite .sr { color: #e6db74 !important } /* Literal.String.Regex */
100 | .codehilite .s1 { color: #e6db74 !important } /* Literal.String.Single */
101 | .codehilite .ss { color: #e6db74 !important } /* Literal.String.Symbol */
102 | .codehilite .bp { color: #f8f8f2 !important } /* Name.Builtin.Pseudo */
103 | .codehilite .vc { color: #f8f8f2 !important } /* Name.Variable.Class */
104 | .codehilite .vg { color: #f8f8f2 !important } /* Name.Variable.Global */
105 | .codehilite .vi { color: #f8f8f2 !important } /* Name.Variable.Instance */
106 | .codehilite .il { color: #ae81ff !important } /* Literal.Number.Integer.Long */
107 |
108 | .codehilite .gh { } /* Generic Heading & Diff Header */
109 | .codehilite .gu { color: #75715e !important ; } /* Generic.Subheading & Diff Unified/Comment? */
110 | .codehilite .gd { color: #f92672 !important ; } /* Generic.Deleted & Diff Deleted */
111 | .codehilite .gi { color: #a6e22e !important ; } /* Generic.Inserted & Diff Inserted */
112 |
113 | .codehilite .md-clipboard:before { color: rgba(255, 255, 255, 0.07) } /* Clipboard button (no hover) */
114 | .codehilite:hover .md-clipboard:before { color: rgba(255, 255, 255, 0.54) } /* Clipboard button (hovered) */
115 |
--------------------------------------------------------------------------------
/docs/stylesheets/dark_theme.css:
--------------------------------------------------------------------------------
1 | /*
2 | //////////////////
3 | // Main content //
4 | //////////////////
5 | */
6 |
7 | /*
8 | Default text color
9 | and background color
10 | */
11 | .md-main {
12 | color: #F5F5F5 !important;
13 | background-color: #212121 !important;
14 | }
15 |
16 | /*
17 | Main headlines
18 | */
19 | .md-main h1 {
20 | color: white !important;
21 | }
22 |
23 | /*
24 | Tables
25 | */
26 | table {
27 | background-color: #616161 !important;
28 | }
29 |
30 | tbody {
31 | background-color: #484848 !important;
32 | }
33 |
34 | /*
35 | Blockquotes
36 | */
37 | .md-typeset blockquote {
38 | color: rgba(255,255,255,0.8) !important;
39 | border-color: rgba(255,255,255,0.54) !important;
40 | }
41 |
42 | /*
43 | ////////////////////
44 | // Navigation bar //
45 | ////////////////////
46 | */
47 |
48 | /*
49 | Left and right toc scrollbar
50 | */
51 | .md-sidebar__scrollwrap::-webkit-scrollbar-thumb {
52 | background-color: #E0E0E0 !important;
53 | }
54 |
55 |
56 |
57 | .md-nav {
58 | color: #F5F5F5 !important;
59 | background-color: #212121 !important;
60 | }
61 |
62 | /*
63 | Arrow Left Icon
64 | */
65 | html .md-nav--primary .md-nav__title:before {
66 | color: #FAFAFA !important;
67 | }
68 |
69 | .md-nav__title {
70 | color: rgba(255,255,255,1) !important;
71 | background-color: #212121 !important;
72 | }
73 |
74 | /*
75 | Arrow Right Icon
76 | */
77 | .md-nav--primary .md-nav__link:after {
78 | color: #FAFAFA !important;
79 | }
80 |
81 | .md-nav__list {
82 | color: rgba(255,255,255,1) !important;
83 | background-color: #212121 !important;
84 | }
85 |
86 | .md-nav__item {
87 | color: rgba(255,255,255,1) !important;
88 | background-color: #212121 !important;
89 | }
90 |
91 | .md-nav__link[data-md-state=blur] {
92 | color: rgba(255,255,255,0.54) !important;
93 | }
94 |
95 | /*
96 | ////////////
97 | // Search //
98 | ////////////
99 | */
100 |
101 | /*
102 | scroll bar
103 |
104 | attention:
105 | background is scroll handle color!
106 | */
107 | .md-search__scrollwrap::-webkit-scrollbar-thumb {
108 | background-color: #E0E0E0 !important;
109 | }
110 | /*
111 | scroll bar background color
112 | */
113 | .md-search__scrollwrap {
114 | background-color: #424242 !important;
115 | }
116 |
117 | /*
118 | Icon color
119 | */
120 | .md-search-result__article--document:before {
121 | color: #EEEEEE !important;
122 | }
123 |
124 | /*
125 | headline color and
126 | result list background
127 | */
128 | .md-search-result__list {
129 | color: #EEEEEE !important;
130 | background-color: #212121 !important;
131 | }
132 |
133 | /*
134 | result info/count
135 | */
136 | .md-search-result__meta {
137 | background-color: #EEEEEE !important;
138 | }
139 |
140 | /*
141 | article preview text color
142 | */
143 | .md-search-result__teaser {
144 | color: #BDBDBD !important;
145 | }
146 |
--------------------------------------------------------------------------------
/geeadd/__init__.py:
--------------------------------------------------------------------------------
1 | # -*- coding: utf-8 -*-
2 |
3 | __author__ = "Samapriya Roy"
4 | __email__ = "samapriya.roy@gmail.com"
5 | __version__ = "1.2.1"
6 |
--------------------------------------------------------------------------------
/geeadd/acl_changer.py:
--------------------------------------------------------------------------------
1 | __copyright__ = """
2 |
3 | Copyright 2024 Samapriya Roy
4 |
5 | Licensed under the Apache License, Version 2.0 (the "License");
6 | you may not use this file except in compliance with the License.
7 | You may obtain a copy of the License at
8 |
9 | http://www.apache.org/licenses/LICENSE-2.0
10 |
11 | Unless required by applicable law or agreed to in writing, software
12 | distributed under the License is distributed on an "AS IS" BASIS,
13 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 | See the License for the specific language governing permissions and
15 | limitations under the License.
16 |
17 | """
18 | __license__ = "Apache 2.0"
19 | import itertools
20 | import json
21 |
22 | import ee
23 |
24 | # Empty Lists
25 |
26 | image_list = []
27 | collection_list = []
28 | table_list = []
29 |
30 | # Recursive folder paths
31 | folder_list = []
32 |
33 |
34 | def get_folder(path):
35 | parser = ee.data.getAsset(path)
36 | if parser["type"].lower() == "folder":
37 | folder_list.append(parser["name"])
38 | recursive(parser["name"])
39 |
40 |
41 | def recursive(path):
42 | path = ee.data.getAsset(path)
43 | if path["type"].lower() == "folder":
44 | path = path["name"]
45 | folder_list.append(path)
46 | children = ee.data.listAssets({"parent": path})
47 | for child in children["assets"]:
48 | if not child["name"] in folder_list:
49 | get_folder(child["name"])
50 | return folder_list
51 |
52 |
53 | # folder parse
54 | def fparse(path):
55 | ee.Initialize()
56 | if ee.data.getAsset(path)["type"].lower() == "folder":
57 | gee_folder_path = recursive(path)
58 | gee_folder_path = sorted(list(set(gee_folder_path)))
59 | for folders in gee_folder_path:
60 | children = ee.data.listAssets({"parent": ee.data.getAsset(folders)["name"]})
61 | for child in children["assets"]:
62 | if child["type"].lower() == "image_collection":
63 | collection_list.append(child["id"])
64 | elif child["type"].lower() == "image":
65 | image_list.append(child["id"])
66 | elif child["type"].lower() == "table":
67 | table_list.append(child["id"])
68 | elif child["type"].lower() == "feature_view":
69 | table_list.append(child["id"])
70 | elif ee.data.getAsset(path)["type"].lower() == "image":
71 | image_list.append(path)
72 | elif ee.data.getAsset(path)["type"].lower() == "image_collection":
73 | collection_list.append(path)
74 | elif ee.data.getAsset(path)["type"].lower() == "table":
75 | table_list.append(path)
76 | elif ee.data.getAsset(path)["type"].lower() == "feature_view":
77 | table_list.append(path)
78 | else:
79 | print(ee.data.getAsset(path)["type"].lower())
80 | return [collection_list, table_list, image_list, folder_list]
81 |
82 |
83 | ##request type of asset, asset path and user to give permission
84 | def access(collection_path, user, role):
85 | ee.Initialize()
86 | if user.endswith("googlegroups.com") and not user.startswith("group:"):
87 | user = f"group:{user}"
88 | elif user.endswith("gserviceaccount.com") and not user.startswith(
89 | "serviceAccount"
90 | ):
91 | user = f"serviceAccount:{user}"
92 | elif user == "allUsers" or user == "allusers":
93 | user = "allUsers"
94 | else:
95 | user = f"user:{user}"
96 | asset_list = fparse(collection_path)
97 | asset_names = list(set(itertools.chain(*asset_list)))
98 | print(f"Changing permission for a total of {len(asset_names)} items...")
99 |
100 | for count, init in enumerate(asset_names):
101 | acl = ee.data.getAssetAcl(init)
102 | if role == "reader":
103 | target_list = acl["readers"]
104 | target_permission = "reader"
105 | if user in target_list:
106 | print(f"{user} already has {role} access to {init} asset: SKIPPING")
107 | else:
108 | acl["readers"].append(user)
109 | try:
110 | ee.data.setAssetAcl(init, json.dumps(acl))
111 | print(f"Added {user} as {target_permission} for {init}")
112 | except Exception as error:
113 | print(error)
114 | elif role == "writer":
115 | target_list = acl["writers"]
116 | target_permission = "writer"
117 | if user in target_list:
118 | print(f"{user} already has {role} access to {init} asset: SKIPPING")
119 | else:
120 | acl["writers"].append(user)
121 | try:
122 | ee.data.setAssetAcl(init, json.dumps(acl))
123 | print(f"Added {user} as {target_permission} for {init}")
124 | except Exception as error:
125 | print(error)
126 | elif role == "delete":
127 | if user == "allUsers" and "all_users_can_read" in acl:
128 | acl.pop("all_users_can_read")
129 | if user in acl["readers"]:
130 | acl["readers"].remove(user)
131 | if user in acl["writers"]:
132 | acl["writers"].remove(user)
133 | if user in acl["owners"]:
134 | acl["owners"].remove(user)
135 | try:
136 | ee.data.setAssetAcl(init, json.dumps(acl))
137 | print(f"Removed permission for {user} from {init}")
138 | except Exception as e:
139 | print(e)
140 |
--------------------------------------------------------------------------------
/geeadd/app2script.py:
--------------------------------------------------------------------------------
1 | __copyright__ = """
2 |
3 | Copyright 2024 Samapriya Roy
4 |
5 | Licensed under the Apache License, Version 2.0 (the "License");
6 | you may not use this file except in compliance with the License.
7 | You may obtain a copy of the License at
8 |
9 | http://www.apache.org/licenses/LICENSE-2.0
10 |
11 | Unless required by applicable law or agreed to in writing, software
12 | distributed under the License is distributed on an "AS IS" BASIS,
13 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 | See the License for the specific language governing permissions and
15 | limitations under the License.
16 |
17 | """
18 | __license__ = "Apache 2.0"
19 | import requests
20 | import jsbeautifier
21 |
22 |
23 | def jsext(url, outfile):
24 | """
25 | Retrieve the application script from an Earth Engine app.
26 |
27 | Args:
28 | url (str): URL of the Earth Engine app.
29 | outfile (str, optional): Output file path for saving the JavaScript code.
30 |
31 | """
32 | head, tail = url.split("/view/")
33 | fetch_url = f"{head}/javascript/{tail}-modules.json"
34 | source_fetch = requests.get(
35 | fetch_url,
36 | )
37 | if source_fetch.status_code == 200:
38 | js_code = source_fetch.json()["dependencies"]
39 | script_path = source_fetch.json()["path"]
40 | js_code = js_code[script_path]
41 |
42 | formatted_code = jsbeautifier.beautify(js_code)
43 | try:
44 | if outfile == None:
45 | print(formatted_code)
46 | else:
47 | with open(outfile, 'w') as f:
48 | f.write(formatted_code)
49 | except Exception as e:
50 | print(e)
51 |
52 |
53 | # jsext(url='https://bullocke.users.earthengine.app/view/amazon',outfile=None)
54 | # jsext(url='https://bullocke.users.earthengine.app/view/amazon')
55 |
--------------------------------------------------------------------------------
/geeadd/batch_copy.py:
--------------------------------------------------------------------------------
1 | """Optimized batch asset copying module for Google Earth Engine."""
2 |
3 | __copyright__ = """
4 |
5 | Copyright 2025 Samapriya Roy
6 |
7 | Licensed under the Apache License, Version 2.0 (the "License");
8 | you may not use this file except in compliance with the License.
9 | You may obtain a copy of the License at
10 |
11 | http://www.apache.org/licenses/LICENSE-2.0
12 |
13 | Unless required by applicable law or agreed to in writing, software
14 | distributed under the License is distributed on an "AS IS" BASIS,
15 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 | See the License for the specific language governing permissions and
17 | limitations under the License.
18 |
19 | """
20 | __license__ = "Apache 2.0"
21 |
22 | import concurrent.futures
23 | import os
24 | import signal
25 | import sys
26 |
27 | import ee
28 | from tqdm import tqdm
29 |
30 |
31 | def camel_case(s):
32 | """Convert string to camel case (Title Case)."""
33 | words = s.split()
34 | return ' '.join(word.title() for word in words)
35 |
36 |
37 | def create_folder(folder_path):
38 | """Create a folder if it doesn't exist, handling both cloud and legacy folders."""
39 | try:
40 | if ee.data.getAsset(folder_path):
41 | return True
42 | except Exception:
43 | try:
44 | ee.data.createAsset({"type": ee.data.ASSET_TYPE_FOLDER_CLOUD}, folder_path)
45 | except Exception:
46 | try:
47 | ee.data.createAsset({"type": ee.data.ASSET_TYPE_FOLDER}, folder_path)
48 | except Exception as e:
49 | print(f"Error creating folder {folder_path}: {e}")
50 | return False
51 | return True
52 |
53 |
54 | def copy_asset(source, destination, asset_type):
55 | """Copy a single asset with appropriate error handling and messaging."""
56 | try:
57 | # Check if destination already exists
58 | if ee.data.getAsset(destination):
59 | print(f"{camel_case(asset_type)} already copied: {destination}")
60 | return False
61 | except Exception:
62 | try:
63 | print(f"Copying {camel_case(asset_type)} to {destination}")
64 | ee.data.copyAsset(source, destination)
65 | return True
66 | except Exception as e:
67 | print(f"Error copying {source} to {destination}: {e}")
68 | return False
69 |
70 |
71 | def copy_image_collection(source, destination, max_workers=10):
72 | """Copy an image collection with parallel execution for speed."""
73 | # Global variable to track if interrupt occurred
74 | global interrupt_received
75 |
76 | try:
77 | # Create the collection if it doesn't exist
78 | try:
79 | if ee.data.getAsset(destination):
80 | print(f"Collection exists: {ee.data.getAsset(destination)['id']}")
81 | except Exception:
82 | print(f"Collection does not exist: Creating {destination}")
83 | try:
84 | ee.data.createAsset({"type": ee.data.ASSET_TYPE_IMAGE_COLL_CLOUD}, destination)
85 | except Exception:
86 | ee.data.createAsset({"type": ee.data.ASSET_TYPE_IMAGE_COLL}, destination)
87 |
88 | # Get list of source images
89 | source_list = ee.data.listAssets({"parent": source})
90 | source_names = [os.path.basename(asset["name"]) for asset in source_list["assets"]]
91 |
92 | # Get list of destination images
93 | collection_path = ee.data.getAsset(destination)["name"]
94 | destination_list = ee.data.listAssets({"parent": collection_path})
95 | destination_names = [os.path.basename(asset["name"]) for asset in destination_list["assets"]]
96 |
97 | # Find images that need to be copied
98 | images_to_copy = set(source_names) - set(destination_names)
99 | total_images = len(images_to_copy)
100 |
101 | if not images_to_copy:
102 | print("All images already exist in destination collection. Nothing to copy.")
103 | return
104 |
105 | print(f"Copying {total_images} images in parallel...")
106 |
107 | # Use ThreadPoolExecutor for parallel copying
108 | results = []
109 | processed_count = 0
110 | executor = concurrent.futures.ThreadPoolExecutor(max_workers=max_workers)
111 | futures = {}
112 |
113 | try:
114 | # Submit all tasks
115 | for item in images_to_copy:
116 | if interrupt_received:
117 | break
118 | source_path = f"{source}/{item}"
119 | dest_path = f"{collection_path}/{item}"
120 | future = executor.submit(ee.data.copyAsset, source_path, dest_path)
121 | futures[future] = item
122 |
123 | # Process results with progress bar
124 | with tqdm(total=total_images, desc="Copying images") as pbar:
125 | for future in concurrent.futures.as_completed(futures):
126 | item = futures[future]
127 |
128 | # Check if we've received an interrupt
129 | if interrupt_received:
130 | pbar.write("\nInterrupt received, cancelling remaining copy operations...")
131 | # Cancel remaining futures
132 | for f in futures:
133 | if not f.done():
134 | f.cancel()
135 | break
136 |
137 | try:
138 | future.result()
139 | results.append(True)
140 | except Exception as e:
141 | pbar.write(f"Error copying {item}: {e}")
142 | results.append(False)
143 |
144 | processed_count += 1
145 | pbar.update(1)
146 |
147 | finally:
148 | # Always shut down the executor
149 | executor.shutdown(wait=False)
150 |
151 | if interrupt_received:
152 | print(f"\nOperation interrupted. Copied {processed_count} of {total_images} images.")
153 | else:
154 | success_count = sum(results)
155 | print(f"Successfully copied {success_count} of {total_images} images")
156 |
157 | except Exception as e:
158 | print(f"Error in collection copy: {e}")
159 |
160 |
161 | def get_folder_structure(path):
162 | """Recursively gather all folders under a path."""
163 | folder_list = []
164 |
165 | def recursive_get_folders(current_path):
166 | try:
167 | asset_info = ee.data.getAsset(current_path)
168 | if asset_info["type"].lower() == "folder":
169 | folder_list.append(asset_info["name"])
170 | children = ee.data.listAssets({"parent": current_path})
171 | for child in children["assets"]:
172 | if child["type"].lower() == "folder":
173 | recursive_get_folders(child["name"])
174 | except Exception as e:
175 | print(f"Error accessing {current_path}: {e}")
176 |
177 | recursive_get_folders(path)
178 | return sorted(list(set(folder_list)))
179 |
180 |
181 | # Global flag to track interrupt status
182 | interrupt_received = False
183 |
184 | # Signal handler for keyboard interrupts
185 | def handle_interrupt(sig, frame):
186 | """Handle interrupt signals gracefully."""
187 | global interrupt_received
188 | if not interrupt_received:
189 | print("\nInterrupt received! Gracefully shutting down... (This may take a moment)")
190 | print("Press Ctrl+C again to force immediate exit (may leave tasks in an inconsistent state)")
191 | interrupt_received = True
192 | else:
193 | print("\nForced exit requested. Exiting immediately.")
194 | sys.exit(1)
195 |
196 | def copy(path, fpath, max_workers=10):
197 | """Copy Earth Engine assets from path to fpath.
198 |
199 | Args:
200 | path: Source asset path
201 | fpath: Destination asset path
202 | max_workers: Maximum number of parallel workers for collection copying
203 | """
204 | global interrupt_received
205 | interrupt_received = False
206 |
207 | # Set up signal handler for graceful interrupts
208 | original_sigint_handler = signal.getsignal(signal.SIGINT)
209 | signal.signal(signal.SIGINT, handle_interrupt)
210 |
211 | try:
212 | ee.Initialize()
213 |
214 | asset_info = ee.data.getAsset(path)
215 | if not asset_info:
216 | print(f"Source asset not found: {path}")
217 | return
218 |
219 | asset_type = asset_info["type"].lower()
220 |
221 | if asset_type == "folder":
222 | # Copy folder structure
223 | folders = get_folder_structure(path)
224 | print(f"Found {len(folders)} folders to copy")
225 |
226 | # Get path prefixes for replacement
227 | initial_path_suffix = path.split("/")[-1]
228 | replace_string = "/".join(ee.data.getAsset(path)["name"].split("/")[:-1]) + "/" + initial_path_suffix
229 |
230 | final_path_suffix = fpath.split("/")[-1]
231 | replaced_string = ee.data.getAsset("/".join(fpath.split("/")[:-1]) + "/")["name"] + "/" + final_path_suffix
232 |
233 | # Create folder structure
234 | for folder in folders:
235 | if interrupt_received:
236 | print("Operation interrupted. Stopping folder creation.")
237 | break
238 |
239 | new_folder = folder.replace(replace_string, replaced_string)
240 | create_folder(new_folder)
241 |
242 | # Process assets in this folder
243 | try:
244 | children = ee.data.listAssets({"parent": folder})
245 | for child in children["assets"]:
246 | if interrupt_received:
247 | print("Operation interrupted. Stopping asset processing.")
248 | break
249 |
250 | child_type = child["type"].lower()
251 | child_path = child["name"]
252 | child_dest = child_path.replace(replace_string, replaced_string)
253 |
254 | if child_type == "image_collection":
255 | copy_image_collection(child_path, child_dest, max_workers)
256 | elif child_type == "image":
257 | copy_asset(child_path, child_dest, "image")
258 | elif child_type == "table":
259 | copy_asset(child_path, child_dest, "table")
260 | elif child_type == "feature_view":
261 | copy_asset(child_path, child_dest, "feature view")
262 |
263 | # Check interrupt again after each asset
264 | if interrupt_received:
265 | break
266 | except Exception as e:
267 | print(f"Error processing assets in folder {folder}: {e}")
268 |
269 | elif asset_type == "image":
270 | # Copy individual image
271 | destination = fpath
272 | copy_asset(path, destination, "image")
273 |
274 | elif asset_type == "image_collection":
275 | # Copy collection
276 | copy_image_collection(path, fpath, max_workers)
277 |
278 | elif asset_type in ("table", "feature_view"):
279 | # Copy table or feature view
280 | copy_asset(path, fpath, asset_type)
281 |
282 | else:
283 | print(f"Unsupported asset type: {asset_type}")
284 |
285 | except Exception as e:
286 | if not interrupt_received:
287 | print(f"Error copying assets: {e}")
288 | finally:
289 | # Restore original signal handler
290 | signal.signal(signal.SIGINT, original_sigint_handler)
291 |
292 | if interrupt_received:
293 | print("\nCopy operation was interrupted and has been stopped.")
294 | print("Some assets may have been copied while others were not.")
295 | else:
296 | print("Copy operation completed.")
297 |
--------------------------------------------------------------------------------
/geeadd/batch_delete.py:
--------------------------------------------------------------------------------
1 | import concurrent.futures
2 | import signal
3 | import sys
4 | import time
5 |
6 | import ee
7 | from tqdm import tqdm
8 |
9 | # Global flag to track interrupt status
10 | interrupt_received = False
11 |
12 | # Global list to track assets
13 | asset_list = []
14 | ee.Initialize()
15 |
16 |
17 | def handle_interrupt(sig, frame):
18 | global interrupt_received
19 | if not interrupt_received:
20 | print(
21 | "\nInterrupt received! Gracefully shutting down... (This may take a moment)"
22 | )
23 | print(
24 | "Press Ctrl+C again to force immediate exit (may leave tasks in an inconsistent state)"
25 | )
26 | interrupt_received = True
27 | else:
28 | print("\nForced exit requested. Exiting immediately.")
29 | sys.exit(1)
30 |
31 |
32 | def get_asset(path):
33 | """Get asset information and add to asset_list."""
34 | try:
35 | parser = ee.data.getAsset(path)
36 | asset_type = parser["type"].lower()
37 | asset_list.append({"path": parser["name"], "type": asset_type})
38 | return parser["name"], asset_type
39 | except Exception as e:
40 | print(f"Error processing {path}: {e}")
41 | return None, None
42 |
43 |
44 | def recursive_parallel(path):
45 | """Recursively gather all assets under a path using parallel execution."""
46 | try:
47 | path_info = ee.data.getAsset(path)
48 | asset_type = path_info["type"].lower()
49 |
50 | if asset_type in ["folder", "image_collection"]:
51 | children = ee.data.listAssets({"parent": path})
52 | paths = [child["name"] for child in children["assets"]]
53 |
54 | with concurrent.futures.ThreadPoolExecutor() as executor:
55 | results = list(executor.map(get_asset, paths))
56 | new_folders = [
57 | res[0]
58 | for res in results
59 | if res[1] in ["folder", "image_collection"] and res[0] is not None
60 | ]
61 | executor.map(recursive_parallel, new_folders)
62 | except Exception as e:
63 | print(f"Error in recursive processing of {path}: {e}")
64 |
65 | return asset_list
66 |
67 |
68 | def delete_with_retry(asset_path, max_retries=3):
69 | for attempt in range(max_retries):
70 | try:
71 | ee.data.deleteAsset(asset_path)
72 | return True
73 | except ee.EEException as e:
74 | error_msg = str(e).lower()
75 | if "not found" in error_msg or "permission denied" in error_msg:
76 | print(f"Cannot delete {asset_path}: {e}")
77 | return True
78 | if "delete its children" in error_msg:
79 | print(f"Asset {asset_path} has children; delaying deletion")
80 | time.sleep(1)
81 | return (
82 | delete_with_retry(asset_path, max_retries - 1)
83 | if max_retries > 1
84 | else False
85 | )
86 | if attempt < max_retries - 1:
87 | time.sleep(2**attempt)
88 | else:
89 | print(
90 | f"Failed to delete {asset_path} after {max_retries} attempts: {e}"
91 | )
92 | return False
93 | except Exception as e:
94 | print(f"Unexpected error deleting {asset_path}: {e}")
95 | return False
96 | return False
97 |
98 |
99 | def delete(ids, max_workers=10, max_retries=3):
100 | global interrupt_received, asset_list
101 | interrupt_received = False
102 | asset_list = []
103 | original_sigint_handler = signal.getsignal(signal.SIGINT)
104 | signal.signal(signal.SIGINT, handle_interrupt)
105 | try:
106 | try:
107 | initial_asset_info = ee.data.getAsset(ids)
108 | if not initial_asset_info:
109 | print(f"Asset not found: {ids}")
110 | return
111 | except Exception as e:
112 | print(f"Error getting asset info: {e}")
113 | return
114 |
115 | print(f"Gathering all assets under {ids}...")
116 | asset_list = recursive_parallel(ids)
117 | asset_list.sort(key=lambda x: x["path"].count("/"), reverse=True)
118 |
119 | if not asset_list:
120 | print("No assets found to delete.")
121 | return
122 | print(f"Found {len(asset_list)} assets to delete")
123 |
124 | successful_deletions = 0
125 | with tqdm(total=len(asset_list), desc="Deleting assets") as pbar:
126 | with concurrent.futures.ThreadPoolExecutor(
127 | max_workers=max_workers
128 | ) as executor:
129 | future_to_asset = {
130 | executor.submit(
131 | delete_with_retry, asset["path"], max_retries
132 | ): asset
133 | for asset in asset_list
134 | }
135 | for future in concurrent.futures.as_completed(future_to_asset):
136 | asset = future_to_asset[future]
137 | if interrupt_received:
138 | pbar.write(
139 | "\nInterrupt received, cancelling remaining delete operations..."
140 | )
141 | for f in future_to_asset:
142 | if not f.done():
143 | f.cancel()
144 | break
145 | try:
146 | if future.result():
147 | successful_deletions += 1
148 | except Exception as e:
149 | pbar.write(f"Error deleting {asset['path']}: {e}")
150 | pbar.update(1)
151 | print(f"\nSuccessfully deleted {successful_deletions}/{len(asset_list)} assets")
152 | except Exception as e:
153 | print(f"Unexpected error in deletion process: {e}")
154 | finally:
155 | signal.signal(signal.SIGINT, original_sigint_handler)
156 | if interrupt_received:
157 | print("\nDelete operation was interrupted and has been stopped.")
158 | print("Some assets may have been deleted while others were not.")
159 |
160 |
161 | #delete("users/samapriya/LA_Datasets_Move")
162 |
--------------------------------------------------------------------------------
/geeadd/batch_mover.py:
--------------------------------------------------------------------------------
1 | """Optimized batch asset moving module for Google Earth Engine."""
2 |
3 | __copyright__ = """
4 |
5 | Copyright 2025 Samapriya Roy
6 |
7 | Licensed under the Apache License, Version 2.0 (the "License");
8 | you may not use this file except in compliance with the License.
9 | You may obtain a copy of the License at
10 |
11 | http://www.apache.org/licenses/LICENSE-2.0
12 |
13 | Unless required by applicable law or agreed to in writing, software
14 | distributed under the License is distributed on an "AS IS" BASIS,
15 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 | See the License for the specific language governing permissions and
17 | limitations under the License.
18 |
19 | """
20 | __license__ = "Apache 2.0"
21 |
22 | import concurrent.futures
23 | import os
24 | import signal
25 | import sys
26 |
27 | import ee
28 | from tqdm import tqdm
29 |
30 | # Global flag to track interrupt status
31 | interrupt_received = False
32 |
33 | # Global list to track folders
34 | folder_list = []
35 |
36 |
37 | # Signal handler for keyboard interrupts
38 | def handle_interrupt(sig, frame):
39 | """Handle interrupt signals gracefully."""
40 | global interrupt_received
41 | if not interrupt_received:
42 | print(
43 | "\nInterrupt received! Gracefully shutting down... (This may take a moment)"
44 | )
45 | print(
46 | "Press Ctrl+C again to force immediate exit (may leave tasks in an inconsistent state)"
47 | )
48 | interrupt_received = True
49 | else:
50 | print("\nForced exit requested. Exiting immediately.")
51 | sys.exit(1)
52 |
53 |
54 | def camel_case(s):
55 | """Convert string to camel case (Title Case)."""
56 | words = s.split()
57 | return " ".join(word.title() for word in words)
58 |
59 |
60 | def create_folder(folder_path, replace_string, replaced_string):
61 | """Create a folder if it doesn't exist, handling both cloud and legacy folders."""
62 | folder_path = folder_path.replace(replace_string, replaced_string)
63 | try:
64 | if ee.data.getAsset(folder_path):
65 | print(f"Folder exists: {ee.data.getAsset(folder_path)['id']}")
66 | except Exception:
67 | print(f"Folder does not exist: Creating {folder_path}")
68 | try:
69 | ee.data.createAsset({"type": ee.data.ASSET_TYPE_FOLDER_CLOUD}, folder_path)
70 | except Exception:
71 | try:
72 | ee.data.createAsset({"type": ee.data.ASSET_TYPE_FOLDER}, folder_path)
73 | except Exception as e:
74 | print(f"Error creating folder {folder_path}: {e}")
75 |
76 |
77 | def move_asset(source, replace_string, replaced_string, fpath, ftype="asset"):
78 | """Move a single asset with appropriate error handling and messaging."""
79 | if replace_string == replaced_string or replace_string is None:
80 | final = fpath
81 | else:
82 | final = source.replace(replace_string, replaced_string)
83 |
84 | try:
85 | # Check if destination already exists
86 | if ee.data.getAsset(final):
87 | print(f"{camel_case(ftype)} already moved: {final}")
88 | return False
89 | except Exception:
90 | try:
91 | print(f"Moving {camel_case(ftype)} to {final}")
92 | ee.data.renameAsset(source, final)
93 | return True
94 | except Exception as e:
95 | print(f"Error moving {source} to {final}: {e}")
96 | return False
97 |
98 |
99 | def move_image_collection(
100 | source, replace_string, replaced_string, fpath, max_workers=10
101 | ):
102 | """Move an image collection with parallel execution for speed."""
103 | # Global variable to track if interrupt occurred
104 | global interrupt_received
105 |
106 | if replace_string == replaced_string or replace_string is None:
107 | collection_path = fpath
108 | else:
109 | collection_path = source.replace(replace_string, replaced_string)
110 |
111 | try:
112 | # Create the collection if it doesn't exist
113 | try:
114 | if ee.data.getAsset(collection_path):
115 | print(f"Collection exists: {ee.data.getAsset(collection_path)['id']}")
116 | except Exception:
117 | print(f"Collection does not exist: Creating {collection_path}")
118 | try:
119 | ee.data.createAsset(
120 | {"type": ee.data.ASSET_TYPE_IMAGE_COLL_CLOUD}, collection_path
121 | )
122 | except Exception:
123 | ee.data.createAsset(
124 | {"type": ee.data.ASSET_TYPE_IMAGE_COLL}, collection_path
125 | )
126 |
127 | # Get list of source images
128 | source_list = ee.data.listAssets({"parent": source})
129 | source_names = [
130 | os.path.basename(asset["name"]) for asset in source_list["assets"]
131 | ]
132 |
133 | # Get list of destination images
134 | collection_path = ee.data.getAsset(collection_path)["name"]
135 | final_list = ee.data.listAssets({"parent": collection_path})
136 | final_names = [
137 | os.path.basename(asset["name"]) for asset in final_list["assets"]
138 | ]
139 |
140 | # Find images that need to be moved
141 | diff = set(source_names) - set(final_names)
142 | total_images = len(diff)
143 |
144 | if not diff:
145 | print(
146 | "All images already exist in destination collection. Nothing to move."
147 | )
148 | return
149 |
150 | print(f"Moving a total of {total_images} images...")
151 |
152 | # Use ThreadPoolExecutor for parallel moving
153 | results = []
154 | processed_count = 0
155 | executor = concurrent.futures.ThreadPoolExecutor(max_workers=max_workers)
156 | futures = {}
157 |
158 | try:
159 | # Submit all tasks
160 | for item in diff:
161 | if interrupt_received:
162 | break
163 | source_path = f"{source}/{item}"
164 | dest_path = f"{collection_path}/{item}"
165 | future = executor.submit(ee.data.renameAsset, source_path, dest_path)
166 | futures[future] = item
167 |
168 | # Process results with progress bar
169 | with tqdm(total=total_images, desc="Moving images") as pbar:
170 | for future in concurrent.futures.as_completed(futures):
171 | item = futures[future]
172 |
173 | # Check if we've received an interrupt
174 | if interrupt_received:
175 | pbar.write(
176 | "\nInterrupt received, cancelling remaining move operations..."
177 | )
178 | # Cancel remaining futures
179 | for f in futures:
180 | if not f.done():
181 | f.cancel()
182 | break
183 |
184 | try:
185 | future.result()
186 | results.append(True)
187 | except Exception as e:
188 | pbar.write(f"Error moving {item}: {e}")
189 | results.append(False)
190 |
191 | processed_count += 1
192 | pbar.update(1)
193 |
194 | finally:
195 | # Always shut down the executor
196 | executor.shutdown(wait=False)
197 |
198 | if interrupt_received:
199 | print(
200 | f"\nOperation interrupted. Moved {processed_count} of {total_images} images."
201 | )
202 | else:
203 | success_count = sum(results)
204 | print(f"Successfully moved {success_count} of {total_images} images")
205 |
206 | except Exception as e:
207 | print(f"Error in collection move: {e}")
208 |
209 |
210 | def get_folder(path):
211 | """Get folder information and add to folder_list."""
212 | parser = ee.data.getAsset(path)
213 | if parser["type"].lower() == "folder":
214 | folder_list.append(parser["name"])
215 | recursive(parser["name"])
216 |
217 |
218 | def recursive(path):
219 | """Recursively gather all folders under a path."""
220 | path_info = ee.data.getAsset(path)
221 | if path_info["type"].lower() == "folder":
222 | path = path_info["name"]
223 | folder_list.append(path)
224 | children = ee.data.listAssets({"parent": path})
225 | for child in children["assets"]:
226 | if not child["name"] in folder_list:
227 | get_folder(child["name"])
228 | return folder_list
229 |
230 |
231 | def mover(path, fpath, max_workers=10):
232 | """Move Earth Engine assets from path to fpath.
233 |
234 | Args:
235 | path: Source asset path
236 | fpath: Destination asset path
237 | max_workers: Maximum number of parallel workers for collection moving
238 | """
239 | global interrupt_received
240 | global folder_list
241 |
242 | interrupt_received = False
243 | folder_list = []
244 |
245 | # Set up signal handler for graceful interrupts
246 | original_sigint_handler = signal.getsignal(signal.SIGINT)
247 | signal.signal(signal.SIGINT, handle_interrupt)
248 |
249 | try:
250 | ee.Initialize()
251 |
252 | try:
253 | if not ee.data.getAsset(path):
254 | print(f"Initial path {path} not found")
255 | return
256 |
257 | asset_info = ee.data.getAsset(path)
258 | asset_type = asset_info["type"].lower()
259 |
260 | if asset_type == "folder":
261 | # Move folder structure
262 | gee_folder_path = recursive(path)
263 | gee_folder_path = sorted(list(set(folder_list)))
264 | print(f"Total folders: {len(set(folder_list))}")
265 |
266 | # Get path prefixes for replacement
267 | initial_path_suffix = path.split("/")[-1]
268 | replace_string = (
269 | "/".join(ee.data.getAsset(path + "/")["name"].split("/")[:-1])
270 | + "/"
271 | + initial_path_suffix
272 | )
273 |
274 | # Get the final path
275 | final_path_suffix = fpath.split("/")[-1]
276 | replaced_string = (
277 | ee.data.getAsset(("/".join(fpath.split("/")[:-1]) + "/"))["name"]
278 | + "/"
279 | + final_path_suffix
280 | )
281 |
282 | # Create folder structure
283 | for folder in gee_folder_path:
284 | if interrupt_received:
285 | print("Operation interrupted. Stopping folder creation.")
286 | break
287 |
288 | create_folder(folder, replace_string, replaced_string)
289 |
290 | # Process assets in this folder
291 | try:
292 | children = ee.data.listAssets({"parent": folder})
293 | for child in children["assets"]:
294 | if interrupt_received:
295 | print(
296 | "Operation interrupted. Stopping asset processing."
297 | )
298 | break
299 |
300 | child_type = child["type"].lower()
301 | child_path = child["name"]
302 |
303 | if child_type == "image_collection":
304 | move_image_collection(
305 | child_path,
306 | replace_string,
307 | replaced_string,
308 | fpath,
309 | max_workers,
310 | )
311 | elif child_type == "image":
312 | move_asset(
313 | child_path,
314 | replace_string,
315 | replaced_string,
316 | fpath,
317 | "image",
318 | )
319 | elif child_type == "table":
320 | move_asset(
321 | child_path,
322 | replace_string,
323 | replaced_string,
324 | fpath,
325 | "table",
326 | )
327 | elif child_type == "feature_view":
328 | move_asset(
329 | child_path,
330 | replace_string,
331 | replaced_string,
332 | fpath,
333 | "feature view",
334 | )
335 |
336 | # Check interrupt again after each asset
337 | if interrupt_received:
338 | break
339 | except Exception as e:
340 | print(f"Error processing assets in folder {folder}: {e}")
341 | print("")
342 |
343 | elif asset_type == "image":
344 | # Move individual image
345 | path = ee.data.getAsset(path)["name"]
346 | initial_path_suffix = path.split("/")[-1]
347 | replace_string = (
348 | "/".join(ee.data.getAsset(path)["name"].split("/")[:-1])
349 | + "/"
350 | + initial_path_suffix
351 | )
352 |
353 | final_path_suffix = fpath.split("/")[-1]
354 | replaced_string = (
355 | ee.data.getAsset("/".join(fpath.split("/")[:-1]))["name"]
356 | + "/"
357 | + final_path_suffix
358 | )
359 |
360 | move_asset(path, replace_string, replaced_string, fpath, "image")
361 |
362 | elif asset_type == "image_collection":
363 | # Move collection
364 | path = ee.data.getAsset(path)["name"]
365 | initial_list = ee.data.listAssets({"parent": path})
366 | assets_names = [
367 | os.path.basename(asset["name"]) for asset in initial_list["assets"]
368 | ]
369 |
370 | initial_path_suffix = path.split("/")[-1]
371 | replace_string = (
372 | "/".join(path.split("/")[:-1]) + "/" + initial_path_suffix
373 | )
374 |
375 | move_image_collection(path, replace_string, None, fpath, max_workers)
376 |
377 | elif asset_type == "table":
378 | # Move table
379 | path = ee.data.getAsset(path)["name"]
380 | replace_string = None
381 | replaced_string = ee.data.getAsset(
382 | "/".join(fpath.split("/")[:-1]) + "/"
383 | )["name"]
384 | move_asset(path, replace_string, replaced_string, fpath, "table")
385 |
386 | elif asset_type == "feature_view":
387 | # Move feature view
388 | path = ee.data.getAsset(path)["name"]
389 | replace_string = None
390 | replaced_string = ee.data.getAsset(
391 | "/".join(fpath.split("/")[:-1]) + "/"
392 | )["name"]
393 | move_asset(path, replace_string, replaced_string, fpath, "feature view")
394 |
395 | else:
396 | print(f"Unsupported asset type: {asset_type}")
397 |
398 | except Exception as e:
399 | print(e)
400 | print(f"Initial path {path} not found")
401 |
402 | except Exception as e:
403 | if not interrupt_received:
404 | print(f"Error moving assets: {e}")
405 | finally:
406 | # Restore original signal handler
407 | signal.signal(signal.SIGINT, original_sigint_handler)
408 |
409 | if interrupt_received:
410 | print("\nMove operation was interrupted and has been stopped.")
411 | print("Some assets may have been moved while others were not.")
412 |
--------------------------------------------------------------------------------
/geeadd/ee_del_meta.py:
--------------------------------------------------------------------------------
1 | from __future__ import print_function
2 |
3 | __copyright__ = """
4 |
5 | Copyright 2024 Samapriya Roy
6 |
7 | Licensed under the Apache License, Version 2.0 (the "License");
8 | you may not use this file except in compliance with the License.
9 | You may obtain a copy of the License at
10 |
11 | http://www.apache.org/licenses/LICENSE-2.0
12 |
13 | Unless required by applicable law or agreed to in writing, software
14 | distributed under the License is distributed on an "AS IS" BASIS,
15 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 | See the License for the specific language governing permissions and
17 | limitations under the License.
18 |
19 | """
20 | __license__ = "Apache 2.0"
21 | import os
22 |
23 | import ee
24 |
25 |
26 | def delprop(collection_path, property):
27 | ee.Initialize()
28 | lst = []
29 | header = ee.data.getInfo(collection_path)["type"]
30 | if header == "IMAGE":
31 | lst.append(collection_path)
32 | assets_names = lst
33 | if header == "IMAGE_COLLECTION":
34 | assets_list = ee.data.getList(params={"id": collection_path})
35 | assets_names = [os.path.basename(asset["id"]) for asset in assets_list]
36 | print("Deleting metadata for total of " + str(len(assets_names)) + " assets.....")
37 | for count, items in enumerate(assets_names):
38 | if header == "IMAGE_COLLECTION":
39 | nullgrid = {property: None}
40 | init = collection_path + "/" + items
41 | try:
42 | print(
43 | "Processing " + str(count + 1) + " of " + str(len(assets_names)),
44 | end="\r",
45 | )
46 | ee.data.setAssetProperties(init, nullgrid)
47 | except Exception as e:
48 | print(
49 | "Could not run " + str(count + 1) + " of " + str(len(assets_names))
50 | )
51 | print(e)
52 | if header == "IMAGE":
53 | nullgrid = {property: None}
54 | init = collection_path + "/" + items
55 | try:
56 | print(
57 | "Processing " + str(count + 1) + " of " + str(len(assets_names)),
58 | end="\r",
59 | )
60 | ee.data.setAssetProperties(init, nullgrid)
61 | except Exception as e:
62 | print(
63 | "Could not run " + str(count + 1) + " of " + str(len(assets_names))
64 | )
65 | print(e)
66 |
67 |
68 | # delprop(collection_path='users/samapriya/LA_ASSET_EXP/LS_UNMX_OTSU_MEDOID',property='gridded')
69 | # ee.data.setAssetProperties(args.asset_id, properties)
70 |
--------------------------------------------------------------------------------
/geeadd/ee_projects.py:
--------------------------------------------------------------------------------
1 | __copyright__ = """
2 |
3 | Copyright 2024 Samapriya Roy
4 |
5 | Licensed under the Apache License, Version 2.0 (the "License");
6 | you may not use this file except in compliance with the License.
7 | You may obtain a copy of the License at
8 |
9 | http://www.apache.org/licenses/LICENSE-2.0
10 |
11 | Unless required by applicable law or agreed to in writing, software
12 | distributed under the License is distributed on an "AS IS" BASIS,
13 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 | See the License for the specific language governing permissions and
15 | limitations under the License.
16 |
17 | """
18 |
19 | import json
20 | import shlex
21 | import subprocess
22 | import sys
23 |
24 |
25 | def is_gcloud_installed():
26 | try:
27 | # Run the 'gcloud --version' command to check if gcloud is installed
28 | result = subprocess.run(['gcloud', '--version'], shell=True,capture_output=True, text=True)
29 | if result.returncode == 0:
30 | return True
31 | else:
32 | return False
33 | except FileNotFoundError:
34 | return False
35 |
36 | def is_gcloud_authenticated():
37 | try:
38 | # Run the 'gcloud auth list' command to check if gcloud is authenticated
39 | result = subprocess.run(['gcloud', 'auth', 'list'], shell=True,capture_output=True, text=True)
40 | if result.returncode == 0:
41 | output = result.stdout.strip()
42 | if "ACTIVE" in output:
43 | return True
44 | else:
45 | return False
46 | else:
47 | return False
48 | except FileNotFoundError:
49 | return False
50 |
51 | def check_gcloud():
52 | if is_gcloud_installed() and is_gcloud_authenticated():
53 | return True
54 | else:
55 | return False
56 |
57 | def list_enabled_services(pname):
58 | """Lists enabled services for a given project.
59 |
60 | Args:
61 | pname: The project name.
62 |
63 | Returns:
64 | A list of enabled services as a JSON object, or None if an error occurs.
65 | """
66 | command = f"gcloud services list --project={pname} --enabled --format=json"
67 | try:
68 | output = subprocess.run(shlex.split(command), shell=True, capture_output=True, text=True)
69 | return json.loads(output.stdout)
70 | except subprocess.CalledProcessError as e:
71 | print(f"Error: gcloud command failed with code {e.returncode}.")
72 | return None
73 | except Exception as e:
74 | return None
75 | def project_permissions(pname):
76 | """Checks if Earth Engine API is enabled for a project.
77 |
78 | Args:
79 | pname: The project name.
80 | """
81 | enabled_services = list_enabled_services(pname)
82 | data = enabled_services
83 | enabled_api_list = []
84 | if data is not None:
85 | for item in data:
86 | project_id = item["parent"]
87 | enabled_api_list.append(item['name'].split('services/')[-1])
88 | enabled_api_list = [item['name'].split('services/')[-1] for item in data if data is not None]
89 | if "earthengine.googleapis.com" in enabled_api_list:
90 | print(f"Project Name: {pname} Project Number: {project_id.split('/')[-1]}")
91 |
92 | def get_projects():
93 | """Retrieves project list and checks Earth Engine permissions."""
94 | if not check_gcloud():
95 | sys.exit("gcloud is either not installed or not authenticated.")
96 | else:
97 | print("\n"+"gcloud is installed and authenticated")
98 | command = f"gcloud projects list --format=json"
99 | try:
100 | print("\n"+"Checking Earth Engine permissions for all projects..."+"\n")
101 | services_json = subprocess.check_output(command, shell=True, text=True)
102 | project_json = json.loads(services_json)
103 | for item in project_json:
104 | project_permissions(item['projectId'])
105 | except subprocess.CalledProcessError as e:
106 | print("Error:", e.output)
107 |
108 | #get_projects()
109 |
--------------------------------------------------------------------------------
/geeadd/ee_report.py:
--------------------------------------------------------------------------------
1 | from __future__ import print_function
2 |
3 | __copyright__ = """
4 |
5 | Copyright 2024 Samapriya Roy
6 |
7 | Licensed under the Apache License, Version 2.0 (the "License");
8 | you may not use this file except in compliance with the License.
9 | You may obtain a copy of the License at
10 |
11 | http://www.apache.org/licenses/LICENSE-2.0
12 |
13 | Unless required by applicable law or agreed to in writing, software
14 | distributed under the License is distributed on an "AS IS" BASIS,
15 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 | See the License for the specific language governing permissions and
17 | limitations under the License.
18 |
19 | """
20 | __license__ = "Apache 2.0"
21 | import csv
22 | import random
23 | import subprocess
24 | import sys
25 |
26 | import ee
27 | from logzero import logger
28 |
29 | # Empty Lists
30 | folder_paths = []
31 | image_list = []
32 | collection_list = []
33 | table_list = []
34 |
35 | suffixes = ["B", "KB", "MB", "GB", "TB", "PB"]
36 |
37 |
38 | def humansize(nbytes):
39 | i = 0
40 | while nbytes >= 1024 and i < len(suffixes) - 1:
41 | nbytes /= 1024.0
42 | i += 1
43 | f = ("%.2f" % nbytes).rstrip("0").rstrip(".")
44 | return "%s %s" % (f, suffixes[i])
45 |
46 |
47 | def recprocess(gee_type, location):
48 | try:
49 | if gee_type == "collection":
50 | own = ee.data.getAssetAcl(location)
51 | o = ",".join(own["owners"])
52 | r = ",".join(own["readers"])
53 | w = ",".join(own["writers"])
54 | return [o, r, w]
55 | elif gee_type == "image":
56 | own = ee.data.getAssetAcl(location)
57 | o = ",".join(own["owners"])
58 | r = ",".join(own["readers"])
59 | w = ",".join(own["writers"])
60 | return [o, r, w]
61 | elif gee_type == "table":
62 | own = ee.data.getAssetAcl(location)
63 | o = ",".join(own["owners"])
64 | r = ",".join(own["readers"])
65 | w = ",".join(own["writers"])
66 | return [o, r, w]
67 | elif gee_type == "folder":
68 | own = ee.data.getAssetAcl(location)
69 | o = ",".join(own["owners"])
70 | r = ",".join(own["readers"])
71 | w = ",".join(own["writers"])
72 | # print(o,r,w)
73 | return [o, r, w]
74 | except Exception as e:
75 | print(e)
76 |
77 |
78 | # Recursive folder paths
79 | def recursive(path):
80 | if ee.data.getAsset(path)["type"].lower() == "folder":
81 | children = ee.data.listAssets({"parent": path})
82 | folder_paths.append(path)
83 | val = [child["type"].lower() == "folder" for child in children["assets"]]
84 | while len(val) > 0 and True in val:
85 | for child in children["assets"]:
86 | if child["type"].lower() == "folder":
87 | folder_paths.append(child["name"])
88 | children = ee.data.listAssets({"parent": child["name"]})
89 | val = [child["type"].lower() == "folder" for child in children["assets"]]
90 | return folder_paths
91 |
92 |
93 | def assetsize(asset):
94 | ee.Initialize()
95 | header = ee.data.getAsset(asset)["type"]
96 | if header == "IMAGE_COLLECTION":
97 | collc = ee.ImageCollection(asset)
98 | size = collc.aggregate_array("system:asset_size")
99 | return [str(humansize(sum(size.getInfo()))), str(collc.size().getInfo())]
100 | elif header == "IMAGE":
101 | collc = ee.Image(asset)
102 | return [str(humansize(collc.get("system:asset_size").getInfo())), 1]
103 | elif header == "TABLE":
104 | collc = ee.FeatureCollection(asset)
105 | return [str(humansize(collc.get("system:asset_size").getInfo())), 1]
106 | elif header == "FOLDER":
107 | b = subprocess.Popen(
108 | "earthengine du {} -s".format(asset), shell=True, stdout=subprocess.PIPE
109 | )
110 | out, err = b.communicate()
111 | val = [item for item in out.decode("ascii").split(" ") if item.isdigit()]
112 | size = humansize(float(val[0]))
113 | num = subprocess.Popen(
114 | "earthengine ls {}".format(asset), shell=True, stdout=subprocess.PIPE
115 | )
116 | out, err = num.communicate()
117 | out = out.decode("ascii")
118 | num = [
119 | i for i in out.split("\n") if i if len(i) > 1 if not i.startswith("Running")
120 | ]
121 | return [str(size), str(len(num))]
122 |
123 |
124 | # folder parse
125 |
126 |
127 | def fparse(path):
128 | ee.Initialize()
129 | if ee.data.getAsset(path)["type"].lower() == "folder":
130 | gee_folder_path = recursive(path)
131 | for folders in gee_folder_path:
132 | children = ee.data.listAssets({"parent": folders})
133 | for child in children["assets"]:
134 | if child["type"].lower() == "image_collection":
135 | collection_list.append(child["id"])
136 | elif child["type"].lower() == "image":
137 | image_list.append(child["id"])
138 | elif child["type"].lower() == "table":
139 | table_list.append(child["id"])
140 | elif ee.data.getAsset(path)["type"].lower() == "image":
141 | image_list.append(path)
142 | elif ee.data.getAsset(path)["type"].lower() == "image_collection":
143 | collection_list.append(path)
144 | elif ee.data.getAsset(path)["type"].lower() == "table":
145 | table_list.append(path)
146 | else:
147 | print(ee.data.getAsset(path)["type"].lower())
148 | return [collection_list, table_list, image_list, folder_paths]
149 |
150 |
151 | # request type of asset, asset path and user to give permission
152 | def ee_report(output, path):
153 | choicelist = [
154 | "Go grab some tea.....",
155 | "Go Stretch.....",
156 | "Go take a walk.....",
157 | "Go grab some coffee.....",
158 | ] # adding something fun
159 | path_list = []
160 | logger.debug("This might take sometime. {}".format(random.choice(choicelist)))
161 | ee.Initialize()
162 | with open(output, "w") as csvfile:
163 | writer = csv.DictWriter(
164 | csvfile,
165 | fieldnames=[
166 | "type",
167 | "path",
168 | "No of Assets",
169 | "size",
170 | "owner",
171 | "readers",
172 | "writers",
173 | ],
174 | delimiter=",",
175 | lineterminator="\n",
176 | )
177 | writer.writeheader()
178 | if path is not None:
179 | if not path.endswith("/") and path.endswith("assets"):
180 | path = path + "/"
181 | parser = ee.data.getAsset(path)
182 | if parser["type"].lower() == "folder" and path.startswith("user"):
183 | path = parser["name"]
184 | path_list.append(path)
185 | logger.debug(f"Processing your folder: {path}")
186 | collection_list, table_list, image_list, folder_paths = fparse(path)
187 | else:
188 | collection_path = ee.data.getAssetRoots()
189 | for roots in collection_path:
190 | path_list.append(roots["id"])
191 | logger.debug("Processing your root folder: {}".format(roots["id"]))
192 | collection_list, table_list, image_list, folder_paths = fparse(roots["id"])
193 | logger.debug(
194 | "Processing a total of: {} folders {} collections {} images {} tables".format(
195 | len(folder_paths),
196 | len(collection_list),
197 | len(image_list),
198 | len(table_list),
199 | )
200 | + "\n"
201 | )
202 | if folder_paths:
203 | for folder in folder_paths:
204 | if not folder in path_list:
205 | gee_id = ee.data.getAsset(folder)["name"]
206 | gee_type = "folder"
207 | logger.info("Processing Folder {}".format(gee_id))
208 | total_size, total_count = assetsize(gee_id)
209 | o, r, w = recprocess(gee_type, gee_id)
210 | try:
211 | with open(output, "a") as csvfile:
212 | writer = csv.writer(csvfile, delimiter=",", lineterminator="\n")
213 | writer.writerow(
214 | [gee_type, gee_id, total_count, total_size, o, r, w]
215 | )
216 | csvfile.close()
217 | except Exception as e:
218 | print(e)
219 | if collection_list:
220 | for collection in collection_list:
221 | gee_id = ee.data.getAsset(collection)["name"]
222 | gee_type = "collection"
223 | logger.info("Processing Collection {}".format(gee_id))
224 | total_size, total_count = assetsize(gee_id)
225 | o, r, w = recprocess(gee_type, gee_id)
226 | # print(gee_id,gee_type,total_size,total_count,o,r,w)
227 | with open(output, "a") as csvfile:
228 | writer = csv.writer(csvfile, delimiter=",", lineterminator="\n")
229 | writer.writerow([gee_type, gee_id, total_count, total_size, o, r, w])
230 | csvfile.close()
231 | if table_list:
232 | for table in table_list:
233 | gee_id = ee.data.getAsset(table)["name"]
234 | gee_type = "table"
235 | logger.info("Processing table {}".format(gee_id))
236 | total_size, total_count = assetsize(gee_id)
237 | o, r, w = recprocess(gee_type, gee_id)
238 | # print(gee_id,gee_type,total_size,total_count,o,r,w)
239 | with open(output, "a") as csvfile:
240 | writer = csv.writer(csvfile, delimiter=",", lineterminator="\n")
241 | writer.writerow([gee_type, gee_id, total_count, total_size, o, r, w])
242 | csvfile.close()
243 | if image_list:
244 | for image in image_list:
245 | gee_id = ee.data.getAsset(image)["name"]
246 | gee_type = "image"
247 | logger.info("Processing image {}".format(gee_id))
248 | total_size, total_count = assetsize(gee_id)
249 | o, r, w = recprocess(gee_type, gee_id)
250 | # print(gee_id,gee_type,total_size,total_count,o,r,w)
251 | with open(output, "a") as csvfile:
252 | writer = csv.writer(csvfile, delimiter=",", lineterminator="\n")
253 | writer.writerow([gee_type, gee_id, total_count, total_size, o, r, w])
254 | csvfile.close()
255 |
--------------------------------------------------------------------------------
/geeadd/geeadd.py:
--------------------------------------------------------------------------------
1 | #! /usr/bin/env python
2 |
3 | from .acl_changer import access
4 | from .app2script import jsext
5 | from .batch_copy import copy
6 | from .batch_delete import delete
7 | from .batch_mover import mover
8 | from .ee_del_meta import delprop
9 | from .ee_projects import get_projects
10 | from .ee_report import ee_report
11 |
12 | __copyright__ = """
13 |
14 | Copyright 2025 Samapriya Roy
15 |
16 | Licensed under the Apache License, Version 2.0 (the "License");
17 | you may not use this file except in compliance with the License.
18 | You may obtain a copy of the License at
19 |
20 | http://www.apache.org/licenses/LICENSE-2.0
21 |
22 | Unless required by applicable law or agreed to in writing, software
23 | distributed under the License is distributed on an "AS IS" BASIS,
24 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
25 | See the License for the specific language governing permissions and
26 | limitations under the License.
27 |
28 | """
29 | __license__ = "Apache 2.0"
30 |
31 | import argparse
32 | import concurrent.futures
33 | import importlib.metadata
34 | import json
35 | import os
36 | import subprocess
37 | import sys
38 | import time
39 | import webbrowser
40 | from datetime import datetime
41 | from importlib.metadata import version
42 |
43 | import ee
44 | import pkg_resources
45 | import requests
46 | from colorama import Fore, Style, init
47 | from packaging import version as pkg_version
48 | from tqdm import tqdm
49 |
50 | os.chdir(os.path.dirname(os.path.realpath(__file__)))
51 | lpath = os.path.dirname(os.path.realpath(__file__))
52 | sys.path.append(lpath)
53 |
54 | now = datetime.now()
55 |
56 | if len(sys.argv) > 1 and sys.argv[1] != "-h":
57 | ee.Initialize()
58 | # Initialize colorama for cross-platform colored terminal output
59 | init(autoreset=True)
60 |
61 |
62 | def compare_version(version1, version2):
63 | """
64 | Compare two version strings using the packaging.version module.
65 | Returns: 1 if version1 > version2, -1 if version1 < version2, 0 if equal
66 | """
67 | v1 = pkg_version.parse(version1)
68 | v2 = pkg_version.parse(version2)
69 |
70 | if v1 > v2:
71 | return 1
72 | elif v1 < v2:
73 | return -1
74 | else:
75 | return 0
76 |
77 |
78 | def get_latest_version(package):
79 | """Get the latest version of a package from PyPI."""
80 | try:
81 | response = requests.get(f"https://pypi.org/pypi/{package}/json", timeout=5)
82 | response.raise_for_status()
83 | return response.json()["info"]["version"]
84 | except (requests.RequestException, KeyError) as e:
85 | print(f"Error fetching version for {package}: {e}")
86 | return None
87 |
88 |
89 | def get_installed_version(package):
90 | """Get the installed version of a package using importlib.metadata."""
91 | try:
92 | return importlib.metadata.version(package)
93 | except importlib.metadata.PackageNotFoundError:
94 | try:
95 | # Fallback to pkg_resources
96 | return pkg_resources.get_distribution(package).version
97 | except pkg_resources.DistributionNotFound:
98 | print(f"Package {package} is not installed")
99 | return None
100 |
101 |
102 | def check_package_version(package_name):
103 | """Check if the installed version of a package is the latest."""
104 | installed_version = get_installed_version(package_name)
105 | latest_version = get_latest_version(package_name)
106 |
107 | if not installed_version or not latest_version:
108 | return
109 |
110 | result = compare_version(latest_version, installed_version)
111 | border = Style.BRIGHT + "========================================================================="
112 |
113 | if result == 1:
114 | print(f"\n{border}")
115 | print(Fore.RED + f"Current version of {package_name} is {installed_version} "
116 | f"upgrade to latest version: {latest_version}" + Style.RESET_ALL)
117 | print(f"{border}")
118 | elif result == -1:
119 | print(f"\n{border}")
120 | print(Fore.YELLOW + f"Possibly running staging code {installed_version} "
121 | f"compared to PyPI release {latest_version}" + Style.RESET_ALL)
122 | print(f"{border}")
123 | # Removed the else branch to avoid printing "up to date" message
124 |
125 | check_package_version("geeadd")
126 |
127 | # Go to the readMe
128 | def readme():
129 | try:
130 | a = webbrowser.open(
131 | "https://samapriya.github.io/gee_asset_manager_addon/", new=2
132 | )
133 | if a == False:
134 | print("Your setup does not have a monitor to display the webpage")
135 | print(
136 | " Go to {}".format(
137 | "https://samapriya.github.io/gee_asset_manager_addon/"
138 | )
139 | )
140 | except Exception as e:
141 | print(e)
142 |
143 |
144 | suffixes = ["B", "KB", "MB", "GB", "TB", "PB"]
145 |
146 |
147 | def humansize(nbytes):
148 | i = 0
149 | while nbytes >= 1024 and i < len(suffixes) - 1:
150 | nbytes /= 1024.0
151 | i += 1
152 | f = ("%.2f" % nbytes).rstrip("0").rstrip(".")
153 | return "%s %s" % (f, suffixes[i])
154 |
155 |
156 | def cancel_tasks(tasks):
157 | """
158 | Cancels Earth Engine tasks based on specified criteria.
159 |
160 | Args:
161 | tasks: Can be "all", "running", "pending", or a specific task ID
162 | """
163 | try:
164 | if tasks == "all":
165 | # Use the task list approach from TaskCancelCommand
166 | print("Attempting to cancel all tasks")
167 | statuses = ee.data.getTaskList()
168 | cancelled_count = 0
169 |
170 | with tqdm(total=len(statuses), desc="Cancelling tasks") as pbar:
171 | for status in statuses:
172 | state = status['state']
173 | task_id = status['id']
174 |
175 | if state == 'READY' or state == 'RUNNING':
176 | try:
177 | ee.data.cancelTask(task_id)
178 | cancelled_count += 1
179 | except ee.EEException as e:
180 | print(f"Error cancelling task {task_id}: {e}")
181 | pbar.update(1)
182 |
183 | if cancelled_count > 0:
184 | print(f"Successfully cancelled {cancelled_count} tasks")
185 | else:
186 | print("No running or pending tasks found to cancel")
187 |
188 | elif tasks == "running":
189 | print("Attempting to cancel running tasks")
190 | statuses = ee.data.getTaskList()
191 | running_tasks = [status for status in statuses if status['state'] == 'RUNNING']
192 |
193 | if running_tasks:
194 | with tqdm(total=len(running_tasks), desc="Cancelling running tasks") as pbar:
195 | cancelled_count = 0
196 | for status in running_tasks:
197 | try:
198 | ee.data.cancelTask(status['id'])
199 | cancelled_count += 1
200 | except ee.EEException as e:
201 | print(f"Error cancelling task {status['id']}: {e}")
202 | pbar.update(1)
203 | print(f"Successfully cancelled {cancelled_count} running tasks")
204 | else:
205 | print("No running tasks found")
206 |
207 | elif tasks == "pending":
208 | print("Attempting to cancel pending tasks")
209 | statuses = ee.data.getTaskList()
210 | pending_tasks = [status for status in statuses if status['state'] == 'READY']
211 |
212 | if pending_tasks:
213 | with tqdm(total=len(pending_tasks), desc="Cancelling pending tasks") as pbar:
214 | cancelled_count = 0
215 | for status in pending_tasks:
216 | try:
217 | ee.data.cancelTask(status['id'])
218 | cancelled_count += 1
219 | except ee.EEException as e:
220 | print(f"Error cancelling task {status['id']}: {e}")
221 | pbar.update(1)
222 | print(f"Successfully cancelled {cancelled_count} pending tasks")
223 | else:
224 | print("No pending tasks found")
225 |
226 | elif tasks is not None:
227 | # Check if it's a valid task ID
228 | print(f"Attempting to cancel task with ID: {tasks}")
229 |
230 | try:
231 | statuses = ee.data.getTaskStatus([tasks])
232 | if not statuses:
233 | print(f"Task {tasks} not found")
234 | return
235 |
236 | status = statuses[0]
237 | state = status['state']
238 |
239 | if state == 'UNKNOWN':
240 | print(f"Unknown task ID: {tasks}")
241 | elif state in ['READY', 'RUNNING']:
242 | ee.data.cancelTask(tasks)
243 | print(f"Successfully cancelled task {tasks}")
244 | else:
245 | print(f"Task {tasks} is already in state '{state}' and cannot be cancelled")
246 | except ee.EEException as e:
247 | print(f"Error accessing task {tasks}: {e}")
248 | else:
249 | print("Please specify 'all', 'running', 'pending', or a specific task ID")
250 |
251 | except Exception as e:
252 | print(f"Error in cancel_tasks: {e}")
253 |
254 | def quota(project_path=None):
255 | """
256 | Display quota information for Earth Engine assets with a visual progress bar.
257 |
258 | Args:
259 | project_path: Optional. Specific project path to check.
260 | If None, attempts to get all available projects.
261 |
262 | Examples:
263 | - "users/username"
264 | - "projects/earthengine-legacy/assets/users/username"
265 | - "projects/my-project-id"
266 | - "my-project-id" (will try to detect the correct format)
267 | """
268 | try:
269 | def draw_bar(percent, width=30):
270 | """Draw a simple progress bar"""
271 | filled = int(width * percent / 100)
272 | bar = "█" * filled + "░" * (width - filled)
273 | return f"[{bar}] {percent:.1f}%"
274 |
275 | def display_quota_info(project_name, info, project_type="Project"):
276 | """Display quota info uniformly"""
277 | if "quota" in info:
278 | quota_info = info["quota"]
279 | print(f"\n{project_type}: {project_name}")
280 |
281 | # Size quota
282 | used_size = int(quota_info.get("sizeBytes", 0))
283 | max_size = int(quota_info.get("maxSizeBytes", 1))
284 | percent = (used_size / max_size * 100) if max_size > 0 else 0
285 | print(f"Storage: {humansize(used_size)} of {humansize(max_size)}")
286 | print(f" {draw_bar(percent)}")
287 |
288 | # Asset count quota
289 | used_assets = int(quota_info.get("assetCount", 0))
290 | max_assets = int(quota_info.get("maxAssets", 1))
291 | percent = (used_assets / max_assets * 100) if max_assets > 0 else 0
292 | print(f"Assets: {used_assets:,} of {max_assets:,}")
293 | print(f" {draw_bar(percent)}")
294 | return True
295 | elif "asset_size" in info:
296 | # Legacy format
297 | print(f"\n{project_type}: {project_name}")
298 |
299 | size_usage = info["asset_size"]["usage"]
300 | size_limit = info["asset_size"]["limit"]
301 | size_percent = (size_usage / size_limit * 100) if size_limit > 0 else 0
302 |
303 | count_usage = info["asset_count"]["usage"]
304 | count_limit = info["asset_count"]["limit"]
305 | count_percent = (
306 | (count_usage / count_limit * 100) if count_limit > 0 else 0
307 | )
308 |
309 | print(f"Storage: {humansize(size_usage)} of {humansize(size_limit)}")
310 | print(f" {draw_bar(size_percent)}")
311 | print(f"Assets: {count_usage:,} of {count_limit:,}")
312 | print(f" {draw_bar(count_percent)}")
313 | return True
314 | else:
315 | print(f"No quota information available for {project_name}")
316 | return False
317 |
318 | # If no path provided, try to get all projects
319 | if project_path is None:
320 | try:
321 | roots = ee.data.getAssetRoots()
322 | if not roots:
323 | print("No accessible projects found.")
324 | return
325 |
326 | # Group projects by parent to avoid showing the same quota multiple times
327 | processed_projects = set()
328 |
329 | for root in roots:
330 | root_path = root["id"]
331 |
332 | # Skip if we've already processed this project
333 | parent_project = (
334 | root_path.split("/assets/")[0]
335 | if "/assets/" in root_path
336 | else root_path
337 | )
338 | if parent_project in processed_projects:
339 | continue
340 |
341 | processed_projects.add(parent_project)
342 |
343 | # First try the direct getInfo approach
344 | try:
345 | asset_info = ee.data.getInfo(parent_project)
346 | if asset_info and "quota" in asset_info:
347 | # This is a project with quota info
348 | project_type = (
349 | "Legacy project"
350 | if "earthengine-legacy" in str(asset_info)
351 | else "Cloud project"
352 | )
353 | display_quota_info(parent_project, asset_info, project_type)
354 | continue
355 | except:
356 | pass
357 |
358 | # Try legacy approach
359 | if root_path.startswith("users/"):
360 | try:
361 | quota_info = ee.data.getAssetRootQuota(root_path)
362 | display_quota_info(root_path, quota_info, "Legacy project")
363 | continue
364 | except:
365 | pass
366 |
367 | # Try cloud approach with multiple formats
368 | if root_path.startswith("projects/"):
369 | cloud_formats = [
370 | # Try with /assets/
371 | f"{parent_project}/assets/",
372 | # Try with just /assets
373 | f"{parent_project}/assets",
374 | # Try original
375 | parent_project,
376 | ]
377 |
378 | success = False
379 | for path_format in cloud_formats:
380 | try:
381 | project_detail = ee.data.getAsset(path_format)
382 | if "quota" in project_detail:
383 | display_quota_info(
384 | parent_project, project_detail, "Cloud project"
385 | )
386 | success = True
387 | break
388 | except:
389 | # Silently try next format
390 | continue
391 |
392 | if success:
393 | continue
394 |
395 | # As a last resort, try getInfo on the original path
396 | try:
397 | info = ee.data.getInfo(root_path)
398 | if info and ("quota" in info or "asset_size" in info):
399 | project_type = (
400 | "Legacy project"
401 | if root_path.startswith("users/")
402 | else "Cloud project"
403 | )
404 | display_quota_info(root_path, info, project_type)
405 | except:
406 | pass
407 |
408 | return
409 | except Exception as e:
410 | print(f"Could not list projects: {str(e)}")
411 | print("Falling back to current user quota.")
412 | try:
413 | # Try to get default user quota
414 | quota_info = ee.data.getAssetRootQuota("users/")
415 | print("Current user quota:")
416 | display_quota_info("users/", quota_info, "User quota")
417 | return
418 | except Exception as e2:
419 | print(f"Could not get user quota: {str(e2)}")
420 | return
421 |
422 | # Handle the case where user provided just a name without prefix
423 | if not project_path.startswith("projects/") and not project_path.startswith(
424 | "users/"
425 | ):
426 | # Try to detect if it's a project ID
427 | try:
428 | # First try as a cloud project
429 | cloud_path = f"projects/{project_path}"
430 | asset_info = ee.data.getInfo(cloud_path)
431 | if asset_info:
432 | print(f"Detected project ID format, using: {cloud_path}")
433 | project_path = cloud_path
434 | except:
435 | # If that fails, try as a legacy user
436 | try:
437 | legacy_path = f"users/{project_path}"
438 | quota_info = ee.data.getAssetRootQuota(legacy_path)
439 | if quota_info:
440 | print(f"Detected legacy user format, using: {legacy_path}")
441 | project_path = legacy_path
442 | except:
443 | # Keep original if both fail
444 | pass
445 |
446 | # Try direct getInfo approach first (works for projects/sat-io style paths)
447 | try:
448 | asset_info = ee.data.getInfo(project_path)
449 | if asset_info and "quota" in asset_info:
450 | project_type = (
451 | "Legacy project"
452 | if "earthengine-legacy" in str(asset_info)
453 | else "Cloud project"
454 | )
455 | return display_quota_info(project_path, asset_info, project_type)
456 | except:
457 | pass
458 |
459 | # Try with /assets suffix for cloud projects
460 | if (
461 | project_path.startswith("projects/")
462 | and not project_path.endswith("/assets")
463 | and "/assets" not in project_path
464 | ):
465 | try:
466 | asset_info = ee.data.getInfo(f"{project_path}/assets")
467 | if asset_info and "quota" in asset_info:
468 | return display_quota_info(project_path, asset_info, "Cloud project")
469 | except:
470 | pass
471 |
472 | # Try legacy approach
473 | try:
474 | quota_info = ee.data.getAssetRootQuota(project_path)
475 | return display_quota_info(project_path, quota_info, "Legacy project")
476 | except:
477 | pass
478 |
479 | # Try cloud approach with multiple formats
480 | cloud_formats = [
481 | f"{project_path}/assets/"
482 | if not project_path.endswith("/assets/") and "/assets/" not in project_path
483 | else project_path,
484 | f"{project_path}/assets"
485 | if not project_path.endswith("/assets") and "/assets" not in project_path
486 | else project_path,
487 | project_path,
488 | ]
489 |
490 | for path_format in cloud_formats:
491 | try:
492 | project_detail = ee.data.getAsset(path_format)
493 | if "quota" in project_detail:
494 | parent_path = (
495 | path_format.split("/assets")[0]
496 | if "/assets" in path_format
497 | else path_format
498 | )
499 | return display_quota_info(
500 | parent_path, project_detail, "Cloud project"
501 | )
502 | except:
503 | continue
504 |
505 | print(f"Could not retrieve quota information for {project_path}")
506 | return False
507 |
508 | except Exception as e:
509 | # print(f"Error retrieving quota: {str(e)}")
510 | return False
511 |
512 | def epoch_convert_time(epoch_timestamp):
513 | dt_object = datetime.fromtimestamp(epoch_timestamp/1000)
514 | formatted_date_time = dt_object.strftime("%Y-%m-%dT%H:%M:%S.%fZ")
515 | return formatted_date_time
516 |
517 | def tasks(state,id):
518 | if state is not None:
519 | task_bundle = []
520 | operations = [
521 | status
522 | for status in ee.data.getTaskList()
523 | if status["state"] == state.upper()
524 | ]
525 | for operation in operations:
526 | task_id = operation["id"]
527 | description = operation["description"].split(":")[0]
528 | op_type = operation["task_type"]
529 | attempt_count = operation["attempt"]
530 | date_format = "%Y-%m-%dT%H:%M:%S.%fZ"
531 | start = datetime.strptime(epoch_convert_time(operation["start_timestamp_ms"]),date_format)
532 | end = datetime.strptime(epoch_convert_time(operation["update_timestamp_ms"]),date_format)
533 | time_difference = end - start
534 | item = {
535 | "task_id": task_id,
536 | "operation_type": op_type,
537 | "description": description,
538 | "run_time": str(time_difference),
539 | "attempt": attempt_count,
540 | }
541 | if 'destination_uris' in operation:
542 | item['item_path']=operation['destination_uris'][0].replace('https://code.earthengine.google.com/?asset=','')
543 | if 'batch_eecu_usage_seconds' in operation:
544 | item['eecu_usage'] = operation['batch_eecu_usage_seconds']
545 | task_bundle.append(item)
546 | print(json.dumps(task_bundle, indent=2))
547 | elif id is not None:
548 | operations = [
549 | status
550 | for status in ee.data.getTaskList()
551 | if status["id"] == id
552 | ]
553 | for operation in operations:
554 | task_id = operation["id"]
555 | description = operation["description"].split(":")[0]
556 | op_type = operation["task_type"]
557 | attempt_count = operation["attempt"]
558 | date_format = "%Y-%m-%dT%H:%M:%S.%fZ"
559 | start = datetime.strptime(epoch_convert_time(operation["start_timestamp_ms"]),date_format)
560 | end = datetime.strptime(epoch_convert_time(operation["update_timestamp_ms"]),date_format)
561 | time_difference = end - start
562 | item = {
563 | "task_id": task_id,
564 | "operation_type": op_type,
565 | "description": description,
566 | "run_time": str(time_difference),
567 | "attempt": attempt_count,
568 | }
569 | if 'destination_uris' in operation:
570 | item['item_path']=operation['destination_uris'][0].replace('https://code.earthengine.google.com/?asset=','')
571 | if 'batch_eecu_usage_seconds' in operation:
572 | item['eecu_usage'] = operation['batch_eecu_usage_seconds']
573 | print(json.dumps(item, indent=2))
574 | else:
575 | statuses = ee.data.getTaskList()
576 | st = []
577 | for status in statuses:
578 | st.append(status["state"])
579 | print(f"Tasks Running: {st.count('RUNNING')}")
580 | print(f"Tasks Pending: {st.count('READY')}")
581 | print(f"Tasks Completed: {st.count('COMPLETED')+st.count('SUCCEEDED')}")
582 | print(f"Tasks Failed: {st.count('FAILED')}")
583 | print(f"Tasks Cancelled: {st.count('CANCELLED') + st.count('CANCELLING')}")
584 |
585 | def assetsize(asset):
586 | """
587 | Print the size and item count of an Earth Engine asset.
588 |
589 | Args:
590 | asset (str): The Earth Engine asset path.
591 |
592 | """
593 |
594 | asset_info = ee.data.getAsset(asset)
595 |
596 | header = asset_info["type"]
597 |
598 | if header in ["IMAGE_COLLECTION", "IMAGE", "TABLE","FEATURE_VIEW"]:
599 | if header == "IMAGE_COLLECTION":
600 | collc = ee.ImageCollection(asset)
601 | size = sum(collc.aggregate_array("system:asset_size").getInfo())
602 | item_count = collc.size().getInfo()
603 | elif header == "IMAGE":
604 | collc = ee.ImageCollection.fromImages([ee.Image(asset)])
605 | size = sum(collc.aggregate_array("system:asset_size").getInfo())
606 | item_count = 1
607 | elif header == "TABLE":
608 | collc = ee.FeatureCollection(asset)
609 | size = float(collc.get("system:asset_size").getInfo())
610 | item_count = collc.size().getInfo()
611 | elif header == "FEATURE_VIEW":
612 | collc = ee.data.getAsset(asset)
613 | size = float(collc['sizeBytes'])
614 | item_count = collc['featureCount']
615 | print(f"\n{asset} ===> {humansize(size)}")
616 | print(f"Total number of items in {header.title()}: {item_count}")
617 |
618 | elif header == "FOLDER":
619 | out = subprocess.check_output(f"earthengine du {asset} -s", shell=True).decode(
620 | "ascii"
621 | )
622 | size = humansize(float(out.split()[0]))
623 | num = subprocess.check_output(f"earthengine ls -r {asset}", shell=True).decode(
624 | "ascii"
625 | )
626 | num = [
627 | i
628 | for i in num.split("\n")
629 | if i and len(i) > 1 and not i.startswith("Running")
630 | ]
631 |
632 | print(f"\n{asset} ===> {size}")
633 | print(f"Total number of items including all folders: {len(num)}")
634 |
635 |
636 | def search(mname, source):
637 | gee_bundle = []
638 | if source is not None and source == "community":
639 | r = requests.get(
640 | "https://raw.githubusercontent.com/samapriya/awesome-gee-community-datasets/master/community_datasets.json"
641 | )
642 | community_list = r.json()
643 | print("Looking within {} community datasets".format(len(community_list)))
644 | i = 1
645 | for rows in community_list:
646 | if mname.lower() in str(rows["title"]).lower():
647 | try:
648 | if rows["type"] == "image_collection":
649 | rows["id"] = "ee.ImageCollection('{}')".format(rows["id"])
650 | elif rows["type"] == "image":
651 | rows["id"] = "ee.Image('{}')".format(rows["id"])
652 | elif rows["type"] == "table":
653 | rows["id"] = "ee.FeatureCollection('{}')".format(rows["id"])
654 | item = {
655 | "index": i,
656 | "title": rows["title"],
657 | "ee_id_snippet": rows["id"],
658 | "provider": rows["provider"],
659 | "tags": rows["tags"],
660 | "license": rows["license"],
661 | "sample_code": rows["sample_code"],
662 | }
663 |
664 | gee_bundle.append(item)
665 | i = i + 1
666 | except Exception as e:
667 | print(e)
668 | elif mname.lower() in str(rows["id"]).lower():
669 | try:
670 | if rows["type"] == "image_collection":
671 | rows["id"] = "ee.ImageCollection('{}')".format(rows["id"])
672 | elif rows["type"] == "image":
673 | rows["id"] = "ee.Image('{}')".format(rows["id"])
674 | elif rows["type"] == "table":
675 | rows["id"] = "ee.FeatureCollection('{}')".format(rows["id"])
676 | item = {
677 | "index": i,
678 | "title": rows["title"],
679 | "ee_id_snippet": rows["id"],
680 | "provider": rows["provider"],
681 | "tags": rows["tags"],
682 | "license": rows["license"],
683 | "sample_code": rows["sample_code"],
684 | }
685 | gee_bundle.append(item)
686 | i = i + 1
687 | except Exception as e:
688 | print(e)
689 | elif mname.lower() in str(rows["provider"]).lower():
690 | try:
691 | if rows["type"] == "image_collection":
692 | rows["id"] = "ee.ImageCollection('{}')".format(rows["id"])
693 | elif rows["type"] == "image":
694 | rows["id"] = "ee.Image('{}')".format(rows["id"])
695 | elif rows["type"] == "table":
696 | rows["id"] = "ee.FeatureCollection('{}')".format(rows["id"])
697 | item = {
698 | "index": i,
699 | "title": rows["title"],
700 | "ee_id_snippet": rows["id"],
701 | "provider": rows["provider"],
702 | "tags": rows["tags"],
703 | "license": rows["license"],
704 | "sample_code": rows["sample_code"],
705 | }
706 | gee_bundle.append(item)
707 | i = i + 1
708 | except Exception as e:
709 | print(e)
710 | elif mname.lower() in str(rows["tags"]).lower():
711 | try:
712 | if rows["type"] == "image_collection":
713 | rows["id"] = "ee.ImageCollection('{}')".format(rows["id"])
714 | elif rows["type"] == "image":
715 | rows["id"] = "ee.Image('{}')".format(rows["id"])
716 | elif rows["type"] == "table":
717 | rows["id"] = "ee.FeatureCollection('{}')".format(rows["id"])
718 | item = {
719 | "index": i,
720 | "title": rows["title"],
721 | "ee_id_snippet": rows["id"],
722 | "provider": rows["provider"],
723 | "tags": rows["tags"],
724 | "license": rows["license"],
725 | "sample_code": rows["sample_code"],
726 | }
727 | gee_bundle.append(item)
728 | i = i + 1
729 | except Exception as e:
730 | print(e)
731 | elif source is None:
732 | r = requests.get(
733 | "https://raw.githubusercontent.com/samapriya/Earth-Engine-Datasets-List/master/gee_catalog.json"
734 | )
735 | catalog_list = r.json()
736 | print("Looking within {} gee catalog datasets".format(len(catalog_list)))
737 | i = 1
738 | for rows in catalog_list:
739 | if mname.lower() in str(rows["title"]).lower():
740 | try:
741 | if rows["type"] == "image_collection":
742 | rows["id"] = "ee.ImageCollection('{}')".format(rows["id"])
743 | elif rows["type"] == "image":
744 | rows["id"] = "ee.Image('{}')".format(rows["id"])
745 | elif rows["type"] == "table":
746 | rows["id"] = "ee.FeatureCollection('{}')".format(rows["id"])
747 | item = {
748 | "index": i,
749 | "title": rows["title"],
750 | "ee_id_snippet": rows["id"],
751 | "start_date": rows["start_date"],
752 | "end_date": rows["end_date"],
753 | "asset_url": rows["asset_url"],
754 | "thumbnail_url": rows["thumbnail_url"],
755 | }
756 | gee_bundle.append(item)
757 | i = i + 1
758 | except Exception as e:
759 | print(e)
760 | elif mname.lower() in str(rows["id"]).lower():
761 | try:
762 | if rows["type"] == "image_collection":
763 | rows["id"] = "ee.ImageCollection('{}')".format(rows["id"])
764 | elif rows["type"] == "image":
765 | rows["id"] = "ee.Image('{}')".format(rows["id"])
766 | elif rows["type"] == "table":
767 | rows["id"] = "ee.FeatureCollection('{}')".format(rows["id"])
768 | item = {
769 | "index": i,
770 | "title": rows["title"],
771 | "ee_id_snippet": rows["id"],
772 | "start_date": rows["start_date"],
773 | "end_date": rows["end_date"],
774 | "asset_url": rows["asset_url"],
775 | "thumbnail_url": rows["thumbnail_url"],
776 | }
777 | gee_bundle.append(item)
778 | i = i + 1
779 | except Exception as e:
780 | print(e)
781 | elif mname.lower() in str(rows["provider"]).lower():
782 | try:
783 | if rows["type"] == "image_collection":
784 | rows["id"] = "ee.ImageCollection('{}')".format(rows["id"])
785 | elif rows["type"] == "image":
786 | rows["id"] = "ee.Image('{}')".format(rows["id"])
787 | elif rows["type"] == "table":
788 | rows["id"] = "ee.FeatureCollection('{}')".format(rows["id"])
789 | item = {
790 | "index": i,
791 | "title": rows["title"],
792 | "ee_id_snippet": rows["id"],
793 | "start_date": rows["start_date"],
794 | "end_date": rows["end_date"],
795 | "asset_url": rows["asset_url"],
796 | "thumbnail_url": rows["thumbnail_url"],
797 | }
798 | gee_bundle.append(item)
799 | i = i + 1
800 | except Exception as e:
801 | print(e)
802 | elif mname.lower() in str(rows["tags"]).lower():
803 | try:
804 | if rows["type"] == "image_collection":
805 | rows["id"] = "ee.ImageCollection('{}')".format(rows["id"])
806 | elif rows["type"] == "image":
807 | rows["id"] = "ee.Image('{}')".format(rows["id"])
808 | elif rows["type"] == "table":
809 | rows["id"] = "ee.FeatureCollection('{}')".format(rows["id"])
810 | item = {
811 | "index": i,
812 | "title": rows["title"],
813 | "ee_id_snippet": rows["id"],
814 | "start_date": rows["start_date"],
815 | "end_date": rows["end_date"],
816 | "asset_url": rows["asset_url"],
817 | "thumbnail_url": rows["thumbnail_url"],
818 | }
819 | gee_bundle.append(item)
820 | i = i + 1
821 | except Exception as e:
822 | print(e)
823 | print("")
824 | print(json.dumps(gee_bundle, indent=4, sort_keys=False))
825 |
826 |
827 | def quota_from_parser(args):
828 | quota(project_path=args.project)
829 |
830 |
831 | def ee_report_from_parser(args):
832 | ee_report(output=args.outfile, path=args.path)
833 |
834 |
835 | def move_from_parser(args):
836 | mover(path=args.initial, fpath=args.final)
837 |
838 |
839 | def copy_from_parser(args):
840 | copy(path=args.initial, fpath=args.final)
841 |
842 |
843 | def access_from_parser(args):
844 | access(collection_path=args.asset, user=args.user, role=args.role)
845 |
846 |
847 | def delete_metadata_from_parser(args):
848 | delprop(collection_path=args.asset, property=args.property)
849 |
850 |
851 | def app2script_from_parser(args):
852 | jsext(url=args.url, outfile=args.outfile)
853 |
854 |
855 | def read_from_parser(args):
856 | readme()
857 |
858 | def projects_from_parser(args):
859 | get_projects()
860 |
861 | def cancel_tasks_from_parser(args):
862 | cancel_tasks(tasks=args.tasks)
863 |
864 |
865 | def delete_collection_from_parser(args):
866 | delete(ids=args.id)
867 |
868 |
869 | def tasks_from_parser(args):
870 | tasks(state=args.state,id=args.id)
871 |
872 |
873 | def assetsize_from_parser(args):
874 | assetsize(asset=args.asset)
875 |
876 |
877 | def search_from_parser(args):
878 | search(mname=args.keywords, source=args.source)
879 |
880 |
881 | def main(args=None):
882 | parser = argparse.ArgumentParser(
883 | description="Google Earth Engine Batch Asset Manager with Addons"
884 | )
885 | subparsers = parser.add_subparsers()
886 |
887 | parser_read = subparsers.add_parser(
888 | "readme", help="Go the web based geeadd readme page"
889 | )
890 | parser_read.set_defaults(func=read_from_parser)
891 |
892 | parser_projects = subparsers.add_parser(
893 | "projects", help="Prints a list of Google Cloud Projects you own with Earth Engine API enabled"
894 | )
895 | parser_projects.set_defaults(func=projects_from_parser)
896 |
897 | parser_quota = subparsers.add_parser(
898 | "quota", help="Print Earth Engine total quota and used quota"
899 | )
900 | optional_named = parser_quota.add_argument_group("Optional named arguments")
901 | optional_named.add_argument(
902 | "--project",
903 | help="Project Name usually in format projects/project-name/assets/",
904 | default=None,
905 | )
906 | parser_quota.set_defaults(func=quota_from_parser)
907 |
908 | parser_app2script = subparsers.add_parser(
909 | "app2script", help="Get underlying script for public Google earthengine app"
910 | )
911 | required_named = parser_app2script.add_argument_group("Required named arguments.")
912 | required_named.add_argument("--url", help="Earthengine app url", required=True)
913 | optional_named = parser_app2script.add_argument_group("Optional named arguments")
914 | optional_named.add_argument(
915 | "--outfile",
916 | help="Write the script out to a .js file: Open in any text editor",
917 | default=None,
918 | )
919 | parser_app2script.set_defaults(func=app2script_from_parser)
920 |
921 | parser_search = subparsers.add_parser(
922 | "search", help="Search public GEE catalog using keywords"
923 | )
924 | required_named = parser_search.add_argument_group("Required named arguments.")
925 | required_named.add_argument(
926 | "--keywords",
927 | help="Keywords to search for can be id, provider, tag and so on",
928 | required=True,
929 | )
930 | optional_named = parser_search.add_argument_group("Optional named arguments")
931 | optional_named.add_argument(
932 | "--source",
933 | help="Type community to search within the Community Dataset Catalog",
934 | default=None,
935 | )
936 | parser_search.set_defaults(func=search_from_parser)
937 |
938 | parser_ee_report = subparsers.add_parser(
939 | "ee_report",
940 | help="Prints a detailed report of all Earth Engine Assets includes Asset Type, Path,Number of Assets,size(MB),unit,owner,readers,writers",
941 | )
942 | required_named = parser_ee_report.add_argument_group("Required named arguments.")
943 | required_named.add_argument(
944 | "--outfile", help="This it the location of your report csv file ", required=True
945 | )
946 | optional_named = parser_ee_report.add_argument_group("Optional named arguments")
947 | optional_named.add_argument(
948 | "--path",
949 | help="Path to any folder including project folders",
950 | default=None,
951 | )
952 | parser_ee_report.set_defaults(func=ee_report_from_parser)
953 |
954 | parser_assetsize = subparsers.add_parser(
955 | "assetsize",
956 | help="Prints any asset size (folders,collections,images or tables) in Human Readable form & Number of assets included",
957 | )
958 | required_named = parser_assetsize.add_argument_group("Required named arguments.")
959 | required_named.add_argument(
960 | "--asset",
961 | help="Earth Engine Asset for which to get size properties",
962 | required=True,
963 | )
964 | parser_assetsize.set_defaults(func=assetsize_from_parser)
965 |
966 | parser_tasks = subparsers.add_parser(
967 | "tasks",
968 | help="Queries current task status [completed,running,ready,failed,cancelled]",
969 | )
970 | optional_named = parser_tasks.add_argument_group("Optional named arguments")
971 | optional_named.add_argument(
972 | "--state",
973 | help="Query by state type COMPLETED|READY|RUNNING|FAILED",
974 | )
975 | optional_named.add_argument(
976 | "--id",
977 | help="Query by task id",
978 | )
979 | parser_tasks.set_defaults(func=tasks_from_parser)
980 |
981 | parser_cancel = subparsers.add_parser(
982 | "cancel", help="Cancel all, running or ready tasks or task ID"
983 | )
984 | required_named = parser_cancel.add_argument_group("Required named arguments.")
985 | required_named.add_argument(
986 | "--tasks",
987 | help="You can provide tasks as running or pending or all or even a single task id",
988 | required=True,
989 | default=None,
990 | )
991 | parser_cancel.set_defaults(func=cancel_tasks_from_parser)
992 |
993 | parser_copy = subparsers.add_parser(
994 | "copy", help="Copies entire folders, collections, images or tables"
995 | )
996 | required_named = parser_copy.add_argument_group("Required named arguments.")
997 | required_named.add_argument("--initial", help="Existing path of assets")
998 | required_named.add_argument("--final", help="New path for assets")
999 | parser_copy.set_defaults(func=copy_from_parser)
1000 |
1001 | parser_move = subparsers.add_parser(
1002 | "move", help="Moves entire folders, collections, images or tables"
1003 | )
1004 | required_named = parser_move.add_argument_group("Required named arguments.")
1005 | required_named.add_argument("--initial", help="Existing path of assets")
1006 | required_named.add_argument("--final", help="New path for assets")
1007 | parser_move.set_defaults(func=move_from_parser)
1008 |
1009 | parser_access = subparsers.add_parser(
1010 | "access",
1011 | help="Sets Permissions for entire folders, collections, images or tables",
1012 | )
1013 | required_named = parser_access.add_argument_group("Required named arguments.")
1014 | required_named.add_argument(
1015 | "--asset",
1016 | help="This is the path to the earth engine asset whose permission you are changing folder/collection/image",
1017 | required=True,
1018 | )
1019 | required_named.add_argument(
1020 | "--user",
1021 | help='Can be user email or serviceAccount like account@gserviceaccount.com or groups like group@googlegroups.com or try using "allUsers" to make it public',
1022 | required=True,
1023 | default=False,
1024 | )
1025 | required_named.add_argument(
1026 | "--role", help="Choose between reader, writer or delete", required=True
1027 | )
1028 | parser_access.set_defaults(func=access_from_parser)
1029 |
1030 | parser_delete = subparsers.add_parser(
1031 | "delete", help="Deletes folders or collections recursively"
1032 | )
1033 | required_named = parser_delete.add_argument_group("Required named arguments.")
1034 | required_named.add_argument(
1035 | "--id",
1036 | help="Full path to asset for deletion. Recursively removes all folders, collections and images.",
1037 | required=True,
1038 | )
1039 | parser_delete.set_defaults(func=delete_collection_from_parser)
1040 |
1041 | parser_delete_metadata = subparsers.add_parser(
1042 | "delete_metadata",
1043 | help="Use with caution: delete any metadata from collection or image",
1044 | )
1045 | required_named = parser_delete_metadata.add_argument_group(
1046 | "Required named arguments."
1047 | )
1048 | required_named.add_argument(
1049 | "--asset",
1050 | help="This is the path to the earth engine asset whose permission you are changing collection/image",
1051 | required=True,
1052 | )
1053 | required_named.add_argument(
1054 | "--property",
1055 | help="Metadata name that you want to delete",
1056 | required=True,
1057 | default=False,
1058 | )
1059 | parser_delete_metadata.set_defaults(func=delete_metadata_from_parser)
1060 |
1061 | args = parser.parse_args()
1062 |
1063 | try:
1064 | func = args.func
1065 | except AttributeError:
1066 | parser.error("too few arguments")
1067 | func(args)
1068 |
1069 |
1070 | if __name__ == "__main__":
1071 | main()
1072 |
--------------------------------------------------------------------------------
/mkdocs.yml:
--------------------------------------------------------------------------------
1 | site_name: geeadd docs
2 | site_description: Google Earth Engine Asset Manager with Addons
3 |
4 | site_author: Samapriya Roy
5 | site_url: https://github.com/samapriya
6 |
7 | # Repository
8 | repo_name: gee_asset_manager_addon
9 | repo_url: https://github.com/samapriya/gee_asset_manager_addon
10 |
11 | # Copyright
12 | copyright: "Copyright © 2021 - 2024 Samapriya Roy"
13 |
14 | # Configuration
15 | theme:
16 | name: "material"
17 | custom_dir: overrides
18 | features:
19 | - announce.dismiss
20 | - content.action.edit
21 | - content.action.view
22 | - content.code.annotate
23 | - content.code.copy
24 | - content.tooltips
25 | - search.highlight
26 | - search.share
27 | - search.suggest
28 | - toc.follow
29 |
30 | # 404 page
31 | static_templates:
32 | - 404.html
33 |
34 | # Don't include MkDocs' JavaScript
35 | include_search_page: false
36 | search_index_only: true
37 |
38 | # Default values, taken from mkdocs_theme.yml
39 | language: en
40 | palette:
41 | - scheme: default
42 | primary: yellow
43 | accent: indigo
44 | toggle:
45 | icon: material/toggle-switch-off-outline
46 | name: Switch to dark mode
47 | - scheme: slate
48 | primary: red
49 | accent: red
50 | toggle:
51 | icon: material/toggle-switch
52 | name: Switch to light mode
53 | font:
54 | text: Google Sans
55 | code: Regular
56 | favicon: assets/favicon.png
57 | icon:
58 | logo: logo
59 |
60 | # Options
61 | extra:
62 | social:
63 | - icon: fontawesome/brands/github
64 | link: https://github.com/samapriya
65 | - icon: fontawesome/brands/medium
66 | link: https://medium.com/@samapriyaroy
67 | - icon: fontawesome/brands/mastodon
68 | link: https://mapstodon.space/@samapriya
69 | - icon: fontawesome/brands/twitter
70 | link: https://twitter.com/samapriyaroy
71 | - icon: fontawesome/brands/linkedin
72 | link: https://www.linkedin.com/in/samapriya
73 | analytics:
74 | provider: google
75 | property: G-P5Q8CHNTVP
76 |
77 | plugins:
78 | - search
79 | - git-revision-date-localized:
80 | enable_creation_date: true
81 | type: timeago
82 | - minify:
83 | minify_html: true
84 |
85 | # Extensions
86 | markdown_extensions:
87 | - admonition
88 | - abbr
89 | - attr_list
90 | - def_list
91 | - footnotes
92 | - meta
93 | - md_in_html
94 | - toc:
95 | permalink: true
96 | - pymdownx.arithmatex:
97 | generic: true
98 | - pymdownx.betterem:
99 | smart_enable: all
100 | - pymdownx.caret
101 | - pymdownx.critic
102 | - pymdownx.details
103 | - pymdownx.emoji:
104 | emoji_index: !!python/name:materialx.emoji.twemoji
105 | emoji_generator: !!python/name:materialx.emoji.to_svg
106 | options:
107 | custom_icons:
108 | - overrides/.icons
109 | - pymdownx.highlight
110 | - pymdownx.inlinehilite
111 | - pymdownx.keys
112 | - pymdownx.magiclink:
113 | repo_url_shorthand: true
114 | user: squidfunk
115 | repo: mkdocs-material
116 | - pymdownx.mark
117 | - pymdownx.smartsymbols
118 | - pymdownx.superfences:
119 | custom_fences:
120 | - name: mermaid
121 | class: mermaid
122 | format: !!python/name:pymdownx.superfences.fence_code_format
123 | - pymdownx.tabbed
124 | - pymdownx.tasklist:
125 | custom_checkbox: true
126 | - pymdownx.tilde
127 |
128 | # Page tree
129 | nav:
130 | - Google Earth Engine Batch Asset Manager with Addons: index.md
131 | - License: license.md
132 | - Prerequisites and Installation: installation.md
133 | - Setup and Search Tools:
134 | - Quota tool: projects/quota.md
135 | - EE API Enabled Projects tool: projects/projects.md
136 | - GEE App to Script tool: projects/app2script.md
137 | - Search tool: projects/search.md
138 | - Asset management tools:
139 | - Asset Size tool: projects/size.md
140 | - Copy Assets tool: projects/copy.md
141 | - Move Assets tool: projects/move.md
142 | - Change asset permissions: projects/access.md
143 | - Delete Assets tool: projects/delete.md
144 | - Assets Report tool: projects/report.md
145 | - Tasks and Metadata tools:
146 | - Tasks tool: projects/task_status.md
147 | - Cancel Tasks tool: projects/cancel_tasks.md
148 | - Delete metadata tool: projects/delete_metadata.md
149 | - Changelog: changelog.md
150 |
--------------------------------------------------------------------------------
/overrides/assets/stylesheets/custom.f7ec4df2.min.css:
--------------------------------------------------------------------------------
1 | @keyframes heart{0%,40%,80%,to{transform:scale(1)}20%,60%{transform:scale(1.15)}}.md-typeset .twitter{color:#00acee}.md-typeset .mastodon{color:#897ff8}.md-typeset .mdx-video{width:auto}.md-typeset .mdx-video__inner{height:0;padding-bottom:56.138%;position:relative;width:100%}.md-typeset .mdx-video iframe{border:none;height:100%;left:0;overflow:hidden;position:absolute;top:0;width:100%}.md-typeset .mdx-heart{animation:heart 1s infinite}.md-typeset .mdx-insiders{color:#e91e63}.md-typeset .mdx-switch button{cursor:pointer;transition:opacity .25s}.md-typeset .mdx-switch button:focus,.md-typeset .mdx-switch button:hover{opacity:.75}.md-typeset .mdx-switch button>code{background-color:var(--md-primary-fg-color);color:var(--md-primary-bg-color);display:block}.md-typeset .mdx-deprecated{opacity:.5;transition:opacity .25s}.md-typeset .mdx-deprecated:focus-within,.md-typeset .mdx-deprecated:hover{opacity:1}.md-typeset .mdx-columns ol,.md-typeset .mdx-columns ul{-moz-columns:2;column-count:2}@media screen and (max-width:29.9375em){.md-typeset .mdx-columns ol,.md-typeset .mdx-columns ul{-moz-columns:initial;columns:initial}}.md-typeset .mdx-columns li{-moz-column-break-inside:avoid;break-inside:avoid}.md-typeset .mdx-flags{margin:2em auto}.md-typeset .mdx-flags ol{list-style:none}.md-typeset .mdx-flags ol li{margin-bottom:1em}.md-typeset .mdx-flags__item{display:flex;gap:.6rem}.md-typeset .mdx-flags__content{display:flex;flex:1;flex-direction:column}.md-typeset .mdx-flags__content span{align-items:baseline;display:inline-flex;justify-content:space-between}.md-typeset .mdx-flags__content>span:nth-child(2){font-size:80%}.md-typeset .mdx-flags__content code{float:right}.md-typeset .mdx-author{display:flex;font-size:.68rem}.md-typeset .mdx-author img{border-radius:100%;height:2rem}.md-typeset .mdx-author p:first-child{flex-shrink:0;margin-right:.8rem}.md-typeset .mdx-author p>span{display:block}.md-typeset .mdx-social{height:min(27rem,80vw);position:relative}.md-typeset .mdx-social:hover .mdx-social__image{background-color:#e4e4e40d}.md-typeset .mdx-social__layer{margin-top:4rem;position:absolute;transform-style:preserve-3d;transition:.25s cubic-bezier(.7,0,.3,1)}.md-typeset .mdx-social__layer:hover .mdx-social__label{opacity:1}.md-typeset .mdx-social__layer:hover .mdx-social__image{background-color:#7f7f7ffc}.md-typeset .mdx-social__layer:hover~.mdx-social__layer{opacity:0}.md-typeset .mdx-social__image{box-shadow:-.25rem .25rem .5rem #0000000d;transform:rotate(-40deg) skew(15deg,15deg) scale(.7);transition:all .25s}.md-typeset .mdx-social__image img{display:block}.md-typeset .mdx-social__label{background-color:var(--md-default-fg-color--light);color:var(--md-default-bg-color);display:block;opacity:0;padding:.2rem .4rem;position:absolute;transition:all .25s}.md-typeset .mdx-social:hover .mdx-social__layer:nth-child(6){transform:translateY(-30px)}.md-typeset .mdx-social:hover .mdx-social__layer:nth-child(5){transform:translateY(-20px)}.md-typeset .mdx-social:hover .mdx-social__layer:nth-child(4){transform:translateY(-10px)}.md-typeset .mdx-social:hover .mdx-social__layer:nth-child(3){transform:translateY(0)}.md-typeset .mdx-social:hover .mdx-social__layer:nth-child(2){transform:translateY(10px)}.md-typeset .mdx-social:hover .mdx-social__layer:first-child{transform:translateY(20px)}.md-typeset .mdx-social:hover .mdx-social__layer:nth-child(0){transform:translateY(30px)}.md-banner{color:var(--md-footer-fg-color--lighter)}.md-banner strong{white-space:nowrap}.md-banner a,.md-banner strong{color:var(--md-footer-fg-color)}.md-banner a:focus,.md-banner a:hover{color:currentcolor}.md-banner a:focus .twemoji,.md-banner a:hover .twemoji{background-color:var(--md-footer-fg-color);box-shadow:none}.md-banner .twemoji{border-radius:100%;box-shadow:inset 0 0 0 .05rem currentcolor;display:inline-block;height:1.2rem;padding:.25rem;transition:all .25s;vertical-align:bottom;width:1.2rem}.md-banner .twemoji svg{display:block;max-height:none}.mdx-container{background:url("data:image/svg+xml;utf8,
27 |
54 |
55 | Support
56 |
57 |
60 | Sponsor
61 |
62 |
63 |