├── .circleci └── config.yml ├── .gitignore ├── .pre-commit-config.yaml ├── CHANGES.txt ├── LICENSE ├── README.md ├── codecov.yml ├── example └── rio-tiler-mosaic-Custom.ipynb ├── requirements.txt ├── rio_tiler_mosaic ├── __init__.py ├── methods │ ├── __init__.py │ ├── base.py │ └── defaults.py └── mosaic.py ├── setup.py ├── tests ├── fixtures │ ├── cog1.tif │ └── cog2.tif ├── test_benchmarks.py └── test_mosaic.py └── tox.ini /.circleci/config.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | common: &common 3 | working_directory: ~/rio-tiler-mosaic 4 | steps: 5 | - checkout 6 | - run: 7 | name: install dependencies 8 | command: pip install tox codecov pre-commit --user 9 | - run: 10 | name: run tox 11 | command: ~/.local/bin/tox 12 | - run: 13 | name: run pre-commit 14 | command: | 15 | if [[ "$CIRCLE_JOB" == "python-3.7" ]]; then 16 | ~/.local/bin/pre-commit run --all-files 17 | fi 18 | - run: 19 | name: upload coverage report 20 | command: | 21 | if [[ "$UPLOAD_COVERAGE" == 1 ]]; then 22 | ~/.local/bin/coverage xml 23 | ~/.local/bin/codecov 24 | fi 25 | when: always 26 | 27 | jobs: 28 | "python-3.6": 29 | <<: *common 30 | docker: 31 | - image: circleci/python:3.6.5 32 | environment: 33 | - TOXENV=py36 34 | - UPLOAD_COVERAGE=1 35 | 36 | "python-3.7": 37 | <<: *common 38 | docker: 39 | - image: circleci/python:3.7.2 40 | environment: 41 | - TOXENV=py37 42 | 43 | benchmark: 44 | docker: 45 | - image: circleci/python:3.7.2 46 | working_directory: ~/rio-tiler-mosaic 47 | steps: 48 | - checkout 49 | - run: pip install -e .[test] --user 50 | - run: | 51 | python -m pytest \ 52 | --benchmark-only \ 53 | --benchmark-autosave \ 54 | --benchmark-columns 'min, max, mean, median' \ 55 | --benchmark-sort 'min' 56 | 57 | deploy: 58 | docker: 59 | - image: circleci/python:3.7.2 60 | environment: 61 | - TOXENV=release 62 | working_directory: ~/rio-tiler-mosaic 63 | steps: 64 | - checkout 65 | - run: 66 | command: pip install -e . --user 67 | - run: 68 | name: verify git tag vs. version 69 | command: | 70 | VERSION=$(python -c 'import rio_tiler_mosaic; print(rio_tiler_mosaic.version)') 71 | if [ "$VERSION" = "$CIRCLE_TAG" ]; then exit 0; else exit 3; fi 72 | - run: 73 | name: install dependencies 74 | command: pip install tox --user 75 | - run: 76 | name: init .pypirc 77 | command: | 78 | echo -e "[pypi]" >> ~/.pypirc 79 | echo -e "username = $PYPI_USER" >> ~/.pypirc 80 | echo -e "password = $PYPI_PASSWORD" >> ~/.pypirc 81 | - run: 82 | name: run tox 83 | command: ~/.local/bin/tox 84 | 85 | workflows: 86 | version: 2 87 | build_and_deploy: 88 | jobs: 89 | - benchmark 90 | - "python-3.6" 91 | - "python-3.7": 92 | filters: # required since `deploy` has tag filters AND requires `build` 93 | tags: 94 | only: /.*/ 95 | - deploy: 96 | requires: 97 | - "python-3.7" 98 | filters: 99 | tags: 100 | only: /^[0-9]+.*/ 101 | branches: 102 | ignore: /.*/ 103 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | *.egg-info/ 24 | .installed.cfg 25 | *.egg 26 | MANIFEST 27 | 28 | # PyInstaller 29 | # Usually these files are written by a python script from a template 30 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 31 | *.manifest 32 | *.spec 33 | 34 | # Installer logs 35 | pip-log.txt 36 | pip-delete-this-directory.txt 37 | 38 | # Unit test / coverage reports 39 | htmlcov/ 40 | .tox/ 41 | .coverage 42 | .coverage.* 43 | .cache 44 | nosetests.xml 45 | coverage.xml 46 | *.cover 47 | .hypothesis/ 48 | .pytest_cache/ 49 | 50 | # Translations 51 | *.mo 52 | *.pot 53 | 54 | # Django stuff: 55 | *.log 56 | local_settings.py 57 | db.sqlite3 58 | 59 | # Flask stuff: 60 | instance/ 61 | .webassets-cache 62 | 63 | # Scrapy stuff: 64 | .scrapy 65 | 66 | # Sphinx documentation 67 | docs/_build/ 68 | 69 | # PyBuilder 70 | target/ 71 | 72 | # Jupyter Notebook 73 | .ipynb_checkpoints 74 | 75 | # pyenv 76 | .python-version 77 | 78 | # celery beat schedule file 79 | celerybeat-schedule 80 | 81 | # SageMath parsed files 82 | *.sage.py 83 | 84 | # Environments 85 | .env 86 | .venv 87 | env/ 88 | venv/ 89 | ENV/ 90 | env.bak/ 91 | venv.bak/ 92 | 93 | # Spyder project settings 94 | .spyderproject 95 | .spyproject 96 | 97 | # Rope project settings 98 | .ropeproject 99 | 100 | # mkdocs documentation 101 | /site 102 | 103 | # mypy 104 | .mypy_cache/ -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | repos: 2 | - 3 | repo: https://github.com/pre-commit/mirrors-isort 4 | rev: v4.3.21 5 | hooks: 6 | - id: isort 7 | language_version: python3.7 8 | - 9 | repo: 'https://github.com/psf/black' 10 | rev: stable 11 | hooks: 12 | - id: black 13 | args: ['--safe'] 14 | language_version: python3.7 15 | - 16 | repo: 'https://github.com/pre-commit/pre-commit-hooks' 17 | rev: v2.4.0 18 | hooks: 19 | - id: flake8 20 | language_version: python3.7 21 | args: [ 22 | # E501 let black handle all line length decisions 23 | # W503 black conflicts with "line break before operator" rule 24 | # E203 black conflicts with "whitespace before ':'" rule 25 | '--ignore=E501,W503,E203'] 26 | - 27 | repo: 'https://github.com/chewse/pre-commit-mirrors-pydocstyle' 28 | # 2.1.1 29 | rev: 22d3ccf6cf91ffce3b16caa946c155778f0cb20f 30 | hooks: 31 | - id: pydocstyle 32 | language_version: python3.7 33 | args: [ 34 | # Check for docstring presence only 35 | '--select=D1', 36 | # Don't require docstrings for tests 37 | '--match=(?!test).*\.py'] 38 | 39 | - 40 | repo: https://github.com/pre-commit/mirrors-mypy 41 | rev: 'v0.770' 42 | hooks: 43 | - id: mypy 44 | args: [--no-strict-optional, --ignore-missing-imports] -------------------------------------------------------------------------------- /CHANGES.txt: -------------------------------------------------------------------------------- 1 | 0.0.1dev5 (2020-07-24) 2 | ---------------------- 3 | - Threading is optional (#13 and #15, co-author with @kylebarron) 4 | - Update for rio-tiler 2.0b1 5 | - add logging for errors 6 | - switch to python 3 and use type hints 7 | - add benchmark 8 | 9 | 0.0.1dev4 (2020-04-22) 10 | ---------------------- 11 | - allow rio-tiler >=1.2 12 | 13 | 0.0.1dev3 (2019-07-25) 14 | ---------------------- 15 | - Fix bad tile stacking creating invalid tile shapes for mean and median method. 16 | - Add stdev method (#7) 17 | - Allow "chunck_size" option to control the number of asset to process per loop (#6) 18 | 19 | 0.0.1dev2 (2019-07-18) 20 | -------------------- 21 | - Force output data type to be the same as the input datatype 22 | from Mean and Median method#4 23 | 24 | 0.0.1dev1 (2019-07-18) 25 | -------------------- 26 | **Breacking Changes** 27 | 28 | - refactor pixel selection method to use MosaicMethodBase abstract class (#1) 29 | - change method names (#2) 30 | 31 | 0.0.1dev0 (2019-05-23) 32 | ------------------ 33 | - Initial release -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2019 Vincent Sarago 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | 2 | This is now directly integrated in rio-tiler~=2.0: https://github.com/cogeotiff/rio-tiler/pull/204 3 | 4 | # rio-tiler-mosaic 5 | 6 | [![Packaging status](https://badge.fury.io/py/rio-tiler-mosaic.svg)](https://badge.fury.io/py/rio-tiler-mosaic) 7 | [![CircleCI](https://circleci.com/gh/cogeotiff/rio-tiler-mosaic.svg?style=svg)](https://circleci.com/gh/cogeotiff/rio-tiler-mosaic) 8 | [![codecov](https://codecov.io/gh/cogeotiff/rio-tiler-mosaic/branch/master/graph/badge.svg)](https://codecov.io/gh/cogeotiff/rio-tiler-mosaic) 9 | 10 | A rio-tiler plugin for creating tiles from multiple observations. 11 | 12 | ![](https://user-images.githubusercontent.com/10407788/57466726-304f5880-724f-11e9-9969-bec4ce940e07.png) 13 | 14 | ## Install 15 | 16 | ```bash 17 | $ pip install rio-tiler-mosaic 18 | ``` 19 | Or 20 | ```bash 21 | $ git clone http://github.com/cogeotiff/rio-tiler-mosaic 22 | $ cd rio-tiler-mosaic 23 | $ pip install -e . 24 | ``` 25 | 26 | ## Rio-tiler + Mosaic 27 | 28 | ![](https://user-images.githubusercontent.com/10407788/57467798-30505800-7251-11e9-9bde-6f50801dc851.png) 29 | 30 | The goal of this rio-tiler plugin is to create tiles from multiple observations. 31 | 32 | Because user might want to choose which pixel goes on top of the tile, this plugin comes with 5 differents `pixel selection` algorithms: 33 | - **First**: takes the first pixel received 34 | - **Highest**: loop though all the assets and return the highest value 35 | - **Lowest**: loop though all the assets and return the lowest value 36 | - **Mean**: compute the mean value of the whole stack 37 | - **Median**: compute the median value of the whole stack 38 | 39 | ### API 40 | 41 | `mosaic_tiler(assets, tile_x, tile_y, tile_z, tiler, pixel_selection=None, chunk_size=5, kwargs)` 42 | 43 | Inputs: 44 | - assets : list, tuple of rio-tiler compatible assets (url or sceneid) 45 | - tile_x : Mercator tile X index. 46 | - tile_y : Mercator tile Y index. 47 | - tile_z : Mercator tile ZOOM level. 48 | - tiler: Rio-tiler's tiler function (e.g rio_tiler.landsat8.tile) 49 | - pixel_selection : optional **pixel selection** algorithm (default: "first"). 50 | - chunk_size: optional, control the number of asset to process per loop. 51 | - kwargs: Rio-tiler tiler module specific otions. 52 | 53 | Returns: 54 | - tile, mask : tuple of ndarray Return tile and mask data. 55 | 56 | #### Examples 57 | 58 | ```python 59 | from rio_tiler.io import COGReader 60 | from rio_tiler_mosaic.mosaic import mosaic_tiler 61 | from rio_tiler_mosaic.methods import defaults 62 | 63 | 64 | def tiler(src_path: str, *args, **kwargs) -> Tuple[numpy.ndarray, numpy.ndarray]: 65 | with COGReader(src_path) as cog: 66 | return cog.tile(*args, **kwargs) 67 | 68 | assets = ["mytif1.tif", "mytif2.tif", "mytif3.tif"] 69 | tile = (1000, 1000, 9) 70 | x, y, z = tile 71 | 72 | # Use Default First value method 73 | mosaic_tiler(assets, x, y, z, tiler) 74 | 75 | # Use Highest value: defaults.HighestMethod() 76 | mosaic_tiler( 77 | assets, 78 | x, 79 | y, 80 | z, 81 | tiler, 82 | pixel_selection=defaults.HighestMethod() 83 | ) 84 | 85 | # Use Lowest value: defaults.LowestMethod() 86 | mosaic_tiler( 87 | assets, 88 | x, 89 | y, 90 | z, 91 | tiler, 92 | pixel_selection=defaults.LowestMethod() 93 | ) 94 | ``` 95 | 96 | ### The `MosaicMethod` interface 97 | 98 | the `rio-tiler-mosaic.methods.base.MosaicMethodBase` class defines an abstract 99 | interface for all `pixel selection` methods allowed by `rio-tiler-mosaic`. its methods and properties are: 100 | 101 | - `is_done`: property, returns a boolean indicating if the process is done filling the tile 102 | - `data`: property, returns the output **tile** and **mask** numpy arrays 103 | - `feed(tile: numpy.ma.ndarray)`: method, update the tile 104 | 105 | The MosaicMethodBase class is not intended to be used directly but as an abstract base class, a template for concrete implementations. 106 | 107 | #### Writing your own Pixel Selection method 108 | 109 | The rules for writing your own `pixel selection algorithm` class are as follows: 110 | 111 | - Must inherit from MosaicMethodBase 112 | - Must provide concrete implementations of all the above methods. 113 | 114 | See [rio_tiler_mosaic.methods.defaults](/rio_tiler_mosaic/defaults.py) classes for examples. 115 | 116 | #### Smart Multi-Threading 117 | 118 | When dealing with an important number of image, you might not want to process the whole stack, especially if the pixel selection method stops when the tile is filled. To allow better optimization, `rio-tiler-mosaic` is fetching the tiles in parallel (threads) but to limit the number of files we also embeded the fetching in a loop (creating 2 level of processing): 119 | 120 | ```python 121 | assets = ["1.tif", "2.tif", "3.tif", "4.tif", "5.tif", "6.tif"] 122 | 123 | # 1st level loop - Creates chuncks of assets 124 | for chunks in _chunks(assets, chunk_size): 125 | # 2nd level loop - Uses threads for process each `chunck` 126 | with futures.ThreadPoolExecutor(max_workers=max_threads) as executor: 127 | future_tasks = [executor.submit(_tiler, asset) for asset in chunks] 128 | ``` 129 | By default the chunck_size is equal to the number or threads (or the number of assets if no threads=0) 130 | 131 | #### More on threading 132 | 133 | The number of threads used can be set in the function call with the `threads=` options. By default it will be equal to `multiprocessing.cpu_count() * 5` or to the MAX_THREADS environment variable. 134 | In some case, threading can slow down your application. You can set threads to `0` to run the tiler in a loop without using a ThreadPool. 135 | 136 | ## Example 137 | 138 | See [/example](/example) 139 | 140 | ## Contribution & Development 141 | 142 | Issues and pull requests are more than welcome. 143 | 144 | **dev install** 145 | 146 | ```bash 147 | $ git clone https://github.com/cogeotiff/rio-tiler-mosaic.git 148 | $ cd rio-tiler-mosaic 149 | $ pip install -e .[dev] 150 | ``` 151 | 152 | **Python3.6 only** 153 | 154 | This repo is set to use `pre-commit` to run *flake8*, *pydocstring* and *black* ("uncompromising Python code formatter") when commiting new code. 155 | 156 | ```bash 157 | $ pre-commit install 158 | ``` 159 | 160 | 161 | ## Implementation 162 | [cogeo-mosaic](http://github.com/developmentseed/cogeo-mosaic.git) 163 | -------------------------------------------------------------------------------- /codecov.yml: -------------------------------------------------------------------------------- 1 | comment: off 2 | 3 | coverage: 4 | status: 5 | project: 6 | default: 7 | target: auto 8 | threshold: 5 9 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | rio-tiler~=2.0a -------------------------------------------------------------------------------- /rio_tiler_mosaic/__init__.py: -------------------------------------------------------------------------------- 1 | """rio-tiler-mosaic: Create tiles from multiple observations.""" 2 | 3 | import pkg_resources 4 | 5 | version = pkg_resources.get_distribution(__package__).version 6 | -------------------------------------------------------------------------------- /rio_tiler_mosaic/methods/__init__.py: -------------------------------------------------------------------------------- 1 | """rio-tiler-mosaic.methods: Mosaic filling methods.""" 2 | -------------------------------------------------------------------------------- /rio_tiler_mosaic/methods/base.py: -------------------------------------------------------------------------------- 1 | """rio-tiler-mosaic.methods abc class.""" 2 | 3 | import abc 4 | 5 | import numpy 6 | 7 | 8 | class MosaicMethodBase(abc.ABC): 9 | """Abstract base class for rio-tiler-mosaic methods objects.""" 10 | 11 | def __init__(self): 12 | """Init backend.""" 13 | self.tile = None 14 | self.exit_when_filled = False 15 | 16 | @property 17 | def is_done(self): 18 | """ 19 | Check if the tile filling is done. 20 | 21 | Returns 22 | ------- 23 | bool 24 | 25 | """ 26 | if self.tile is None: 27 | return False 28 | 29 | if self.exit_when_filled and not numpy.ma.is_masked(self.tile): 30 | return True 31 | 32 | return False 33 | 34 | @property 35 | def data(self): 36 | """ 37 | Return data and mask. 38 | 39 | Returns 40 | ------- 41 | tile: numpy.ndarray 42 | mask: numpy.ndarray 43 | 44 | """ 45 | if self.tile is not None: 46 | return self.tile.data, ~self.tile.mask[0] * 255 47 | else: 48 | return None, None 49 | 50 | @abc.abstractmethod 51 | def feed(self, tile): 52 | """ 53 | Fill mosaic tile. 54 | 55 | Parameters 56 | ---------- 57 | tile: numpy.ma.ndarray 58 | 59 | """ 60 | -------------------------------------------------------------------------------- /rio_tiler_mosaic/methods/defaults.py: -------------------------------------------------------------------------------- 1 | """rio_tiler_mosaic.methods.defaults: default mosaic filling methods.""" 2 | 3 | import numpy 4 | 5 | from rio_tiler_mosaic.methods.base import MosaicMethodBase 6 | 7 | 8 | class FirstMethod(MosaicMethodBase): 9 | """Feed the mosaic tile with the first pixel available.""" 10 | 11 | def __init__(self): 12 | """Overwrite base and init First method.""" 13 | super(FirstMethod, self).__init__() 14 | self.exit_when_filled = True 15 | 16 | def feed(self, tile): 17 | """Add data to tile.""" 18 | if self.tile is None: 19 | self.tile = tile 20 | pidex = self.tile.mask & ~tile.mask 21 | 22 | mask = numpy.where(pidex, tile.mask, self.tile.mask) 23 | self.tile = numpy.ma.where(pidex, tile, self.tile) 24 | self.tile.mask = mask 25 | 26 | 27 | class HighestMethod(MosaicMethodBase): 28 | """Feed the mosaic tile with the highest pixel values.""" 29 | 30 | def feed(self, tile): 31 | """Add data to tile.""" 32 | if self.tile is None: 33 | self.tile = tile 34 | 35 | pidex = ( 36 | numpy.bitwise_and(tile.data > self.tile.data, ~tile.mask) | self.tile.mask 37 | ) 38 | 39 | mask = numpy.where(pidex, tile.mask, self.tile.mask) 40 | self.tile = numpy.ma.where(pidex, tile, self.tile) 41 | self.tile.mask = mask 42 | 43 | 44 | class LowestMethod(MosaicMethodBase): 45 | """Feed the mosaic tile with the lowest pixel values.""" 46 | 47 | def feed(self, tile): 48 | """Add data to tile.""" 49 | if self.tile is None: 50 | self.tile = tile 51 | 52 | pidex = ( 53 | numpy.bitwise_and(tile.data < self.tile.data, ~tile.mask) | self.tile.mask 54 | ) 55 | 56 | mask = numpy.where(pidex, tile.mask, self.tile.mask) 57 | self.tile = numpy.ma.where(pidex, tile, self.tile) 58 | self.tile.mask = mask 59 | 60 | 61 | class MeanMethod(MosaicMethodBase): 62 | """Stack the tiles and return the Mean pixel value.""" 63 | 64 | def __init__(self, enforce_data_type=True): 65 | """Overwrite base and init Mean method.""" 66 | super(MeanMethod, self).__init__() 67 | self.enforce_data_type = enforce_data_type 68 | self.tile = [] 69 | 70 | @property 71 | def data(self): 72 | """Return data and mask.""" 73 | if self.tile: 74 | tile = numpy.ma.mean(numpy.ma.stack(self.tile, axis=0), axis=0) 75 | if self.enforce_data_type: 76 | tile = tile.astype(self.tile[0].dtype) 77 | return tile.data, ~tile.mask[0] * 255 78 | else: 79 | return None, None 80 | 81 | def feed(self, tile): 82 | """Add data to tile.""" 83 | self.tile.append(tile) 84 | 85 | 86 | class MedianMethod(MosaicMethodBase): 87 | """Stack the tiles and return the Median pixel value.""" 88 | 89 | def __init__(self, enforce_data_type=True): 90 | """Overwrite base and init Median method.""" 91 | super(MedianMethod, self).__init__() 92 | self.enforce_data_type = enforce_data_type 93 | self.tile = [] 94 | 95 | @property 96 | def data(self): 97 | """Return data and mask.""" 98 | if self.tile: 99 | tile = numpy.ma.median(numpy.ma.stack(self.tile, axis=0), axis=0) 100 | if self.enforce_data_type: 101 | tile = tile.astype(self.tile[0].dtype) 102 | return tile.data, ~tile.mask[0] * 255 103 | else: 104 | return None, None 105 | 106 | def feed(self, tile): 107 | """Create a stack of tile.""" 108 | self.tile.append(tile) 109 | 110 | 111 | class StdevMethod(MosaicMethodBase): 112 | """Stack the tiles and return the Standart Deviation value.""" 113 | 114 | def __init__(self, enforce_data_type=True): 115 | """Overwrite base and init Stdev method.""" 116 | super(StdevMethod, self).__init__() 117 | self.tile = [] 118 | 119 | @property 120 | def data(self): 121 | """Return data and mask.""" 122 | if self.tile: 123 | tile = numpy.ma.std(numpy.ma.stack(self.tile, axis=0), axis=0) 124 | return tile.data, ~tile.mask[0] * 255 125 | else: 126 | return None, None 127 | 128 | def feed(self, tile): 129 | """Add data to tile.""" 130 | self.tile.append(tile) 131 | -------------------------------------------------------------------------------- /rio_tiler_mosaic/mosaic.py: -------------------------------------------------------------------------------- 1 | """rio_tiler_mosaic.mosaic: create tile from multiple assets.""" 2 | 3 | import logging 4 | from concurrent import futures 5 | from typing import Callable, Generator, Optional, Sequence, Tuple, Union 6 | 7 | import numpy 8 | 9 | from rio_tiler.constants import MAX_THREADS 10 | from rio_tiler.utils import _chunks 11 | from rio_tiler_mosaic.methods.base import MosaicMethodBase 12 | from rio_tiler_mosaic.methods.defaults import FirstMethod 13 | 14 | logger = logging.getLogger() 15 | logger.setLevel(logging.ERROR) 16 | TaskType = Union[Generator[Callable, None, None], Sequence[futures.Future]] 17 | 18 | 19 | def _filter_tasks(tasks: TaskType): 20 | """ 21 | Filter tasks to remove Exceptions. 22 | 23 | Attributes 24 | ---------- 25 | tasks : list or tuple 26 | Sequence of 'concurrent.futures._base.Future' or 'partial' 27 | 28 | Yields 29 | ------ 30 | Successful task's result 31 | 32 | """ 33 | for future in tasks: 34 | try: 35 | if isinstance(future, futures.Future): 36 | yield future.result() 37 | else: 38 | yield future 39 | except Exception as err: 40 | logging.error(err) 41 | pass 42 | 43 | 44 | def mosaic_tiler( 45 | assets: Sequence[str], 46 | tile_x: int, 47 | tile_y: int, 48 | tile_z: int, 49 | tiler: Callable, 50 | pixel_selection: Optional[MosaicMethodBase] = None, 51 | chunk_size: Optional[int] = None, 52 | threads: int = MAX_THREADS, 53 | **kwargs, 54 | ) -> Tuple[numpy.ndarray, numpy.ndarray]: 55 | """ 56 | Create mercator tile from multiple observations. 57 | 58 | Attributes 59 | ---------- 60 | assets: list or tuple 61 | List of tiler compatible asset. 62 | tile_x: int 63 | Mercator tile X index. 64 | tile_y: int 65 | Mercator tile Y index. 66 | tile_z: int 67 | Mercator tile ZOOM level. 68 | tiler: callable 69 | tiler function. The function MUST take asset, x, y, z, **kwargs as arguments, 70 | and MUST return a tuple with tile data and mask 71 | e.g: 72 | def tiler(asset: str, x: int, y: int, z: int, **kwargs) -> Tuple[numpy.ndarray, numpy.ndarray]: 73 | with COGReader(asset) as cog: 74 | return cog.tile(x, y, z, **kwargs) 75 | pixel_selection: MosaicMethod, optional 76 | Instance of MosaicMethodBase class. 77 | default: "rio_tiler_mosaic.methods.defaults.FirstMethod". 78 | chunk_size: int, optional 79 | Control the number of asset to process per loop (default = threads). 80 | threads: int, optional 81 | Number of threads to use. If <=1, runs single threaded without an event 82 | loop. By default reads from the MAX_THREADS environment variable, and if 83 | not found defaults to multiprocessing.cpu_count() * 5. 84 | kwargs: dict, optional 85 | tiler specific options. 86 | 87 | Returns 88 | ------- 89 | tile, mask : tuple of ndarray 90 | Return tile and mask data. 91 | 92 | """ 93 | if pixel_selection is None: 94 | pixel_selection = FirstMethod() 95 | 96 | if not isinstance(pixel_selection, MosaicMethodBase): 97 | raise Exception( 98 | "Mosaic filling algorithm should be an instance of" 99 | "'rio_tiler_mosaic.methods.base.MosaicMethodBase'" 100 | ) 101 | 102 | if not chunk_size: 103 | chunk_size = threads or len(assets) 104 | 105 | tasks: TaskType 106 | 107 | for chunks in _chunks(assets, chunk_size): 108 | if threads: 109 | with futures.ThreadPoolExecutor(max_workers=threads) as executor: 110 | tasks = [ 111 | executor.submit(tiler, asset, tile_x, tile_y, tile_z, **kwargs) 112 | for asset in chunks 113 | ] 114 | else: 115 | tasks = (tiler(asset, tile_x, tile_y, tile_z, **kwargs) for asset in chunks) 116 | 117 | for t, m in _filter_tasks(tasks): 118 | t = numpy.ma.array(t) 119 | t.mask = m == 0 120 | 121 | pixel_selection.feed(t) 122 | if pixel_selection.is_done: 123 | return pixel_selection.data 124 | 125 | return pixel_selection.data 126 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | """Setup for rio-tiler-mosaic.""" 2 | 3 | from setuptools import find_packages, setup 4 | 5 | # Runtime requirements. 6 | inst_reqs = ["rio-tiler~=2.0a"] 7 | 8 | extra_reqs = { 9 | "test": ["pytest", "pytest-cov", "pytest-benchmark"], 10 | "dev": ["pytest", "pytest-cov", "pytest-benchmark", "pre-commit"], 11 | } 12 | 13 | with open("README.md") as f: 14 | long_description = f.read() 15 | 16 | setup( 17 | name="rio-tiler-mosaic", 18 | version="0.0.1dev5", 19 | python_requires=">=3", 20 | long_description=long_description, 21 | long_description_content_type="text/markdown", 22 | description=u"""A rio-tiler plugin to create mosaic tiles.""", 23 | classifiers=[ 24 | "Intended Audience :: Information Technology", 25 | "Intended Audience :: Science/Research", 26 | "License :: OSI Approved :: BSD License", 27 | "Programming Language :: Python :: 3.6", 28 | "Topic :: Scientific/Engineering :: GIS", 29 | ], 30 | keywords="COG Mosaic GIS", 31 | author=u"Vincent Sarago", 32 | author_email="vincent@developmentseed.org", 33 | url="https://github.com/cogeotiff/rio-tiler-mosaic", 34 | license="MIT", 35 | packages=find_packages(exclude=["ez_setup", "examples", "tests"]), 36 | include_package_data=True, 37 | zip_safe=False, 38 | install_requires=inst_reqs, 39 | extras_require=extra_reqs, 40 | ) 41 | -------------------------------------------------------------------------------- /tests/fixtures/cog1.tif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cogeotiff/rio-tiler-mosaic/d39f820acb646c6238f4a97fea8e2c63c20d3b69/tests/fixtures/cog1.tif -------------------------------------------------------------------------------- /tests/fixtures/cog2.tif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cogeotiff/rio-tiler-mosaic/d39f820acb646c6238f4a97fea8e2c63c20d3b69/tests/fixtures/cog2.tif -------------------------------------------------------------------------------- /tests/test_benchmarks.py: -------------------------------------------------------------------------------- 1 | """Benchmark.""" 2 | 3 | import os 4 | 5 | import pytest 6 | import rasterio 7 | 8 | from rio_tiler.io import COGReader 9 | from rio_tiler_mosaic import mosaic 10 | 11 | asset1 = os.path.join(os.path.dirname(__file__), "fixtures", "cog1.tif") 12 | asset2 = os.path.join(os.path.dirname(__file__), "fixtures", "cog2.tif") 13 | assets = [asset1, asset2, asset1, asset2, asset1, asset2] 14 | 15 | 16 | def _tiler(asset, x, y, z): 17 | with COGReader(asset) as cog: 18 | return cog.tile(x, y, z) 19 | 20 | 21 | def read_tile(threads, asset_list): 22 | """Benchmark rio-tiler.utils._tile_read.""" 23 | with rasterio.Env( 24 | GDAL_CACHEMAX=0, GDAL_DISABLE_READDIR_ON_OPEN="EMPTY_DIR", 25 | ): 26 | return mosaic.mosaic_tiler(asset_list, 150, 180, 9, _tiler, threads=threads) 27 | 28 | 29 | @pytest.mark.parametrize("threads", [0, 1, 2, 3, 4, 5]) 30 | @pytest.mark.parametrize("nb_image", [1, 2, 3, 4, 5]) 31 | def test_threads(benchmark, nb_image, threads): 32 | """Test tile read for multiple combination of datatype/mask/tile extent.""" 33 | benchmark.name = f"{nb_image}images-{threads}threads" 34 | benchmark.group = f"{nb_image}images" 35 | 36 | asset_list = assets[:nb_image] 37 | 38 | tile, _ = benchmark(read_tile, threads, asset_list) 39 | assert tile.shape == (3, 256, 256) 40 | -------------------------------------------------------------------------------- /tests/test_mosaic.py: -------------------------------------------------------------------------------- 1 | """tests ard_tiler.mosaic.""" 2 | 3 | import os 4 | from typing import Tuple 5 | 6 | import numpy 7 | import pytest 8 | 9 | from rio_tiler.io import COGReader 10 | from rio_tiler_mosaic import mosaic 11 | from rio_tiler_mosaic.methods import defaults 12 | 13 | asset1 = os.path.join(os.path.dirname(__file__), "fixtures", "cog1.tif") 14 | asset2 = os.path.join(os.path.dirname(__file__), "fixtures", "cog2.tif") 15 | assets = [asset1, asset2] 16 | assets_order = [asset2, asset1] 17 | 18 | # Full covered tile 19 | x = 150 20 | y = 182 21 | z = 9 22 | 23 | # Partially covered tile 24 | xp = 150 25 | yp = 180 26 | zp = 9 27 | 28 | # Outsize tile 29 | xout = 140 30 | yout = 130 31 | zout = 9 32 | 33 | 34 | def _read_tile(src_path: str, *args, **kwargs) -> Tuple[numpy.ndarray, numpy.ndarray]: 35 | """Read tile from an asset""" 36 | with COGReader(src_path) as cog: 37 | tile, mask = cog.tile(*args, **kwargs) 38 | return tile, mask 39 | 40 | 41 | def test_mosaic_tiler(): 42 | """Test mosaic tiler.""" 43 | # should return (None, None) with tile outside bounds 44 | t, m = mosaic.mosaic_tiler(assets, xout, yout, zout, _read_tile) 45 | assert not t 46 | assert not m 47 | 48 | # test with default and full covered tile and default options 49 | t, m = mosaic.mosaic_tiler(assets, x, y, z, _read_tile) 50 | assert t.shape == (3, 256, 256) 51 | assert m.shape == (256, 256) 52 | assert m.all() 53 | assert t[0][-1][-1] == 8682 54 | 55 | # Test last pixel selection 56 | assetsr = list(reversed(assets)) 57 | t, m = mosaic.mosaic_tiler(assetsr, x, y, z, _read_tile) 58 | assert t.shape == (3, 256, 256) 59 | assert m.shape == (256, 256) 60 | assert m.all() 61 | assert t[0][-1][-1] == 8057 62 | 63 | t, m = mosaic.mosaic_tiler(assets, x, y, z, _read_tile, indexes=1) 64 | assert t.shape == (1, 256, 256) 65 | assert m.shape == (256, 256) 66 | assert t.all() 67 | assert m.all() 68 | assert t[0][-1][-1] == 8682 69 | 70 | # Test darkest pixel selection 71 | t, m = mosaic.mosaic_tiler( 72 | assets, x, y, z, _read_tile, pixel_selection=defaults.LowestMethod() 73 | ) 74 | assert m.all() 75 | assert t[0][-1][-1] == 8057 76 | 77 | to, mo = mosaic.mosaic_tiler( 78 | assets_order, x, y, z, _read_tile, pixel_selection=defaults.LowestMethod() 79 | ) 80 | numpy.testing.assert_array_equal(t[0, m], to[0, mo]) 81 | 82 | # Test brightest pixel selection 83 | t, m = mosaic.mosaic_tiler( 84 | assets, x, y, z, _read_tile, pixel_selection=defaults.HighestMethod() 85 | ) 86 | assert m.all() 87 | assert t[0][-1][-1] == 8682 88 | 89 | to, mo = mosaic.mosaic_tiler( 90 | assets_order, x, y, z, _read_tile, pixel_selection=defaults.HighestMethod() 91 | ) 92 | numpy.testing.assert_array_equal(to, t) 93 | numpy.testing.assert_array_equal(mo, m) 94 | 95 | # test with default and partially covered tile 96 | t, m = mosaic.mosaic_tiler( 97 | assets, xp, yp, zp, _read_tile, pixel_selection=defaults.HighestMethod() 98 | ) 99 | assert t.any() 100 | assert not m.all() 101 | 102 | # test when tiler raise errors (outside bounds) 103 | t, m = mosaic.mosaic_tiler(assets, 150, 300, 9, _read_tile) 104 | assert not t 105 | assert not m 106 | 107 | # Test mean pixel selection 108 | t, m = mosaic.mosaic_tiler( 109 | assets, x, y, z, _read_tile, pixel_selection=defaults.MeanMethod() 110 | ) 111 | assert t.shape == (3, 256, 256) 112 | assert m.shape == (256, 256) 113 | assert m.all() 114 | assert t[0][-1][-1] == 8369 115 | 116 | # Test mean pixel selection 117 | t, m = mosaic.mosaic_tiler( 118 | assets, 119 | x, 120 | y, 121 | z, 122 | _read_tile, 123 | pixel_selection=defaults.MeanMethod(enforce_data_type=False), 124 | ) 125 | assert m.all() 126 | assert t[0][-1][-1] == 8369.5 127 | 128 | # Test median pixel selection 129 | t, m = mosaic.mosaic_tiler( 130 | assets, x, y, z, _read_tile, pixel_selection=defaults.MedianMethod() 131 | ) 132 | assert t.shape == (3, 256, 256) 133 | assert m.shape == (256, 256) 134 | assert m.all() 135 | assert t[0][-1][-1] == 8369 136 | 137 | # Test median pixel selection 138 | t, m = mosaic.mosaic_tiler( 139 | assets, 140 | x, 141 | y, 142 | z, 143 | _read_tile, 144 | pixel_selection=defaults.MedianMethod(enforce_data_type=False), 145 | ) 146 | assert m.all() 147 | assert t[0][-1][-1] == 8369.5 148 | 149 | # Test invalid Pixel Selection class 150 | with pytest.raises(Exception): 151 | 152 | class aClass(object): 153 | pass 154 | 155 | mosaic.mosaic_tiler(assets, x, y, z, _read_tile, pixel_selection=aClass()) 156 | 157 | 158 | def test_mosaic_tiler_Stdev(): 159 | """Test Stdev mosaic methods.""" 160 | tile1, _ = _read_tile(assets[0], x, y, z) 161 | tile2, _ = _read_tile(assets[1], x, y, z) 162 | 163 | t, m = mosaic.mosaic_tiler( 164 | assets, x, y, z, _read_tile, pixel_selection=defaults.StdevMethod() 165 | ) 166 | assert t.shape == (3, 256, 256) 167 | assert m.shape == (256, 256) 168 | assert m.all() 169 | assert t[0][-1][-1] == numpy.std([tile1[0][-1][-1], tile2[0][-1][-1]]) 170 | assert t[1][-1][-1] == numpy.std([tile1[1][-1][-1], tile2[1][-1][-1]]) 171 | assert t[2][-1][-1] == numpy.std([tile1[2][-1][-1], tile2[2][-1][-1]]) 172 | 173 | 174 | def test_threads(): 175 | """Test mosaic tiler.""" 176 | assets = [asset1, asset2, asset1, asset2, asset1, asset2] 177 | 178 | tnothread, _ = mosaic.mosaic_tiler(assets, x, y, z, _read_tile, threads=0) 179 | tmulti_threads, _ = mosaic.mosaic_tiler(assets, x, y, z, _read_tile, threads=1) 180 | numpy.testing.assert_array_equal(tnothread, tmulti_threads) 181 | 182 | t, _ = mosaic.mosaic_tiler(assets, x, y, z, _read_tile, threads=0, chunk_size=2) 183 | assert t.shape == (3, 256, 256) 184 | t, _ = mosaic.mosaic_tiler(assets, x, y, z, _read_tile, threads=2, chunk_size=4) 185 | assert t.shape == (3, 256, 256) 186 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | envlist = py36,py37 3 | 4 | [testenv] 5 | extras = test 6 | commands= 7 | python -m pytest --cov rio_tiler_mosaic --cov-report term-missing --ignore=venv --benchmark-skip 8 | deps= 9 | numpy 10 | 11 | # Lint 12 | [flake8] 13 | exclude = .git,__pycache__,docs/source/conf.py,old,build,dist 14 | max-line-length = 90 15 | 16 | [mypy] 17 | no_strict_optional = True 18 | ignore_missing_imports = True 19 | 20 | [tool:isort] 21 | include_trailing_comma = True 22 | multi_line_output = 3 23 | line_length = 90 24 | known_first_party = rio_tiler,rio_tiler_mosaic 25 | known_third_party = rasterio,mercantile,supermercado,affine 26 | default_section = THIRDPARTY 27 | 28 | 29 | # Release tooling 30 | [testenv:build] 31 | basepython = python3 32 | skip_install = true 33 | deps = 34 | wheel 35 | setuptools 36 | commands = 37 | python setup.py sdist 38 | 39 | [testenv:release] 40 | basepython = python3 41 | skip_install = true 42 | deps = 43 | {[testenv:build]deps} 44 | twine >= 1.5.0 45 | commands = 46 | {[testenv:build]commands} 47 | twine upload --skip-existing dist/* 48 | --------------------------------------------------------------------------------