├── .gitignore ├── .pre-commit-config.yaml ├── .travis.yml ├── HACKING ├── MANIFEST.in ├── Makefile ├── README.md ├── plyflatten ├── __about__.py ├── __init__.py ├── cli.py ├── rasterization.py └── utils.py ├── pyproject.toml ├── setup.cfg ├── setup.py ├── src ├── fail.c ├── plyflatten.c └── xmalloc.c └── tests ├── data ├── 1.ply ├── 2.ply ├── crs.ply ├── result.tiff └── std.tiff └── test_plyflatten.py /.gitignore: -------------------------------------------------------------------------------- 1 | __pycache__/ 2 | build/ 3 | dist/ 4 | *.egg-info/ 5 | .coverage 6 | 7 | # This is created when installing a package in editable mode 8 | # and when there is a pyproject.toml file 9 | # It is solved in pip >= 19.3 10 | # https://github.com/pypa/pip/blob/master/NEWS.rst#193-2019-10-14 11 | pip-wheel-metadata/ 12 | 13 | logs 14 | 15 | lib/ 16 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | repos: 2 | - repo: https://github.com/timothycrosley/isort 3 | rev: 4.3.21 4 | hooks: 5 | - id: isort 6 | additional_dependencies: ["toml"] 7 | - repo: https://github.com/psf/black 8 | rev: stable 9 | hooks: 10 | - id: black 11 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | os: linux 2 | dist: xenial 3 | language: python 4 | python: 3.7 5 | 6 | install: 7 | - pip install -e ".[test]" --upgrade --upgrade-strategy eager 8 | 9 | script: pytest tests 10 | 11 | jobs: 12 | include: 13 | - stage: formatting 14 | name: black 15 | install: 16 | - pip install black 17 | script: 18 | - black --check . 19 | - stage: formatting 20 | name: isort 21 | install: 22 | - pip install "isort[pyproject]" 23 | script: 24 | - isort --check-only . 25 | 26 | - stage: packaging 27 | name: "Python sdist installation" 28 | install: skip 29 | script: 30 | - python setup.py sdist 31 | - pip install dist/*.tar.gz 32 | 33 | - stage: test 34 | 35 | - stage: deploy 36 | if: tag IS present AND repo = cmla/plyflatten 37 | install: skip 38 | script: skip 39 | deploy: 40 | - provider: pypi 41 | user: carlodef 42 | password: 43 | - secure: "DdaWq1uqrfrr+VrWcu7Mtmvkm0PeUOsVZa/JymJulCFSEuekKQ/sfD142/CUYzECLJfaXGgfbihPl4QtnzIgaHmfKsVfIoIAkdbvfhNMj9C0bysRZ/hohsgDOL5PWh8ioGwhPh663IW9oAN5HcODK7Op+7Z3iHXQJwCJaNcd134+L3JemQr76ZV1VoJh+RN1ilv0v/4AyXEGXBkd9F7aQvdy8Xbv0XdcSYNHShiBWJDQ/yuSxEKEQ8tF57pUzdHIJMMP+u+1evkX71l1oqrAp11M5SHZXlUDlUls+gx2+4iobXxaRYTSJp5dwqqUcUps7d4HUQBxtujlr6EyFn0MrCZZ2aA8wGZR7e/Tsccx83flr3OsOEfrMHYaMG4yu6gCUqj3z8+lwU4oYMTwZABaR6HWyhKU4nnFvaotJp/KtYidZTubU+nZChnF3J5Zm7HBD0Z2L8n2Z/OssGjYVuJA3XGmnNQ6duxuYYDNY4SlTzbrSVsZmUInRUA7YDAguLt/JKIFFrotB7WR7t8VzJ0LGMfnroxbu78x7yA2gQuH0H86OhgEZ2ph+R0E5RyocwrQJqohERYyNuFd4weyPyRv1BZ/R3E5/dKq62mW2SvdOyxeq0+LoeSda8Km/6tmSoGMnr93K74sD6v9sRB8BbpqwEWxKuMLby5A2BUAoYaKzRc=" 44 | distributions: sdist 45 | on: 46 | tags: true 47 | -------------------------------------------------------------------------------- /HACKING: -------------------------------------------------------------------------------- 1 | python binding to plyflatten installable via "pip install plyflatten" 2 | 3 | To update the pypi hub, you need an account on pypi, and then run the following 4 | commands: 5 | 6 | # update version number on file plyflatten/__about__.py 7 | python setup.py sdist bdist_wheel 8 | python -m twine upload dist/plyflatten-X.tar.gz 9 | rm -rf build dist plyflatten.egg-info 10 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include Makefile 2 | 3 | recursive-include src *.c 4 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | .PHONY: help install test release-patch release-minor release-major clean 2 | 3 | .DEFAULT: help 4 | help: 5 | @echo "make install" 6 | @echo " Install the package in editable mode with the \`dev\` and" 7 | @echo " \`test\` extras and run \`pre-commit install\`," 8 | @echo " and compile the C code." 9 | @echo 10 | @echo "make test" 11 | @echo " Run tests with \`pytest\`" 12 | @echo 13 | @echo "make release-{patch,minor,major}" 14 | @echo " Increment the version number to the next patch/minor/major" 15 | @echo " version (following semanting versioning) using \`bumpversion\`" 16 | 17 | 18 | lib: lib/libplyflatten.so 19 | 20 | lib/libplyflatten.so: src/plyflatten.c src/*.c 21 | mkdir -p lib 22 | $(CC) $(CFLAGS) -fPIC -shared -o $@ $< 23 | 24 | install: 25 | pip install -e ".[dev,test]" 26 | pre-commit install 27 | 28 | BOLD=$$(tput bold) 29 | NORMAL=$$(tput sgr0) 30 | 31 | test: 32 | PYTHONPATH=. pytest . 33 | 34 | release-patch: 35 | @# Remove the trailing -dev0 from version number and tag the new version 36 | bumpversion release --tag 37 | @echo "${BOLD}Tagged version: $$(bumpversion --dry-run --list patch | grep current_version | cut -d= -f2)${NORMAL}" 38 | @make dev 39 | 40 | release-minor: 41 | @# Increment to the new minor 42 | bumpversion minor --no-commit 43 | @# Remove the trailing -dev0 from version number 44 | bumpversion release --allow-dirty --tag 45 | @echo "${BOLD}Tagged version: $$(bumpversion --dry-run --list patch | grep current_version | cut -d= -f2)${NORMAL}" 46 | @make dev 47 | 48 | release-major: 49 | @# Increment to the new major 50 | bumpversion major --no-commit 51 | @# Remove the trailing -dev0 from version number 52 | bumpversion release --allow-dirty --tag 53 | @echo "${BOLD}Tagged version: $$(bumpversion --dry-run --list patch | grep current_version | cut -d= -f2)${NORMAL}" 54 | @make dev 55 | 56 | dev: 57 | @# Increment to the new patch (automatically adds -dev0 to it) 58 | bumpversion patch --message "Post-release: {new_version}" 59 | @echo "${BOLD}Current version: $$(bumpversion --dry-run --list patch | grep current_version | cut -d= -f2)${NORMAL}" 60 | 61 | clean: 62 | find . -name '*.pyc' -delete 63 | find . -name '__pycache__' -type d | xargs rm -fr 64 | rm -rf lib/ 65 | rm -rf pip-wheel-metadata/ 66 | rm -rf *.egg-info/ 67 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # PlyFlatten 2 | 3 | Take a series of ply files and produce a digital elevation map 4 | 5 | ## Installation 6 | 7 | ``` 8 | pip install plyflatten 9 | ``` 10 | 11 | ## Usage 12 | 13 | This package comes with a command-line tool: 14 | ``` 15 | $ plyflatten --help 16 | usage: plyflatten [-h] [--resolution RESOLUTION] 17 | list_plys [list_plys ...] dsm_path 18 | 19 | plyflatten: Take a series of ply files and produce a digital elevation map 20 | 21 | positional arguments: 22 | list_plys Space-separated list of .ply files 23 | dsm_path Path to output DSM file 24 | 25 | optional arguments: 26 | -h, --help show this help message and exit 27 | --resolution RESOLUTION 28 | Resolution of the DSM in meters (defaults to 1m) 29 | ``` 30 | 31 | Try using it on the test data provided with the repository: 32 | ``` 33 | plyflatten tests/data/{1,2}.ply out.tiff --resolution 2 34 | ``` 35 | 36 | ## Contributing 37 | 38 | To work on this project, install the development requirements by running: 39 | ``` 40 | make install 41 | ``` 42 | 43 | The tests can be run with: 44 | ``` 45 | make test 46 | ``` 47 | -------------------------------------------------------------------------------- /plyflatten/__about__.py: -------------------------------------------------------------------------------- 1 | __title__ = "plyflatten" 2 | __description__ = "Take a series of ply files and produce a digital elevation map" 3 | __url__ = "https://github.com/cmla/plyflatten" 4 | __author__ = """Carlo de Franchis""" 5 | __author_email__ = "carlo.de-franchis@ens-cachan.fr" 6 | __version__ = "0.2.3" 7 | -------------------------------------------------------------------------------- /plyflatten/__init__.py: -------------------------------------------------------------------------------- 1 | from plyflatten.__about__ import __version__ # noqa 2 | from plyflatten.rasterization import plyflatten, plyflatten_from_plyfiles_list # noqa 3 | -------------------------------------------------------------------------------- /plyflatten/cli.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import sys 3 | 4 | import rasterio 5 | 6 | from plyflatten import plyflatten_from_plyfiles_list 7 | from plyflatten.__about__ import __description__, __title__ 8 | 9 | 10 | def main(): 11 | parser = argparse.ArgumentParser(description=(f"{__title__}: {__description__}")) 12 | parser.add_argument("list_plys", nargs="+", help=("Space-separated list of .ply files")) 13 | parser.add_argument("dsm_path", help=("Path to output average height DSM file (average heights)")) 14 | parser.add_argument("--std", help=("Path to (optional) output standard deviation DSM file")) 15 | parser.add_argument("--min", help=("Path to (optional) output minimum height DSM file")) 16 | parser.add_argument("--max", help=("Path to (optional) output maximum height DSM file")) 17 | parser.add_argument( 18 | "--resolution", 19 | default=1, 20 | type=float, 21 | help=("Resolution of the DSM in meters (defaults to 1m)"), 22 | ) 23 | args = parser.parse_args() 24 | std_ = args.std is not None 25 | min_ = args.min is not None 26 | max_ = args.min is not None 27 | 28 | raster, profile = plyflatten_from_plyfiles_list( 29 | args.list_plys, args.resolution, std=std_, min=min_, max=max_ 30 | ) 31 | profile["dtype"] = raster.dtype 32 | profile["height"] = raster.shape[0] 33 | profile["width"] = raster.shape[1] 34 | profile["count"] = 1 35 | profile["driver"] = "GTiff" 36 | 37 | # 1. write avg DSM 38 | # plyflatten outputs a height DSM by default, but extra_col_idx can be hardcoded to change the magnitude 39 | # e.g. if the input clouds have format [x y z r g b], set extra_col_idx = 1 to compute avg/std/min/max of r channel 40 | extra_col_idx = 0 41 | with rasterio.open(args.dsm_path, "w", **profile) as f: 42 | f.write(raster[:, :, extra_col_idx], 1) 43 | 44 | # 2. (optional) write std, min and max DSM 45 | extra_stats_total = 1 + std_ + min_ + max_ 46 | nb_extra_columns = raster.shape[2] // extra_stats_total 47 | assert raster.shape[2] % extra_stats_total == 0 48 | extra_stats_count = 1 49 | for b, extra_stat_path in zip([std_, min_, max_], [args.std, args.min, args.max]): 50 | if b: 51 | with rasterio.open(extra_stat_path, "w", **profile) as f: 52 | f.write(raster[:, :, extra_stats_count * nb_extra_columns + extra_col_idx], 1) 53 | extra_stats_count += 1 54 | 55 | if __name__ == "__main__": 56 | sys.exit(main()) # pragma: no cover 57 | -------------------------------------------------------------------------------- /plyflatten/rasterization.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2019, David Youssefi (CNES) 2 | 3 | 4 | import ctypes 5 | import os 6 | 7 | import affine 8 | import numpy as np 9 | from numpy.ctypeslib import ndpointer 10 | 11 | from plyflatten import utils 12 | 13 | # TODO: This is kind of ugly. Cleaner way to do this is to update 14 | # LD_LIBRARY_PATH, which we should do once we have a proper config file 15 | parent_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) 16 | lib = ctypes.CDLL(os.path.join(parent_dir, "lib", "libplyflatten.so")) 17 | 18 | class Raster: 19 | 20 | def __init__(self, xoff, yoff, resolution, xsize, ysize, radius, sigma, nb_extra_columns): 21 | 22 | # roi, resolution, etc 23 | self.xoff = xoff 24 | self.yoff = yoff 25 | self.resolution = resolution 26 | self.xsize = xsize 27 | self.ysize = ysize 28 | self.radius = radius 29 | self.sigma = sigma 30 | self.nb_extra_columns = nb_extra_columns 31 | 32 | # statistics we want to extract from the point clouds 33 | raster_shape = (xsize * ysize, nb_extra_columns) 34 | self.avg = np.zeros(raster_shape, dtype="float32") 35 | self.std = np.zeros(raster_shape, dtype="float32") 36 | self.min = np.inf * np.ones(raster_shape, dtype="float32") 37 | self.max = -np.inf * np.ones(raster_shape, dtype="float32") 38 | self.cnt = np.zeros((xsize * ysize, 1), dtype="float32") 39 | 40 | 41 | def compute_roi_from_ply_list(clouds_list, resolution): 42 | 43 | xmin, xmax = np.inf, -np.inf 44 | ymin, ymax = np.inf, -np.inf 45 | 46 | for cloud in clouds_list: 47 | cloud_data, _ = utils.read_3d_point_cloud_from_ply(cloud) 48 | current_cloud = cloud_data.astype(np.float64) 49 | xx, yy = current_cloud[:, 0], current_cloud[:, 1] 50 | xmin = np.min((xmin, np.amin(xx))) 51 | ymin = np.min((ymin, np.amin(yy))) 52 | xmax = np.max((xmax, np.amax(xx))) 53 | ymax = np.max((ymax, np.amax(yy))) 54 | 55 | xoff = np.floor(xmin / resolution) * resolution 56 | xsize = int(1 + np.floor((xmax - xoff) / resolution)) 57 | 58 | yoff = np.ceil(ymax / resolution) * resolution 59 | ysize = int(1 - np.floor((ymin - yoff) / resolution)) 60 | 61 | return xoff, yoff, xsize, ysize 62 | 63 | 64 | def plyflatten(cloud, xoff, yoff, resolution, xsize, ysize, radius, sigma, raster=None): 65 | """ 66 | Projects a points cloud into the raster band(s) of a raster image 67 | 68 | Args: 69 | cloud: A nb_points x (2+nb_extra_columns) numpy array: 70 | | x0 y0 [z0 r0 g0 b0 ...] | 71 | | x1 y1 [z1 r1 g1 b1 ...] | 72 | | ... | 73 | | xN yN [zN rN gN bN ...] | 74 | x, y give positions of the points into the final raster, the "extra 75 | columns" give the values 76 | xoff, yoff: offset position (upper left corner) considering the georeferenced image 77 | resolution: resolution of the output georeferenced image 78 | xsize, ysize: size of the georeferenced image 79 | radius: controls the spread of the blob from each point 80 | sigma: radius of influence for each point (unit: pixel) 81 | std (bool): if True, return additional channels with standard deviations 82 | 83 | Returns; 84 | A numpy array of shape (ysize, xsize, n) where n is nb_extra_columns if 85 | std=False and 2*nb_extra_columns if std=True 86 | """ 87 | nb_points, nb_extra_columns = cloud.shape[0], cloud.shape[1] - 2 88 | if raster is None: 89 | raster = Raster(xoff, yoff, resolution, xsize, ysize, radius, sigma, nb_extra_columns) 90 | else: 91 | assert nb_extra_columns == raster.nb_extra_columns 92 | 93 | # Set expected args and return types 94 | raster_shape = (raster.xsize * raster.ysize, raster.nb_extra_columns) 95 | lib.rasterize_cloud.argtypes = ( 96 | ndpointer(dtype=ctypes.c_double, shape=np.shape(cloud)), 97 | ndpointer(dtype=ctypes.c_float, shape=raster_shape), 98 | ndpointer(dtype=ctypes.c_float, shape=raster_shape), 99 | ndpointer(dtype=ctypes.c_float, shape=raster_shape), 100 | ndpointer(dtype=ctypes.c_float, shape=raster_shape), 101 | ndpointer(dtype=ctypes.c_float, shape=(raster.xsize * raster.ysize, 1)), 102 | ctypes.c_int, 103 | ctypes.c_int, 104 | ctypes.c_double, 105 | ctypes.c_double, 106 | ctypes.c_double, 107 | ctypes.c_int, 108 | ctypes.c_int, 109 | ctypes.c_int, 110 | ctypes.c_float, 111 | ) 112 | 113 | # Call rasterize_cloud function from libplyflatten.so 114 | lib.rasterize_cloud( 115 | np.ascontiguousarray(cloud.astype(np.float64)), 116 | raster.avg, 117 | raster.std, 118 | raster.min, 119 | raster.max, 120 | raster.cnt, 121 | nb_points, 122 | raster.nb_extra_columns, 123 | raster.xoff, 124 | raster.yoff, 125 | raster.resolution, 126 | raster.xsize, 127 | raster.ysize, 128 | raster.radius, 129 | raster.sigma, 130 | ) 131 | 132 | return raster 133 | 134 | 135 | def plyflatten_from_plyfiles_list( 136 | clouds_list, resolution, radius=0, roi=None, sigma=None, std=False, amin=False, amax=False 137 | ): 138 | """ 139 | Projects a points cloud into the raster band(s) of a raster image (points clouds as files) 140 | 141 | Args: 142 | clouds_list: list of cloud.ply files 143 | resolution: resolution of the georeferenced output raster file 144 | roi: region of interest: (xoff, yoff, xsize, ysize), compute plyextrema if None 145 | std (bool): if True, return additional channels with standard deviations 146 | 147 | Returns: 148 | raster: georeferenced raster 149 | profile: profile for rasterio 150 | """ 151 | 152 | # region of interest (compute plyextrema if roi is None) 153 | xoff, yoff, xsize, ysize = compute_roi_from_ply_list(clouds_list, resolution) if roi is None else roi 154 | 155 | raster = None 156 | for cloud in clouds_list: 157 | cloud_data, _ = utils.read_3d_point_cloud_from_ply(cloud) 158 | current_cloud = cloud_data.astype(np.float64) 159 | 160 | # The copy() method will reorder to C-contiguous order by default: 161 | cloud = current_cloud.copy() 162 | sigma = float("inf") if sigma is None else sigma 163 | raster = plyflatten(cloud, xoff, yoff, resolution, xsize, ysize, radius, sigma, raster) 164 | 165 | raster_shape = (raster.xsize * raster.ysize, raster.nb_extra_columns) 166 | lib.finishing_touches.argtypes = ( 167 | ndpointer(dtype=ctypes.c_float, shape=raster_shape), 168 | ndpointer(dtype=ctypes.c_float, shape=raster_shape), 169 | ndpointer(dtype=ctypes.c_float, shape=raster_shape), 170 | ndpointer(dtype=ctypes.c_float, shape=raster_shape), 171 | ndpointer(dtype=ctypes.c_float, shape=(raster.xsize * raster.ysize, 1)), 172 | ctypes.c_int, 173 | ctypes.c_int, 174 | ctypes.c_int, 175 | ) 176 | 177 | lib.finishing_touches( 178 | raster.avg, 179 | raster.std, 180 | raster.min, 181 | raster.max, 182 | raster.cnt, 183 | raster.nb_extra_columns, 184 | raster.xsize, 185 | raster.ysize, 186 | ) 187 | 188 | # Transform result into a numpy array 189 | raster_ = raster.avg.reshape((raster.ysize, raster.xsize, raster.nb_extra_columns)) 190 | if std: 191 | raster_std = raster.std.reshape((raster.ysize, raster.xsize, raster.nb_extra_columns)) 192 | raster_ = np.dstack((raster_, raster_std)) 193 | if amin: 194 | raster_min = raster.min.reshape((raster.ysize, raster.xsize, raster.nb_extra_columns)) 195 | raster_ = np.dstack((raster_, raster_min)) 196 | if amax: 197 | raster_max = raster.max.reshape((raster.ysize, raster.xsize, raster.nb_extra_columns)) 198 | raster_ = np.dstack((raster_, raster_max)) 199 | 200 | crs, crs_type = utils.crs_from_ply(clouds_list[0]) 201 | crs_proj = utils.rasterio_crs(utils.crs_proj(crs, crs_type)) 202 | 203 | # construct profile dict 204 | profile = dict() 205 | profile["tiled"] = True 206 | profile["compress"] = "deflate" 207 | profile["predictor"] = 2 208 | profile["nodata"] = float("nan") 209 | profile["crs"] = crs_proj 210 | profile["transform"] = affine.Affine(resolution, 0.0, xoff, 0.0, -resolution, yoff) 211 | 212 | return raster_, profile 213 | -------------------------------------------------------------------------------- /plyflatten/utils.py: -------------------------------------------------------------------------------- 1 | import re 2 | from packaging import version 3 | 4 | import numpy as np 5 | import plyfile 6 | import pyproj 7 | import rasterio 8 | from pyproj.enums import WktVersion 9 | from rasterio.crs import CRS as RioCRS 10 | 11 | 12 | class InvalidPlyCommentsError(Exception): 13 | pass 14 | 15 | 16 | def rasterio_crs(proj_crs): 17 | """ 18 | Return a rasterio.crs.CRS object that corresponds to the given parameters. 19 | See: https://pyproj4.github.io/pyproj/stable/crs_compatibility.html#converting-from-pyproj-crs-crs-to-rasterio-crs-crs 20 | 21 | Args: 22 | proj_crs (pyproj.crs.CRS): pyproj CRS object 23 | 24 | Returns: 25 | rasterio.crs.CRS: object that can be used with rasterio 26 | """ 27 | if version.parse(rasterio.__gdal_version__) < version.parse("3.0.0"): 28 | rio_crs = RioCRS.from_wkt(proj_crs.to_wkt(WktVersion.WKT1_GDAL)) 29 | else: 30 | rio_crs = RioCRS.from_wkt(proj_crs.to_wkt()) 31 | return rio_crs 32 | 33 | 34 | def crs_proj(crs_params, crs_type="UTM"): 35 | """ 36 | Return a pyproj.Proj object that corresponds 37 | to the given UTM zone string or the CRS parameters 38 | 39 | Args: 40 | crs_params (int, str, dict): UTM zone number + hemisphere (eg: '30N') or an authority 41 | string (EPSG:xxx) or any other format supported by pyproj 42 | crs_type (str): 'UTM' (default) or 'CRS' 43 | 44 | Returns: 45 | pyproj.Proj or pyproj.crs.CRS: object that can be used to transform coordinates 46 | """ 47 | if crs_type == "UTM": 48 | zone_number = crs_params[:-1] 49 | hemisphere = crs_params[-1] 50 | crs_params = { 51 | "proj": "utm", 52 | "zone": zone_number, 53 | "ellps": "WGS84", 54 | "datum": "WGS84", 55 | "south": (hemisphere == "S"), 56 | } 57 | elif crs_type == "CRS": 58 | if isinstance(crs_params, str): 59 | try: 60 | crs_params = int(crs_params) 61 | except (ValueError, TypeError): 62 | pass 63 | return pyproj.crs.CRS(crs_params) 64 | 65 | 66 | def crs_code_from_comments(comments, crs_type="UTM"): 67 | re_type = "[0-9]{1,2}[NS]" if crs_type == "UTM" else ".*" 68 | regex = r"^projection: {} ({})".format(crs_type, re_type) 69 | crs_code = None 70 | for comment in comments: 71 | s = re.search(regex, comment) 72 | if s: 73 | crs_code = s.group(1) 74 | return crs_code 75 | 76 | 77 | def crs_from_ply(ply_path): 78 | _, comments = read_3d_point_cloud_from_ply(ply_path) 79 | crs_params = crs_code_from_comments(comments, crs_type="CRS") 80 | 81 | utm_zone = None 82 | if not crs_params: 83 | utm_zone = crs_code_from_comments(comments, crs_type="UTM") 84 | 85 | if not crs_params and not utm_zone: 86 | raise InvalidPlyCommentsError( 87 | "Invalid header comments {} for ply file {}".format(comments, ply_path) 88 | ) 89 | 90 | crs_type = "CRS" if crs_params else "UTM" 91 | 92 | return crs_params or utm_zone, crs_type 93 | 94 | 95 | def read_3d_point_cloud_from_ply(path_to_ply_file): 96 | """ 97 | Read a 3D point cloud from a ply file and return a numpy array. 98 | 99 | Args: 100 | path_to_ply_file (str): path to a .ply file 101 | 102 | Returns: 103 | numpy array with the list of 3D points, one point per line 104 | list of strings with the ply header comments 105 | """ 106 | plydata = plyfile.PlyData.read(path_to_ply_file) 107 | d = np.asarray(plydata["vertex"].data) 108 | array = np.column_stack([d[p.name] for p in plydata["vertex"].properties]) 109 | return array, plydata.comments 110 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [tool.black] 2 | line-length=100 3 | 4 | [tool.isort] 5 | default_section="THIRDPARTY" 6 | known_first_party=["plyflatten"] 7 | multi_line_output=3 8 | include_trailing_comma=true 9 | force_grid_wrap=0 10 | use_parentheses=true 11 | line_length=100 12 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [bumpversion] 2 | current_version = 0.2.1dev 3 | parse = (?P\d+)\.(?P\d+)\.(?P\d+)((?P[a-z]+))? 4 | serialize = 5 | {major}.{minor}.{patch}{release} 6 | {major}.{minor}.{patch} 7 | commit = True 8 | message = Release: {new_version} 9 | 10 | [bumpversion:part:release] 11 | optional_value = placeholder 12 | first_value = dev 13 | values = 14 | dev 15 | placeholder 16 | 17 | [bumpversion:file:plyflatten/__about__.py] 18 | 19 | [tool:pytest] 20 | addopts = --cov plyflatten --cov-report term-missing 21 | 22 | [coverage:run] 23 | branch = True 24 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | import os 2 | import subprocess 3 | from codecs import open 4 | 5 | from setuptools import setup 6 | from setuptools.command import build_py, develop 7 | 8 | here = os.path.abspath(os.path.dirname(__file__)) 9 | 10 | package = "plyflatten" 11 | 12 | about = {} 13 | with open(os.path.join(here, package, "__about__.py"), "r", "utf-8") as f: 14 | exec(f.read(), about) 15 | 16 | 17 | def readme(): 18 | with open(os.path.join(here, "README.md"), "r", "utf-8") as f: 19 | return f.read() 20 | 21 | 22 | class CustomDevelop(develop.develop): 23 | """ 24 | Class needed for "pip install -e ." 25 | """ 26 | 27 | def run(self): 28 | subprocess.check_call("make lib", shell=True) 29 | super(CustomDevelop, self).run() 30 | 31 | 32 | class CustomBuildPy(build_py.build_py): 33 | """ 34 | Class needed for "pip install plyflatten" 35 | """ 36 | 37 | def run(self): 38 | super(CustomBuildPy, self).run() 39 | subprocess.check_call("make lib", shell=True) 40 | subprocess.check_call("cp -r lib build/lib/", shell=True) 41 | 42 | 43 | install_requires = ["affine", "numpy", "plyfile", "pyproj", "rasterio"] 44 | 45 | extras_require = { 46 | "dev": ["bump2version", "pre-commit"], 47 | "test": ["pytest", "pytest-cov"], 48 | } 49 | 50 | setup( 51 | name=about["__title__"], 52 | version=about["__version__"], 53 | description=about["__description__"], 54 | long_description=readme(), 55 | long_description_content_type="text/markdown", 56 | url=about["__url__"], 57 | author=about["__author__"], 58 | author_email=about["__author_email__"], 59 | packages=[package], 60 | install_requires=install_requires, 61 | extras_require=extras_require, 62 | entry_points=""" 63 | [console_scripts] 64 | plyflatten=plyflatten.cli:main 65 | """, 66 | cmdclass={"develop": CustomDevelop, "build_py": CustomBuildPy}, 67 | ) 68 | -------------------------------------------------------------------------------- /src/fail.c: -------------------------------------------------------------------------------- 1 | // Copied from https://github.com/mnhrdt/imscript/blob/master/src/fail.c 2 | 3 | #ifndef _FAIL_C 4 | #define _FAIL_C 5 | 6 | #include 7 | #include 8 | 9 | #ifdef __linux 10 | # include 11 | # include 12 | static const char *emptystring = ""; 13 | static const char *myname(void) 14 | { 15 | # define n 0x29a 16 | //const int n = 0x29a; 17 | static char buf[n]; 18 | pid_t p = getpid(); 19 | snprintf(buf, n, "/proc/%d/cmdline", p); 20 | FILE *f = fopen(buf, "r"); 21 | if (!f) return emptystring; 22 | int c, i = 0; 23 | while ((c = fgetc(f)) != EOF && i < n) { 24 | # undef n 25 | buf[i] = c ? c : ' '; 26 | i += 1; 27 | } 28 | if (i) buf[i-1] = '\0'; 29 | fclose(f); 30 | return buf; 31 | } 32 | #else 33 | static const char *myname(void) { return ""; } 34 | #endif//__linux 35 | 36 | #ifdef DOTRACE 37 | #include 38 | #endif 39 | 40 | #ifndef BACKTRACE_SYMBOLS 41 | #define BACKTRACE_SYMBOLS 50 42 | #endif 43 | 44 | static void print_trace(FILE *f) 45 | { 46 | (void)f; 47 | #ifdef DOTRACE 48 | void *array[BACKTRACE_SYMBOLS]; 49 | size_t size, i; 50 | char **strings; 51 | 52 | size = backtrace (array, BACKTRACE_SYMBOLS); 53 | strings = backtrace_symbols (array, size); 54 | 55 | fprintf (f, "Obtained %zu stack frames.\n", size); 56 | 57 | for (i = 0; i < size; i++) 58 | fprintf (f, "%s\n", strings[i]); 59 | 60 | free (strings); 61 | #endif 62 | } 63 | 64 | #include 65 | 66 | //static void fail(const char *fmt, ...) __attribute__((noreturn,format(printf,1,2))); 67 | static void fail(const char *fmt, ...) __attribute__((noreturn)); 68 | static void fail(const char *fmt, ...) 69 | { 70 | va_list argp; 71 | fprintf(stderr, "\nFAIL(\"%s\"): ", myname()); 72 | va_start(argp, fmt); 73 | vfprintf(stderr, fmt, argp); 74 | va_end(argp); 75 | fprintf(stderr, "\n\n"); 76 | fflush(NULL); 77 | print_trace(stderr); 78 | #ifdef NDEBUG 79 | exit(-1); 80 | #else//NDEBUG 81 | exit(*(volatile int *)0x43); 82 | #endif//NDEBUG 83 | } 84 | 85 | #endif//_FAIL_C 86 | -------------------------------------------------------------------------------- /src/plyflatten.c: -------------------------------------------------------------------------------- 1 | // take a series of ply files and produce a digital elevation map 2 | 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | 10 | #include "xmalloc.c" 11 | 12 | // rescale a double: 13 | // min: start of interval 14 | // resolution: spacing between values 15 | static int rescale_double_to_int(double x, double min, double resolution) 16 | { 17 | int r = floor( (x - min) / resolution); 18 | return r; 19 | } 20 | 21 | static float recenter_double(int i, double xmin, double resolution) 22 | { 23 | return xmin + resolution * (0.5 + i); 24 | } 25 | 26 | struct accumulator_image { 27 | float *min; 28 | float *max; 29 | float *cnt; 30 | float *avg; 31 | float *std; 32 | int w, h; 33 | }; 34 | 35 | // update the output images with a new height 36 | static void accumulate_height( 37 | struct accumulator_image *x, 38 | int i, int j, // position of the new height 39 | const double *v, // new height (and r, b, g...) 40 | int v_size, // nb "extra" columns (height, r, g, b...) 41 | float weight) // relative weight 42 | { 43 | uint64_t k = (uint64_t) x->w * j + i; 44 | uint64_t k2; 45 | for (int l = 0; l < v_size; l++) { 46 | k2 = v_size*k+l; 47 | x->avg[k2] = (v[l] * weight + x->cnt[k] * x->avg[k2]) / (weight + x->cnt[k]); 48 | x->std[k2] = (pow(v[l], 2) * weight + x->cnt[k] * x->std[k2]) / (weight + x->cnt[k]); 49 | x->min[k2] = fmin(x->min[k2], v[l]); 50 | x->max[k2] = fmax(x->max[k2], v[l]); 51 | } 52 | x->cnt[k] += weight; 53 | } 54 | 55 | // check whether a point is inside the image domain 56 | static int insideP(int w, int h, int i, int j) 57 | { 58 | return i>=0 && j>=0 && iavg) is given as an argument as a linear buffer of float of size (xsize*ysize*nb_extra_columns). */ 74 | /* This buffer is filled by the function. */ 75 | 76 | void rasterize_cloud( 77 | const double * input_buffer, 78 | float * raster_avg, 79 | float * raster_std, 80 | float * raster_min, 81 | float * raster_max, 82 | float * raster_cnt, 83 | const int nb_points, 84 | const int nb_extra_columns, // z, r, g, b, ... 85 | const double xoff, const double yoff, 86 | const double resolution, 87 | const int xsize, const int ysize, 88 | const int radius, const float sigma) 89 | { 90 | 91 | // get current accumulator status 92 | struct accumulator_image x[1]; 93 | x->w = xsize; 94 | x->h = ysize; 95 | x->min = raster_min; 96 | x->max = raster_max; 97 | x->cnt = raster_cnt; 98 | x->avg = raster_avg; 99 | x->std = raster_std; 100 | 101 | double sigma2mult2 = 2*sigma*sigma; 102 | 103 | // accumulate points of cloud to the image 104 | for (uint64_t k = 0; k < (uint64_t) nb_points; k++) { 105 | int ind = k * (2 + nb_extra_columns); 106 | double xx = input_buffer[ind]; 107 | double yy = input_buffer[ind+1]; 108 | int i = rescale_double_to_int(xx, xoff, resolution); 109 | int j = rescale_double_to_int(-yy, -yoff, resolution); 110 | 111 | for (int k1 = -radius; k1 <= radius; k1++) 112 | for (int k2 = -radius; k2 <= radius; k2++) { 113 | int ii = i + k1; 114 | int jj = j + k2; 115 | float dist_x = xx - recenter_double(ii, xoff, resolution); 116 | float dist_y = yy - recenter_double(jj, yoff, -resolution); 117 | float dist = hypot(dist_x, dist_y); 118 | float weight = distance_weight(sigma, dist); 119 | 120 | if (insideP(x->w, x->h, ii, jj)) { 121 | accumulate_height(x, ii, jj, 122 | &(input_buffer[ind+2]), 123 | nb_extra_columns, weight); 124 | assert(isfinite(input_buffer[ind+2])); 125 | } 126 | } 127 | } 128 | 129 | } 130 | 131 | 132 | void finishing_touches( 133 | float * raster_avg, 134 | float * raster_std, 135 | float * raster_min, 136 | float * raster_max, 137 | float * raster_cnt, 138 | const int nb_extra_columns, 139 | const int xsize, const int ysize) 140 | { 141 | 142 | // get current accumulator status 143 | struct accumulator_image x[1]; 144 | x->w = xsize; 145 | x->h = ysize; 146 | x->min = raster_min; 147 | x->max = raster_max; 148 | x->cnt = raster_cnt; 149 | x->avg = raster_avg; 150 | x->std = raster_std; 151 | 152 | // set unknown values to NAN 153 | for (uint64_t i = 0; i < (uint64_t) xsize*ysize; i++) { 154 | if (!x->cnt[i]){ 155 | for (uint64_t j = 0; j < (uint64_t) nb_extra_columns; j++){ 156 | x->avg[nb_extra_columns*i+j] = NAN; 157 | x->min[nb_extra_columns*i+j] = NAN; 158 | x->max[nb_extra_columns*i+j] = NAN; 159 | } 160 | } 161 | if (x->cnt[i]<2) { 162 | for (uint64_t j = 0; j < (uint64_t) nb_extra_columns; j++) 163 | x->std[nb_extra_columns*i+j] = NAN; 164 | } 165 | else { 166 | // so far x->std contains E[x^2] and x->avg contains E[x] 167 | // the standard deviation is then computed std = sqrt(E[x^2] - E[x]^2) 168 | for (uint64_t j = 0; j < (uint64_t) nb_extra_columns; j++) 169 | x->std[nb_extra_columns*i+j] = sqrt( x->std[nb_extra_columns*i+j] - pow(x->avg[nb_extra_columns*i+j], 2) ); 170 | } 171 | } 172 | } 173 | -------------------------------------------------------------------------------- /src/xmalloc.c: -------------------------------------------------------------------------------- 1 | // Copied from https://github.com/mnhrdt/imscript/blob/master/src/xmalloc.c 2 | 3 | #ifndef _XMALLOC_C 4 | #define _XMALLOC_C 5 | 6 | #include 7 | 8 | #include "fail.c" 9 | 10 | static void *xmalloc(size_t size) 11 | { 12 | #ifndef NDEBUG 13 | { 14 | double sm = size / (0x100000 * 1.0); 15 | if (sm > 1000) 16 | fprintf(stderr, "WARNING: large malloc" 17 | " %zu bytes (%gMB)\n", size, sm); 18 | } 19 | #endif 20 | if (size == 0) 21 | fail("xmalloc: zero size"); 22 | void *new = malloc(size); 23 | if (!new) 24 | { 25 | double sm = size / (0x100000 * 1.0); 26 | fail("xmalloc: out of memory when requesting " 27 | "%zu bytes (%gMB)",//:\"%s\"", 28 | size, sm);//, strerror(errno)); 29 | } 30 | return new; 31 | } 32 | 33 | inline // to avoid unused warnings 34 | static void *xrealloc(void *p, size_t s) 35 | { 36 | void *r = realloc(p, s); 37 | if (!r) fail("realloc failed"); 38 | return r; 39 | } 40 | 41 | inline // to avoid unused warnings 42 | static void xfree(void *p) 43 | { 44 | if (!p) 45 | fail("thou shalt not free a null pointer!"); 46 | free(p); 47 | } 48 | 49 | #endif//_XMALLOC_C 50 | -------------------------------------------------------------------------------- /tests/data/1.ply: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/centreborelli/plyflatten/98af8f7dee180d56a19b8cd88ad004f68aa425e7/tests/data/1.ply -------------------------------------------------------------------------------- /tests/data/2.ply: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/centreborelli/plyflatten/98af8f7dee180d56a19b8cd88ad004f68aa425e7/tests/data/2.ply -------------------------------------------------------------------------------- /tests/data/crs.ply: -------------------------------------------------------------------------------- 1 | ply 2 | format binary_little_endian 1.0 3 | comment created by S2P 4 | comment projection: CRS epsg:32740 5 | element vertex 3 6 | property double x 7 | property double y 8 | property double z 9 | property uchar red 10 | property uchar green 11 | property uchar blue 12 | end_header 13 | ��`���AO�h�0]AS�A>3��@www��.B��A$����0]AE�|�4��@aaaX/&���Aq�rÓ0]Au*��:��@nnn 14 | -------------------------------------------------------------------------------- /tests/data/result.tiff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/centreborelli/plyflatten/98af8f7dee180d56a19b8cd88ad004f68aa425e7/tests/data/result.tiff -------------------------------------------------------------------------------- /tests/data/std.tiff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/centreborelli/plyflatten/98af8f7dee180d56a19b8cd88ad004f68aa425e7/tests/data/std.tiff -------------------------------------------------------------------------------- /tests/test_plyflatten.py: -------------------------------------------------------------------------------- 1 | # pylint: disable=redefined-outer-name, missing-docstring 2 | import os 3 | 4 | import numpy as np 5 | import pyproj 6 | import pytest 7 | import rasterio 8 | 9 | from plyflatten import plyflatten_from_plyfiles_list, utils 10 | 11 | 12 | @pytest.fixture() 13 | def clouds_list(): 14 | here = os.path.abspath(os.path.dirname(__file__)) 15 | 16 | clouds_list = [] 17 | for i in [1, 2]: 18 | clouds_list.append(os.path.join(here, "data", f"{i}.ply")) 19 | 20 | return clouds_list 21 | 22 | 23 | @pytest.fixture() 24 | def expected_dsm(): 25 | here = os.path.abspath(os.path.dirname(__file__)) 26 | 27 | with rasterio.open(os.path.join(here, "data", "result.tiff")) as f: 28 | raster = f.read(1) 29 | gsd = f.res 30 | 31 | return raster, gsd 32 | 33 | 34 | @pytest.fixture() 35 | def expected_std(): 36 | here = os.path.abspath(os.path.dirname(__file__)) 37 | 38 | with rasterio.open(os.path.join(here, "data", "std.tiff")) as f: 39 | raster = f.read(1) 40 | 41 | return raster 42 | 43 | 44 | @pytest.fixture() 45 | def ply_file_with_crs(): 46 | here = os.path.abspath(os.path.dirname(__file__)) 47 | return os.path.join(here, "data", "crs.ply") 48 | 49 | 50 | def test_plyflatten_from_plyfiles_list(clouds_list, expected_dsm): 51 | raster, _ = plyflatten_from_plyfiles_list(clouds_list, resolution=2) 52 | raster = raster[:, :, 0] 53 | 54 | expected_raster, _ = expected_dsm 55 | np.testing.assert_allclose(expected_raster, raster, equal_nan=True) 56 | 57 | 58 | def test_std(clouds_list, expected_std): 59 | raster, _ = plyflatten_from_plyfiles_list(clouds_list, resolution=1, std=True) 60 | assert raster.shape[2] == 2 61 | raster = raster[:, :, 1] 62 | 63 | np.testing.assert_allclose(expected_std, raster, equal_nan=True) 64 | 65 | 66 | def test_resolution(clouds_list, expected_dsm, r=4): 67 | raster, _ = plyflatten_from_plyfiles_list(clouds_list, resolution=r) 68 | raster = raster[:, :, 0] 69 | 70 | reference, (rx, ry) = expected_dsm 71 | assert abs(raster.shape[0] * r - reference.shape[0] * rx) <= 2 * max(r, rx) 72 | assert abs(raster.shape[1] * r - reference.shape[1] * ry) <= 2 * max(r, ry) 73 | 74 | 75 | def test_ply_comment_crs(ply_file_with_crs): 76 | crs_params, crs_type = utils.crs_from_ply(ply_file_with_crs) 77 | assert crs_type == "CRS" 78 | 79 | pyproj_crs = utils.crs_proj(crs_params, crs_type) 80 | assert isinstance(pyproj_crs, pyproj.crs.CRS) 81 | --------------------------------------------------------------------------------