├── modules ├── XYZ │ ├── XYZ │ ├── REQUIRE │ ├── test │ │ ├── 3d.jl │ │ └── runtests.jl │ ├── LICENSE.md │ ├── src │ │ ├── XYZ.jl │ │ ├── profiler.jl │ │ ├── cloudfilter.jl │ │ ├── pointfilters.jl │ │ ├── cloud.jl │ │ ├── lasio.jl │ │ ├── writers.jl │ │ ├── cloudutils.jl │ │ ├── cloudclassifier.jl │ │ └── rasterize.jl │ └── README.md ├── PeatUtils │ ├── README.md │ ├── LICENSE.md │ ├── test │ │ └── runtests.jl │ └── src │ │ └── PeatUtils.jl └── GridOperations │ ├── test │ ├── REQUIRE │ └── runtests.jl │ ├── REQUIRE │ ├── README.md │ ├── src │ ├── GridOperations.jl │ ├── color.jl │ ├── segments.jl │ ├── utils.jl │ ├── interpolation.jl │ └── filter.jl │ └── LICENSE.md ├── data └── small.laz ├── figures ├── displaz.png └── pmf_example.png ├── install.jl ├── LICENSE ├── README.md └── lidar_pipeline.jl /modules/XYZ/XYZ: -------------------------------------------------------------------------------- 1 | XYZ -------------------------------------------------------------------------------- /modules/PeatUtils/README.md: -------------------------------------------------------------------------------- 1 | # PeatUtils 2 | -------------------------------------------------------------------------------- /modules/GridOperations/test/REQUIRE: -------------------------------------------------------------------------------- 1 | ColorTypes 2 | -------------------------------------------------------------------------------- /data/small.laz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Deltares/als2dtm/master/data/small.laz -------------------------------------------------------------------------------- /figures/displaz.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Deltares/als2dtm/master/figures/displaz.png -------------------------------------------------------------------------------- /figures/pmf_example.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Deltares/als2dtm/master/figures/pmf_example.png -------------------------------------------------------------------------------- /modules/GridOperations/REQUIRE: -------------------------------------------------------------------------------- 1 | julia 0.6 2 | NearestNeighbors 3 | Combinatorics 4 | GeoStats 5 | Images 6 | ColorSchemesLight 7 | -------------------------------------------------------------------------------- /modules/GridOperations/README.md: -------------------------------------------------------------------------------- 1 | # GridOperations 2 | 3 | A collection of various operations on grids. 4 | 5 | Currently contains functions for interpolation and colorization. 6 | -------------------------------------------------------------------------------- /modules/XYZ/REQUIRE: -------------------------------------------------------------------------------- 1 | julia 0.6 2 | ColorTypes 3 | Colors 4 | DataArrays 5 | GDAL 6 | LasIO 7 | NearestNeighbors 8 | PeatUtils 9 | ProgressMeter 10 | SortingAlgorithms 11 | StaticArrays 12 | -------------------------------------------------------------------------------- /install.jl: -------------------------------------------------------------------------------- 1 | Pkg.add("Colors") 2 | Pkg.add("ColorSchemes") 3 | Pkg.add("NearestNeighbors") 4 | Pkg.add("GeoStats") 5 | Pkg.add("Images") 6 | Pkg.add("Clustering") 7 | Pkg.add("GeoJSON") 8 | Pkg.add("LibGEOS") 9 | Pkg.add("ArgParse") 10 | Pkg.add("Lumberjack") 11 | Pkg.add("SortingAlgorithms") 12 | Pkg.add("ProgressMeter") 13 | Pkg.add("DataArrays") 14 | Pkg.add("Glob") 15 | Pkg.add("LasIO") 16 | Pkg.add("GDAL") 17 | -------------------------------------------------------------------------------- /modules/XYZ/test/3d.jl: -------------------------------------------------------------------------------- 1 | using XYZ 2 | using StaticArrays 3 | using BenchmarkTools, Compat 4 | 5 | a = collect(1:16) 6 | a = reshape(a, (2,2,4)) 7 | XYZ.offsets(a) 8 | 9 | bbox = XYZ.NBoundingBox(xmin=542900.0, xmax=544100.0, ymin=9679900.0, ymax=9681077.6, zmin=1.06, zmax=329.09) 10 | dims = SVector(5, 5, 5) 11 | coords = MVector(1.0,2.0,3.0) 12 | coords = MVector(1,-1,2) 13 | @btime invalid_coords(dims, coords) 14 | @btime sub2ind(dims, coords[1], coords[2], coords[3]) 15 | -------------------------------------------------------------------------------- /modules/GridOperations/src/GridOperations.jl: -------------------------------------------------------------------------------- 1 | __precompile__() 2 | 3 | module GridOperations 4 | 5 | # export interp_missing, 6 | export interp_missing!, create_mask, interp2d, 7 | colorize, 8 | hist_filter, pmf_filter, vosselman_filter, 9 | im_thresholding, im_segments_cleanup, im_segments_reduce 10 | 11 | using Colors 12 | using ColorSchemes 13 | using Compat 14 | using GeoStats 15 | using ImageFiltering 16 | using Images 17 | using NearestNeighbors 18 | using ProgressMeter 19 | 20 | include("utils.jl") 21 | include("interpolation.jl") 22 | include("segments.jl") 23 | include("filter.jl") 24 | include("color.jl") 25 | 26 | end 27 | -------------------------------------------------------------------------------- /modules/GridOperations/test/runtests.jl: -------------------------------------------------------------------------------- 1 | using GridOperations; const GO = GridOperations 2 | using Base.Test 3 | using ColorTypes 4 | 5 | @testset "Colorize" begin 6 | arr = [3.0, 4.0, 5.0, NaN] 7 | colored = GO.colorize(arr) 8 | @test colored == [ 9 | RGB{Float64}(0.267004, 0.004874, 0.329415), 10 | RGB{Float64}(0.1281485, 0.565107, 0.5508925), 11 | RGB{Float64}(0.993248, 0.906157, 0.143936), 12 | RGB{Float64}(1.0, 1.0, 1.0) 13 | ] 14 | end 15 | 16 | @testset "Progressive morphological filter" begin 17 | srand(1) 18 | zmin = rand(3.0f0:0.1f0:5.0f0, 30, 40) 19 | max_window_radius = 9.0 20 | slope = 0.151234 21 | dh_max = 2.0 22 | dh_min = 0.4 23 | nodata = -9999.0 24 | boundarymask = trues(zmin) 25 | zmax_pmf, flags = GO.pmf_filter(zmin, boundarymask, max_window_radius, slope, dh_max, dh_min, 1.0) 26 | @test count(flags) === 754 27 | @test minimum(zmax_pmf) ≈ 3.4 28 | @test maximum(zmax_pmf) ≈ 3.702468 # large Float32 inaccuracy, should be 3.7 29 | @test hash(zmax_pmf) === 0x520f02f7bbe47b0e 30 | end 31 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2019 Deltares 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /modules/XYZ/LICENSE.md: -------------------------------------------------------------------------------- 1 | The XYZ.jl package is licensed under the MIT "Expat" License: 2 | 3 | > Copyright (c) 2016: Deltares. 4 | > 5 | > Permission is hereby granted, free of charge, to any person obtaining a copy 6 | > of this software and associated documentation files (the "Software"), to deal 7 | > in the Software without restriction, including without limitation the rights 8 | > to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | > copies of the Software, and to permit persons to whom the Software is 10 | > furnished to do so, subject to the following conditions: 11 | > 12 | > The above copyright notice and this permission notice shall be included in all 13 | > copies or substantial portions of the Software. 14 | > 15 | > THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | > IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | > FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | > AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | > LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | > OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | > SOFTWARE. 22 | > 23 | -------------------------------------------------------------------------------- /modules/PeatUtils/LICENSE.md: -------------------------------------------------------------------------------- 1 | The PeatUtils.jl package is licensed under the MIT "Expat" License: 2 | 3 | > Copyright (c) 2015: Deltares. 4 | > 5 | > Permission is hereby granted, free of charge, to any person obtaining 6 | > a copy of this software and associated documentation files (the 7 | > "Software"), to deal in the Software without restriction, including 8 | > without limitation the rights to use, copy, modify, merge, publish, 9 | > distribute, sublicense, and/or sell copies of the Software, and to 10 | > permit persons to whom the Software is furnished to do so, subject to 11 | > the following conditions: 12 | > 13 | > The above copyright notice and this permission notice shall be 14 | > included in all copies or substantial portions of the Software. 15 | > 16 | > THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 17 | > EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 18 | > MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. 19 | > IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY 20 | > CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, 21 | > TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE 22 | > SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 23 | -------------------------------------------------------------------------------- /modules/GridOperations/LICENSE.md: -------------------------------------------------------------------------------- 1 | The GridOperations.jl package is licensed under the MIT "Expat" License: 2 | 3 | > Copyright (c) 2017: Deltares. 4 | > 5 | > 6 | > Permission is hereby granted, free of charge, to any person obtaining a copy 7 | > 8 | > of this software and associated documentation files (the "Software"), to deal 9 | > 10 | > in the Software without restriction, including without limitation the rights 11 | > 12 | > to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 13 | > 14 | > copies of the Software, and to permit persons to whom the Software is 15 | > 16 | > furnished to do so, subject to the following conditions: 17 | > 18 | > 19 | > 20 | > The above copyright notice and this permission notice shall be included in all 21 | > 22 | > copies or substantial portions of the Software. 23 | > 24 | > 25 | > 26 | > THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 27 | > 28 | > IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 29 | > 30 | > FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 31 | > 32 | > AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 33 | > 34 | > LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 35 | > 36 | > OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 37 | > 38 | > SOFTWARE. 39 | > 40 | > 41 | -------------------------------------------------------------------------------- /modules/XYZ/src/XYZ.jl: -------------------------------------------------------------------------------- 1 | __precompile__() 2 | 3 | module XYZ 4 | 5 | # export 6 | # # cloud 7 | # Cloud, BoundingBox, positions, attributes, boundingbox, calc_bbox, update_bbox!, 8 | # # pointfilters 9 | # ground, water, notground, notoutlier, lastreturn, 10 | # # classifiers 11 | # classify_ground!, classify_ground_approx!, classify_outliers!, classify_water!, 12 | # classify_min!, classify_max!, classify!, reset_class!, 13 | # classify_below_surface!, classify_above_surface!, classify_water_approx!, 14 | # # raster 15 | # Raster, rasterize, define_raster, 16 | # # profile 17 | # define_profile, filter_profile, 18 | # # reducers 19 | # reduce_min, reduce_max, reduce_pointfilter, 20 | # # writers 21 | # to_tif, to_las, to_xyz, grid2tif, 22 | # # readers 23 | # read_pointcloud, 24 | # # utils 25 | # describe, paint!, create_kdtree, point_density!, grid2cloud_attribute!, copy_classification! 26 | 27 | using Compat 28 | using Nullables 29 | using ColorTypes 30 | using FixedPointNumbers 31 | using Colors 32 | using GDAL 33 | using FileIO 34 | using LasIO 35 | using NearestNeighbors 36 | using PeatUtils 37 | using ProgressMeter 38 | using SortingAlgorithms 39 | using StaticArrays 40 | 41 | include("cloud.jl") 42 | include("pointfilters.jl") 43 | include("rasterize.jl") 44 | include("cloudutils.jl") 45 | include("profiler.jl") 46 | include("cloudfilter.jl") 47 | include("cloudclassifier.jl") 48 | include("lasio.jl") 49 | include("writers.jl") 50 | 51 | end 52 | -------------------------------------------------------------------------------- /modules/XYZ/README.md: -------------------------------------------------------------------------------- 1 | # XYZ 2 | 3 | Julia package for easily setting up pipelines for processing point clouds. 4 | The goal is to have a simple API that is flexible enough for diverse processing, including processes 5 | that take into account neighbouring points. Such a mockup API is presented below. 6 | 7 | ## High level API 8 | ```julia 9 | using XYZ 10 | 11 | pipeline(infile, process) 12 | ``` 13 | 14 | ## Medium level API 15 | ```julia 16 | using XYZ 17 | 18 | pts = read("las.las") 19 | 20 | pts::PointCloud 21 | 22 | function predicate(p) 23 | true 24 | end 25 | 26 | function classify(p) # or colorize 27 | p′ 28 | end 29 | 30 | filter!(pts, pointfilter) 31 | classify!(pts, predicate, class) 32 | 33 | laspts = (header, Vector{LasPoint}, spatialindex) 34 | laspt = (header, Vector{LasPoint}[i], spatialindex) 35 | 36 | 37 | 38 | custom_env_filter1!(pts) 39 | custom_env_filter2!(pts) 40 | 41 | write_raster(pts::Vector{LasPoint}) = write_raster(xyz) 42 | 43 | write("out.las", pts::PointCloud, predicate, option) 44 | write("out.tif", pts, predicate, option) # 2D grid 45 | write("out.csv", pts, predicate, option) # 2D profile 46 | write("out.csv", pts, predicate, option) # XYZ with extra columns 47 | 48 | 49 | function PointCloud{T <: AbstractFloat}(laspoints::Vector{LasPoint}, header::LasHeader, ) 50 | positions = StaticArray 51 | Dict{Symbol,Vector{Any}}() 52 | PointCloud(positions, BallTree(xy_array(header, lasp), Chebyshev()), attr, header) 53 | 54 | end 55 | 56 | cloud = PointCloud(positions) 57 | 58 | cloud = PointCloud(positions, Chebyshev()) 59 | cloud = PointCloud(positions, Eucldean()) 60 | 61 | type PointCloud{Dim,T,SIndex} 62 | positions::Vector{SVector{Dim,T}} 63 | spatial_index::SIndex 64 | attributes::Dict{Symbol,Vector{Any}} 65 | header::LasHeader 66 | end 67 | ``` 68 | -------------------------------------------------------------------------------- /modules/GridOperations/src/color.jl: -------------------------------------------------------------------------------- 1 | # functions to colorize numeric arrays or grayscale images with color maps 2 | 3 | """Colorize an array by applying a colormap from `ColorSchemes` 4 | 5 | For available colormaps see [this overview](http://juliagraphics.github.io/ColorSchemes.jl/stable/assets/figures/colorschemes.png). 6 | """ 7 | function colorize(A::AbstractArray; 8 | contrast::Bool=false, # enhance contrast by cutting the 2% on each side, like the QGIS default 9 | cmap::Symbol=:viridis, # matplotlib default 10 | nodata=-9999.0, 11 | vmin=nothing, # costly to filter twice 12 | vmax=nothing, 13 | bg=colorant"black") # black, color of nodata 14 | 15 | novalues = (A .== nodata) .| isnan.(A) 16 | values = broadcast(~, novalues) 17 | 18 | # calculate vmin and vmax 19 | # taking into account completely empty arrays 20 | if vmin == nothing || vmax == nothing 21 | if all(novalues) 22 | # return black image 23 | # shortcut because cannot reduce over empty collection 24 | return fill(bg, size(A)) 25 | else 26 | # do it here to prevent passing twice over the data 27 | # if both vmin and vmax are nothing 28 | vmin_calculated, vmax_calculated = if contrast 29 | quantile(A[values], [0.02, 0.98]) 30 | else 31 | extrema(A[values]) 32 | end 33 | end 34 | end 35 | if vmin == nothing 36 | vmin = vmin_calculated 37 | end 38 | if vmax == nothing 39 | vmax = vmax_calculated 40 | end 41 | 42 | # get the right colormap from ColorSchemes 43 | colormap = getfield(ColorSchemes, cmap) 44 | if (vmin ≈ vmax) || (vmin > vmax) 45 | return fill(bg, size(A)) 46 | end 47 | sc_img = (A .- vmin) .* (1.0 / (vmax - vmin)) 48 | sc_img[novalues] = 0.0 # set to background color later, just to avoid get NaN InexactError 49 | color(x) = get(colormap, x) 50 | cimg = color.(sc_img) 51 | cimg[novalues] = bg 52 | cimg 53 | end 54 | 55 | colorize(A::Union{BitMatrix, Matrix{Bool}}) = Gray.(A) 56 | -------------------------------------------------------------------------------- /modules/PeatUtils/test/runtests.jl: -------------------------------------------------------------------------------- 1 | using PeatUtils 2 | using Base.Test 3 | using GDAL 4 | 5 | # reuse GDAL.jl test data 6 | tifpath = joinpath(Pkg.dir("GDAL"), "test", "data", "utmsmall.tif") 7 | tifpath_local = joinpath(dirname(@__FILE__), "data", "utmsmall.tif") 8 | 9 | cp(tifpath, tifpath_local, remove_destination=true) 10 | 11 | @test getnodata(tifpath) == -1.0e10 12 | 13 | nodata = -9999.0 14 | setnodata!(tifpath_local, nodata) 15 | @test getnodata(tifpath_local) == nodata 16 | 17 | # test pointsample() 18 | let 19 | GDAL.allregister() 20 | dataset = GDAL.open(tifpath_local, GDAL.GA_ReadOnly) 21 | geotransform = zeros(6) 22 | GDAL.getgeotransform(dataset, geotransform) 23 | band = GDAL.getrasterband(dataset, 1) 24 | 25 | @test pointsample(band, geotransform, 443334.9, 3750015.9) === UInt8(189) 26 | @test pointsample(band, geotransform, 443167.9, 3749928.3) === UInt8(173) 27 | 28 | GDAL.close(dataset) 29 | GDAL.destroydrivermanager() 30 | end 31 | 32 | 33 | A, x_min, y_max, cellsize, epsg, nodata = read_raster(tifpath_local) 34 | 35 | @test x_min === 440720.0 36 | @test y_max === 3.75132e6 37 | @test cellsize === 60.0 38 | @test epsg === Nullable{Int64}(26711) 39 | @test nodata === -9999.0 40 | 41 | tifpath_local_out = joinpath(dirname(@__FILE__), "data", "utmsmall_out.tif") 42 | write_raster(tifpath_local_out, A, x_min, y_max, cellsize; epsg=epsg) 43 | 44 | ref_wkt_utm48n = """PROJCS["WGS 84 / UTM zone 48N",GEOGCS["WGS 84",DATUM["WGS_1984",SPHEROID["WGS 84",6378137,298.257223563,AUTHORITY["EPSG","7030"]],AUTHORITY["EPSG","6326"]],PRIMEM["Greenwich",0,AUTHORITY["EPSG","8901"]],UNIT["degree",0.0174532925199433,AUTHORITY["EPSG","9122"]],AUTHORITY["EPSG","4326"]],PROJECTION["Transverse_Mercator"],PARAMETER["latitude_of_origin",0],PARAMETER["central_meridian",105],PARAMETER["scale_factor",0.9996],PARAMETER["false_easting",500000],PARAMETER["false_northing",0],UNIT["metre",1,AUTHORITY["EPSG","9001"]],AXIS["Easting",EAST],AXIS["Northing",NORTH],AUTHORITY["EPSG","32648"]]""" 45 | ref_epsg_utm48n = Nullable(32648) 46 | @test epsg2wkt(ref_epsg_utm48n) == ref_wkt_utm48n 47 | @test wkt2epsg(ref_wkt_utm48n) === ref_epsg_utm48n 48 | 49 | rm(tifpath_local) 50 | rm(tifpath_local_out) 51 | 52 | # test without projection information 53 | # A, x_min, y_max, cellsize, epsg, nodata = read_raster("a.tif") 54 | # @test isnull(epsg) 55 | # @test isa(epsg, Nullable{Int}) 56 | -------------------------------------------------------------------------------- /modules/XYZ/test/runtests.jl: -------------------------------------------------------------------------------- 1 | using XYZ 2 | using StaticArrays 3 | using Base.Test 4 | 5 | # use test file from LasIO 6 | filename = "libLAS_1.2.las" # point format 0 7 | filepath = joinpath(Pkg.dir("LasIO"), "test", filename) 8 | workdir = dirname(@__FILE__) 9 | 10 | @testset "Cloud contruction and indexing" begin 11 | cloud, header = XYZ.read_pointcloud(filepath) 12 | 13 | attr = cloud.attributes 14 | @test isa(attr, Dict{Symbol,Vector}) 15 | @test Set(keys(attr)) == Set(Symbol[:number_of_returns, :edge_of_flight_line, :intensity, 16 | :key_point, :scan_direction, :withheld, :pt_src_id, :classification, :scan_angle, :synthetic, 17 | :return_number, :user_data]) 18 | 19 | @test XYZ.boundingbox(cloud) == XYZ.calc_bbox(cloud) 20 | 21 | cloudp1 = cloud[1:1] 22 | @test length(cloudp1) == 1 23 | @test XYZ.positions(cloudp1) == [SVector{3, Float64}(1.44013394e6, 375000.23, 846.66)] 24 | @test cloudp1[:intensity] == [0x00fa] 25 | @test cloudp1[:scan_angle] == [Int8(0)] 26 | @test cloudp1[:user_data] == [0x00] 27 | @test cloudp1[:pt_src_id] == [0x001d] 28 | @test cloudp1[:return_number] == [0x00] 29 | @test cloudp1[:number_of_returns] == [0x00] 30 | @test cloudp1[:scan_direction] == [false] 31 | @test cloudp1[:edge_of_flight_line] == [false] 32 | @test cloudp1[:classification] == [0x02] 33 | @test cloudp1[:synthetic] == [false] 34 | @test cloudp1[:key_point] == [false] 35 | @test cloudp1[:withheld] == [false] 36 | end 37 | 38 | @testset "Outlier classification" begin 39 | cloud, header = XYZ.read_pointcloud(filepath) 40 | 41 | XYZ.reset_class!(cloud) 42 | @test all(c -> c == 0x00, cloud[:classification]) 43 | 44 | XYZ.classify_outliers!(cloud; 45 | cellsize = 100.0, # cellsize used for evaluation of points 46 | dz = 1.0, # threshold in vertical distance between low points 47 | max_outliers = 15, # max number of outliers in one cell = low points to be evaluated 48 | max_height = 100.0) # threshold in vertical distance for high points 49 | 50 | low_noise = UInt8(7) # from ASPRS Standard LIDAR Point Classes 51 | high_noise = UInt8(18) # from ASPRS Standard LIDAR Point Classes (LAS 1.4) 52 | @test count(c -> c == low_noise, cloud[:classification]) == 13 53 | @test count(c -> c == high_noise, cloud[:classification]) == 1948 54 | end 55 | 56 | @testset "Rasterization" begin 57 | cloud, header = XYZ.read_pointcloud(filepath) 58 | cellsize = 10.0 59 | nground = 402812 60 | nodata = -9999.0 61 | rasterindex = XYZ.define_raster(cloud, cellsize) 62 | @test rasterindex.npoints == 497536 63 | @test typeof(rasterindex) == XYZ.Raster{Int32} 64 | @test rasterindex.nrow == rasterindex.ncol == 500 65 | 66 | raster = XYZ.rasterize(cloud, rasterindex; reducer=XYZ.reducer_minz) 67 | @test size(raster) == (500, 500, 1) 68 | @test eltype(raster) == Float32 69 | @test count(x -> x == nodata, raster) == 3336 70 | 71 | rasterindex_ground = XYZ.filter_raster(rasterindex, cloud, XYZ.ground) 72 | @test rasterindex_ground.npoints == nground 73 | 74 | # pointfilter 75 | @test count(i -> XYZ.ground(cloud, i), 1:length(cloud)) == nground 76 | end 77 | -------------------------------------------------------------------------------- /modules/XYZ/src/profiler.jl: -------------------------------------------------------------------------------- 1 | #= 2 | functions to create index points from pointcloud along line defined by start & end point. 3 | 4 | profiler generetes DataFrame with XYZ and attribute information from pointcloud, 5 | where X and Y are defined along line 6 | 7 | =# 8 | 9 | struct Profile{U<:Integer,T<:AbstractFloat} 10 | pointindex::Vector{U} 11 | pointlx::Vector{T} 12 | pointly::Vector{T} 13 | maxdist::T 14 | pstart::Tuple{T, T} 15 | pend::Tuple{T, T} 16 | len::T 17 | end 18 | 19 | # length of profile line segment 20 | linelength(p::Profile) = p.len 21 | pointindex(p::Profile) = p.pointindex 22 | pointlx(p::Profile) = p.pointlx 23 | pointly(p::Profile) = p.pointly 24 | 25 | # Base functions for Profile type 26 | Base.length(p::Profile) = length(p.pointindex) 27 | 28 | "index points along profileline" 29 | function define_profile(cloud::Cloud, ps::Tuple{T,T}, pe::Tuple{T,T}, maxdist::T; 30 | pointfilter = nothing) where T <: Real 31 | 32 | # define line and bbox 33 | x_min, x_max = minmax(ps[1], pe[1]) 34 | y_min, y_max = minmax(ps[2], pe[2]) 35 | x_min -= maxdist 36 | y_min -= maxdist 37 | x_max += maxdist 38 | y_max += maxdist 39 | 40 | # define rotated grid orientation 41 | transrot = segment_transformation(ps, pe) 42 | len = hypot(ps[1] - pe[1], ps[2] - pe[2]) #length(ps, pe) 43 | 44 | # loop through Cloud and store points 45 | n = length(positions(cloud)) 46 | idx = Int32[] # index of points in Cloud 47 | lx = Float64[] # distance along line 48 | ly = Float64[] # distance from line 49 | for i in 1:n 50 | xi, yi, zi = positions(cloud)[i] 51 | outside_bbox(x_min, y_min, x_max, y_max, xi, yi) && continue 52 | (pointfilter != nothing) && (pointfilter(cloud, i) || continue) 53 | 54 | p = SVector(xi, yi) 55 | p′ = transrot(p) 56 | 57 | if (0.0 <= p′[1] < len) && (abs(p′[2]) <= maxdist) 58 | push!(idx, i) 59 | push!(lx, p′[1]) 60 | push!(ly, p′[2]) 61 | end 62 | end 63 | np = length(idx) 64 | # sort points based on distance along line lx 65 | if np >= 100_000 # RadixSort only faster with many points 66 | perm = sortperm(lx, alg=RadixSort) 67 | else 68 | perm = sortperm(lx) 69 | end 70 | 71 | Profile(idx[perm], lx[perm], ly[perm], maxdist, ps, pe, len) 72 | end 73 | 74 | 75 | "create new profile based on pointfilter applied to points in profile" 76 | function filter_profile(cloud::Cloud, prof::Profile, pointfilter) 77 | n = length(prof) 78 | idx = Int32[] 79 | for i in 1:n 80 | idxi = pointindex(prof)[i] 81 | (pointfilter != nothing) && (pointfilter(cloud, i) || continue) 82 | push!(idx, idxi) 83 | end 84 | 85 | Profile( 86 | pointindex(prof)[idx], 87 | pointlx(prof)[idx], 88 | pointly(prof)[idx], 89 | prof.maxwidth, 90 | prof.pstart, 91 | prof.pend, 92 | prof.len) 93 | end 94 | 95 | # utils 96 | """Set up a transformation that will map points to a rotated 97 | and translated coordinate system with the origin on the segment 98 | start, and positive x along the segment.""" 99 | function segment_transformation(ps, pe) 100 | θ = -atan2(pe[2]-ps[2], pe[1]-ps[1]) 101 | rot = SMatrix{2,2}([cos(θ) -sin(θ); sin(θ) cos(θ)]) 102 | trans = SVector(-ps[1], -ps[2]) 103 | transrot(x::SVector{2,Float64}) = *(rot, x+trans) 104 | end 105 | 106 | 107 | "check if a point falls outside a bounding box" 108 | function outside_bbox(x_min, y_min, x_max, y_max, x, y) 109 | x < x_min || y < y_min || x > x_max || y > y_max 110 | end 111 | -------------------------------------------------------------------------------- /modules/XYZ/src/cloudfilter.jl: -------------------------------------------------------------------------------- 1 | #= 2 | filter Clouds based on attributes and surrounding points 3 | Format should be according to: 4 | 5 | classify!cloud::Cloud; kwargs) 6 | cloud[subset] 7 | end 8 | 9 | =# 10 | 11 | "function returns cloud indices based on statistical reduce function " 12 | function reduce_index(cloud::Cloud, r::Raster; 13 | reduceri = reducer_minz_index, # function to find index of min max 14 | pointfilter = nothing, # predicate function to filter individual points 15 | min_dens = 0) # minimum point density in cell to consider for educed cloud) 16 | 17 | subset = Int32[] 18 | # loop through cells 19 | @showprogress 5 "Reducer processing.." for i in 1:length(r) 20 | # find points in range 21 | idx0 = r[i] 22 | # loop through points and filter 23 | if pointfilter != nothing # if filter is given 24 | idx = Int32[] # indices after filter 25 | for i in idx0 26 | pointfilter(cloud, i) || continue 27 | push!(idx, i) 28 | end 29 | else 30 | idx = idx0 # no filter applied 31 | end 32 | 33 | np = length(idx) 34 | # calculate density [points / m2] 35 | if min_dens > 0 36 | di = np / area 37 | di < min_dens && continue # min density threshold 38 | end 39 | 40 | np == 0 && continue # stat functions don't work on empty arrays 41 | # set statistics to grid 42 | push!(subset, idx[reduceri(cloud, idx)]) 43 | end 44 | subset 45 | end 46 | 47 | "function to reduce pointcloud based on statistical min per gridcell" 48 | function reduce_min(cloud::Cloud, cellsize::Real; 49 | pointfilter = nothing, # predicate function to filter individual points 50 | min_dens = 0) # minimum point density in cell to consider for reduced cloud 51 | 52 | r = define_raster(cloud, cellsize; pointfilter = pointfilter) 53 | reduce_min(cloud, r; pointfilter = pointfilter, min_dens = min_dens) 54 | end 55 | 56 | "function to reduce pointcloud based on min per gridcell" 57 | function reduce_min(cloud::Cloud, r::Raster; 58 | pointfilter = nothing, # predicate function to filter individual points 59 | min_dens = 0) # minimum point density in cell to consider for reduced cloud 60 | 61 | subset = reduce_index(cloud, r; reduceri = reducer_minz_index, pointfilter = pointfilter, min_dens = min_dens) 62 | 63 | # return raster with statistics, density is saved to last layer 64 | cloud[subset] 65 | end 66 | 67 | "function to reduce pointcloud based on statistical max per gridcell" 68 | function reduce_max(cloud::Cloud, cellsize::Real; 69 | pointfilter = nothing, # predicate function to filter individual points 70 | min_dens = 0) # minimum point density in cell to consider for educed cloud 71 | 72 | r = define_raster(cloud, cellsize; pointfilter = pointfilter) 73 | reduce_max(cloud, r; pointfilter = pointfilter, min_dens = min_dens) 74 | end 75 | 76 | "function to reduce pointcloud based on statistical max per gridcell" 77 | function reduce_max(cloud::Cloud, r::Raster; 78 | pointfilter = nothing, # predicate function to filter individual points 79 | min_dens = 0) # minimum point density in cell to consider for educed cloud 80 | 81 | subset = reduce_index(cloud, r; reduceri = reducer_maxz_index, pointfilter = pointfilter, min_dens = min_dens) 82 | 83 | # return raster with statistics, density is saved to last layer 84 | cloud[subset] 85 | end 86 | 87 | function reduce_pointfilter(cloud::Cloud, pointfilter) 88 | subset = Int32[] 89 | for i in 1:length(positions(cloud)) 90 | (pointfilter != nothing) && (pointfilter(cloud, i) || continue) 91 | push!(subset, i) 92 | end 93 | 94 | cloud[subset] 95 | end 96 | -------------------------------------------------------------------------------- /modules/XYZ/src/pointfilters.jl: -------------------------------------------------------------------------------- 1 | #= 2 | Predicate functions to filter points in Clouds based on positions or attributes. 3 | A point is defined by a PointCould and index. 4 | 5 | onsistent use of pointfilters: 6 | value in scripts can either be or a function according to format below 7 | --> true: the points are used in the function. 8 | --> False: the points are skipped 9 | 10 | Format should be according to: 11 | 12 | function pointfilter(cloud::Cloud, index::Int) 13 | true 14 | end 15 | 16 | =# 17 | 18 | ground_water(cloud::Cloud, i::Integer) = (cloud[:classification][i] == 2) || (cloud[:classification][i] == 9) 19 | ground(cloud::Cloud, i::Integer) = (cloud[:classification][i] == 2) 20 | notground(cloud::Cloud, i::Integer) = (cloud[:classification][i] != 2) 21 | water(cloud::Cloud, i::Integer) = (cloud[:classification][i] == 9) 22 | notwater(cloud::Cloud, i::Integer) = (cloud[:classification][i] != 9) 23 | notoutlier(cloud::Cloud, i::Integer) = cloud[:classification][i] != 7 && cloud[:classification][i] != 18 24 | outlier(cloud::Cloud, i::Integer) = cloud[:classification][i] == 7 || cloud[:classification][i] == 18 25 | unclassified(cloud::Cloud, i::Integer) = cloud[:classification][i] == 0 26 | unclassified_lastreturn(cloud::Cloud, i::Integer) = lastreturn(cloud, i) && unclassified(cloud, i) 27 | unclassified_firstreturn(cloud::Cloud, i::Integer) = firstreturn(cloud, i) && unclassified(cloud, i) 28 | 29 | highscanangle(cloud::Cloud, i::Integer) = abs(cloud[:scan_angle][i]) > 20.0 30 | lowscanangle(cloud::Cloud, i::Integer) = abs(cloud[:scan_angle][i]) <= 20.0 31 | 32 | # dummy filters 33 | filtertrue(::Cloud, ::Integer) = true 34 | filterfalse(::Cloud, ::Integer) = false 35 | 36 | # ASPRS, but only on normalized pointcloud(!) 37 | low_veg(cloud::Cloud, i::Integer) = 0.5 < getz(cloud, i) <= 2.0 38 | med_veg(cloud::Cloud, i::Integer) = 2.0 < getz(cloud, i) <= 5.0 39 | high_veg(cloud::Cloud, i::Integer) = 5.0 < getz(cloud, i) 40 | 41 | non_tree(cloud::Cloud, i::Integer) = getz(cloud, i) < 50.0 # higher trees don't exist ;) 42 | 43 | # check if last return 44 | lastreturn(cloud::Cloud, i::Integer) = cloud[:return_number][i] == cloud[:number_of_returns][i] 45 | firstreturn(cloud::Cloud, i::Integer) = cloud[:return_number][i] == 1 46 | singlereturn(cloud::Cloud, i::Integer) = cloud[:return_number][i] == cloud[:number_of_returns][i] == 1 47 | 48 | # edge of flightline 49 | flightedge(cloud::Cloud, i::Integer) = cloud[:edge_of_flight_line][i] == 1 50 | 51 | # combined 52 | 53 | "First returns, not ground and not on edge. Default for chm method" 54 | function unclassifiedfirstreturnonedge(cloud::Cloud, i::Integer) 55 | unclassified_firstreturn(cloud, i) && !flightedge(cloud, i) 56 | end 57 | 58 | "First returns, not ground and not with high scan angle." 59 | function unclassifiedfirstreturnlowscanangle(cloud::Cloud, i::Integer) 60 | unclassified_firstreturn(cloud, i) && lowscanangle(cloud, i) 61 | end 62 | 63 | "First returns, not ground, only trees and not with high scan angle." 64 | function unclassifiedfirstreturnnontreelowscanangle(cloud::Cloud, i::Integer) 65 | unclassified_firstreturn(cloud, i) && lowscanangle(cloud, i) && non_tree(cloud, i) 66 | end 67 | 68 | "not outlier and is last return. default pointfilter for classify_ground! method" 69 | function notoutlier_lastreturn(cloud::Cloud, i::Integer) 70 | lastreturn(cloud, i) && notoutlier(cloud, i) 71 | end 72 | 73 | "not outlier and is last return. default pointfilter for classify_ground! method" 74 | function notoutlier_ground(cloud::Cloud, i::Integer) 75 | notoutlier(cloud, i) && ground(cloud, i) 76 | end 77 | 78 | "not outlier and is last return. default pointfilter for classify_ground! method" 79 | function notoutlier_notwater_lastreturn(cloud::Cloud, i::Integer) 80 | lastreturn(cloud, i) && notoutlier(cloud, i) && notwater(cloud, i) 81 | end 82 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [![DOI](https://zenodo.org/badge/178010374.svg)](https://zenodo.org/badge/latestdoi/178010374) 2 | # ALS to DTM 3 | 4 | All code required to create DTMs from raw `.las` or `.laz` files. Uses the morphological filter from *Zhang et al. (2003)*[1] for filtering ground points. 5 | Code used in *Vernimmen et al. 2019. Creating a lowland and peatland landscape DTM from interpolated partial coverage LiDAR data for Central Kalimantan and East Sumatra. Remote Sensing 11, 1152* [2] [![DOI:10.3390/rs11101152](https://zenodo.org/badge/DOI/10.3390/rs11101152.svg)](https://doi.org/10.3390/rs11101152) 6 | 7 | ## Derived packages 8 | This is an archived copy of a part of our complete LiDAR pipeline, running on Julia 0.6 (unmaintained as of 4/2/2019). 9 | We advise to use our derived, more generic open-source packages: 10 | - [GeoArrays.jl](https://github.com/evetion/GeoArrays.jl) For spatial raster creation (replaces PeatUtils). 11 | - [GeoRasterFiltering.jl](https://github.com/Deltares/GeoRasterFiltering.jl) For spatial filters (replaces GridOperations). 12 | - [PointCloudRasterizers.jl](https://github.com/Deltares/PointCloudRasterizers.jl) For rasterizing and filtering pointclouds (replaces XYZ). 13 | - [LasIO.jl](https://github.com/visr/LasIO.jl) Native parser for the .las data format. 14 | - [LazIO.jl](https://github.com/evetion/LazIO.jl) Extends LasIO for using compressed .laz data. 15 | 16 | 17 | ## Installation 18 | Download Julia 0.6 @ https://julialang.org/downloads/oldreleases.html. 19 | Run the `install.jl` script to install necessary packages. 20 | 21 | For using the modules with your own scripts, add the Modules folder to your `LOADPATH` by using: 22 | `push!(LOAD_PATH, "modules")` 23 | 24 | For using `.laz` files, this requires a working version of `laszip` in your path. 25 | 26 | ## Usage 27 | ![pmf](figures/displaz.png) 28 | *ALS data example: data/small.laz Note the low vegetation on the mounds. Visualization in Displaz.* 29 | 30 | 31 | ```julia 32 | push!(LOAD_PATH, "modules") 33 | include("lidar_pipeline.jl") 34 | 35 | infile = "data/small.laz" 36 | outfolder = "output/" 37 | epsg = 32748 38 | 39 | # PMF settings 40 | radius = 16. 41 | slope = 0.6 42 | dhmax = 1.1 43 | dhmin = 0.5 44 | 45 | lidar_pipeline(infile, epsg, outfolder, radius, slope, dhmax, dhmin) 46 | ``` 47 | 48 | The `.tiff` files (`_zmin`, `z_max` and `_zmin_filtered`) in the output folder are used in the following figure: 49 | 50 | ![pmf](figures/pmf_example.png) 51 | *A) Minimum elevation grid. B) Maximum allowed elevations from PMF filter. C) Resulting DTM where A > B are non-ground points.* 52 | 53 | 54 | ## Related packages 55 | Outside these and derived packages, look at these packages: 56 | - [SpatialGrids.jl](https://github.com/FugroRoames/SpatialGrids.jl) provides 2D and 3D grid structures for working with point cloud data. 57 | - [Displaz](https://github.com/FugroRoames/displaz) A hackable lidar viewer 58 | - [PointClouds.jl](https://github.com/FugroRoames/PointClouds.jl) 59 | Point cloud data structures in pure julia 60 | 61 | and also into discussions at 62 | https://github.com/visr/LasIO.jl/issues/4 and https://github.com/FugroRoames/PointClouds.jl/issues/7 in which we actively colloborate to further strengthen the Julia LiDAR ecosystem. 63 | 64 | ## References 65 | [1]: Zhang, Keqi, Shu-Ching Chen, Dean Whitman, Mei-Ling Shyu, Jianhua Yan, and Chengcui Zhang. “A Progressive Morphological Filter for Removing Nonground Measurements from Airborne LIDAR Data.” IEEE Transactions on Geoscience and Remote Sensing 41, no. 4 (2003): 872–82. https://doi.org/10.1109/TGRS.2003.810682. 66 | 67 | [2]: [![DOI:10.3390/rs11101152](https://zenodo.org/badge/DOI/10.3390/rs11101152.svg)](https://doi.org/10.3390/rs11101152) Vernimmen, R., Hooijer, A., Yuherdha, A.T., Visser, M., Pronk, M., Eilander, D., Akmalia, R., Fitranatanegara, N., Mulyadi, D., Andreas, H., Ouellette, J., Hadley, W., 2019. Creating a lowland and peatland landscape DTM from interpolated partial coverage LiDAR data for Central Kalimantan and East Sumatra. Remote Sensing 11, 1152, https://doi.org/10.3390/rs11101152. 68 | -------------------------------------------------------------------------------- /modules/XYZ/src/cloud.jl: -------------------------------------------------------------------------------- 1 | 2 | "bounding box representing the smallest straightup rectangle containing all data " 3 | struct BoundingBox 4 | xmin::Float64 5 | ymin::Float64 6 | zmin::Float64 7 | xmax::Float64 8 | ymax::Float64 9 | zmax::Float64 10 | end 11 | 12 | # construct BoundingBox with keyword argumets to prevent mixups 13 | function BoundingBox(;xmin=0.0::Float64, ymin=0.0::Float64, zmin=0.0::Float64, 14 | xmax=0.0::Float64, ymax=0.0::Float64, zmax=0.0::Float64) 15 | BoundingBox(xmin, ymin, zmin, xmax, ymax, zmax) 16 | end 17 | 18 | function Base.show(io::IO, bbox::BoundingBox) 19 | print(io, @sprintf "xmin=%.1f, " bbox.xmin ) 20 | print(io, @sprintf "xmax=%.1f, " bbox.xmax ) 21 | print(io, @sprintf "ymin=%.1f, " bbox.ymin ) 22 | print(io, @sprintf "ymax=%.1f, " bbox.ymax ) 23 | print(io, @sprintf "zmin=%.2f, " bbox.zmin ) 24 | print(io, @sprintf "zmax=%.2f" bbox.zmax ) 25 | end 26 | 27 | """Cloud is a point cloud container type similar to Clouds' Cloud, without 28 | a spatial index, but with a simple bounding box that is kept up to date.""" 29 | mutable struct Cloud 30 | positions::Vector{MVector{3,Float64}} 31 | attributes::Dict{Symbol,Vector} 32 | boundingbox::BoundingBox 33 | end 34 | 35 | Cloud(pos, attrs) = Cloud(pos, attrs, calc_bbox(pos)) 36 | 37 | function Base.show(io::IO, cloud::Cloud) 38 | print(io, "PointCloud (N=$(length(cloud)), bbox=[$(cloud.boundingbox)], attributes=$(keys(cloud.attributes)))") 39 | end 40 | 41 | 42 | "Get the vector of point cloud positions" 43 | positions(cloud::Cloud) = getfield(cloud, :positions) 44 | 45 | "Get the Dict of point cloud attributes" 46 | attributes(cloud::Cloud) = getfield(cloud, :attributes) 47 | 48 | "Get the bounding box of the point cloud" 49 | boundingbox(cloud::Cloud) = getfield(cloud, :boundingbox) 50 | 51 | "Calculate bounding box of the positions" 52 | function calc_bbox(P::Vector{SVector{3,Float64}}) 53 | xmin, xmax, ymin, ymax, zmin, zmax = Inf, -Inf, Inf, -Inf, Inf, -Inf 54 | @inbounds for p in P 55 | xmin = min(xmin, p[1]) 56 | xmax = max(xmax, p[1]) 57 | ymin = min(ymin, p[2]) 58 | ymax = max(ymax, p[2]) 59 | zmin = min(zmin, p[3]) 60 | zmax = max(zmax, p[3]) 61 | end 62 | BoundingBox(xmin=xmin,ymin=ymin,zmin=zmin,xmax=xmax,ymax=ymax,zmax=zmax) 63 | end 64 | 65 | "Calculate bounding box of the positions" 66 | function calc_bbox(P::Vector{MVector{3,Float64}}) 67 | xmin, xmax, ymin, ymax, zmin, zmax = Inf, -Inf, Inf, -Inf, Inf, -Inf 68 | @inbounds for p in P 69 | xmin = min(xmin, p[1]) 70 | xmax = max(xmax, p[1]) 71 | ymin = min(ymin, p[2]) 72 | ymax = max(ymax, p[2]) 73 | zmin = min(zmin, p[3]) 74 | zmax = max(zmax, p[3]) 75 | end 76 | BoundingBox(xmin=xmin,ymin=ymin,zmin=zmin,xmax=xmax,ymax=ymax,zmax=zmax) 77 | end 78 | 79 | "Calculate bounding box of the point cloud" 80 | function calc_bbox(cloud::Cloud) 81 | P = positions(cloud) 82 | calc_bbox(P) 83 | end 84 | 85 | "Update the bounding box of the point cloud" 86 | function update_bbox!(cloud::Cloud) 87 | cloud.boundingbox = calc_bbox(cloud) 88 | nothing 89 | end 90 | 91 | "Subset a point cloud with specified indices" 92 | function Base.getindex(cloud::Cloud, I) 93 | pos = positions(cloud)[I] 94 | attrs = Dict{Symbol,Vector}() 95 | for (k,v) in attributes(cloud) 96 | attrs[k] = v[I] 97 | end 98 | Cloud(pos, attrs) 99 | end 100 | 101 | "Get a point cloud attribute by indexing with a `Symbol`" 102 | Base.getindex(cloud::Cloud, attr::Symbol) = attributes(cloud)[attr] 103 | 104 | "Set a point cloud attribute assigning to a `Symbol` index" 105 | Base.setindex!(cloud::Cloud, A::AbstractVector, attr::Symbol) = attributes(cloud)[attr] = A 106 | 107 | "Number of points in the point cloud" 108 | Base.length(cloud::Cloud) = length(positions(cloud)) 109 | -------------------------------------------------------------------------------- /modules/GridOperations/src/segments.jl: -------------------------------------------------------------------------------- 1 | #function for segmentation in grids based on thresholding and connectivity 2 | # 3 | # based on functions from Images module 4 | 5 | 6 | function im_labeling(tf::Matrix{U}; connectivity=8) where U <: Integer 7 | # create connectivity array 8 | if connectivity == 8 9 | connect_array = trues(3,3) 10 | elseif connectivity == 4 11 | connect_array = BitMatrix([0 1 0; 1 1 1; 0 1 0]) 12 | end 13 | 14 | # label areas 15 | segments = label_components(tf, connect_array) 16 | end 17 | 18 | # copy of function above, not sure how to catch both types 19 | function im_labeling(tf::BitMatrix; connectivity=8) 20 | # create connectivity array 21 | if connectivity == 8 22 | connect_array = trues(3,3) 23 | elseif connectivity == 4 24 | connect_array = BitMatrix([0 1 0; 1 1 1; 0 1 0]) 25 | end 26 | 27 | # label areas 28 | segments = label_components(tf, connect_array) 29 | end 30 | 31 | """cleanup binary images based on minimal area of segments 32 | Input 33 | segments labeled array 34 | min_area min area of one segment [m2] 35 | res resolution of 1 pixel [m2] 36 | connectivity all surrounding pixels (=8) or only N-E-S-W pixels (=4) (default = 8) 37 | 38 | output 39 | updated label array 40 | """ 41 | function im_segments_cleanup(segments::Array{U, 2}, min_area::Real; 42 | res=1, connectivity=8) where U <: Integer 43 | # remove noise 44 | segments = copy(segments) 45 | segment_areas = component_lengths(segments) 46 | segment_idx = component_indices(segments) 47 | for lab in 1:maximum(segments) # skip background == 0 48 | area = segment_areas[lab+1] 49 | if area < (min_area / (res*res)) 50 | for idx in segment_idx[lab+1] 51 | segments[idx] = 0 52 | end 53 | end 54 | end 55 | # re-label segments 56 | im_labeling(segments; connectivity=connectivity) 57 | end 58 | 59 | 60 | function split_segments(segments::Array{U, 2}, block_size, min_area; 61 | res=1, connectivity=8) where U <: Integer 62 | 63 | # split segments 64 | segments = copy(segments) 65 | segment_idx = component_subscripts(segments) 66 | segment_bbox = component_boxes(segments) 67 | block_size = Float64(block_size) / res 68 | nsegments = maximum(segments) 69 | for lab in 1:nsegments # skip background == 0 70 | (ymin, xmin), (ymax, xmax) = segment_bbox[lab+1] 71 | dx, dy = xmax-xmin+1, ymax-ymin+1 72 | if ((dx > 1.5*block_size) || (dy > 1.5*block_size)) 73 | nrow, ncol = min(Int(cld(dy, block_size)),1), min(Int(cld(dx,block_size)),1) 74 | for (y, x) in segment_idx[lab+1] 75 | # cells include bottom and left boundaries 76 | col = Int(fld(x - xmin, block_size)+1) 77 | row = Int(fld(ymax - y, block_size)+1) 78 | iblock = sub2ind((nrow, ncol), row, col) 79 | segments[y,x] = iblock 80 | end 81 | end 82 | end 83 | # re-label segments 84 | segments = im_labeling(segments; connectivity=connectivity) 85 | 86 | # merge segments with too small area 87 | segment_areas = component_lengths(segments) 88 | segment_idx = component_indices(segments) 89 | for lab in 1:maximum(segments) # skip background == 0 90 | area = segment_areas[lab+1] 91 | if area < (min_area / (res*res)) 92 | b = falses(segments) 93 | for idx in segment_idx[lab+1] 94 | b[idx] = true 95 | end 96 | bd = dilate(b) 97 | neighboring_cells = bd .& broadcast(~, b) 98 | new_lab = maximum(segments[neighboring_cells]) # find label neighboring segment 99 | new_lab == 0 && continue # no neighboring segments 100 | for idx in segment_idx[lab+1] 101 | segments[idx] = new_lab 102 | end 103 | end 104 | end 105 | # re-label segments 106 | im_labeling(segments; connectivity=connectivity) 107 | end 108 | 109 | """reduce pixel in 'A' within a connected segment in binary grid 'tf' using function 'func' 110 | 111 | Input 112 | segments labeled array 113 | A input image with to sample from can [Array{2,Real}] 114 | 115 | output 116 | image with reduced value in all segment pixels""" 117 | function im_segments_reduce(segments::Array{U, 2}, A::Array{T, 2}; 118 | connectivity=8, 119 | func=x -> maximum(x)-minimum(x), 120 | nodata=-9999.0) where U <: Integer where T <: AbstractFloat 121 | @assert size(segments) == size(A) "the size of image A and binary grid should match" 122 | 123 | # get indices 124 | initiate = true 125 | B = 0 126 | 127 | for idx in component_indices(segments) 128 | if initiate # skip background, but initiate output 129 | # initiate output array with output type at first loop 130 | vals = func(T[1, 2]) 131 | @assert isa(vals, Real) "incorrect reducer function" 132 | reduced_type = typeof(vals) 133 | B = fill(reduced_type(nodata), size(A)) 134 | initiate = false 135 | else 136 | vals = [xi for xi=A[idx] if xi != nodata] 137 | if !isempty(vals) 138 | B[idx] = func(vals) 139 | end 140 | end 141 | end 142 | 143 | B 144 | end 145 | -------------------------------------------------------------------------------- /modules/GridOperations/src/utils.jl: -------------------------------------------------------------------------------- 1 | 2 | """tests if point p1 and p2 are on the same side of line ab""" 3 | function same_side(p1::Vector{U},p2::Vector{U}, a::Vector{U}, b::Vector{U}) where U <: AbstractFloat 4 | cp1 = cross(b-a, p1-a) 5 | cp2 = cross(b-a, p2-a) 6 | dot(cp1, cp2) >= 0 7 | end 8 | 9 | """ tests if a point p is Left|On|Right of an infinite line ab 10 | - >0 for p left of the line ab 11 | - =0 for p on the line 12 | - <0 for p right of the line 13 | """ 14 | function is_left(p::Vector{U}, a::Vector{U}, b::Vector{U}) where U <: Real 15 | (b[1]-a[1])*(p[2]-a[2]) - (p[1]-a[1])*(b[2]-a[2]) 16 | end 17 | 18 | """calculates 2D convec hull for set of points 19 | returns points which determine the hull and its indices in the input list 20 | 21 | based on Andrew's Monotone Chain Algorithm (Andrew, 1979), explained here: 22 | http://geomalgorithms.com/a10-_hull-1.html""" 23 | function get_hull(P::Vector{Vector{T}}) where T <: Real 24 | np = length(P) 25 | # sort indices, first by x then y 26 | pm = zeros(T,(np, 3)) 27 | [pm[i,:] = [P[i][1], P[i][2], i] for i in 1:np] # to 2d array 28 | pm = sortrows(pm, by=x->(x[1],x[2])) 29 | P = [pm[i,1:2] for i in 1:np] 30 | idx = Int[pm[i,3] for i in 1:np] 31 | # get indices of points with 1st x min or max and 2nd y min or max 32 | min_min = 1 # min x, min y 33 | i = 0 ## initiate so it is known outside loop, arghh... 34 | for i in 2:np 35 | P[i][1] != P[1][1] && break 36 | end 37 | min_max = i-1 38 | for i in np-1:-1:1 39 | P[i][1] != P[np][1] && break 40 | end 41 | max_min = i+1 42 | max_max = np 43 | # initiate convex hull stack with point indices 44 | stack = Int[] 45 | 46 | ## LOWER HULL 47 | push!(stack, min_min) 48 | top = 1 # no. of points in stack 49 | for i in 2:max_min-1 # loop through points with increasing x sequence 50 | is_left(P[i], P[min_min], P[max_min]) >= 0 && continue # ignore P[i] above or on the lower line 51 | while top >= 2 # at least two points in stack 52 | is_left(P[i], P[stack[top-1]], P[stack[top]]) > 0 && break # P[i] in hull 53 | pop!(stack) 54 | top -= 1 55 | end 56 | push!(stack, i) 57 | top += 1 58 | end 59 | push!(stack, max_min) 60 | top += 1 61 | 62 | ## UPPER HULL 63 | # if distinct xmax points push max_min point onto stack 64 | if max_max != max_min 65 | push!(stack, max_max) 66 | top += 1 67 | end 68 | bot = top # bottom point of upper hull 69 | for i in np-1:-1:2 # loop through points with decreasing x sequence 70 | is_left(P[i], P[max_max], P[min_max]) >= 0 && continue # ignore P[i] below or on the upper line 71 | while bot > top # at least two points in upper stack 72 | is_left(P[i], P[stack[top-1]], P[stack[top]]) > 0 && break # P[i] in hull 73 | pop!(stack) 74 | top -= 1 75 | end 76 | push!(stack, i) 77 | top += 1 78 | end 79 | push!(stack, min_max) 80 | top += 1 81 | 82 | # return points of hull and indices 83 | P[stack], idx[stack] 84 | end 85 | 86 | """calculates bbox coordinates for set of points 87 | returns the list with coordinates the indices of the points which determine the bbox""" 88 | function get_bbox(P::Vector{Vector{T}}) where T <: Real 89 | @assert length(P) >= 2 90 | xmin, xmax, ymin, ymax = Inf, -Inf, Inf, -Inf 91 | xmini, xmaxi, ymini, ymaxi = 0, 0, 0, 0 # declare indices of points that make up bbox 92 | i = 0 93 | @inbounds for p in P 94 | addp = false 95 | i += i # index of point in vector P 96 | if p[1] < xmin 97 | xmin = p[1] 98 | xmini = i 99 | end 100 | if p[1] > xmax 101 | xmax = p[1] 102 | xmaxi = i 103 | end 104 | if p[2] < ymin 105 | ymin = p[2] 106 | ymini = i 107 | end 108 | if p[2] > ymax 109 | ymax = p[2] 110 | ymaxi = p 111 | end 112 | end 113 | 114 | # return bbox and indices that make up bbox 115 | [xmin,xmax,ymin,ymax], [xmini, xmaxi, ymini, ymaxi] 116 | end 117 | 118 | function triangle_area(a::Vector{U}, b::Vector{U}, c::Vector{U}) where U <: AbstractFloat 119 | abs(a[1]*(b[2]-c[2]) + b[1]*(c[2]-a[2]) + c[1]*(a[2]-b[2])) / 2. 120 | end 121 | 122 | """check if point p [x,y] in triangle made out of points a,b,c""" 123 | function in_triangle(p::Vector{U}, a::Vector{U},b::Vector{U},c::Vector{U}) where U <: AbstractFloat 124 | p,a,b,c = [[x[1],x[2],0.] for x in [p,a,b,c]] 125 | same_side(p,a, b,c) && same_side(p,b, a,c) && same_side(p,c, a,b) 126 | end 127 | 128 | """check if point (x,y) in traingle made out of points vector""" 129 | function in_triangle(x::AbstractFloat, y::AbstractFloat, points::Vector{Vector{U}}) where U <: AbstractFloat 130 | length(points) 131 | npoints != 3 && warn("more than three points in list, check based on first three points only") 132 | a,b,c = points[1], points[2], points[3] 133 | xypoint_in_triangle([x, y], a, b,c) 134 | end 135 | 136 | """check if point (x,y) in bbox""" 137 | function in_bbox(x::AbstractFloat, y::AbstractFloat, points::Vector{Vector{U}}) where U <: AbstractFloat 138 | xmin,xmax,ymin,ymax = get_bbox(points)[1] 139 | xmin <= x <= xmax && ymin <= y <= ymax 140 | end 141 | 142 | 143 | """check if point in hull of all points based on hull computation""" 144 | function in_hull(x::Real, y::Real, points::Vector{Vector{U}}) where U <: Real 145 | npoints = length(points) 146 | @assert npoints >= 3 ["point vector should contain at least three points"] 147 | push!(points, [x,y]) 148 | p_hull, idx_hull = get_hull(points) 149 | !in(npoints+1, idx_hull) 150 | end 151 | -------------------------------------------------------------------------------- /lidar_pipeline.jl: -------------------------------------------------------------------------------- 1 | using Compat 2 | using GeoJSON 3 | using GridOperations 4 | using LibGEOS 5 | using PeatUtils 6 | using TiledIteration 7 | using XYZ 8 | using Glob 9 | using Lumberjack 10 | 11 | const overlap = 0. # width of overlap [m] 12 | 13 | # general settings 14 | const high_res = 1.0 # cellsize for water grids & and high res dtm [m] 15 | const nodata = -9999.0 16 | 17 | 18 | function xyvec(cloud::XYZ.Cloud, sampleratio::Float64) 19 | n = length(cloud.positions) 20 | step = round(Int, cld(1, sampleratio)) 21 | # have minimum of points needed 22 | if n < 10000 | fld(n, step) < 1000 23 | step = 1 24 | end 25 | idx_samples = 1:step:n 26 | nsample = length(idx_samples) 27 | A = Vector{Vector{Float64}}() # empy vector of vectors 28 | sizehint!(A, nsample) 29 | for i in idx_samples 30 | p = cloud.positions[i] 31 | push!(A, [p[1], p[2]]) 32 | end 33 | A 34 | end 35 | 36 | # simplify raster from cloud 37 | function cloud2arr(cloud::XYZ.Cloud, tile_bbox::XYZ.BoundingBox, cellsize::Float64; pointfilter=nothing, reducer=XYZ.reducer_medz, interpolate=true) 38 | index = XYZ.define_raster(cloud, tile_bbox, overlap, cellsize; snapgrid=25.0, epsg=epsg, pointfilter=pointfilter) 39 | raster = XYZ.rasterize(cloud, index; reducer=reducer, min_dens = 0)[:,:,1] 40 | if interpolate 41 | raster = GridOperations.interp_missing!(copy(raster), eltype(raster)(nodata); mode="kriging") 42 | end 43 | raster, index 44 | end 45 | 46 | # simplify tif creation 47 | function cloud2tif(folder::String, fn::String, cloud::XYZ.Cloud, tile_bbox::XYZ.BoundingBox, cellsize::Float64, filter, reducer; interpolate=false) 48 | raster, index = cloud2arr(cloud, tile_bbox, cellsize, pointfilter=filter, reducer=reducer, interpolate=interpolate) 49 | XYZ.grid2tif(folder, fn, index, raster; nodata=nodata) 50 | end 51 | 52 | 53 | """pipeline to classify cloud and create dtm""" 54 | function lidar_pipeline( 55 | filename::String, 56 | epsg::Int, 57 | out_dir::String, 58 | gf_radius::Float64, 59 | gf_slope::Float64, 60 | gf_dh_max::Float64, 61 | gf_dh_min::Float64, 62 | ) 63 | 64 | # file admin 65 | fileid = basename(splitext(filename)[1]) 66 | in_dir = dirname(filename) 67 | filen = fileid 68 | 69 | out_dir = joinpath(out_dir, fileid) 70 | isdir(out_dir) || mkpath(out_dir) 71 | info("Ouput directory is $(out_dir)") 72 | 73 | # Set up Logging 74 | add_truck(LumberjackTruck(joinpath(out_dir, "pipeline.log")), "pipelinelogger") 75 | info("Processing $(filen)") 76 | info("with ground parameters r:$(gf_radius) \t s:$(gf_slope) \t min:$(gf_dh_min) \t max:$(gf_dh_max)") 77 | 78 | # read las 79 | info("--> read $(fileid)") 80 | info("--> write results to $(out_dir)") 81 | cloud, header = XYZ.read_pointcloud(filename) 82 | info(" $(cloud)") 83 | 84 | # reset classification 85 | XYZ.reset_class!(cloud) 86 | 87 | tile_bbox = XYZ.BoundingBox(xmin=header.x_min, ymin=header.y_min, 88 | xmax=header.x_max, ymax=header.y_max) 89 | 90 | ## CLASSIFY OUTLIERS 91 | info("--> 1.) classify outliers") 92 | # classify low and high outliers 93 | # evaluate points per in [m] cells 94 | # low outliers if a jump larger than [m] in lowest n [-] points 95 | # high outliers if points more than [m] above lowest non-outlier point 96 | XYZ.classify_outliers!(cloud; 97 | cellsize = 100.0, # cellsize used for evaluation of points 98 | dz = 1.0, # threshold in vertical distance between low points 99 | max_outliers = 15, # max number of outliers in one cell = low points to be evaluated 100 | max_height=100.0) # threshold in vertical distance for high points 101 | 102 | ## CLASSIFY GROUND with Zhang algorithm. use only non-water, non-outlier, last return points 103 | info("--> 1.5) intermediate rasters") 104 | 105 | # make non-filled raster index for high resolution cells with non-outliers and last return points 106 | r = XYZ.define_raster(cloud, tile_bbox, overlap, high_res; pointfilter = XYZ.unclassified_lastreturn, epsg = epsg) 107 | if !(r.nrow >= 2 && r.ncol >= 2) 108 | info("Defined raster is too small to be classified, skipping") 109 | @show r 110 | return 111 | end 112 | 113 | boundarymask_r = trues(r.nrow, r.ncol) 114 | smallboundarymask_r = trues(r.nrow, r.ncol) 115 | 116 | zmin = XYZ.rasterize(cloud, r; reducer = XYZ.reducer_minz, pointfilter=nothing)[:,:,1] 117 | XYZ.grid2tif(out_dir, "$(filen)_zmin.tif", r, zmin; nodata=nodata) 118 | 119 | # create las mask and dropouts & vegetation binary images 120 | haspoints = Array{Bool}(XYZ.rasterize(cloud, r; reducer = XYZ.reducer_count)[:,:,1] .> 0) 121 | las_mask = GridOperations.create_mask(haspoints) 122 | dropouts = Array{Bool}(las_mask .& broadcast(~, haspoints)) # nodata cells within las mask 123 | 124 | # zmin interpolated surface 125 | r_filled = XYZ.inpaint_missings_nn(r, cloud)[1] 126 | zmin_filled = XYZ.rasterize(cloud, r_filled; reducer = XYZ.reducer_minz, pointfilter=nothing)[:,:,1] 127 | XYZ.grid2tif(out_dir, "$(filen)_zmin_nn.tif", r, zmin_filled; nodata=nodata) 128 | 129 | ## CLASSIFY GROUND with Zhang algorithm. use only non-water, non-outlier, last return points 130 | info("--> 2.) classify terrain (pmf)") 131 | 132 | # apply zhang to grid 133 | zmax_pmf, flags = GridOperations.pmf_filter(zmin_filled, boundarymask_r, gf_radius, gf_slope, gf_dh_max, gf_dh_min, r.cellsize) 134 | XYZ.grid2tif(out_dir, "$(filen)_zmax_pmf.tif", r, zmax_pmf; nodata=nodata) 135 | bin_vegetation = Array{Bool}(flags .> 0) .& broadcast(~, dropouts) 136 | 137 | # use maxz grid for classification 138 | XYZ.classify_below_surface!(cloud, r, zmax_pmf, 2) 139 | XYZ.classify_mask!(cloud, r, smallboundarymask_r, 28) # unused class for edge effects 140 | 141 | # filter zmin grid 142 | zmin_filtered = copy(zmin) 143 | zmin_filtered[((bin_vegetation .| dropouts) .| .!smallboundarymask_r)] = nodata 144 | XYZ.grid2tif(out_dir, "$(filen)_zmin_filtered.tif", r, zmin_filtered; nodata=nodata) 145 | 146 | nothing 147 | end 148 | -------------------------------------------------------------------------------- /modules/GridOperations/src/interpolation.jl: -------------------------------------------------------------------------------- 1 | """interpolate missing values (nodata) in a grid 2 | methods available: 3 | nearest neighbor 4 | ordinary kriging 5 | inverste distance interpolation 6 | 7 | inputs 8 | img the image to be interpolated 9 | nodata nodata value in img 10 | mask boolean array. only interpolate where true 11 | mode "nn", "kriging", "idw" 12 | k number of neighbors (default=10; for kriging and idw only) 13 | p idw power (default=10.0; for idw only) 14 | extrapolate check if point inside triangle of neighbors (default=false; for kriging and idw only) 15 | """ 16 | function interp_missing!( 17 | img::Array{T,2}, 18 | nodata::T; 19 | mask=trues(size(img)), 20 | mode="nn", 21 | k=10, # number of neighboring points used in kringing and idw 22 | p=2.0, # idw power 23 | extrapolate=true) where T <: AbstractFloat 24 | 25 | # initiate interpolators or spatial index 26 | xy_p, z_p = xy_z_grid(img, nodata=nodata) 27 | 28 | # skip images with only no-data values 29 | if length(xy_p) > 0 30 | if mode in ["nn", "kriging", "idw"] 31 | tree = KDTree(xy_p) 32 | else 33 | error("mode $(mode) not implemented") 34 | end 35 | 36 | # interpolation 37 | ismissing = (img .== nodata) .& mask 38 | dims = size(img) 39 | for I in 1:length(img) 40 | ismissing[I] || continue 41 | x,y = ind2sub(dims, I) 42 | if mode == "nn" 43 | img[I] = interpolator_nn(x, y, xy_p, z_p, tree) 44 | elseif mode == "kriging" 45 | img[I] = interpolator_kriging(x, y, xy_p, z_p, tree; k=k, extrapolate=extrapolate) 46 | elseif mode == "idw" 47 | img[I] = interpolator_idw(x, y, xy_p, z_p, tree; p=p, k=k, extrapolate=extrapolate) 48 | end 49 | end 50 | end 51 | 52 | img 53 | end 54 | 55 | """interpolate z at xy based on values z_p at xy_p""" 56 | function interp2d(xy::Matrix{U}, xy_p::Matrix{U}, z_p::Vector{T}; 57 | mode="nn", 58 | k=10, # number of neighboring points used in kringing and idw 59 | p=2.0, # idw power 60 | extrapolate=true, 61 | tree::NearestNeighbors.NNTree=KDTree(xy_p)) where T <: AbstractFloat where U <: Real 62 | 63 | # output 64 | n = size(xy,2) 65 | z = zeros(T,length(xy)) 66 | 67 | # interpolation 68 | for I in 1:n 69 | x,y = xy[:,I] 70 | if mode == "nn" 71 | z[I] = interpolator_nn(x, y, xy_p, z_p, tree) 72 | elseif mode == "kriging" 73 | z[I] = interpolator_kriging(x, y, xy_p, z_p, tree; k=k, extrapolate=extrapolate) 74 | elseif mode == "idw" 75 | z[I] = interpolator_idw(x, y, xy_p, z_p, tree; p=p, k=k, extrapolate=extrapolate) 76 | end 77 | end 78 | 79 | z 80 | end 81 | 82 | function interpolator_kriging(x::Real, y::Real, 83 | xy_p::Array{U,2}, z_p::Array{T,1}, 84 | tree::NearestNeighbors.NNTree=KDTree(xy_p); 85 | k=10, # number of neighboring points used in kriging and idw 86 | extrapolate=true, 87 | nodata=-9999.0) where T <: AbstractFloat where U <: Real 88 | 89 | idxs, dists = knn(tree, [x, y], k, true) 90 | if !extrapolate && !in_hull(x, y, [xy_p[:,i] for i in idxs]) 91 | T(nodata) 92 | elseif length(idxs) > 0 93 | # define a covariance model 94 | γ = GaussianVariogram(sill=1.0f0, range=1.0f0, nugget=0.0f0) 95 | # define an estimator (i.e. build the Kriging system) 96 | ordkrig = OrdinaryKriging(xy_p[:,idxs], z_p[idxs], γ) 97 | # estimate at target location 98 | μ, _ = estimate(ordkrig, T[x,y]) # not using σ² 99 | μ 100 | else 101 | error("no points to interpolate") 102 | end 103 | end 104 | 105 | function interpolator_idw(x::Real, y::Real, 106 | xy_p::Array{U,2}, z_p::Array{T,1}, 107 | tree::NearestNeighbors.NNTree=KDTree(xy_p); 108 | p=2.0, # idw power 109 | k=10, # number of neighboring points used in kringing and idw 110 | extrapolate=true, 111 | nodata=-9999.0) where T <: AbstractFloat where U <: Real 112 | 113 | idxs, dists = knn(tree, [x, y], k, true) 114 | if !extrapolate && !in_hull(x, y, [xy_p[:,i] for i in idxs]) 115 | T(nodata) 116 | elseif length(idxs) > 0 117 | idw_interpolation(z_p[idxs], dists; power=p) 118 | else 119 | error("no points to interpolate") 120 | end 121 | end 122 | 123 | function interpolator_nn(x::Real, y::Real, 124 | xy_p::Array{U,2}, z_p::Array{T,1}, 125 | tree::NearestNeighbors.NNTree=KDTree(xy_p)) where T <: AbstractFloat where U <: Real 126 | 127 | idx, dists = knn(tree, Float32[x, y], 1) 128 | z_p[idx[1]] 129 | end 130 | 131 | 132 | """loop over image to find missings & make arrays with datapoints""" 133 | function xy_z_grid(img::Array{T,2}; nodata::Real=-9999.0) where T 134 | n = count(x -> x != nodata, img) 135 | xy = zeros(Float32, (2, n)) 136 | z = zeros(Float32, n) 137 | dims = size(img) 138 | i = 0 139 | for I in 1:length(img) 140 | if img[I] != nodata 141 | i += 1 142 | x,y = ind2sub(dims, I) 143 | # build xy and z arrays for KDTree 144 | xy[1, i] = x 145 | xy[2, i] = y 146 | z[i] = img[I] 147 | end 148 | end 149 | xy, z 150 | end 151 | 152 | 153 | "fill las dropouts, but keep no data areas next to strips open" 154 | function create_mask(img::Array; nodata::Real=-9999.0, n=10) 155 | mask = img .!= nodata 156 | for i in 1:n 157 | ImageMorphology.dilate!(mask) 158 | end 159 | for i in 1:n 160 | ImageMorphology.erode!(mask) 161 | end 162 | Array{Bool,2}(mask) 163 | end 164 | 165 | """calculate the Inverse Distance Weighting interpolation for one point""" 166 | function idw_interpolation(z::Vector{U}, dis::Vector{U}; power=10.) where U <: AbstractFloat 167 | invdisp = 1.0 ./ (dis .^ power) 168 | weights = invdisp ./ sum(invdisp) 169 | dot(weights, z) 170 | end 171 | 172 | """linear interpolation based fox xy withing triangle in 3D plane""" 173 | function linear_interpolation(x::Real, y::Real, a::Vector{U}, 174 | b::Vector{U}, c::Vector{U}) where U <: AbstractFloat 175 | 176 | t1 = ((b[1]-a[1])*(c[3]-a[3])-(c[1]-a[1])*(b[3]-a[3])) 177 | t2 = ((b[2]-a[2])*(c[3]-a[3])-(c[2]-a[2])*(b[3]-a[3])) 178 | n = ((b[1]-a[1])*(c[2]-a[2])-(c[1]-a[1])*(b[2]-a[2])) 179 | z = a[3] + t1/n*(y-a[2]) - t2/n*(x-a[1]) 180 | end 181 | -------------------------------------------------------------------------------- /modules/XYZ/src/lasio.jl: -------------------------------------------------------------------------------- 1 | """Prepare and allocate an empty dict to be used as a Cloud attribute table 2 | based on a list of needed attributes""" 3 | function prepare_attributes(pointtype::Type{T}, n::Integer) where T <: LasPoint 4 | n = Int(n) # cannot construct BitVector(n) with n <: UInt32 (fix in julia) 5 | # create dict with all Las point format 0 properties 6 | attr = Dict{Symbol, Vector}( 7 | :intensity => zeros(UInt16, n), 8 | :return_number => zeros(UInt8, n), 9 | :number_of_returns => zeros(UInt8, n), 10 | :scan_direction => zeros(Bool, n), 11 | :edge_of_flight_line => zeros(Bool, n), 12 | :classification => zeros(UInt8, n), 13 | :synthetic => zeros(Bool, n), 14 | :key_point => zeros(Bool, n), 15 | :withheld => zeros(Bool, n), 16 | :scan_angle => zeros(Int8, n), # -90 to +90 17 | :user_data => zeros(UInt8, n), 18 | :pt_src_id => zeros(UInt16, n), 19 | ) 20 | # add additional attributes based on availability 21 | if pointtype <: LasIO.LasPointTime 22 | attr[:time] = zeros(Float64, n) 23 | end 24 | if pointtype <: LasIO.LasPointColor 25 | attr[:color] = Vector{RGB{N0f16}}(n) 26 | end 27 | attr 28 | end 29 | 30 | # this function is a performance bottleneck 31 | function add_point!( 32 | lasp::LasPoint, 33 | header::LasHeader, 34 | pointdata::Vector{SVector{3,Float64}}, 35 | attr::Dict{Symbol, Vector}, 36 | i::Integer, 37 | pointtype::Type{T}) where T <: LasPoint 38 | 39 | pointdata[i] = SVector{3,Float64}( 40 | xcoord(lasp, header), 41 | ycoord(lasp, header), 42 | zcoord(lasp, header)) 43 | attr[:intensity][i] = intensity(lasp) 44 | attr[:return_number][i] = return_number(lasp) 45 | attr[:number_of_returns][i] = number_of_returns(lasp) 46 | attr[:scan_direction][i] = scan_direction(lasp) 47 | attr[:edge_of_flight_line][i] = edge_of_flight_line(lasp) 48 | attr[:classification][i] = classification(lasp) 49 | attr[:synthetic][i] = synthetic(lasp) 50 | attr[:key_point][i] = key_point(lasp) 51 | attr[:withheld][i] = withheld(lasp) 52 | attr[:scan_angle][i] = reinterpret(Int8, scan_angle(lasp)) # fix laszip reader 53 | attr[:user_data][i] = user_data(lasp) 54 | attr[:pt_src_id][i] = pt_src_id(lasp) 55 | # add additional attributes based on availability 56 | if pointtype <: LasIO.LasPointTime 57 | attr[:time][i] = gps_time(lasp) 58 | end 59 | if pointtype <: LasIO.LasPointColor 60 | attr[:color][i] = RGB(lasp) 61 | end 62 | end 63 | 64 | function read_pointcloud(s::Union{Stream{format"LAS"}, Pipe}) 65 | LasIO.skiplasf(s) 66 | header = read(s, LasHeader) 67 | 68 | n = Int(header.records_count) 69 | pointtype = LasIO.pointformat(header) 70 | 71 | # pre allocate the parts that will go into the Cloud 72 | pointdata = Vector{SVector{3,Float64}}(n) 73 | attr = prepare_attributes(pointtype, n) 74 | 75 | @showprogress 1 "Reading pointcloud..." for i=1:n 76 | lasp = read(s, pointtype) 77 | add_point!(lasp, header, pointdata, attr, i, pointtype) 78 | end 79 | header, pointdata, attr 80 | end 81 | 82 | function read_pointcloud(filepath::File{format"LAS"}) 83 | header, pointdata, attr = open(filepath) do io 84 | read_pointcloud(io) 85 | end 86 | cloud = Cloud(pointdata, attr) 87 | cloud, header 88 | end 89 | 90 | function read_pointcloud(filepath::File{format"LAZ"}) 91 | fn = filename(filepath) 92 | header, pointdata, attr = open(`laszip -olas -stdout -i "$fn"`) do s 93 | read_pointcloud(s) 94 | end 95 | cloud = Cloud(pointdata, attr) 96 | cloud, header 97 | end 98 | 99 | """ 100 | read_pointcloud(filepath::AbstractString) 101 | 102 | Read a pointcloud stored in LAS or LAZ format to a Cloud 103 | """ 104 | function read_pointcloud(filepath::AbstractString) 105 | # converts to FileIO File LAS or LAZ 106 | # such that it dispatches to the right read_pointcloud 107 | fio = query(filepath) 108 | read_pointcloud(fio) 109 | end 110 | 111 | function laspoint_shared(cloud::Cloud, i::Integer, h::LasHeader) 112 | # fields shared among all point formats 113 | pos = positions(cloud)[i] 114 | x = xcoord(pos[1], h) 115 | y = ycoord(pos[2], h) 116 | z = zcoord(pos[3], h) 117 | intensity = cloud[:intensity][i] 118 | flagb = flag_byte(cloud[:return_number][i], cloud[:number_of_returns][i], 119 | cloud[:scan_direction][i], cloud[:edge_of_flight_line][i]) 120 | rawcl = raw_classification(cloud[:classification][i], cloud[:synthetic][i], 121 | cloud[:key_point][i], cloud[:withheld][i]) 122 | scan_angle = cloud[:scan_angle][i] 123 | user_data = cloud[:user_data][i] 124 | pt_src_id = cloud[:pt_src_id][i] 125 | 126 | x, y, z, intensity, flagb, rawcl, scan_angle, user_data, pt_src_id 127 | end 128 | 129 | "Cloud point to LasPoint0" 130 | function laspoint(pointtype::Type{LasPoint0}, cloud::Cloud, i::Integer, h::LasHeader) 131 | x, y, z, intensity, flagb, rawcl, scan_angle, user_data, pt_src_id = laspoint_shared(cloud, i, h) 132 | LasPoint0(x, y, z, intensity, flagb, rawcl, scan_angle, user_data, pt_src_id) 133 | end 134 | 135 | "Cloud point to LasPoint1" 136 | function laspoint(pointtype::Type{LasPoint1}, cloud::Cloud, i::Integer, h::LasHeader) 137 | x, y, z, intensity, flagb, rawcl, scan_angle, user_data, pt_src_id = laspoint_shared(cloud, i, h) 138 | gpst = cloud[:time][i] 139 | LasPoint1(x, y, z, intensity, flagb, rawcl, scan_angle, user_data, pt_src_id, 140 | gpst) 141 | end 142 | 143 | "Cloud point to LasPoint2" 144 | function laspoint(pointtype::Type{LasPoint2}, cloud::Cloud, i::Integer, h::LasHeader) 145 | x, y, z, intensity, flagb, rawcl, scan_angle, user_data, pt_src_id = laspoint_shared(cloud, i, h) 146 | r, g, b = rgb_components(cloud[:color][i]) 147 | LasPoint2(x, y, z, intensity, flagb, rawcl, scan_angle, user_data, pt_src_id, 148 | r, g, b) 149 | end 150 | 151 | "Cloud point to LasPoint3" 152 | function laspoint(pointtype::Type{LasPoint3}, cloud::Cloud, i::Integer, h::LasHeader) 153 | x, y, z, intensity, flagb, rawcl, scan_angle, user_data, pt_src_id = laspoint_shared(cloud, i, h) 154 | gpst = cloud[:time][i] 155 | r, g, b = rgb_components(cloud[:color][i]) 156 | LasPoint3(x, y, z, intensity, flagb, rawcl, scan_angle, user_data, pt_src_id, 157 | gpst, r, g, b) 158 | end 159 | 160 | "Update the header bounding box and counts based on point data. 161 | Extends the LasIO.update! to also work on `Cloud`." 162 | function LasIO.update!(h::LasHeader, cloud::Cloud) 163 | n = length(cloud) 164 | x_min, y_min, z_min = Inf, Inf, Inf 165 | x_max, y_max, z_max = -Inf, -Inf, -Inf 166 | point_return_count = zeros(UInt32, 5) 167 | for (i, p) in enumerate(positions(cloud)) 168 | x, y, z = p 169 | if x < x_min 170 | x_min = x 171 | end 172 | if y < y_min 173 | y_min = y 174 | end 175 | if z < z_min 176 | z_min = z 177 | end 178 | if x > x_max 179 | x_max = x 180 | end 181 | if y > y_max 182 | y_max = y 183 | end 184 | if z > z_max 185 | z_max = z 186 | end 187 | # add max statemate to ensure no accidental zeros are passed 188 | point_return_count[max(cloud[:return_number][i], 1)] += 1 189 | end 190 | h.x_min = x_min 191 | h.y_min = y_min 192 | h.z_min = z_min 193 | h.x_max = x_max 194 | h.y_max = y_max 195 | h.z_max = z_max 196 | h.records_count = n 197 | h.point_return_count = point_return_count 198 | nothing 199 | end 200 | -------------------------------------------------------------------------------- /modules/PeatUtils/src/PeatUtils.jl: -------------------------------------------------------------------------------- 1 | __precompile__() 2 | 3 | module PeatUtils 4 | using Nullables 5 | using Compat 6 | import Compat.String 7 | using JSON 8 | using SortingAlgorithms 9 | # using DataArrays 10 | using GeoJSON 11 | using GDAL 12 | 13 | export getnodata, setnodata!, rio_info 14 | export wkt, gdalarray 15 | export uint_mapping 16 | export fraction_na 17 | export pointsample 18 | export driver_list 19 | export outside 20 | export read_raster 21 | export write_raster 22 | export wkt2epsg, epsg2wkt 23 | 24 | """ 25 | Load a `NULL`-terminated list of strings 26 | 27 | That is it expects a "StringList", in the sense of the CPL functions, as a 28 | NULL terminated array of strings. 29 | 30 | Function copied over from ArchGDAL.jl 31 | """ 32 | function unsafe_loadstringlist(pstringlist::Ptr{Cstring}) 33 | stringlist = Vector{String}() 34 | (pstringlist == C_NULL) && return stringlist 35 | i = 1 36 | item = unsafe_load(pstringlist, i) 37 | while Ptr{UInt8}(item) != C_NULL 38 | push!(stringlist, unsafe_string(item)) 39 | i += 1 40 | item = unsafe_load(pstringlist, i) 41 | end 42 | stringlist 43 | end 44 | 45 | "Get the WKT of an Integer EPSG code" 46 | function epsg2wkt(epsg::Nullable{Int}) 47 | if isnull(epsg) 48 | return "" # missing projections are represented as empty strings 49 | else 50 | epsgcode = get(epsg) 51 | srs = GDAL.newspatialreference(C_NULL) 52 | GDAL.importfromepsg(srs, epsgcode) 53 | wkt_ptr = Ref(Cstring(C_NULL)) 54 | GDAL.exporttowkt(srs, wkt_ptr) 55 | return unsafe_string(wkt_ptr[]) 56 | end 57 | end 58 | 59 | epsg2wkt(epsg::Integer) = epsg2wkt(Nullable{Int}(epsg)) 60 | 61 | """For EPSG strings like "4326" or "EPSG:4326" """ 62 | function epsg2wkt(epsg::String) 63 | if isempty(epsg) 64 | return "" 65 | end 66 | i = findlast(epsg, ':') + 1 # also works if : is not there 67 | epsgcode = Nullable{Int}(parse(Int, epsg[i:end])) 68 | epsg2wkt(epsgcode) 69 | end 70 | 71 | "Get the Nullable{Int} EPSG code from a WKT string" 72 | function wkt2epsg(wkt::String) 73 | if isempty(wkt) 74 | return Nullable{Int}() # no projection 75 | else 76 | # no projection 77 | srs = GDAL.newspatialreference(C_NULL) 78 | GDAL.importfromwkt(srs, [wkt]) 79 | epsg = parse(Int, GDAL.getauthoritycode(srs, C_NULL)) 80 | return Nullable{Int}(epsg) 81 | end 82 | end 83 | 84 | # "Convert a DataArray to an ordinary array inplace" 85 | # function convert!(Array, da::DataArray, nodata) 86 | # da.data[da.na] = nodata 87 | # da.data 88 | # end 89 | 90 | "fallthrough" 91 | convert!(Array, arr::AbstractArray, nodata) = arr 92 | 93 | # turn 2D DataArray into regular 1 band array 94 | function gdalarray(da::AbstractArray{T, 2}, nodata) where T 95 | if eltype(da) in (Int32, Int64) # not supported by GDAL 96 | GDT, totype = GDAL.GDT_Int32, Int32 97 | elseif eltype(da) == Float32 98 | GDT, totype = GDAL.GDT_Float32, Float32 99 | else # fallthrough: convert to Float64 100 | GDT, totype = GDAL.GDT_Float64, Float64 101 | end 102 | arr = convert!(Array, da, nodata) # see fallthrough above 103 | assert(eltype(da) == totype) 104 | arr, GDT 105 | end 106 | 107 | const gdt_lookup = Dict{DataType, GDAL.GDALDataType}( 108 | UInt8 => GDAL.GDT_Byte, 109 | UInt16 => GDAL.GDT_UInt16, 110 | Int16 => GDAL.GDT_Int16, 111 | UInt32 => GDAL.GDT_UInt32, 112 | Int32 => GDAL.GDT_Int32, 113 | Float32 => GDAL.GDT_Float32, 114 | Float64 => GDAL.GDT_Float64 115 | ) 116 | 117 | const type_lookup = Dict{GDAL.GDALDataType, DataType}( 118 | GDAL.GDT_Byte => UInt8, 119 | GDAL.GDT_UInt16 => UInt16, 120 | GDAL.GDT_Int16 => Int16, 121 | GDAL.GDT_UInt32 => UInt32, 122 | GDAL.GDT_Int32 => Int32, 123 | GDAL.GDT_Float32 => Float32, 124 | GDAL.GDT_Float64 => Float64 125 | ) 126 | 127 | 128 | "Get the nodata value from the raster metadata" 129 | function getnodata(fname::AbstractString) 130 | GDAL.allregister() 131 | dataset = GDAL.open(fname, GDAL.GA_ReadOnly) 132 | band = GDAL.getrasterband(dataset, 1) 133 | nodata = GDAL.getrasternodatavalue(band, C_NULL) 134 | GDAL.close(dataset) 135 | GDAL.destroydrivermanager() 136 | nodata 137 | end 138 | 139 | "Set a new nodata value in the raster metadata" 140 | function setnodata!(fname::AbstractString, nodata::Real) 141 | GDAL.allregister() 142 | dataset = GDAL.open(fname, GDAL.GA_Update) 143 | band = GDAL.getrasterband(dataset, 1) 144 | GDAL.setrasternodatavalue(band, nodata) 145 | GDAL.close(dataset) 146 | GDAL.destroydrivermanager() 147 | nodata 148 | end 149 | 150 | "Returns the output of rio info as a Dict" 151 | function rio_info(fname::AbstractString) 152 | infojson = readall(`rio info $fname`) 153 | JSON.parse(infojson) 154 | end 155 | 156 | "SortingAlgorithms.jl only has uint_mapping(o::Perm, i::Int)" 157 | SortingAlgorithms.uint_mapping(o::Base.Order.Perm{Base.Order.ForwardOrdering,Array{UInt32,1}}, i::UInt32) = SortingAlgorithms.uint_mapping(o.order, o.data[i]) 158 | 159 | # "Returns the fraction of NA cells in a DataArray" 160 | # fraction_na(da::DataArray) = sum(da.na) / length(da.na) 161 | 162 | function driver_list() 163 | driverlist = String[] 164 | for i = 0:(GDAL.getdrivercount() - 1) 165 | driver = GDAL.getdriver(i) 166 | if driver != C_NULL 167 | push!(driverlist, GDAL.getdrivershortname(driver)) 168 | end 169 | end 170 | return driverlist 171 | end 172 | 173 | "check if a point falls outside a bounding box" 174 | function outside(x_min, y_min, x_max, y_max, x, y) 175 | x < x_min || y < y_min || x > x_max || y > y_max 176 | end 177 | 178 | "check if a point falls outside a bounding box" 179 | function outside(bbox, x, y) 180 | x < bbox[1] || y < bbox[2] || x > bbox[3] || y > bbox[4] 181 | end 182 | 183 | "Indexing with points into a GDAL RasterBand, use like this: band[x, y]" 184 | function pointsample(band::Ptr{GDAL.GDALRasterBandH}, geotransform::Vector{Float64}, x::AbstractFloat, y::AbstractFloat) 185 | x_min = geotransform[1] 186 | x_res = geotransform[2] 187 | y_max = geotransform[4] 188 | y_res = geotransform[6] 189 | 190 | gdt = GDAL.getrasterdatatype(band) 191 | T = type_lookup[gdt] 192 | valref = Ref(zero(T)) # black 193 | 194 | nxoff = Int(fld(x - x_min, x_res)) 195 | nyoff = Int(fld(y - y_max, y_res)) 196 | 197 | GDAL.rasterio(band, GDAL.GF_Read, nxoff, nyoff, 1, 1, 198 | valref, 1, 1, gdt, 0, 0) 199 | valref[] 200 | end 201 | 202 | "Indexing with points into a GDAL RasterBand, use like this: band[x, y]" 203 | function pointsample(T::DataType, band::Ptr{GDAL.GDALRasterBandH}, geotransform::Vector{Float64}, x::AbstractFloat, y::AbstractFloat) 204 | x_min = geotransform[1] 205 | x_res = geotransform[2] 206 | y_max = geotransform[4] 207 | y_res = geotransform[6] 208 | 209 | gdt = gdt_lookup[T] 210 | valref = Ref(zero(T)) # black 211 | 212 | nxoff = Int(fld(x - x_min, x_res)) 213 | nyoff = Int(fld(y - y_max, y_res)) 214 | 215 | GDAL.rasterio(band, GDAL.GF_Read, nxoff, nyoff, 1, 1, 216 | valref, 1, 1, gdt, 0, 0) 217 | valref[] 218 | end 219 | 220 | "Get the data, origin" 221 | function read_raster(fname::String) 222 | GDAL.allregister() 223 | dataset = GDAL.open(fname, GDAL.GA_ReadOnly) 224 | band = GDAL.getrasterband(dataset, 1) # supports single band only 225 | nodata = GDAL.getrasternodatavalue(band, C_NULL) 226 | 227 | # initialize array to read in with correct dimensions and datatype 228 | xsize = GDAL.getrasterxsize(dataset) 229 | ysize = GDAL.getrasterysize(dataset) 230 | gdt = GDAL.getrasterdatatype(band) 231 | arrtype = type_lookup[gdt] 232 | A = zeros(arrtype, xsize, ysize) # dimensions reversed such that we can transpose it back 233 | # read complete band 234 | GDAL.rasterio(band, GDAL.GF_Read, 0, 0, size(A,1), size(A,2), 235 | A, size(A,1), size(A,2), gdt, 0, 0) 236 | 237 | transform = zeros(6) 238 | GDAL.getgeotransform(dataset, transform) 239 | x_min = transform[1] 240 | y_max = transform[4] 241 | cellsize = transform[2] 242 | 243 | GDAL.close(dataset) 244 | GDAL.destroydrivermanager() 245 | 246 | Float64.(A'), x_min, y_max, cellsize, nodata, transform 247 | end 248 | 249 | "Create a GDAL raster dataset" 250 | function create_raster( 251 | fname::AbstractString, 252 | A::Matrix{T}, 253 | x_min::Real, 254 | y_max::Real, 255 | cellsize::Real, 256 | gdaldriver::Ptr{GDAL.GDALDriverH}; 257 | epsg::Nullable{Int}=Nullable{Int}(), 258 | nodata::Union{Real, Void}=nothing) where T 259 | 260 | bandcount = 1 # this function supports only 1 band rasters 261 | gdt = gdt_lookup[T] 262 | 263 | # Set compression options for GeoTIFFs 264 | if gdaldriver == GDAL.getdriverbyname("GTiff") 265 | options = ["COMPRESS=DEFLATE","TILED=YES"] 266 | else 267 | options = String[] 268 | end 269 | 270 | dstdataset = GDAL.create(gdaldriver, fname, size(A,2), size(A,1), bandcount, gdt, options) 271 | 272 | transform = Float64[x_min, cellsize, 0.0, y_max, 0.0, -cellsize] 273 | GDAL.setgeotransform(dstdataset,transform) 274 | 275 | projection = epsg2wkt(epsg) 276 | GDAL.setprojection(dstdataset, projection) 277 | 278 | dstband = GDAL.getrasterband(dstdataset,1) 279 | GDAL.rasterio(dstband, GDAL.GF_Write, 0, 0, size(A,2), size(A,1), 280 | A', size(A,2), size(A,1), gdt, 0, 0) 281 | 282 | nodata == nothing || GDAL.setrasternodatavalue(dstband, nodata) 283 | dstdataset 284 | end 285 | 286 | "Write an array to a geospatial GDAL raster" 287 | function write_raster(fname::String, A::Matrix{T}, x_min::Real, y_max::Real, cellsize::Real; 288 | epsg::Nullable{Int}=Nullable{Int}(), 289 | driver::String="GTiff", 290 | nodata::Union{Real, Void}=nothing) where T 291 | 292 | # trying to catch some errors before entering unsafe GDAL territory 293 | # for safer alternatives look at ArchGDAL.jl 294 | dir = dirname(fname) 295 | if isfile(fname) 296 | rm(fname) 297 | elseif !isdir(dir) 298 | mkpath(dir) 299 | end 300 | 301 | GDAL.allregister() 302 | gdaldriver = GDAL.getdriverbyname(driver) 303 | 304 | if GDAL.getmetadataitem(gdaldriver, "DCAP_CREATE", "") == "YES" 305 | ds = create_raster(fname, A, x_min, y_max, cellsize, gdaldriver; epsg=epsg, nodata=nodata) 306 | GDAL.close(ds) 307 | elseif GDAL.getmetadataitem(driver, "DCAP_CREATECOPY", "") == "YES" 308 | memdriver = GDAL.getdriverbyname("MEM") 309 | dsmem = create_raster(fname, A, x_min, y_max, cellsize, memdriver; epsg=epsg, nodata=nodata) 310 | progressfunc = convert(Ptr{GDAL.GDALProgressFunc}, C_NULL) 311 | ds = GDAL.createcopy(gdaldriver, fname, dsmem, 0, C_NULL, progressfunc, C_NULL) 312 | GDAL.close(dsmem) 313 | GDAL.close(ds) 314 | else 315 | throw(DomainError("GDAL driver $driver does not support CREATE or CREATECOPY")) 316 | end 317 | 318 | GDAL.destroydrivermanager() 319 | nothing 320 | end 321 | 322 | "Write an array to a geospatial GDAL raster" 323 | function write_raster(fname::String, A::Matrix{T}, x_min::Real, y_max::Real, 324 | cellsize::Real, epsg::Int; 325 | kwargs...) where T 326 | write_raster(fname, A, x_min, y_max, cellsize; epsg=Nullable{Int}(epsg), kwargs...) 327 | end 328 | 329 | # function write_raster(fname::String, A::DataMatrix{T}, x_min::Real, y_max::Real, cellsize::Real; 330 | # nodata::Union{Real, Void}=nothing, 331 | # kwargs...) where T 332 | # ndval = nodata == nothing ? 0 : nodata 333 | # # convert missing data to a normal Matrix 334 | # B = convert(Matrix, A, ndval) 335 | # write_raster(fname, B, x_min, y_max, cellsize; nodata=ndval, kwargs...) 336 | # end 337 | 338 | end 339 | -------------------------------------------------------------------------------- /modules/XYZ/src/writers.jl: -------------------------------------------------------------------------------- 1 | "Decompose a ColorTypes.RGB{N0f16} into red, green and blue color channels" 2 | rgb_components(c::RGB{N0f16}) = red(c), green(c), blue(c) 3 | 4 | "write tiffile from Cloud" 5 | function to_tif( 6 | outDir::String, 7 | filenames::Vector{String}, 8 | cloud::Cloud, 9 | cellsize::Real, 10 | epsg::Int; 11 | reducer = minz, 12 | pointfilter = nothing, 13 | min_dens = 0, 14 | nodata = -9999, 15 | return_density = false) 16 | 17 | # create grid definition 18 | r = define_raster(cloud, cellsize; epsg=epsg, pointfilter = pointfilter) 19 | 20 | to_tif(outDir, filenames, cloud, r; 21 | reducer = reducer, pointfilter = nothing, # filter in define_raster cellsize 22 | min_dens = min_dens, nodata = nodata, return_density = return_density) 23 | end 24 | 25 | "write tiffile from Cloud with given raster definition" 26 | function to_tif(outDir::String, filenames::Vector{String}, 27 | cloud::Cloud, r::Raster; 28 | reducer = minz, pointfilter = nothing, 29 | min_dens = 0, nodata = -9999, return_density = false) 30 | 31 | # make raster from Cloud based on statistical summary per griddcell and point filter 32 | grid = rasterize(cloud, r; 33 | reducer = reducer, 34 | pointfilter = pointfilter, 35 | min_dens = min_dens, 36 | nodata = nodata, 37 | return_density = return_density) 38 | 39 | # write to file 40 | grid2tif(outDir, filenames, r, grid; 41 | nodata=nodata) 42 | end 43 | 44 | """util function to save a grid to tif using properties from raster defintion r 45 | for 2d and 3d (multiple layers) grids; 3d grids are written to multiple files""" 46 | function grid2tif(outDir::String, filenames::Vector{String}, r::Raster, grid::Array; nodata=-9999) 47 | ndim = ndims(grid) 48 | grid = copy(grid) 49 | if ndim == 3 50 | nlayers = size(grid, 3) 51 | for i in 1:nlayers 52 | fn = joinpath(outDir, filenames[i]) 53 | grid[:, :, i][r.mask] = nodata 54 | write_raster(fn, grid[:, :, i], r.bbox.xmin, r.bbox.ymax, r.cellsize; epsg=XYZ.epsg(r), nodata=nodata) 55 | end 56 | elseif ndim == 2 57 | fn = joinpath(outDir, filenames[1]) 58 | grid[r.mask] = nodata 59 | write_raster(fn, grid, r.bbox.xmin, r.bbox.ymax, r.cellsize; epsg=XYZ.epsg(r), nodata=nodata) 60 | end 61 | end 62 | "for 2d arrays" 63 | function grid2tif(outDir::String, filename::String, r::Raster, grid::Array; nodata=-9999) 64 | @assert (ndims(grid) == 2) "Invalid grid size, use a vector of filenames to save layers to multiple files" 65 | fn = joinpath(outDir, filename) 66 | grid = copy(grid) 67 | grid[r.mask] = nodata 68 | write_raster(fn, grid, r.bbox.xmin, r.bbox.ymax, r.cellsize; epsg=XYZ.epsg(r), nodata=nodata) 69 | end 70 | 71 | "merge two headers, rejecting incompatible headers, and favoring the first" 72 | function Base.merge(h1::LasHeader, h2::LasHeader) 73 | # check compatibility 74 | msg = "Cannot merge files, " 75 | h1.version_major === h2.version_major || error(msg * "LAS major version mismatch") 76 | h1.version_minor === h2.version_minor || error(msg * "LAS minor version mismatch") 77 | h1.data_format_id === h2.data_format_id || error(msg * "Point format mismatch") 78 | h1.data_format_id === h2.data_format_id || error(msg * "Point format mismatch") 79 | h1.data_record_length === h2.data_record_length || error(msg * "Point record length mismatch") 80 | 81 | # merge selected fields 82 | records_count = h1.records_count + h2.records_count 83 | point_return_count = h1.point_return_count + h2.point_return_count 84 | x_max = max(h1.x_max, h2.x_max) 85 | x_min = min(h1.x_min, h2.x_min) 86 | y_max = max(h1.y_max, h2.y_max) 87 | y_min = min(h1.y_min, h2.y_min) 88 | z_max = max(h1.z_max, h2.z_max) 89 | z_min = min(h1.z_min, h2.z_min) 90 | 91 | # Note that because this function is used for appending LAS files, the size of the header 92 | # must keep the same size. Therefore the VLRs are currently not extended, and neither are 93 | # the user defined bytes. 94 | # Furthermore it is assumed that the new coordinates will fit in with the existing h1 scale 95 | # and offset values. 96 | 97 | # create merged header 98 | LasHeader( 99 | h1.file_source_id, 100 | h1.global_encoding, 101 | h1.guid_1, 102 | h1.guid_2, 103 | h1.guid_3, 104 | h1.guid_4, 105 | h1.version_major, 106 | h1.version_minor, 107 | h1.system_id, 108 | h1.software_id, 109 | h1.creation_doy, 110 | h1.creation_year, 111 | h1.header_size, 112 | h1.data_offset, 113 | h1.n_vlr, 114 | h1.data_format_id, 115 | h1.data_record_length, 116 | records_count, 117 | point_return_count, 118 | h1.x_scale, 119 | h1.y_scale, 120 | h1.z_scale, 121 | h1.x_offset, 122 | h1.y_offset, 123 | h1.z_offset, 124 | x_max, 125 | x_min, 126 | y_max, 127 | y_min, 128 | z_max, 129 | z_min, 130 | h1.variable_length_records, # VLRs currently not extended, would need to be merged but not duplicated 131 | h1.user_defined_bytes 132 | ) 133 | end 134 | 135 | "write LAS points" 136 | function writepoints(io::IO, header::LasHeader, cloud::Cloud) 137 | n = length(positions(cloud)) 138 | pointtype = LasIO.pointformat(header) 139 | for i in 1:n 140 | lasp = laspoint(pointtype, cloud, i, header) 141 | write(io, lasp) 142 | end 143 | end 144 | 145 | "Construct a LasIO Vector{LasPoint} from a XYZ.Cloud" 146 | function lasio_vector(header::LasHeader, cloud::Cloud) 147 | n = length(positions(cloud)) 148 | pointtype = LasIO.pointformat(header) 149 | pointdata = Vector{pointtype}(n) 150 | for i in 1:n 151 | pointdata[i] = laspoint(pointtype, cloud, i, header) 152 | end 153 | pointdata 154 | end 155 | 156 | "write LAS/LAZ file from Cloud" 157 | function to_las(outDir::String, filename::String, 158 | cloud::Cloud, header::LasHeader) 159 | 160 | fn = joinpath(outDir, filename) 161 | 162 | header = deepcopy(header) # prevent unwanted side effects 163 | LasIO.update!(header, cloud) 164 | 165 | # converts to FileIO File LAS or LAZ 166 | # such that it dispatches to the right save 167 | fio = query(fn) 168 | save(fio, header, cloud) 169 | nothing 170 | end 171 | 172 | # Add save on Cloud be able to write LAS from Cloud without copies 173 | function save(f::File{format"LAS"}, header::LasHeader, cloud::Cloud) 174 | # convert File to String to get a normal IO object 175 | # that the other functions can use 176 | fn = filename(f) 177 | open(fn, "w") do io 178 | write(io, "LASF") 179 | write(io, header) 180 | writepoints(io, header, cloud) 181 | end 182 | end 183 | 184 | # Add save on Cloud be able to write LAZ from Cloud without copies 185 | function save(f::FileIO.File{FileIO.DataFormat{:LAZ}}, header::LasHeader, cloud::Cloud) 186 | # pipes las to laszip to write laz 187 | fn = filename(f) 188 | open(`laszip -olaz -stdin -o "$fn"`, "w") do s 189 | savebuf(s, header, cloud) 190 | end 191 | end 192 | 193 | # Add savebuf on Cloud be able to write LAZ from Cloud without copies 194 | function savebuf(s::IO, header::LasHeader, cloud::Cloud) 195 | # checks 196 | header_n = header.records_count 197 | n = length(cloud.positions) 198 | msg = "number of records in header ($header_n) does not match data length ($n)" 199 | @assert header_n == n msg 200 | pointtype = LasIO.pointformat(header) 201 | 202 | # write header 203 | write(s, magic(format"LAS")) 204 | write(s, header) 205 | 206 | # 2048 points seemed to be an optimum for the libLAS_1.2.las testfile 207 | npoints_buffered = 2048 208 | bufsize = header.data_record_length * npoints_buffered 209 | buf = IOBuffer(bufsize) 210 | # write points 211 | for i in 1:n 212 | p = laspoint(pointtype, cloud, i, header) 213 | write(buf, p) 214 | if rem(i, npoints_buffered) == 0 215 | write(s, take!(buf)) 216 | elseif i == n 217 | write(s, take!(buf)) 218 | end 219 | end 220 | end 221 | 222 | "general XYZ (csv) writer" 223 | function to_xyz(outDir::String, filename::String, cloud::Cloud; 224 | precision = 2, delimiter=",", 225 | attributes = Symbol[], 226 | pointfilter = nothing) 227 | 228 | fn = joinpath(outDir, filename) 229 | nrow = length(positions(cloud)) 230 | write_attributes = length(attributes) > 0 231 | header = xyz_header(attributes; delimiter=delimiter) 232 | 233 | # open file 234 | open(fn, "w") do f 235 | # write header 236 | write(f, header) 237 | 238 | # loop through points and write a line per point to file 239 | for i in 1:nrow 240 | (pointfilter != nothing) && (pointfilter(cloud, i) || continue) 241 | line = String[] 242 | # read and position point data and format 243 | for p in positions(cloud)[i] 244 | push!(line, format(p, precision=precision)) 245 | push!(line, delimiter) 246 | end 247 | # read and attribute point data and format 248 | if write_attributes 249 | for key in attributes 250 | if typeof(cloud[key][i]) <: Integer 251 | push!(line, format(Int(cloud[key][i]), precision=0)) 252 | else 253 | push!(line, format(cloud[key][i], precision=precision)) 254 | end 255 | push!(line, delimiter) 256 | end 257 | end 258 | 259 | # write formatted line 260 | write(f, string(join(line)[1:end-length(delimiter)], "\n")) 261 | end 262 | end # close file 263 | end 264 | 265 | "XYZ (csv) writer for points along profile. The coordinates in profile orientation 266 | are written in extra columns px and py to xyz file" 267 | function to_xyz(outDir::String, filename::String, cloud::Cloud, prof::Profile; 268 | precision = 2, delimiter=",", 269 | attributes = Symbol[], 270 | pointfilter = nothing) 271 | 272 | fn = joinpath(outDir, filename) 273 | nrow = length(prof) 274 | write_attributes = length(attributes) > 0 275 | header = [] 276 | header = xyz_header(cat(1, ["px", "py"], attributes); delimiter=delimiter) 277 | 278 | # open file 279 | open(fn, "w") do f 280 | # write header 281 | write(f, header) 282 | 283 | # loop through points and write a line per point to file 284 | ip = 0 285 | for i in pointindex(prof) 286 | ip += 1 287 | (pointfilter != nothing) && (pointfilter(cloud, i) || continue) 288 | line = String[] # create vector with formated point data 289 | # read profile coordinates and format 290 | # read and position point data and format 291 | for p in positions(cloud)[i] 292 | push!(line, format(p, precision=precision)) 293 | push!(line, delimiter) 294 | end 295 | pos = Float64[pointlx(prof)[ip], pointly(prof)[ip]] 296 | for p in pos 297 | push!(line, format(p, precision=precision)) 298 | push!(line, delimiter) 299 | end 300 | # read and attribute point data and format 301 | if write_attributes 302 | for key in attributes 303 | if typeof(cloud[key][i]) <: Integer 304 | push!(line, format(Int(cloud[key][i]), precision=0)) 305 | else 306 | push!(line, format(cloud[key][i], precision=precision)) 307 | end 308 | push!(line, delimiter) 309 | end 310 | end 311 | 312 | # write formatted line 313 | write(f, string(join(line)[1:end-length(delimiter)], "\n")) 314 | end 315 | end # close file 316 | end 317 | 318 | "create header for xyz files" 319 | function xyz_header(attributes::Vector{}; delimiter=",") 320 | header = ["x,","y,","z,"] 321 | if length(attributes) > 0 322 | for key in attributes 323 | push!(header, string(key)) 324 | push!(header, delimiter) 325 | end 326 | end 327 | string(join(header)[1:end-length(delimiter)], "\n") 328 | end 329 | -------------------------------------------------------------------------------- /modules/XYZ/src/cloudutils.jl: -------------------------------------------------------------------------------- 1 | using StatsBase 2 | 3 | # get 2D xy array from pointcloud 4 | "read xy from Cloud" 5 | function xy_array(cloud::Cloud, index::AbstractVector) 6 | n = length(index) 7 | XY = zeros(2, n) 8 | for i = 1:n 9 | p = positions(cloud)[index[i]] 10 | XY[1, i] = p[1] 11 | XY[2, i] = p[2] 12 | end 13 | XY 14 | end 15 | xy_array(cloud::Cloud) = xy_array(cloud, 1:length(cloud)) 16 | 17 | # get arrays of x/y/z coordinates 18 | getx(cloud::Cloud, index::Vector{U}) where U <: Integer = getindex.(positions(cloud)[index], 1) 19 | getx(cloud::Cloud, index::Integer) = positions(cloud)[index][1] 20 | getx(cloud::Cloud) = getindex.(positions(cloud), 1) 21 | gety(cloud::Cloud, index::Vector{U}) where U <: Integer = getindex.(positions(cloud)[index], 2) 22 | gety(cloud::Cloud, index::Integer) = positions(cloud)[index][2] 23 | gety(cloud::Cloud) = getindex.(positions(cloud), 2) 24 | getz(cloud::Cloud, index::Vector{U}) where U <: Integer = getindex.(positions(cloud)[index], 3) 25 | getz(cloud::Cloud, index::Integer) = positions(cloud)[index][3] 26 | getz(cloud::Cloud) = getindex.(positions(cloud), 3) 27 | 28 | # set arrays of x/y/z coordinates 29 | setx!(cloud::Cloud, index::Vector{U}, value::Vector{Float64}) where U <: Integer = setindex!.(positions(cloud)[index], value, 1) 30 | sety!(cloud::Cloud, index::Vector{U}, value::Vector{Float64}) where U <: Integer = setindex!.(positions(cloud)[index], value, 2) 31 | setz!(cloud::Cloud, index::Vector{U}, value::Vector{Float64}) where U <: Integer = setindex!.(positions(cloud)[index], value, 3) 32 | setx!(cloud::Cloud, index::Integer, value::Float64) = setindex!(positions(cloud)[index], value, 1) 33 | sety!(cloud::Cloud, index::Integer, value::Float64) = setindex!(positions(cloud)[index], value, 2) 34 | setz!(cloud::Cloud, index::Integer, value::Float64) = setindex!(positions(cloud)[index], value, 3) 35 | 36 | # normalize z from given z value 37 | function normalizez!(cloud::Cloud, index::Vector{U}, value::Float64) where U <: Integer 38 | z = getz(cloud, index) 39 | setz!(cloud, index, z - value) 40 | end 41 | 42 | # statistical functions 43 | reducer_minz(cloud::Cloud, index::Vector{U}) where U <: Integer = minimum(getz(cloud, index)) 44 | reducer_maxz(cloud::Cloud, index::Vector{U}) where U <: Integer = maximum(getz(cloud, index)) 45 | reducer_medz(cloud::Cloud, index::Vector{U}) where U <: Integer = median(getz(cloud, index)) 46 | reducer_meanz(cloud::Cloud, index::Vector{U}) where U <: Integer = mean(getz(cloud, index)) 47 | reducer_madz(cloud::Cloud, index::Vector{U}) where U <: Integer = mad(getz(cloud, index), normalize=false) 48 | reducer_stdz(cloud::Cloud, index::Vector{U}) where U <: Integer = std(getz(cloud, index)) 49 | reducer_99pz(cloud::Cloud, index::Vector{U}) where U <: Integer = maximum(trim(getz(cloud, index), prop=0.01)) 50 | 51 | # reducer for vegetation histogram splits 52 | function reducer_veg_below_2m(cloud::Cloud, index::Vector{U}) where U <: Integer 53 | heights = getz(cloud, index) 54 | below = heights .< 2. 55 | return sum(below) / length(heights) * 100. 56 | end 57 | function reducer_veg_below_5m(cloud::Cloud, index::Vector{U}) where U <: Integer 58 | heights = getz(cloud, index) 59 | below = heights .< 5. 60 | return sum(below) / length(heights) * 100. 61 | end 62 | function reducer_veg_above_25m(cloud::Cloud, index::Vector{U}) where U <: Integer 63 | heights = getz(cloud, index) 64 | above = heights .> 25. 65 | return sum(above) / length(heights) * 100. 66 | end 67 | function reducer_veg_above_20m(cloud::Cloud, index::Vector{U}) where U <: Integer 68 | heights = getz(cloud, index) 69 | above = heights .> 20. 70 | return sum(above) / length(heights) * 100. 71 | end 72 | 73 | function reducer_ranz(cloud::Cloud, index::Vector{U}) where U <: Integer 74 | z = getz(cloud, index) 75 | maximum(z) - minimum(z) 76 | end 77 | 78 | function reducer_pctz(cloud::Cloud, index::Vector{U}) where U <: Integer 79 | quantile(getz(cloud, index), [0.1, 0.25, 0.5, 0.75, 0.9]) 80 | end 81 | 82 | # reducer function to count number of points per cell 83 | reducer_count(cloud::XYZ.Cloud, index::Vector{U}) where U <: Integer = length(index) 84 | reducer_dens = reducer_count # backwards compatibility 85 | 86 | # reducer functions for any point attribute 87 | function reducer_min_attr(field::Symbol) 88 | reducer(cloud::XYZ.Cloud, index::Vector{U}) where U <: Integer = minimum(cloud[field][index]) 89 | end 90 | function reducer_max_attr(field::Symbol) 91 | reducer(cloud::XYZ.Cloud, index::Vector{U}) where U <: Integer = maximum(cloud[field][index]) 92 | end 93 | function reducer_med_attr(field::Symbol) 94 | reducer(cloud::XYZ.Cloud, index::Vector{U}) where U <: Integer = median(cloud[field][index]) 95 | end 96 | function reducer_min_abs_attr(field::Symbol) 97 | reducer(cloud::XYZ.Cloud, index::Vector{U}) where U <: Integer = minimum(abs.(cloud[field][index])) 98 | end 99 | 100 | # reducer function to get any attribute of the lowest or highest point 101 | function reducer_lowestpoint_attr(field::Symbol) 102 | reducer(cloud::XYZ.Cloud, index::Vector{U}) where U <: Integer = cloud[field][index[indmin(getz(cloud, index))]] 103 | end 104 | function reducer_highestpoint_attr(field::Symbol) 105 | reducer(cloud::XYZ.Cloud, index::Vector{U}) where U <: Integer = cloud[field][index[indmax(getz(cloud, index))]] 106 | end 107 | 108 | # reduce_index, returns local(!) index of minimum / maximum 109 | reducer_minz_index(cloud::Cloud, index::Vector{U}) where U <: Integer = indmin(getz(cloud, index)) 110 | reducer_maxz_index(cloud::Cloud, index::Vector{U}) where U <: Integer = indmax(getz(cloud, index)) 111 | 112 | 113 | const class_description = Dict{Int, String}( 114 | 0 => "Created, never classified", 115 | 1 => "Unclassified", 116 | 2 => "Ground", 117 | 3 => "Low Vegetation", 118 | 4 => "Medium Vegetation", 119 | 5 => "High Vegetation", 120 | 6 => "Building", 121 | 7 => "Low Point (noise)", 122 | 8 => "Model Key-point (mass point)", 123 | 9 => "Water", 124 | 12 => "Overlap Points", 125 | 126 | 18 => "High noise" # LAS 1.4 127 | ) 128 | 129 | "Describe a `Cloud` with a number of statistics" 130 | function describe(io::IO, cloud::Cloud) 131 | classes = sort!(unique(cloud[:classification])) 132 | quantvals = [0.0, 0.1, 0.25, 0.5, 0.75, 0.9, 1.0] 133 | print(io, "quantiles,") 134 | join(io, quantvals, ",") 135 | println(io, "") 136 | for class in classes 137 | idxs = cloud[:classification] .== class 138 | println(io, " class, $class, $(class_description[class])") 139 | print(io, " elevation,") 140 | join(io, quantile(getz(cloud)[idxs], quantvals), ",") 141 | print(io, "\n intensity,") 142 | join(io, quantile(cloud[:intensity][idxs], quantvals), ",") 143 | print(io, "\n") 144 | end 145 | end 146 | 147 | function describe(filepath::String, cloud::Cloud) 148 | open(filepath, "w") do io 149 | describe(io, cloud) 150 | end 151 | end 152 | 153 | "Paint a Cloud efficiently with a Raster index and RGB bands from an orthophoto. 154 | Does not load the entire orthophoto into memory, but one block at a time." 155 | function paint!(pc::Cloud, r::Raster, 156 | rband::Ptr{GDAL.GDALRasterBandH}, 157 | gband::Ptr{GDAL.GDALRasterBandH}, 158 | bband::Ptr{GDAL.GDALRasterBandH}) 159 | 160 | # 257 would be a full mapping, but 256 is in the standard, 161 | # see https://groups.google.com/d/msg/lasroom/Th2_-ywc2q8/4JRajnxxO0wJ 162 | const colorscale = 0x0100 # = 256 163 | # assume all bands have same block sizes and data types 164 | @assert GDAL.getrasterdatatype(rband) === GDAL.GDT_Byte 165 | ncolblock, nrowblock = Ref(Cint(-1)), Ref(Cint(-1)) 166 | GDAL.getblocksize(rband, ncolblock, nrowblock) # size of single blocks 167 | ncolblock, nrowblock = ncolblock[], nrowblock[] # get values from Ref 168 | ncol = GDAL.getrasterbandxsize(rband) 169 | nrow = GDAL.getrasterbandysize(rband) 170 | # number of blocks in the x and y direction 171 | nblockx = cld(ncol, ncolblock) 172 | nblocky = cld(nrow, nrowblock) 173 | # number of rows ans columns in the last block of every row or column 174 | ncollastblock = rem(ncol, ncolblock) == 0 ? ncolblock : rem(ncol, ncolblock) 175 | nrowlastblock = rem(nrow, nrowblock) == 0 ? nrowblock : rem(nrow, nrowblock) 176 | # asserted it is GDAL.GDT_Byte 177 | # switched dimensions instead of transposing 178 | rblock = zeros(UInt8, ncolblock, nrowblock) 179 | gblock = zeros(UInt8, ncolblock, nrowblock) 180 | bblock = zeros(UInt8, ncolblock, nrowblock) 181 | # iterate over blocks 182 | for blockrow = 1:nblocky 183 | for blockcol = 1:nblockx 184 | nxblockoff = blockcol-1 185 | nyblockoff = blockrow-1 186 | # possible optimization: only read blocks that contain points? 187 | GDAL.readblock(rband, nxblockoff, nyblockoff, rblock) 188 | GDAL.readblock(gband, nxblockoff, nyblockoff, gblock) 189 | GDAL.readblock(bband, nxblockoff, nyblockoff, bblock) 190 | nrowthisblock = blockrow == nblocky ? nrowlastblock : nrowblock 191 | ncolthisblock = blockcol == nblockx ? ncollastblock : ncolblock 192 | # calculate the global indices for Cloud Raster index 193 | startrow = (blockrow - 1) * nrowblock 194 | startcol = (blockcol - 1) * ncolblock 195 | # iterate over block cells 196 | for i in 1:nrowthisblock 197 | for j in 1:ncolthisblock 198 | row = startrow + i 199 | col = startcol + j 200 | idxs = r[row,col] 201 | isempty(idxs) && continue # big speedup if many pixels don't have points 202 | color = RGB{N0f16}( 203 | reinterpret(N0f16, rblock[j,i] * colorscale), 204 | reinterpret(N0f16, gblock[j,i] * colorscale), 205 | reinterpret(N0f16, bblock[j,i] * colorscale)) 206 | pc[:color][idxs] = color 207 | end 208 | end 209 | end 210 | end 211 | nothing 212 | end 213 | 214 | function paint!(pc::Cloud, orthofile::AbstractString) 215 | # GDAL setup 216 | GDAL.allregister() 217 | ds = GDAL.open(orthofile, GDAL.GA_ReadOnly) 218 | geotransform = zeros(6) 219 | GDAL.getgeotransform(ds, geotransform) 220 | epsg = wkt2epsg(GDAL.getprojectionref(ds)) 221 | rband = GDAL.getrasterband(ds, 1) 222 | gband = GDAL.getrasterband(ds, 2) 223 | bband = GDAL.getrasterband(ds, 3) 224 | ncol = Int(GDAL.getrasterbandxsize(rband)) 225 | nrow = Int(GDAL.getrasterbandysize(rband)) 226 | 227 | # create raster index that is guaranteed to align with the orthophoto pixels 228 | r = define_raster(pc, nrow, ncol, geotransform, epsg=epsg, pointfilter=nothing) 229 | paint!(pc, r, rband, gband, bband) 230 | GDAL.close(ds) 231 | nothing 232 | end 233 | 234 | function create_kdtree(cloud::Cloud; 235 | dim=2, 236 | metric=Euclidean(), 237 | index = 1:length(cloud)) 238 | 239 | # create tree 240 | if dim == 2 241 | tree = KDTree(xy_array(cloud, index), metric) 242 | elseif dim == 3 243 | tree = KDTree(positions(cloud)[index], metric) 244 | end 245 | 246 | tree 247 | end 248 | 249 | function point_density!(cloud::Cloud, spatialindex::NearestNeighbors.NNTree; 250 | range=(1.0 / (4/3.0 * pi ))^(1.0/3), 251 | pointfilter=nothing) # vol = 1m3 252 | @assert (spatialindex.metric == Euclidean()) && (length(spatialindex.data[1]) == 3) 253 | n = length(cloud) 254 | cloud[:point_density] = zeros(Float32, n) 255 | vol = 4/3.0 * pi * range^3 256 | 257 | # loop through points and calculate point density 258 | @showprogress 5 "looping through las points..." for i in 1:n 259 | (pointfilter != nothing) && (pointfilter(cloud, i) || continue) 260 | # find number of points in range 261 | neighbors = inrange(spatialindex, positions(cloud)[i], range) 262 | # set point source attribute to visualize no. of neighbors 263 | cloud[:point_density][i] = length(neighbors) / vol 264 | end 265 | 266 | nothing 267 | end 268 | point_density!(cloud::Cloud; range=(1.0 / (4/3.0 * pi ))^(1.0/3)) = point_density!(cloud, create_kdtree(cloud; dim=3); range=range) 269 | 270 | unique_id!(cloud::Cloud) = cloud[:id] = 1:length(cloud); 271 | 272 | """copy classification based on unique id, 273 | assumes the field 'id' in cloud copy_from has the index of the same point in cloud copy_to""" 274 | function copy_attributes!(cloud_copy_to::Cloud, cloud_copy_from::Cloud; field=:classification) 275 | @assert haskey(cloud_copy_from.attributes, :id) "both point clouds should have an 'id' field" 276 | @assert haskey(cloud_copy_from.attributes, field) "the 'copy_from' point cloud does not have a field $(field)" 277 | 278 | n = length(cloud_copy_to) 279 | if !haskey(cloud_copy_to.attributes, field) 280 | cloud_copy_to[field] = zeros(eltype(cloud_copy_from[field]), n) 281 | end 282 | 283 | # copy attribute valeu back to original cloud 284 | for (idx, v) in zip(cloud_copy_from[:id], cloud_copy_from[field]) 285 | cloud_copy_to[field][idx] = v 286 | end 287 | nothing 288 | end 289 | 290 | """project the values of grid to point in pointcloud based on a raster index""" 291 | function grid2cloud_attribute!(cloud::Cloud, grid::Array{T,2}, r::Raster, name::Symbol) where T <: Real 292 | @assert size(grid) == (r.nrow, r.ncol) "raster definition 'r' should have same size as 'grid'" 293 | cloud[name] = zeros(T,length(cloud)) 294 | for icell in 1:length(r) 295 | v = grid[icell] 296 | for i in r[icell] 297 | cloud[name][i] = v 298 | end 299 | end 300 | nothing 301 | end 302 | -------------------------------------------------------------------------------- /modules/XYZ/src/cloudclassifier.jl: -------------------------------------------------------------------------------- 1 | #= 2 | Classify Clouds based on attributes and surrounding points 3 | Format should be according to: 4 | 5 | classify!cloud::Cloud; kwargs) 6 | cloud' 7 | end 8 | 9 | =# 10 | 11 | ## BASICS 12 | " classification of point in Cloud" 13 | function classify!(cloud::Cloud, i::Integer, class::UInt8) 14 | cloud[:classification][i] = class 15 | nothing 16 | end 17 | 18 | 19 | " classification of of Cloud based on pointfilter" 20 | function classify!(cloud::Cloud, pointfilter, class::UInt8) 21 | for i in 1:length(cloud) 22 | (pointfilter(cloud, i) || continue) 23 | cloud[:classification][i] = class 24 | end 25 | nothing 26 | end 27 | 28 | "set classification of a point to zero" 29 | function reset_class!(cloud::Cloud, i::Integer) 30 | classify!(cloud, i, UInt8(0)) 31 | nothing 32 | end 33 | 34 | "set classification of all points in Cloud to zero" 35 | function reset_class!(cloud::Cloud; 36 | pointfilter = nothing) 37 | 38 | for i in 1:length(cloud) 39 | (pointfilter != nothing) && (pointfilter(cloud, i) || continue) 40 | classify!(cloud, i, UInt8(0)) 41 | end 42 | nothing 43 | end 44 | 45 | ## Raster based 46 | "function to classify minima in pointcloud based per gridcell" 47 | function classify_min!(cloud::Cloud, cellsize::Real, class::Int; 48 | pointfilter = nothing, # predicate function to filter individual points 49 | min_dens = 0) # minimum point density in cell to consider for educed cloud 50 | 51 | r = define_raster(cloud, cellsize; pointfilter = pointfilter) 52 | classify_min!(cloud, r, class; pointfilter = pointfilter, min_dens = min_dens) 53 | end 54 | 55 | "function to classify minima in pointcloud based per gridcell" 56 | function classify_min!(cloud::Cloud, r::Raster, class::Int; 57 | pointfilter = nothing, # predicate function to filter individual points 58 | min_dens = 0) # minimum point density in cell to consider for educed cloud 59 | 60 | subset = reduce_index(cloud, r; reduceri = reducer_minz_index, pointfilter = pointfilter, min_dens = min_dens) 61 | 62 | # return raster with statistics, density is saved to last layer 63 | for i in subset 64 | classify!(cloud, i, UInt8(class)) 65 | end 66 | 67 | nothing 68 | end 69 | 70 | "function to classify maxima in pointcloud based per gridcell" 71 | function classify_max!(cloud::Cloud, cellsize::Real, class::Int; 72 | pointfilter = nothing, # predicate function to filter individual points 73 | min_dens = 0) # minimum point density in cell to consider for educed cloud 74 | 75 | r = define_raster(cloud, cellsize; pointfilter = pointfilter) 76 | classify_max!(cloud, r, class; pointfilter = pointfilter, min_dens = min_dens) 77 | end 78 | 79 | "function to classify maxima in pointcloud based per gridcell" 80 | function classify_max!(cloud::Cloud, r::Raster, class::Int; 81 | pointfilter = nothing, # predicate function to filter individual points 82 | min_dens = 0) # minimum point density in cell to consider for educed cloud 83 | 84 | subset = reduce_index(cloud, r; reduceri = reducer_maxz_index, pointfilter = pointfilter, min_dens = min_dens) 85 | 86 | # return raster with statistics, density is saved to last layer 87 | for i in subset 88 | classify!(cloud, i, UInt8(class)) 89 | end 90 | 91 | nothing 92 | end 93 | 94 | "classifies points with a cell if z <= to the z-value in surface + tolerance" 95 | function classify_below_surface!( 96 | cloud::Cloud, r::Raster, surf::AbstractArray{T, 2}, class::Integer; 97 | tolerance = 0.0, 98 | pointfilter = nothing, 99 | nodata = -9999.0) where T <: Real 100 | 101 | ((size(r)[1] == size(surf)[1]) && (size(r)[2] == size(surf)[2])) || error("raster and surface size mismatch") 102 | for icell in 1:length(r) 103 | z_max = surf[icell] + tolerance 104 | z_max == nodata && continue 105 | for i in r[icell] # indices Cloud 106 | positions(cloud)[i][3] <= z_max || continue 107 | (pointfilter != nothing) && (pointfilter(cloud, i) || continue) 108 | classify!(cloud, i, UInt8(class)) 109 | end 110 | end 111 | 112 | nothing 113 | end 114 | 115 | function classify_below_surface!(cloud::Cloud, surf::Real, class::Integer; 116 | tolerance = 0.0, 117 | pointfilter = nothing) 118 | 119 | z_max = surf + tolerance 120 | for i in 1:length(position(cloud)) # indices Cloud 121 | positions(cloud)[i][3] <= z_max || continue 122 | (pointfilter != nothing) && (pointfilter(cloud, i) || continue) 123 | classify!(cloud, i, UInt8(class)) 124 | end 125 | 126 | nothing 127 | end 128 | 129 | "classifies points with a cell if z > to the z-value in surface + buffer" 130 | function classify_above_surface!(cloud::Cloud, r::Raster, 131 | surf::AbstractArray{T, 2}, class::Integer; 132 | buffer = 0.0, 133 | pointfilter = nothing, 134 | nodata = -9999.0) where T <: Real 135 | 136 | ((size(r)[1] == size(surf)[1]) && (size(r)[2] == size(surf)[2])) || error("raster and surface size mismatch") 137 | for icell in 1:length(r) 138 | z_min = surf[icell] + buffer 139 | z_min == nodata && continue 140 | for i in r[icell] # indices Cloud 141 | positions(cloud)[i][3] > z_min || continue 142 | (pointfilter != nothing) && (pointfilter(cloud, i) || continue) 143 | classify!(cloud, i, UInt8(class)) 144 | end 145 | end 146 | 147 | nothing 148 | end 149 | 150 | "classify points outside of raster mask where true indicates data not to classify" 151 | function classify_mask!(cloud::Cloud, r::Raster, mask::Union{BitArray, Array{Bool}}, class::Integer) 152 | size(r) == size(mask) || error("raster and mask size mismatch") 153 | for icell in 1:length(r) 154 | mask[icell] && continue # skip points inside mask 155 | for i in r[icell] # pointcloud indices 156 | classify!(cloud, i, UInt8(class)) 157 | end 158 | end 159 | nothing 160 | end 161 | 162 | function classify_above_surface!(cloud::Cloud, surf::Real, class::Integer; 163 | buffer = 0.0, 164 | pointfilter = nothing) 165 | 166 | z_min = surf + buffer 167 | for i in 1:length(position(cloud)) # indices Cloud 168 | positions(cloud)[i][3] > z_min || continue 169 | (pointfilter != nothing) && (pointfilter(cloud, i) || continue) 170 | classify!(cloud, i, UInt8(class)) 171 | end 172 | 173 | nothing 174 | end 175 | 176 | ## Slope based 177 | """Adapted from Vosselman (2000) 178 | slope based classification of pointdata from point cloud""" 179 | function vosselman(cloud::Cloud, i::Integer, neighbors::Vector{U}; 180 | max_slope=0.2, 181 | tolerance=0.0) where U <: Integer 182 | 183 | if length(neighbors) > 1 184 | # get positions point i 185 | xi, yi, zi = positions(cloud)[i] 186 | 187 | # get z values 188 | z_vec = getz(cloud, neighbors) # z only 189 | idx_order = sortperm(z_vec) # loop through points from min to max elevation 190 | 191 | # initiate 192 | isclass = false 193 | for j in neighbors[idx_order] 194 | cloud[:classification][j] == 7 && continue # skip comparison with low outlier 195 | # 1) get horizontal and vertical distance 196 | xj, yj, zj = positions(cloud)[j] 197 | di = hypot(xi - xj, yi - yj) 198 | dz = zj - zi + tolerance 199 | # 2) calculate max z difference based on distance 200 | thresh = -1 * di * max_slope 201 | # 3) compare with actual z diff; if z diff too large -> no ground point 202 | dz < thresh && break 203 | # 4) classify if all points below point i have been checked without break 204 | if dz > 0 205 | isclass = true 206 | break 207 | end 208 | end 209 | else 210 | isclass = true # no neighbors 211 | end 212 | 213 | isclass 214 | end 215 | 216 | ## applied 217 | "classify outliers" 218 | function classify_outliers!(cloud::Cloud; 219 | cellsize=100.0, # cellsize size for regural grid 220 | dz=1.0, # min distance for outlier classification 221 | max_outliers=15, # max number of outlier per cell 222 | max_height=Inf) # maximum heigh above lowest non outlier in cell 223 | 224 | low_noise = UInt8(7) # from ASPRS Standard LIDAR Point Classes 225 | high_noise = UInt8(18) # from ASPRS Standard LIDAR Point Classes (LAS 1.4) 226 | do_high_noise = !isinf(max_height) 227 | # all points below -5 m are always set to low noise 228 | const cutoff_low = -5.0 229 | too_low(cloud::Cloud, i::Integer) = getz(cloud, i) < cutoff_low 230 | classify!(cloud, too_low, low_noise) 231 | # all points above 100 m are always set to high noise 232 | const cutoff_high = 100.0 233 | too_high(cloud::Cloud, i::Integer) = getz(cloud, i) > cutoff_high 234 | classify!(cloud, too_high, high_noise) 235 | # points outside the cutoff should not be included in the outlier analysis 236 | not_cut_off(cloud::Cloud, i::Integer) = cutoff_low <= getz(cloud, i) <= cutoff_high 237 | r = define_raster(cloud, cellsize, pointfilter=not_cut_off) 238 | # create dz Vector with length of pointcloud to store as new attribute later 239 | dz_outliers = zeros(length(positions(cloud))) 240 | # loop through cells 241 | @showprogress 5 "Classifying outliers.." for icell in 1:length(r) 242 | # find points in range 243 | idxs = r[icell] # get points in rastercell 244 | np = length(idxs) 245 | np < (max_outliers*10) && continue # max 10% outliers. make sure is in balance with 246 | 247 | # get z values of points in cell 248 | z_vec = getz(cloud, idxs) 249 | if np >= 1e5 # RadixSort only faster with many points 250 | idx_order = sortperm(z_vec, alg=RadixSort) 251 | else 252 | idx_order = sortperm(z_vec) 253 | end 254 | 255 | # get difference in z for lowest max_outliers+1 points 256 | dz_vec = z_vec[idx_order[2:max_outliers+1]] - z_vec[idx_order[1:max_outliers]] 257 | if sum(dz_vec .>= dz) == 0 && !do_high_noise 258 | continue # no low outliers 259 | end 260 | ilast = findlast(dz_vec .>= dz) # find the index of the last jump 261 | zmin = z_vec[idx_order[ilast+1]] # lowest non outlier 262 | 263 | if do_high_noise 264 | # classify points higher than max_height above lowest non outlier as high noise 265 | zdif = z_vec - zmin 266 | toohi = zdif .> max_height 267 | itoohi = idxs[toohi] 268 | dz_outliers[itoohi] = zdif[toohi] 269 | for i in itoohi 270 | classify!(cloud, i, high_noise) 271 | end 272 | end 273 | 274 | for i in idxs[idx_order[1:ilast]] 275 | dz_outliers[i] = zmin - getz(cloud, i) # difference in elevation with lowes non outlier 276 | classify!(cloud, i, low_noise) 277 | end 278 | end 279 | # update Cloud with dz of outliers 280 | cloud[:dz] = dz_outliers 281 | nothing 282 | end 283 | 284 | "Apply Vosselman (2000) to classify ground points" 285 | function classify_ground!(cloud::Cloud, spatialindex::NearestNeighbors.NNTree; 286 | radius=3.0, # radius to check slope condition [m] 287 | max_slope=0.3, # max allowd slope between ground points [m/m] 288 | tolerance=0.0, # vertical tolerance [m] 289 | pointfilter=nothing, # only classify points with class 290 | class=2) 291 | 292 | # check if the spatialindex is 2D Euclidean 293 | @assert (spatialindex.metric == Euclidean()) && (length(spatialindex.data[1]) == 2) 294 | n = length(positions(cloud)) 295 | # loop through points 296 | @showprogress 5 "Classifying low surface points (Vosselman).." for i in 1:n 297 | # get xyz & class of point i 298 | (pointfilter != nothing) && (pointfilter(cloud, i) || continue) 299 | 300 | xi, yi, zi = positions(cloud)[i] 301 | # find number of points in range 302 | neighbors = inrange(spatialindex, [xi, yi], radius) 303 | 304 | # classify 305 | isground = vosselman(cloud, i, neighbors; max_slope=max_slope, tolerance=tolerance) 306 | 307 | # classes according to asprs las format 308 | if isground 309 | classify!(cloud, i, UInt8(class)) # ground 310 | end 311 | end 312 | 313 | nothing 314 | end 315 | 316 | """Apply Vosselman (2000) to classify water points 317 | only apply to points below min_z_tolerance from minima in grid with cellsize=radius""" 318 | function classify_water!(cloud::Cloud, spatialindex::NearestNeighbors.NNTree; 319 | radius=50.0, # radius to check slope condition [m] 320 | max_slope=0.01, # max allowd slope between ground points [m/m] 321 | tolerance=0.05, # vertical tolerance vosselman [m] (due to vairance in water level measurements) 322 | min_z_tolerance = 0.3, # only apply to points below min_z_tolerance from minima in grid with cellsize=radius 323 | pointfilter = nothing, # only classify points with class 324 | class = 9) 325 | 326 | r_large = define_raster(cloud, radius; pointfilter = pointfilter) 327 | z_max = rasterize(cloud, r_large, reducer = reducer_minz) + min_z_tolerance 328 | 329 | # check if the spatialindex is 2D Euclidean 330 | @assert (spatialindex.metric == Euclidean()) && (length(spatialindex.data[1]) == 2) 331 | 332 | # loop through points 333 | @showprogress 5 "Classifying water points (Vosselman).." for icell in 1:length(r_large) 334 | max_z = z_max[icell] 335 | idx0 = r_large[icell] 336 | for i in idx0 337 | # get xyz & class of point i 338 | (pointfilter != nothing) && (pointfilter(cloud, i) || continue) 339 | 340 | xi, yi, zi = positions(cloud)[i] 341 | zi <= max_z || continue 342 | 343 | # find number of points in range 344 | neighbors = inrange(spatialindex, [xi, yi], radius) 345 | 346 | # classify 347 | iswater = vosselman(cloud, Int(i), neighbors; max_slope=max_slope, tolerance=tolerance) 348 | 349 | # classes according to asprs las format 350 | if iswater 351 | classify!(cloud, i, UInt8(class)) # ground 352 | end 353 | end 354 | end 355 | 356 | nothing 357 | end 358 | -------------------------------------------------------------------------------- /modules/GridOperations/src/filter.jl: -------------------------------------------------------------------------------- 1 | """percentile filter based on updating histograms while sliding over the image 2 | implementation of Perreault and Hebert (2007) algorithm see: http://nomis80.org/ctmf.pdf 3 | 4 | precision is limited by number of bins. 5 | for a images with a bit depth of 8 bit (e.g. greyscale images), full precision 6 | can be acchieved with a dual-level histogram of 16 bins 7 | For images with a higher bit depth more bins are needed and the algorithm slows, 8 | the precision can be set with the optional precision paramter 9 | 10 | input 11 | A 2d image Array 12 | p percentile value [1-100] 13 | r radius of window [n. cells]: i.e.: the window size is 2r+1 by 2r+1 14 | is8bit true if A has a bit-depth of 8 bits and no scaling needs to be applied (default false) 15 | precision number of decimals to preserve when binning 16 | """ 17 | function hist_filter(A::Array{T,2}, perc::Real, r::Int; 18 | precision=0, 19 | nodata_value=T(-9999.0), 20 | hist_min=nothing, 21 | hist_max=nothing) where T <: Real 22 | scale = T(10^precision) # default scale value 23 | ksize = (2*r+1)^2 # kernel size 24 | nrows, ncols = size(A) # image size 25 | mask = (A .≠ nodata_value) .& isfinite.(A) 26 | if T == UInt8 27 | im_min, im_max = 0, 255 28 | elseif T <: Integer 29 | precision > 0 && info("no scaling applied as A has integer values") 30 | scale = T(1) 31 | im_min, im_max = floor(Int, minimum(A[mask])), ceil(Int, maximum(A[mask])) 32 | else 33 | im_min, im_max = floor(Int, minimum(A[mask]))*scale, ceil(Int, maximum(A[mask]))*scale 34 | end 35 | # scale min max of hist to hist_min or hist_max if given 36 | if hist_min != nothing 37 | im_min = hist_min * scale 38 | end 39 | if hist_max != nothing 40 | im_max = hist_max * scale 41 | end 42 | 43 | nodata_value_scaled = nodata_value*scale 44 | @assert (im_max - im_min) >= 1 "the image bit depth with given 'precision', 'nodata_value', 'hist_min' and 'hist_max' params is zero" 45 | # create histogram bin size 46 | nbins = ceil(Int, sqrt(im_max-im_min)) 47 | # fine level histograms have bins with size 1 48 | # course bin histogram bins are defined by: 49 | edges_course = Int[im_min-1:nbins:nbins*nbins+im_min;] 50 | 51 | # initialization of histograms 52 | # the counts arrays are not setup as offsetarrays because slicing doesn't work on these arrays 53 | # instead the arrays have an offset of r in indices 54 | cnts_course = zeros(Int32, (nrows+2*r, nbins)) 55 | cnts_fine = zeros(Int32, (nrows+2*r, nbins, nbins)) 56 | # create seperate bins for both extremes and nodata 57 | cnts_nodata = zeros(Int32, nrows+2*r) 58 | cnts_min = zeros(Int32, nrows+2*r) 59 | cnts_max = zeros(Int32, nrows+2*r) 60 | 61 | # create output array B 62 | B = similar(A) 63 | # pad A 64 | Apad = padarray(A, Pad(:reflect, r, r)) 65 | 66 | function add_value!(i::Int, v::Real; w=1) 67 | if scale > 1 68 | v = round(v * scale, 0) # scaling for non 8bit images 69 | end 70 | if v ≈ nodata_value_scaled 71 | cnts_nodata[i+r] += w # offset r in cnts arrays 72 | elseif v < im_min 73 | cnts_min[i+r] += w 74 | elseif v >= im_max 75 | cnts_max[i+r] += w 76 | else 77 | icourse = min(Int(div(v-im_min, nbins)) + 1, nbins) # min to include right boundary 78 | ifine = min(Int(div(v-edges_course[icourse], 1)), nbins) # fine bins have size 1 79 | cnts_course[i+r, icourse] += w # offset r in cnts arrays 80 | cnts_fine[i+r, ifine, icourse] += w 81 | end 82 | 83 | nothing 84 | end 85 | 86 | # find bin corresponding to percentile start from FIRST bin 87 | function bin_count_course(i::Int, p_idx::Int, pcum::Int) 88 | pbin, icourse = 0, 0 89 | for icourse in 1:nbins 90 | # use of sum is faster than loop. tested with BenchmarkTools and ProfileView 91 | pbin = sum(cnts_course[i:i+r*2, icourse]) 92 | pcum += pbin #[icourse] 93 | pcum >= p_idx && break 94 | end 95 | pcum -= pbin 96 | pcum, icourse 97 | end 98 | function bin_count_fine(i::Int, p_idx::Int, pcum::Int, icourse::Int) 99 | ifine=0 100 | for ifine in 1:nbins 101 | pcum += sum(cnts_fine[i:i+r*2, ifine, icourse]) 102 | pcum >= p_idx && break 103 | end 104 | pcum, ifine 105 | end 106 | 107 | # find bin corresponding to percentile -> start from LAST bin 108 | function bin_count_course_rev(i::Int, p_idx::Int, pcum::Int) 109 | pbin, icourse = 0, 0 110 | for icourse in nbins:-1:1 111 | pbin = sum(cnts_course[i:i+r*2, icourse]) 112 | pcum += pbin #[icourse] 113 | pcum >= (ksize - p_idx) && break 114 | end 115 | pcum -= pbin 116 | pcum, icourse 117 | end 118 | function bin_count_fine_rev(i::Int, p_idx::Int, pcum::Int, icourse::Int) 119 | ifine = 0 120 | for ifine in nbins:-1:1 121 | pcum += sum(cnts_fine[i:i+r*2, ifine, icourse]) 122 | pcum >= (ksize - p_idx) && break 123 | end 124 | pcum, ifine 125 | end 126 | 127 | # find percentile value from multi level histogram 128 | function calc_perc(i::Int) 129 | # number of nodata_size values in kernel 130 | nodata_size = sum(cnts_nodata[i:i+r*2]) 131 | if nodata_size < ksize # calculate percentile if more than half window with values 132 | p_idx = ceil(Int, max((ksize-nodata_size) * perc / 100., 1)) 133 | # find percentile bin 134 | icourse, ifine = 0, 0 135 | if perc <= 100 136 | hist_min != nothing ? pcum = sum(cnts_min[i:i+r*2]) : pcum = 0 137 | # start from begin of histogram 138 | pcum, icourse = bin_count_course(i, p_idx, pcum) 139 | pcum, ifine = bin_count_fine(i, p_idx, pcum, icourse) 140 | else 141 | hist_max != nothing ? pcum = sum(cnts_max[i:i+r*2]) : pcum = 0 142 | # start from end in histograms 143 | pcum, icourse = bin_count_course_rev(i, p_idx, pcum) 144 | pcum, ifine = bin_count_fine_rev(i, p_idx, pcum, icourse) 145 | end 146 | 147 | # calculate the value of fine bin at 148 | return (edges_course[icourse] + ifine) / scale 149 | else 150 | return nodata_value 151 | end 152 | end 153 | 154 | ## start actual looping 155 | for j = 1:ncols, i in 1:nrows 156 | if j == 1 && i == 1 # initiate top left 157 | for kj = -r:r, ki = -r:r 158 | add_value!(i+ki, Apad[i+ki, j+kj]) 159 | end 160 | elseif j == 1 && i > 1 # initiate first col 161 | for kj in -r:r 162 | add_value!(i+r, Apad[i+r, j+kj]) 163 | end 164 | elseif i == 1 && j > 1 # first row 165 | for ki in -r:r 166 | add_value!(i+ki, Apad[i+ki, j-r-1]; w=-1) # remove old value 167 | add_value!(i+ki, Apad[i+ki, j+r]) 168 | end 169 | else 170 | add_value!(i+r, Apad[i+r, j-r-1]; w=-1) # remove old value 171 | add_value!(i+r, Apad[i+r, j+r]) 172 | end 173 | B[i,j] = calc_perc(i) 174 | end 175 | B 176 | end 177 | 178 | 179 | """apply vosselman criterium to grid, 180 | 181 | vosselman criterium: 182 | z_{i} <= min z[ki] + dist[ki] * max_slope 183 | for ki in i-r:i+r, j-r:j+r 184 | 185 | output 186 | flags true if criterium not met 187 | B max allow value per cell given neighbors 188 | """ 189 | function vosselman_filter(A::Array{T,2}, 190 | radius, # kernel radius [m] 191 | max_slope, # mas tolorated slope [m/m] 192 | tolerance=0; 193 | res=1., # resolution of grid a.k.a. cell size [m] 194 | nodata::T=T(-9999.0)) where T <: AbstractFloat 195 | 196 | r = round(Int, radius/res) # length in # off cells in which to search for neighbours 197 | 198 | # create output array B 199 | nrows, ncols = size(A) 200 | B = ones(T, (nrows, ncols)) * Inf16 201 | flags = falses(A) 202 | # pad A 203 | Apad = ImageFiltering.padarray(A, Pad(:replicate, r, r)) 204 | 205 | # loop offsets 206 | for kj = -r:r, ki = -r:r 207 | k_dist = hypot(ki,kj) 208 | ((k_dist > r) || ((ki==1) && (kj==1))) && continue # skip offset if out of range or zero 209 | k_tol = k_dist .* res .* max_slope + tolerance 210 | ## loop through image 211 | for j = 1:ncols, i = 1:nrows 212 | zi = A[i,j] 213 | zi == nodata && continue 214 | # replace if smaller than current (find min in kernel) 215 | zj = Apad[i+ki,j+kj] + k_tol 216 | zj == nodata && continue # skip nodata values in kernel 217 | if zj < B[i, j] 218 | B[i, j] = zj 219 | end 220 | if zj < zi 221 | flags[i, j] = true 222 | end 223 | end 224 | end 225 | 226 | B, flags 227 | end 228 | 229 | # morphological opening & tophat = img - opening(img) 230 | # square window, no circular kernel yet available! 231 | function morph_tophat(img::Matrix{Float32}, boundarymask::BitMatrix, r::Int) 232 | # img_min = mapwindow(minimum, img, (s, s)) # erosion 233 | # img_open = mapwindow(maximum, img_min, (s, s)) # dilation 234 | img_min = morph_erosion(img, boundarymask, r) 235 | img_open = morph_dilation(img_min, boundarymask, r) 236 | img_open 237 | end 238 | 239 | """Implementation of the Progressive Morphological Filter (Zhang, 2003) 240 | works based on zmin grid where dropouts are filled with nearest neighbor value 241 | NOTE: make sure A has no nodata values 242 | 243 | output 244 | flags true if criterium not met 245 | B max allow value per cell given neighbors 246 | 247 | (Zhang, 2003): http://users.cis.fiu.edu/~chens/PDF/TGRS.pdf """ 248 | function pmf_filter(A::Matrix{Float32}, boundarymask::BitMatrix, 249 | max_window_radius::Float64, # max radius for squared window [m] 250 | slope::Float64, # terrain slope [m/m] 251 | dh_max::Float64, # maximum elevation threshold between subsequent tophat operations [m] 252 | dh_min::Float64, # initial elevation threshold [m] 253 | res::Float64) # cellsize [m] 254 | 255 | @show max_window_radius, slope, dh_min, dh_max, res 256 | 257 | nrow, ncol = size(A) 258 | totcells = nrow * ncol 259 | # maximum window size of kernel in pixels 260 | rad_pix_max = round(Int, max_window_radius / res) 261 | # exponential increase radius; window size = 3,5,9,17,33,..,N 262 | kmax = ceil(Int, log2(rad_pix_max*2)) 263 | w = [Int(exp2(k)) + 1 for k in 1:kmax] 264 | w[end] = rad_pix_max * 2 + 1 265 | # deprecate linear window growth -> does not give satisfying results 266 | # linear increase radius; window size = 3,5,7,9...,N 267 | # w = Int[2*k + 1 for k in 1:rad_pix_max] 268 | 269 | # give a warning when the slope will exceed dh_max 270 | slope * (w[end] - w[end-1]) * res + dh_min > dh_max && warn("dh_max will limit pmf filter") 271 | 272 | # initialize 273 | dh_t = dh_min 274 | Af = copy(A) 275 | # create max allowed surface (use to classify pointcloud) 276 | B = Af + dh_t 277 | # loop through kernel sizes 278 | 279 | @showprogress 1 "Progessive filtering..." for (k, wk) in enumerate(w) 280 | 281 | # threshold 282 | if k > 1 283 | dh_t = min(slope * (wk - w[k-1]) * res + dh_min, dh_max) 284 | end 285 | 286 | # calculate morph opening 287 | Af = morph_tophat(Af, boundarymask, fld(wk, 2)) 288 | 289 | # update max allowed surface 290 | if k == 1 291 | B[:,:] = Af + dh_t 292 | else 293 | for I in eachindex(Af) 294 | # replace if smaller than current z_max 295 | z_max = Af[I] + dh_t 296 | B[I] = min(B[I], z_max) 297 | end 298 | end 299 | 300 | end 301 | # flag non ground points based on dh_t threshold 302 | flags = Array{Bool}(A .> B) 303 | # ncells = count(x -> x == 1, flags) 304 | # debugging print statement to be placed in loop above 305 | # println(" iter $(k): $(ncells)/$(totcells) filtered, window size = $(wk) cells, dh = $(@sprintf("%.2f", dh_t))") 306 | 307 | B, flags 308 | end 309 | 310 | # lee filter 311 | # http://stackoverflow.com/questions/39785970/speckle-lee-filter-in-python 312 | # Lee, J. S. (1980). Digital image enhancement and noise filtering by use of 313 | # local statistics. IEEE transactions on pattern analysis and machine intelligence, (2), 165-168. 314 | # 315 | # enl (equivalen number of looks) -> 1 in LiDAR intensity data 316 | # addititive noise 317 | function lee_filter(img::Array{T,2}, size::Real; nodata=T(-9999.0)) where T <: AbstractFloat 318 | # local statistics 319 | fmean(A::AbstractArray) = mean(filter(x -> x != nodata, A)) 320 | window_mean = mapwindow(fmean, img, (size, size)) 321 | window_sqr_mean = mapwindow(fmean, img.^2, (size, size)) 322 | window_variance = window_sqr_mean - window_mean.^2 323 | # global variance 324 | img_variance = var(filter(x -> x != nodata, img)) 325 | 326 | 327 | # smoothing weights 328 | w = window_variance.^2 ./ (window_variance.^2 + img_variance.^2) 329 | 330 | # smoothed image 331 | img_output = (img .* w) + (window_mean .* (1 - w)) 332 | end 333 | 334 | # enhanced lee filter 335 | # 336 | # Lopes, A., Touzi, R., & Nezry, E. (1990). Adaptive speckle filters and scene 337 | # heterogeneity. IEEE transactions on Geoscience and Remote Sensing, 28(6), 992-1000. 338 | # enl (equivalent number of looks) -> 1.0 for lidar data 339 | function enhanced_lee_filter(img::Array{T,2}, size::Real; 340 | enl=1.0, 341 | k=1.0, 342 | nodata=T(-9999.0)) where T <: AbstractFloat 343 | 344 | max_cof_var = sqrt(1+2/enl) 345 | cu = 0.523/sqrt(enl) 346 | 347 | # local statistics 348 | fmean(A::AbstractArray) = mean(filter(x -> x != nodata, A)) 349 | window_mean = mapwindow(fmean, img, (size, size)) 350 | window_sqr_mean = mapwindow(fmean, img.^2, (size, size)) 351 | window_variance = window_sqr_mean - window_mean.^2 352 | # in case of img being Float32 and having very high values, 353 | # such as sometimes happens with intensity 354 | # slightly negative values for window_variance can arise 355 | # these are here set to 0.0 356 | window_variance = clamp.(window_variance, 0.0, Inf) 357 | window_cof_var = sqrt.(window_variance) ./ window_mean 358 | window_cof_var[window_cof_var .== 0] = 0.001 # avoid zero division 359 | 360 | # smoothing weights 361 | # filter in heterogeneous area 362 | w = T.(exp.(-k .* (window_cof_var - cu) ./ (max_cof_var - window_cof_var))) 363 | # preserver original value for "point target" (e.g around edges) 364 | w[window_cof_var .>= max_cof_var] = 0.0 365 | # return local mean in homogeneous areas 366 | w[window_cof_var .<= cu] = 1.0 367 | 368 | # smoothed image 369 | img_output = (img .* (1-w)) + (window_mean .* w) 370 | end 371 | 372 | # morphological operations with circular kernel 373 | function morph_erosion(A::Matrix{Float32}, boundarymask::BitMatrix, r::Int) 374 | nrows, ncols = size(A) # image size 375 | B = copy(A) # create output array B 376 | Apad = padarray(A, Pad(:replicate, r, r)) # pad A 377 | 378 | # loop offsets 379 | @inbounds for kj = -r:r, ki = -r:r 380 | ((hypot(ki,kj) > r) || ((ki==0) && (kj==0))) && continue # skip offset if out of range or zero 381 | ## loop through image 382 | for j = 1:ncols, i = 1:nrows 383 | boundarymask[i,j] || continue 384 | # replace if smaller than current (find min in kernel) 385 | zo = Apad[i+ki,j+kj] 386 | if zo < B[i, j] 387 | B[i, j] = zo 388 | end 389 | end 390 | end 391 | B 392 | end 393 | 394 | function morph_dilation(A::Matrix{Float32}, boundarymask::BitMatrix, r::Int) 395 | nrows, ncols = size(A) # image size 396 | B = copy(A) # create output array B 397 | Apad = padarray(A, Pad(:replicate, r, r)) # pad A 398 | 399 | # loop offsets 400 | @inbounds for kj = -r:r, ki = -r:r 401 | ((hypot(ki,kj) > r) || ((ki==0) && (kj==0))) && continue # skip offset if out of range or zero 402 | ## loop through image 403 | for j = 1:ncols, i = 1:nrows 404 | boundarymask[i,j] || continue 405 | # replace if larger than current (find max in kernel) 406 | zo = Apad[i+ki,j+kj] 407 | if zo > B[i, j] 408 | B[i, j] = zo 409 | end 410 | end 411 | end 412 | B 413 | end 414 | -------------------------------------------------------------------------------- /modules/XYZ/src/rasterize.jl: -------------------------------------------------------------------------------- 1 | #= 2 | Functions to create a raster from Clouds. 3 | 4 | reg_grid and reg_grid_index create a link between raster cells and Points in the 5 | Cloud. 6 | rasterize summarizes the Points per cell to one statistical value with and creates 7 | the actual rasters 8 | 9 | =# 10 | 11 | struct Raster{U<:Integer} 12 | pointindex::Vector{Vector{U}} 13 | nrow::Int 14 | ncol::Int 15 | bbox::BoundingBox 16 | cellsize::Float64 17 | epsg::Nullable{Int} 18 | npoints::Int # number of unique points in index 19 | mask::BitArray{2} # mask == True -> missing 20 | end 21 | 22 | function Raster( 23 | pointindex::Vector{Vector{U}}, 24 | nrow::Int, 25 | ncol::Int, 26 | bbox::BoundingBox, 27 | cellsize::Float64, 28 | epsg::Nullable{Int}, 29 | npoints::Int 30 | ) where U <: Integer 31 | 32 | Raster(pointindex,nrow,ncol,bbox,cellsize,epsg, npoints, falses(nrow,ncol)) 33 | end 34 | 35 | function Raster( 36 | pointindex::Vector{Vector{U}}, 37 | nrow::Int, 38 | ncol::Int, 39 | bbox::BoundingBox, 40 | cellsize::Float64, 41 | epsg::Int, 42 | npoints::Int, 43 | mask::BitArray{2} 44 | ) where U <: Integer 45 | 46 | Raster(pointindex,nrow,ncol,bbox,cellsize,Nullable{Int}(epsg), npoints, mask) 47 | end 48 | 49 | function Base.show(io::IO, r::Raster) 50 | println(io, "RasterIndex (nrow=$(r.nrow), ncol=$(r.ncol), cellsize=$(r.cellsize), n unique points=$(r.npoints), epsg=$(r.epsg), bbox=[$(r.bbox)])") 51 | end 52 | 53 | # construct coordinates in vector or meshgrid from Raster 54 | epsg(r::Raster) = r.epsg 55 | npoints(r::Raster) = r.npoints # number of unique points in index 56 | 57 | # length is number of cells 58 | Base.length(r::Raster) = length(r.pointindex) 59 | Base.size(r::Raster) = (r.nrow, r.ncol) 60 | Base.eachindex(r::Raster) = eachindex(r.pointindex) 61 | 62 | # ind2sub & sub2ind for raster 63 | Base.ind2sub(r::Raster, i::Integer) = ind2sub((r.nrow, r.ncol), i) 64 | Base.sub2ind(r::Raster, row::Integer, col::Integer) = sub2ind((r.nrow, r.ncol), row, col) 65 | 66 | "transform coordinates to row, col in raster" 67 | function coords2sub(r::Raster, x::AbstractFloat, y::AbstractFloat) 68 | # cells include top en left boundaries 69 | col = Int(fld(x - r.bbox.xmin, r.cellsize)+1) 70 | row = Int(fld(r.bbox.ymax - y, r.cellsize)+1) 71 | # include points on lower and right bbox boundaries 72 | if x == r.bbox.xmax 73 | col = col - 1 74 | end 75 | if y == r.bbox.ymin 76 | row = row - 1 77 | end 78 | ((col <= 0) || (row <= 0) || (row > r.nrow) || (col > r.ncol)) && error("coordinates outside raster domain") 79 | row, col 80 | end 81 | 82 | # based on index of raster pointindex 83 | Base.getindex(r::Raster, i::Integer) = r.pointindex[i] 84 | # based on index of raster, return row, col, pointindex 85 | Base.getindex(r::Raster, row::Integer, col::Integer) = r[sub2ind(r, row, col)] 86 | # based on coordinates raster, return pointindex 87 | function Base.getindex(r::Raster, x::AbstractFloat, y::AbstractFloat) 88 | row, col = coords2sub(r, x, y) 89 | r[row, col] 90 | end 91 | 92 | "transform coordinates to row, col in raster" 93 | function coords2ind(bbox::BoundingBox, 94 | nrow::Integer, 95 | ncol::Integer, 96 | cellsize::Float64, 97 | x::AbstractFloat, 98 | y::AbstractFloat) 99 | 100 | # cells include bottom and left boundaries 101 | col = Int(fld(x - bbox.xmin, cellsize)+1) 102 | row = Int(fld(bbox.ymax - y, cellsize)+1) 103 | # include points on lower and right bbox boundaries 104 | if x == bbox.xmax 105 | col = col - 1 106 | end 107 | if y == bbox.ymin 108 | row = row - 1 109 | end 110 | isinside = !((col <= 0) || (row <= 0) || (row > nrow) || (col > ncol)) 111 | sub2ind((nrow, ncol), row, col), isinside 112 | end 113 | 114 | "transform coordinates to row, col in raster" 115 | function coords2ind_overlap(bbox::BoundingBox, 116 | nrow::Integer, 117 | ncol::Integer, 118 | cellsize::Float64, 119 | x::AbstractFloat, 120 | y::AbstractFloat; 121 | overlap = 100.0) 122 | 123 | # cells include bottom and left boundaries 124 | col = Int(fld(x - bbox.xmin, cellsize)+1) 125 | row = Int(fld(bbox.ymax - y, cellsize)+1) 126 | # include points on lower and right bbox boundaries 127 | # if x == bbox.xmax 128 | # col = col - 1 129 | # end 130 | # if y == bbox.ymin 131 | # row = row - 1 132 | # end 133 | isinside = !((col <= 0) || (row <= 0) || (row > nrow) || (col > ncol)) 134 | # return a list of indices 135 | 136 | # this assumes that the overlap is less than half the cell size 137 | inleft = rem(x - bbox.xmin, cellsize) <= overlap 138 | inbottom = rem(y - bbox.ymin, cellsize) <= overlap 139 | inright = rem(x - bbox.xmin, cellsize) >= (cellsize - overlap) 140 | intop = rem(y - bbox.ymin, cellsize) >= (cellsize - overlap) 141 | 142 | # correct for boundary issues 143 | if !isinside 144 | inleft = false 145 | inbottom = false 146 | inright = false 147 | intop = false 148 | end 149 | if col == 1 150 | inleft = false 151 | end 152 | if row == nrow 153 | inbottom = false 154 | end 155 | if col == ncol 156 | inright = false 157 | end 158 | if row == 1 159 | intop = false 160 | end 161 | 162 | row, col, isinside, inleft, inbottom, inright, intop 163 | end 164 | 165 | "transform row, col in raster to center cell x, y coordinates" 166 | function ind2coords(bbox::BoundingBox, 167 | nrow::Integer, 168 | ncol::Integer, 169 | cellsize::Float64, 170 | icell::Integer) 171 | 172 | row, col = ind2sub((nrow, ncol), icell) 173 | # cells include bottom and left boundaries 174 | x = bbox.xmin + (col - 0.5) * cellsize 175 | y = bbox.ymax - (row - 0.5) * cellsize 176 | 177 | x, y 178 | end 179 | 180 | function rastercellcoordinates(r::Raster) 181 | ncells = r.ncol * r.nrow 182 | cellcoords = Vector{SVector{2, Float64}}() 183 | sizehint!(cellcoords, ncells) 184 | for icell in 1:ncells 185 | coords = SVector(ind2coords(r.bbox, r.nrow, r.ncol, r.cellsize, icell)) 186 | push!(cellcoords, coords) 187 | end 188 | cellcoords 189 | end 190 | 191 | "raster definition constructor from Cloud, cellsize and predefined tiling and epsg" 192 | function define_raster(cloud::Cloud, orig_tile::BoundingBox, overlap::Float64, cellsize::Float64; 193 | snapgrid=cellsize, 194 | epsg=Nullable{Int}(), 195 | pointfilter = nothing) 196 | # determine inner col and rows and its outer col and rows 197 | ocol = round(Int, abs(orig_tile.xmax - orig_tile.xmin) / cellsize) 198 | orow = round(Int, abs(orig_tile.ymax - orig_tile.ymin) / cellsize) 199 | overlapcells = Int(cld(overlap, cellsize)) 200 | ncol = round(Int, ocol + 2 * overlapcells) 201 | nrow = round(Int, orow + 2 * overlapcells) 202 | 203 | # create mask and mask out overlaps 204 | mask = trues(nrow, ncol) 205 | mask[(overlapcells+1):(end-overlapcells), (overlapcells+1):(end-overlapcells)] = false 206 | 207 | # calculate bounding box including overlap 208 | bbox = BoundingBox(xmin=orig_tile.xmin - overlapcells * cellsize, 209 | xmax=orig_tile.xmax + overlapcells * cellsize, 210 | ymin=orig_tile.ymin - overlapcells * cellsize, 211 | ymax=orig_tile.ymax + overlapcells * cellsize 212 | ) 213 | 214 | # create grid definition 215 | xy_idx, npoints = reg_grid_index(cloud, nrow, ncol, bbox, cellsize; 216 | pointfilter = pointfilter) 217 | Raster(xy_idx, nrow, ncol, bbox, cellsize, epsg, npoints, mask) 218 | end 219 | 220 | "raster definition constructor from Cloud, cellsize and epsg" 221 | function define_raster(cloud::Cloud, cellsize::Float64; 222 | snapgrid=cellsize, 223 | epsg=Nullable{Int}(), 224 | pointfilter = nothing) 225 | 226 | # create grid definition 227 | nrow, ncol, bbox = reg_grid(cloud, cellsize; snapgrid=snapgrid) 228 | xy_idx, npoints = reg_grid_index(cloud, nrow, ncol, bbox, cellsize; 229 | pointfilter = pointfilter) 230 | Raster(xy_idx, nrow, ncol, bbox, cellsize, epsg, npoints) 231 | end 232 | 233 | "raster definition constructor from Cloud, and GDAL geotransform" 234 | function define_raster(cloud::Cloud, nrow::Integer, ncol::Integer, geotransform::Vector{Float64}; 235 | epsg=Nullable{Int}(), 236 | pointfilter = nothing) 237 | 238 | # create grid definition 239 | cellsize = Float64(geotransform[2]) 240 | nrow, ncol, bbox = reg_grid(nrow, ncol, geotransform) 241 | xy_idx, npoints = reg_grid_index(cloud, nrow, ncol, bbox, cellsize; 242 | pointfilter = pointfilter) 243 | Raster(xy_idx, nrow, ncol, bbox, cellsize, epsg, npoints) 244 | end 245 | 246 | "raster definition constructor from Cloud, and GDAL geotransform" 247 | function define_raster_overlap(cloud::Cloud, nrow::Integer, ncol::Integer, geotransform::Vector{Float64}; 248 | epsg=Nullable{Int}(), 249 | pointfilter = nothing, 250 | overlap = 100.0) 251 | 252 | # create grid definition 253 | cellsize = Float64(geotransform[2]) 254 | nrow, ncol, bbox = reg_grid(nrow, ncol, geotransform) 255 | xy_idx, npoints = reg_grid_index_overlap(cloud, nrow, ncol, bbox, cellsize; 256 | pointfilter = pointfilter, 257 | overlap = overlap) 258 | Raster(xy_idx, nrow, ncol, bbox, cellsize, epsg, npoints) 259 | end 260 | 261 | "create vectors with bbox from Cloud spatial index based on cell size 262 | snap to utm coordinates by default" 263 | function reg_grid(cloud::Cloud, cellsize::Float64; snapgrid=cellsize) 264 | # snap bbox to cellsize coordinates (to the outside) 265 | # this assumes the coordinates are in cm precision 266 | # do the fld with scaled integers to prevent floating point issues 267 | # like fld(549.0, 0.01) == 54899.0 268 | if snapgrid != 0 269 | snapgrid_int = round(Int32, 100 * snapgrid) 270 | x_min_int = round(Int32, 100 * boundingbox(cloud).xmin) 271 | y_min_int = round(Int32, 100 * boundingbox(cloud).ymin) 272 | x_max_int = round(Int32, 100 * boundingbox(cloud).xmax) 273 | y_max_int = round(Int32, 100 * boundingbox(cloud).ymax) 274 | x_min = fld(x_min_int, snapgrid_int) * snapgrid_int * 0.01 275 | y_min = fld(y_min_int, snapgrid_int) * snapgrid_int * 0.01 276 | x_max = cld(x_max_int, snapgrid_int) * snapgrid_int * 0.01 277 | y_max = cld(y_max_int, snapgrid_int) * snapgrid_int * 0.01 278 | end 279 | 280 | # determine size of grid 281 | dy = y_max - y_min 282 | dx = x_max - x_min 283 | nrow = round(Int, dy / cellsize) 284 | ncol = round(Int, dx / cellsize) 285 | bbox = BoundingBox(xmin=x_min, ymin=y_min, xmax=x_max, ymax=y_max, zmin=NaN, zmax=NaN) 286 | nrow, ncol, bbox 287 | end 288 | 289 | "create vectors with bbox based on a GDAL geotransform vector" 290 | function reg_grid(nrow::Integer, ncol::Integer, geotransform::Vector{Float64}) 291 | # assumes a square grid 292 | cellsize = Float64(geotransform[2]) 293 | x_min = geotransform[1] 294 | y_max = geotransform[4] 295 | x_max = x_min + nrow * cellsize 296 | y_min = y_max - ncol * cellsize 297 | bbox = BoundingBox(xmin=x_min, ymin=y_min, xmax=x_max, ymax=y_max) 298 | 299 | # determine size of grid 300 | dy = y_max - y_min 301 | dx = x_max - x_min 302 | nrow = round(Int, dy / cellsize) 303 | ncol = round(Int, dx / cellsize) 304 | 305 | nrow, ncol, bbox 306 | end 307 | 308 | "Create reg_grid_index from Cloud and vectors of x & y coordinates" 309 | function reg_grid_index( 310 | cloud::Cloud, 311 | nrow::Int, 312 | ncol::Int, 313 | bbox::BoundingBox, 314 | cellsize::Float64; 315 | pointfilter=nothing) 316 | 317 | n = nrow * ncol # number of cells 318 | # the idx array gives per LAS point the linear index into the grid it was assigned to 319 | idx = Array{Vector{Int32}}(n) # some stay zero because of skipped points 320 | for icell in 1:n 321 | idx[icell] = Int32[] 322 | end 323 | 324 | npoints = 0 325 | @showprogress 2 "Indexing grid..." for i in 1:length(cloud) 326 | (pointfilter != nothing) && (pointfilter(cloud, i) || continue) 327 | # calc cellindex 328 | pos = positions(cloud)[i] 329 | xi, yi = pos[1], pos[2] 330 | icell, isinside = coords2ind(bbox, nrow, ncol, cellsize, xi, yi) 331 | isinside && push!(idx[icell], i) 332 | npoints += 1 333 | end 334 | 335 | idx, npoints 336 | end 337 | 338 | "Create reg_grid_index from Cloud and vectors of x & y coordinates" 339 | function reg_grid_index_overlap( 340 | cloud::Cloud, 341 | nrow::Int, 342 | ncol::Int, 343 | bbox::BoundingBox, 344 | cellsize::Float64; 345 | pointfilter = nothing, 346 | overlap = 100.0) 347 | 348 | n = nrow * ncol # number of cells 349 | # the idx array gives per LAS point the linear index into the grid it was assigned to 350 | idx = Array{Vector{Int32}}(n) # some stay zero because of skipped points 351 | for icell in 1:n 352 | idx[icell] = Int32[] 353 | end 354 | 355 | # with overlap the main difference is that each point may not be in 0 or 1 cells, 356 | # but 0 to n (normally n is max 4) 357 | npoints = 0 358 | for i in 1:length(cloud) 359 | (pointfilter != nothing) && (pointfilter(cloud, i) || continue) 360 | # calc cellindex 361 | pos = positions(cloud)[i] 362 | xi, yi = pos[1], pos[2] 363 | row, col, isinside, inleft, inbottom, inright, intop = coords2ind_overlap(bbox, nrow, ncol, cellsize, xi, yi, overlap=overlap) 364 | icell = sub2ind((nrow, ncol), row, col) 365 | 366 | ileft = sub2ind((nrow, ncol), row, col-1) 367 | ibottom = sub2ind((nrow, ncol), row+1, col) 368 | iright = sub2ind((nrow, ncol), row, col+1) 369 | itop = sub2ind((nrow, ncol), row-1, col) 370 | 371 | ileftbottom = sub2ind((nrow, ncol), row+1, col-1) 372 | irightbottom = sub2ind((nrow, ncol), row+1, col+1) 373 | irighttop = sub2ind((nrow, ncol), row-1, col+1) 374 | ilefttop = sub2ind((nrow, ncol), row-1, col-1) 375 | 376 | # center 377 | isinside && push!(idx[icell], i) 378 | inleft && push!(idx[ileft], i) 379 | inbottom && push!(idx[ibottom], i) 380 | inright && push!(idx[iright], i) 381 | intop && push!(idx[itop], i) 382 | 383 | # don't only add overlap to the sides but also away from the corners 384 | inleft && inbottom && push!(idx[ileftbottom], i) 385 | inright && inbottom && push!(idx[irightbottom], i) 386 | inright && intop && push!(idx[irighttop], i) 387 | inleft && intop && push!(idx[ilefttop], i) 388 | 389 | npoints += 1 390 | end 391 | 392 | idx, npoints 393 | end 394 | 395 | """fill empty raster index cells with nearest neighbor point 396 | this method creates a new raster index""" 397 | function inpaint_missings_nn(r::Raster, cloud::Cloud; 398 | fill_max_dist = 10) 399 | # index points that satisfy pointfilter 400 | index = Int32[] 401 | for i=1:length(r) 402 | for idx in r[i] 403 | push!(index, idx) 404 | end 405 | end 406 | 407 | missings = falses((r.nrow, r.ncol)) 408 | idx = deepcopy(r.pointindex) 409 | 410 | # cannot do knn search on empty tree 411 | if isempty(index) 412 | return Raster(idx, r.nrow, r.ncol, r.bbox, r.cellsize, r.epsg, r.npoints), trues(r.nrow, r.ncol) 413 | end 414 | 415 | tree = create_kdtree(cloud; dim=2, index=index) 416 | 417 | if length(tree.data) <= 1 418 | return r, missings 419 | end 420 | 421 | @showprogress 1 "Painting missing data..." for icell in 1:length(r) 422 | isempty(idx[icell]) || continue 423 | missings[icell] = true 424 | # for empty cells find nearest neighbor point 425 | x, y = ind2coords(r.bbox,r.nrow,r.ncol,r.cellsize,icell) 426 | i, d = knn(tree, [x, y], 1) 427 | d[1] <= fill_max_dist && push!(idx[icell], index[i[1]]) 428 | end 429 | 430 | # number of unique points unchanged 431 | Raster(idx, r.nrow, r.ncol, r.bbox, r.cellsize, r.epsg, r.npoints), missings 432 | end 433 | 434 | 435 | """filter a raster index with a pointfilter""" 436 | function filter_raster(r::Raster, cloud::Cloud, pointfilter) 437 | n = length(r) 438 | # create empty index 439 | idx = Array{Vector{Int32}}(n) # some stay zero because of skipped points 440 | for icell in 1:n 441 | idx[icell] = Int32[] 442 | end 443 | 444 | # index points that satisfy pointfilter 445 | npoints = 0 446 | for icell=1:n 447 | for i in r[icell] 448 | (pointfilter != nothing) && (pointfilter(cloud, i) || continue) 449 | push!(idx[icell], i) 450 | npoints += 1 451 | end 452 | end 453 | 454 | # return new index 455 | Raster(idx, r.nrow, r.ncol, r.bbox, r.cellsize, r.epsg, npoints) 456 | end 457 | 458 | "generic function to create 2d array from Cloud 459 | inputs 460 | cloud Cloud 461 | vx, vy x&y coordinate vectors defining the grid 462 | xy_idx Cloud indices per gridcell 463 | reducer statistical function to summarize points per cell 464 | function reducer(cloud::Cloud, index::Vector{Int}) 465 | Vector{Real} 466 | end 467 | pointfilter function to filter points 468 | function pointfilter(cloud::Cloud, index::Int) 469 | true 470 | end 471 | " 472 | function rasterize(cloud::Cloud, r::Raster; 473 | reducer = reducer_minz, # function to calculate value per cell pased on 474 | pointfilter = nothing, # predicate function to filter individual points 475 | min_dens = 0, 476 | nodata = -9999.0, 477 | return_density = false) 478 | 479 | # check dimensions of output 480 | nlayers = 1 + return_density 481 | area = r.cellsize^2 482 | 483 | # setup output arrays 484 | raster = fill(Float32(nodata), r.nrow, r.ncol, nlayers) 485 | 486 | # loop through cells 487 | for i in 1:length(r) 488 | # find points in range 489 | idx0 = r[i] 490 | row, col = ind2sub(r, i) 491 | # loop through points and filter 492 | if pointfilter != nothing # if filter is given 493 | idx = Int[] # indices after filter 494 | for i in idx0 495 | pointfilter(cloud, i) || continue 496 | push!(idx, i) 497 | end 498 | else 499 | idx = idx0 # no filter applied 500 | end 501 | 502 | np = length(idx) 503 | # calculate density [points / m2] 504 | di = Float64(np) / area 505 | if return_density 506 | raster[row, col, end] = di # save to end layer 507 | end 508 | if min_dens > 0 509 | di < min_dens && continue # min density threshold 510 | end 511 | 512 | np < 1 && continue # stat functions don't work on empty arrays 513 | # set statistics to grid 514 | raster[row, col, 1:end-return_density] = reducer(cloud, idx) 515 | end 516 | 517 | # return raster with statistics, density is saved to last layer 518 | raster 519 | end 520 | --------------------------------------------------------------------------------