├── .github └── workflows │ ├── build.yml │ └── tests.yml ├── .gitignore ├── CMakeLists.txt ├── LICENSE ├── README.md ├── environment.yml ├── src ├── alg.cpp ├── alg.hpp ├── boundary.cpp ├── clip.cpp ├── density.cpp ├── info.cpp ├── main.cpp ├── merge.cpp ├── nlohmann │ └── json.hpp ├── thin.cpp ├── tile │ ├── BufferCache.cpp │ ├── BufferCache.hpp │ ├── Cell.cpp │ ├── Cell.hpp │ ├── Common.hpp │ ├── EpfTypes.hpp │ ├── FileDimInfo.hpp │ ├── FileProcessor.cpp │ ├── FileProcessor.hpp │ ├── Las.cpp │ ├── Las.hpp │ ├── Point.hpp │ ├── README.md │ ├── ThreadPool.cpp │ ├── ThreadPool.hpp │ ├── TileGrid.cpp │ ├── TileGrid.hpp │ ├── TileKey.hpp │ ├── Writer.cpp │ ├── Writer.hpp │ └── tile.cpp ├── to_raster.cpp ├── to_raster_tin.cpp ├── to_vector.cpp ├── translate.cpp ├── utils.cpp ├── utils.hpp ├── vpc.cpp └── vpc.hpp ├── tests ├── conftest.py ├── data │ ├── rectangle.gpkg │ ├── rectangle1.gpkg │ ├── rectangle2.gpkg │ ├── rectangle3.gpkg │ └── rectangle4.gpkg ├── test_boundary.py ├── test_clip.py ├── test_merge.py ├── test_thin.py ├── test_to_vector.py ├── test_translate.py └── utils.py └── vpc-spec.md /.github/workflows/build.yml: -------------------------------------------------------------------------------- 1 | name: Build 2 | 3 | on: [push, pull_request] 4 | 5 | 6 | concurrency: 7 | group: ${{ github.workflow }}-${{ github.head_ref || github.ref }} 8 | cancel-in-progress: true 9 | 10 | jobs: 11 | build: 12 | name: Compile on ${{ matrix.os }} 13 | runs-on: ${{ matrix.os }} 14 | strategy: 15 | fail-fast: false 16 | matrix: 17 | os: [macos-latest, windows-latest, ubuntu-latest] 18 | 19 | steps: 20 | - name: Checkout 21 | uses: actions/checkout@v4 22 | 23 | - name: Setup MSVC environment 24 | uses: ilammy/msvc-dev-cmd@v1 25 | if: matrix.os == 'windows-latest' 26 | 27 | - uses: mamba-org/setup-micromamba@v1 28 | name: Setup Conda 29 | with: 30 | init-shell: bash 31 | environment-file: environment.yml 32 | environment-name: "wrench" 33 | cache-environment: true 34 | cache-downloads: true 35 | 36 | - name: Create working directory 37 | shell: bash -l {0} 38 | run: | 39 | mkdir build 40 | 41 | - name: Run cmake 42 | shell: bash -l {0} 43 | working-directory: ./build 44 | run: | 45 | if [ "$RUNNER_OS" == "Windows" ]; then 46 | export CC=cl.exe 47 | export CXX=cl.exe 48 | fi 49 | cmake -G Ninja \ 50 | -DCMAKE_INSTALL_PREFIX=${CONDA_PREFIX} \ 51 | .. 52 | 53 | - name: Build pdal_wrench 54 | shell: bash -l {0} 55 | working-directory: ./build 56 | run: | 57 | ninja 58 | ninja install 59 | -------------------------------------------------------------------------------- /.github/workflows/tests.yml: -------------------------------------------------------------------------------- 1 | name: Test 2 | 3 | on: [push, pull_request] 4 | 5 | concurrency: 6 | group: ${{ github.workflow }}-${{ github.head_ref || github.ref }} 7 | cancel-in-progress: true 8 | 9 | jobs: 10 | build: 11 | name: Run Python Tests 12 | runs-on: ubuntu-latest 13 | 14 | steps: 15 | 16 | - name: Checkout 17 | uses: actions/checkout@v4 18 | 19 | - uses: mamba-org/setup-micromamba@v2 20 | name: Setup Conda 21 | with: 22 | init-shell: bash 23 | environment-file: environment.yml 24 | environment-name: "wrench" 25 | cache-environment: true 26 | cache-downloads: true 27 | create-args: >- 28 | python=3.12 29 | 30 | - name: Add Python packages 31 | shell: bash -el {0} 32 | run: conda install pytest GDAL -y 33 | 34 | - name: Create working directory 35 | shell: bash -l {0} 36 | run: | 37 | mkdir build 38 | 39 | - name: Run cmake 40 | shell: bash -l {0} 41 | working-directory: ./build 42 | run: | 43 | cmake -G Ninja \ 44 | -DCMAKE_INSTALL_PREFIX=${CONDA_PREFIX} \ 45 | .. 46 | 47 | - name: Build pdal_wrench 48 | shell: bash -l {0} 49 | working-directory: ./build 50 | run: | 51 | ninja 52 | ninja install 53 | 54 | - name: Run tests 55 | shell: bash -el {0} 56 | run: | 57 | pytest tests -vv 58 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | CMakeLists.txt.user 2 | .idea 3 | build-*/ 4 | .vscode 5 | build 6 | /tests/data/*.laz 7 | /tests/data/*.las 8 | /tests/data/*.vpc -------------------------------------------------------------------------------- /CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 3.5) 2 | 3 | project(pdal_wrench LANGUAGES CXX) 4 | 5 | set(CMAKE_CXX_STANDARD 17) 6 | set(CMAKE_CXX_STANDARD_REQUIRED ON) 7 | 8 | # Compile flag. Make it possible to turn it off. 9 | set (PEDANTIC TRUE CACHE BOOL "Determines if we should compile in pedantic mode.") 10 | 11 | find_package(PDAL REQUIRED) 12 | find_package(GDAL REQUIRED) 13 | find_package(Threads REQUIRED) 14 | 15 | add_executable(pdal_wrench 16 | src/main.cpp 17 | src/alg.cpp 18 | src/boundary.cpp 19 | src/clip.cpp 20 | src/density.cpp 21 | src/info.cpp 22 | src/merge.cpp 23 | src/thin.cpp 24 | src/to_raster.cpp 25 | src/to_raster_tin.cpp 26 | src/to_vector.cpp 27 | src/translate.cpp 28 | src/utils.cpp 29 | src/vpc.cpp 30 | 31 | src/tile/tile.cpp 32 | src/tile/BufferCache.cpp 33 | src/tile/Cell.cpp 34 | src/tile/FileProcessor.cpp 35 | src/tile/Las.cpp 36 | src/tile/TileGrid.cpp 37 | src/tile/ThreadPool.cpp 38 | src/tile/Writer.cpp 39 | ) 40 | 41 | target_include_directories(pdal_wrench 42 | PRIVATE 43 | #${CMAKE_CURRENT_BINARY_DIR}/include 44 | ${PDAL_INCLUDE_DIRS} 45 | ${GDAL_INCLUDE_DIR} 46 | #${PROJECT_SOURCE_DIR} 47 | ) 48 | target_link_libraries(pdal_wrench 49 | PRIVATE 50 | #${CMAKE_THREAD_LIBS_INIT} 51 | ${PDAL_LIBRARIES} 52 | ${GDAL_LIBRARY} 53 | ${CMAKE_THREAD_LIBS_INIT} 54 | ) 55 | 56 | install(TARGETS pdal_wrench DESTINATION bin) 57 | 58 | ############################################################# 59 | # enable warnings 60 | 61 | if (PEDANTIC) 62 | message (STATUS "Pedantic compiler settings enabled") 63 | 64 | if(WIN32) 65 | add_definitions(-DWIN32_LEAN_AND_MEAN) 66 | endif() 67 | 68 | if(MSVC) 69 | set(_warnings "") 70 | if (NOT USING_NMAKE AND NOT USING_NINJA) 71 | set(_warnings "${_warnings} /W4" ) 72 | endif() 73 | 74 | # disable warnings 75 | set(_warnings "${_warnings} /wd4091 ") # 'typedef': ignored on left of '' when no variable is declared (occurs in MS DbgHelp.h header) 76 | set(_warnings "${_warnings} /wd4100 ") # unused formal parameters 77 | set(_warnings "${_warnings} /wd4127 ") # constant conditional expressions (used in Qt template classes) 78 | set(_warnings "${_warnings} /wd4190 ") # 'identifier' has C-linkage specified, but returns UDT 'identifier2' which is incompatible with C 79 | set(_warnings "${_warnings} /wd4231 ") # nonstandard extension used : 'identifier' before template explicit instantiation (used in Qt template classes) 80 | set(_warnings "${_warnings} /wd4244 ") # conversion from '...' to '...' possible loss of data 81 | set(_warnings "${_warnings} /wd4251 ") # needs to have dll-interface to be used by clients of class (occurs in Qt template classes) 82 | set(_warnings "${_warnings} /wd4267 ") # 'argument': conversion from 'size_t' to 'int', possible loss of data 83 | set(_warnings "${_warnings} /wd4275 ") # non dll-interface class '...' used as base for dll-interface class '...' 84 | set(_warnings "${_warnings} /wd4290 ") # c++ exception specification ignored except to indicate a function is not __declspec(nothrow) (occurs in sip generated bindings) 85 | set(_warnings "${_warnings} /wd4456 ") # declaration of '...' hides previous local declaration 86 | set(_warnings "${_warnings} /wd4457 ") # declaration of '...' hides a function parameter 87 | set(_warnings "${_warnings} /wd4458 ") # declaration of '...' hides class member 88 | set(_warnings "${_warnings} /wd4505 ") # unreferenced local function has been removed (QgsRasterDataProvider::extent) 89 | set(_warnings "${_warnings} /wd4510 ") # default constructor could not be generated (sqlite3_index_info, QMap) 90 | set(_warnings "${_warnings} /wd4512 ") # assignment operator could not be generated (sqlite3_index_info) 91 | set(_warnings "${_warnings} /wd4610 ") # user defined constructor required (sqlite3_index_info) 92 | set(_warnings "${_warnings} /wd4706 ") # assignment within conditional expression (pal) 93 | set(_warnings "${_warnings} /wd4714 ") # function '...' marked as __forceinline not inlined (QString::toLower/toUpper/trimmed) 94 | set(_warnings "${_warnings} /wd4800 ") # 'int' : forcing value to bool 'true' or 'false' (performance warning) 95 | set(_warnings "${_warnings} /wd4996 ") # '...': was declared deprecated (unfortunately triggered when implementing deprecated interfaces even when it is deprecated too) 96 | set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${_warnings}") 97 | set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${_warnings}") 98 | else() 99 | # add warnings via flags (not as definitions as on Mac -Wall can not be overridden per language ) 100 | set(_warnings "-Wall -Wextra -Wno-long-long -Wformat-security -Wno-strict-aliasing") 101 | 102 | set(WERROR FALSE CACHE BOOL "Treat build warnings as errors.") 103 | if (WERROR) 104 | set(_warnings "${_warnings} -Werror") 105 | endif() 106 | set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${_warnings}") 107 | 108 | # c++ only warnings 109 | set(_warnings "${_warnings} -Wnon-virtual-dtor") 110 | 111 | # unavoidable - we can't avoid these, as older, supported compilers do not support removing the redundant move 112 | set(_warnings "${_warnings} -Wno-redundant-move") 113 | 114 | # disable misleading-indentation warning -- it's slow to parse the sip files and not needed since we have the automated code styling rules 115 | set(_warnings "${_warnings} -Wno-misleading-indentation") 116 | 117 | if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU" AND CMAKE_CXX_COMPILER_VERSION VERSION_GREATER 7.9.999) 118 | # heaps of these thrown by Qt headers at the moment (sep 2019) 119 | set(_warnings "${_warnings} -Wno-deprecated-copy") 120 | endif() 121 | 122 | set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${_warnings}") 123 | 124 | if ("${CMAKE_CXX_COMPILER_ID}" MATCHES "Clang") 125 | set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wreturn-type-c-linkage -Woverloaded-virtual -Wimplicit-fallthrough") 126 | endif() 127 | 128 | # add any extra CXXFLAGS flags set by user. can be -D CXX_EXTRA_FLAGS or environment variable 129 | # command line -D option overrides environment variable 130 | # e.g. useful for suppressing transient upstream warnings in dependencies, like Qt 131 | set(CXX_EXTRA_FLAGS "" CACHE STRING "Additional appended CXXFLAGS") 132 | if ("${CXX_EXTRA_FLAGS}" STREQUAL "" AND DEFINED $ENV{CXX_EXTRA_FLAGS}) 133 | set(CXX_EXTRA_FLAGS "$ENV{CXX_EXTRA_FLAGS}") 134 | endif() 135 | if (NOT "${CXX_EXTRA_FLAGS}" STREQUAL "") 136 | message (STATUS "Appending CXX_EXTRA_FLAGS") 137 | set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${CXX_EXTRA_FLAGS}") 138 | endif() 139 | endif() 140 | endif() 141 | 142 | if (CMAKE_CXX_COMPILER_ID MATCHES "Clang") 143 | set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Qunused-arguments") 144 | set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Qunused-arguments") 145 | set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -Qunused-arguments") 146 | set(CMAKE_MODULE_LINKER_FLAGS "${CMAKE_MODULE_LINKER_FLAGS} -Qunused-arguments") 147 | set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Qunused-arguments") 148 | endif() 149 | 150 | if(CMAKE_SYSTEM_PROCESSOR MATCHES "^(powerpc|ppc)") 151 | # spatialite crashes on ppc - see bugs.debian.org/603986 152 | add_definitions( -fno-strict-aliasing ) 153 | endif() 154 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | 2 | # PDAL wrench 3 | 4 | A collection of easy to use command line tools for processing of point cloud data: 5 | 6 | - basic management of data (info, translate, merge, tile, thin, clip, boundary, density, to_vector) 7 | - export of raster grids (to_raster, to_raster_tin) 8 | - handle [virtual point clouds](vpc-spec.md) (build_vpc) 9 | 10 | And more algorithms will be added in the future! 11 | 12 | The whole suite of tools has been integrated into the QGIS Processing framework in QGIS starting from QGIS 3.32 (release in June 2023). 13 | There is no plugin to install - everything is available in QGIS core - just open the Processing toolbox and search for point cloud algorithms. 14 | 15 | Most of the tools are multi-threaded, making good use of all available CPUs for fast processing. 16 | 17 | All tools are based on PDAL pipelines, also using the other usual geospatial libraries: GDAL/OGR, GEOS and PROJ. 18 | 19 | Some may ask why not let users build their pipelines in PDAL directly. There are several reasons: 20 | 21 | - ease of use: PDAL is great for advanced users, but not everyone finds it easy to manually craft JSON files with pipelines, study manuals of the many stages and read details about file formats involved 22 | - parallel execution: PDAL runs pipelines in a single thread - only one CPU gets to do the work normally - and users need to implement their own parallelism if they wish so 23 | 24 | # How to build 25 | 26 | You will need: PDAL >= 2.5 and GDAL >= 3.0, both with development files. 27 | 28 | Building is done with CMake: 29 | ``` 30 | mkdir build 31 | cd build 32 | cmake .. 33 | make 34 | ``` 35 | 36 | If PDAL is not installed in the usual system paths (e.g. in `/usr`), then add `PDAL_DIR` parameter when running CMake - for example if your PDAL installation dir is `/home/martin/pdal-inst`: 37 | ``` 38 | cmake -DPDAL_DIR=/home/martin/pdal-inst/lib/cmake/PDAL .. 39 | ``` 40 | 41 | # Parallel processing 42 | 43 | PDAL runs point cloud pipelines in a single thread and any parallelization is up to users of the library. 44 | PDAL wrench has parallel processing built in and tries to run pipelines in parallel. This generally happens in two scenarios: 45 | 46 | - input dataset is in COPC or EPT format - these formats allow efficient spatial queries (to access only data in a particular bounding box), 47 | so PDAL wrench can split the input into multiple tiles that can be processed independently in parallel 48 | - input dataset is a [virtual point cloud (VPC)](vpc-spec.md) - such datasets are composed of a number of files, so the whole work can be split into jobs 49 | where each parallel job processes one or more input files 50 | 51 | If the input is a single LAS/LAZ file, no parallelization is attempted. This may change in the future with introduction of more complex algorithms (where the cost of reading the input is much lower than the cost of the actual algorithm). 52 | 53 | # Commands 54 | 55 | ## info 56 | 57 | Prints basic metadata from the point cloud file: 58 | 59 | ``` 60 | pdal_wrench info --input=data.las 61 | ``` 62 | 63 | ## translate 64 | 65 | Convert to a different file format (e.g. create compressed LAZ): 66 | 67 | ``` 68 | pdal_wrench translate --input=data.las --output=data.laz 69 | ``` 70 | 71 | Reproject point cloud to a different coordinate reference system: 72 | 73 | ``` 74 | pdal_wrench translate --input=data.las --output=reprojected.las --transform-crs=EPSG:3857 75 | ``` 76 | 77 | Assign coordinate reference system (if not present or wrong): 78 | 79 | ``` 80 | pdal_wrench translate --input=data-with-invalid-crs.las --output=data.las --assign-crs=EPSG:3857 81 | ``` 82 | 83 | 84 | ## boundary 85 | 86 | Exports a polygon file containing boundary. It may contain holes and it may be a multi-part polygon. 87 | 88 | ``` 89 | pdal_wrench boundary --input=data.las --output=boundary.gpkg 90 | ``` 91 | 92 | ## density 93 | 94 | Exports a raster file where each cell contains number of points that are in that cell's area. 95 | 96 | ``` 97 | pdal_wrench density --input=data.las --resolution=1 --output=density.tif 98 | ``` 99 | 100 | ## clip 101 | 102 | Outputs only points that are inside of the clipping polygons. 103 | 104 | ``` 105 | pdal_wrench clip --input=data.las --polygon=clip.gpkg --output=data_clipped.las 106 | ``` 107 | 108 | ## merge 109 | 110 | Merges multiple point cloud files to a single one. 111 | 112 | ``` 113 | pdal_wrench merge --output=merged.las data1.las data2.las data3.las 114 | ``` 115 | 116 | Alternatively, it is possible to merge files whose paths are specified in a text file (one file per line) 117 | 118 | ``` 119 | pdal_wrench merge --output=merged.las --input-file-list=my_list.txt 120 | ``` 121 | 122 | ## tile 123 | 124 | Creates tiles from input data. For example to get tiles sized 100x100: 125 | 126 | ``` 127 | pdal_wrench tile --length=100 --output=/data/tiles data1.las data2.las data3.las 128 | ``` 129 | 130 | This tool can also read input data from a text file (one file per line) 131 | 132 | ``` 133 | pdal_wrench tile --length=100 --output=/data/tiles --input-file-list=my_list.txt 134 | ``` 135 | 136 | ## thin 137 | 138 | Creates a thinned version of the point cloud by only keeping every N-th point (`every-nth` mode) or keep points based on their distance (`sample` mode). 139 | 140 | For example, to only keep every 20th point, so only 5% of points will be in the output: 141 | 142 | ``` 143 | pdal_wrench thin --output=thinned.las --mode=every-nth --step-every-nth=20 --input=data.las 144 | ``` 145 | 146 | Alternatively, to sample points using Poisson sampling of the input: 147 | 148 | ``` 149 | pdal_wrench thin --output=thinned.las --mode=sample --step-sample=20 --input=data.las 150 | ``` 151 | 152 | ## to_raster 153 | 154 | Exports point cloud data to a 2D raster grid, having cell size of given resolution, writing values from the specified attribute. Uses inverse distance weighting. 155 | 156 | ``` 157 | pdal_wrench to_raster --output=raster.tif --resolution=1 --attribute=Z --input=data.las 158 | ``` 159 | 160 | ## to_raster_tin 161 | 162 | Exports point cloud data to a 2D raster grid like `to_raster` does, but using a triangulation of points and then interpolating cell values from triangles. It does not produce any "holes" when some data are missing. Only supports output of Z attribute. 163 | 164 | ``` 165 | pdal_wrench to_raster_tin --output=raster.tif --resolution=1 --input=data.las 166 | ``` 167 | 168 | ## to_vector 169 | 170 | Exports point cloud data to a vector layer with 3D points (a GeoPackage), optionally with extra attributes: 171 | 172 | ``` 173 | pdal_wrench to_vector --output=data.gpkg --input=data.las 174 | ``` 175 | 176 | # Virtual Point Clouds (VPC) 177 | 178 | This is similar to GDAL's VRT - a single file referring to other files that contain actual data. Software then may handle all data as a single dataset. 179 | 180 | Virtual point clouds based on STAC protocol's ItemCollection, which is in fact a GeoJSON feature collection with extra metadata written in a standard way. See [VPC spec](vpc-spec.md) for more details on the file format. 181 | 182 | To create a virtual point cloud: 183 | ``` 184 | pdal_wrench build_vpc --output=hello.vpc data1.las data2.las data3.las 185 | ``` 186 | 187 | Or, if the inputs are listed in a text file: 188 | ``` 189 | data1.las 190 | data2.las 191 | data3.las 192 | ``` 193 | 194 | You can provide that file as input: 195 | ``` 196 | pdal_wrench build_vpc --output=hello.vpc --input-file-list=inputs.txt 197 | ``` 198 | 199 | Afterwards, other algorithms can be applied to a VPC: 200 | ``` 201 | pdal_wrench clip --input=hello.vpc --polygon=clip.gpkg --output=hello_clipped.vpc 202 | ``` 203 | 204 | This will create a grid for each data file separately in parallel and then merge the results to a final GeoTIFF: 205 | ``` 206 | pdal_wrench density --input=hello.vpc --resolution=1 --output=density.tif 207 | ``` 208 | 209 | When algorithms create derived VPCs, by default they use uncompressed LAS, but `--output-format=laz` option can switch to compressed LAZ. 210 | 211 | ## VPC support in algorithms 212 | 213 | | Algorithm | VPC | Notes | 214 | |--------------|-----------|--| 215 | | info | yes | | 216 | | boundary | multi-threaded | per file | 217 | | density | multi-threaded | spatial tiling | 218 | | clip | multi-threaded | per file | 219 | | merge | single-threaded | | 220 | | tile | multi-threaded | per file | 221 | | thin | multi-threaded | per file | 222 | | to_raster | multi-threaded | spatial tiling | 223 | | to_raster_tin | multi-threaded | spatial tiling | 224 | | to_vector | multi-threaded | per file | 225 | | translate | multi-threaded | per file | 226 | -------------------------------------------------------------------------------- /environment.yml: -------------------------------------------------------------------------------- 1 | name: wrench 2 | channels: 3 | - conda-forge 4 | dependencies: 5 | - conda 6 | - mamba 7 | - compilers 8 | - ninja 9 | - cmake 10 | - pdal 11 | 12 | -------------------------------------------------------------------------------- /src/alg.cpp: -------------------------------------------------------------------------------- 1 | /***************************************************************************** 2 | * Copyright (c) 2023, Lutra Consulting Ltd. and Hobu, Inc. * 3 | * * 4 | * All rights reserved. * 5 | * * 6 | * This program is free software; you can redistribute it and/or modify * 7 | * it under the terms of the GNU General Public License as published by * 8 | * the Free Software Foundation; either version 3 of the License, or * 9 | * (at your option) any later version. * 10 | * * 11 | ****************************************************************************/ 12 | 13 | #include "alg.hpp" 14 | 15 | #include "utils.hpp" 16 | #include "vpc.hpp" 17 | 18 | #include 19 | 20 | #include 21 | #include 22 | 23 | using namespace pdal; 24 | 25 | 26 | bool runAlg(std::vector args, Alg &alg) 27 | { 28 | 29 | try 30 | { 31 | if ( !alg.parseArgs(args) ) 32 | return false; 33 | } 34 | catch (const pdal::arg_error& err) 35 | { 36 | std::cerr << "Failed to parse arguments: " << err.what() << std::endl; 37 | return false; 38 | } 39 | 40 | if (alg.hasSingleInput) 41 | { 42 | if (ends_with(alg.inputFile, ".vpc")) 43 | { 44 | VirtualPointCloud vpc; 45 | if (!vpc.read(alg.inputFile)) 46 | return false; 47 | alg.totalPoints = vpc.totalPoints(); 48 | alg.bounds = vpc.box3d(); 49 | if (!alg.needsSingleCrs) 50 | alg.crs = SpatialReference(vpc.crsWkt); 51 | 52 | if (alg.needsSingleCrs && vpc.crsWkt == "_mix_") 53 | { 54 | std::cerr << "Algorithm requires that all inputs are in the same CRS. Please transform them to a single CRS first." << std::endl; 55 | return false; 56 | } 57 | } 58 | else 59 | { 60 | QuickInfo qi = getQuickInfo(alg.inputFile); 61 | alg.totalPoints = qi.m_pointCount; 62 | alg.bounds = qi.m_bounds; 63 | alg.crs = qi.m_srs; 64 | } 65 | } 66 | 67 | std::vector> pipelines; 68 | 69 | alg.preparePipelines(pipelines); 70 | 71 | if (pipelines.empty()) 72 | return false; 73 | 74 | runPipelineParallel(alg.totalPoints, alg.isStreaming, pipelines, alg.max_threads, alg.verbose); 75 | 76 | alg.finalize(pipelines); 77 | 78 | return true; 79 | } 80 | 81 | 82 | bool Alg::parseArgs(std::vector args) 83 | { 84 | pdal::Arg* argInput = nullptr; 85 | if (hasSingleInput) 86 | { 87 | argInput = &programArgs.add("input,i", "Input point cloud file", inputFile); 88 | } 89 | 90 | (void)programArgs.add("filter,f", "Filter expression for input data", filterExpression); 91 | (void)programArgs.add("bounds", "Filter by rectangle", filterBounds); 92 | 93 | addArgs(); // impl in derived 94 | 95 | // parallel run support (generic) 96 | pdal::Arg& argThreads = programArgs.add("threads", "Max number of concurrent threads for parallel runs", max_threads); 97 | 98 | programArgs.add("verbose", "Print extra debugging output", verbose); 99 | 100 | try 101 | { 102 | programArgs.parseSimple(args); 103 | } 104 | catch(pdal::arg_error err) 105 | { 106 | std::cerr << "failed to parse arguments: " << err.what() << std::endl; 107 | return false; 108 | } 109 | 110 | // TODO: ProgramArgs does not support required options 111 | if (argInput && !argInput->set()) 112 | { 113 | std::cerr << "missing input" << std::endl; 114 | return false; 115 | } 116 | 117 | if (!filterBounds.empty()) 118 | { 119 | try 120 | { 121 | parseBounds(filterBounds); 122 | } 123 | catch (pdal::Bounds::error& err) 124 | { 125 | std::cerr << "invalid bounds: " << err.what() << std::endl; 126 | return false; 127 | } 128 | } 129 | 130 | if (!checkArgs()) // impl in derived class 131 | return false; 132 | 133 | if (!args.empty()) 134 | { 135 | std::cerr << "unexpected args!" << std::endl; 136 | for ( auto & a : args ) 137 | std::cerr << " - " << a << std::endl; 138 | return false; 139 | } 140 | 141 | if (!argThreads.set()) // in such case our value is reset to zero 142 | { 143 | // use number of cores if not specified by the user 144 | max_threads = std::thread::hardware_concurrency(); 145 | if (max_threads == 0) 146 | { 147 | // in case the value can't be detected, use something reasonable... 148 | max_threads = 4; 149 | } 150 | } 151 | 152 | return true; 153 | } 154 | 155 | void removeFiles(const std::vector &tileOutputFiles, bool removeParentDirIfEmpty) 156 | { 157 | if (tileOutputFiles.empty()) 158 | { 159 | return; 160 | } 161 | 162 | fs::path outputDir = fs::path(tileOutputFiles[0]).parent_path(); 163 | for (const std::string &f : tileOutputFiles) 164 | { 165 | if (fs::exists(fs::path(f))) 166 | { 167 | fs::remove(f); 168 | } 169 | } 170 | 171 | if (removeParentDirIfEmpty && fs::is_empty(outputDir)) 172 | { 173 | fs::remove(outputDir); 174 | } 175 | } 176 | -------------------------------------------------------------------------------- /src/alg.hpp: -------------------------------------------------------------------------------- 1 | /***************************************************************************** 2 | * Copyright (c) 2023, Lutra Consulting Ltd. and Hobu, Inc. * 3 | * * 4 | * All rights reserved. * 5 | * * 6 | * This program is free software; you can redistribute it and/or modify * 7 | * it under the terms of the GNU General Public License as published by * 8 | * the Free Software Foundation; either version 3 of the License, or * 9 | * (at your option) any later version. * 10 | * * 11 | ****************************************************************************/ 12 | 13 | #pragma once 14 | 15 | #include 16 | 17 | #include 18 | #include 19 | 20 | #include "utils.hpp" 21 | 22 | using namespace pdal; 23 | 24 | struct ParallelJobInfo; 25 | 26 | namespace fs = std::filesystem; 27 | 28 | /** 29 | * Base class for algorithms. The general pattern is that: 30 | * 1. algorithm defines arguments (addArgs()) and checks if the values from user are valid (checkArgs()) 31 | * 2. prepare PDAL pipelines (preparePipelines()) and get them executed in multiple threads 32 | * 3. run finalization code (finalize()) 33 | */ 34 | struct Alg 35 | { 36 | // parallel runs (generic) 37 | int max_threads = -1; 38 | 39 | // a hint whether the pipelines will be executed in streaming mode 40 | bool isStreaming = true; 41 | 42 | // all algs should have some input... 43 | bool hasSingleInput = true; // some algs need multiple inputs - they should set this flag to false 44 | std::string inputFile; 45 | 46 | std::string filterExpression; // optional argument to limit input points 47 | 48 | std::string filterBounds; // optional clipping rectangle for input (pdal::Bounds) 49 | 50 | bool needsSingleCrs = true; // most algs assume that all input files in VPC are in the same CRS, 51 | // and only few exceptions (e.g. info) tolerate mixture of multiple CRS 52 | 53 | bool verbose = false; // write extra debugging output from the algorithm 54 | 55 | point_count_t totalPoints = 0; // calculated number of points from the input data 56 | BOX3D bounds; // calculated 3D bounding box from the input data 57 | SpatialReference crs; // CRS of the input data (only valid when needsSingleCrs==true) 58 | 59 | 60 | pdal::ProgramArgs programArgs; 61 | 62 | Alg() = default; 63 | virtual ~Alg() = default; 64 | 65 | // no copying 66 | Alg(const Alg &other) = delete; 67 | Alg& operator=(const Alg &other) = delete; 68 | 69 | bool parseArgs(std::vector args); 70 | 71 | // interface 72 | 73 | /** 74 | * Adds required and optional arguments to "programArgs" member variable. 75 | */ 76 | virtual void addArgs() = 0; 77 | /** 78 | * Called after argument parsing - evaluates whether the input is correct, returns false if not. 79 | */ 80 | virtual bool checkArgs() = 0; 81 | /** 82 | * Prepares pipelines that the algorithm needs to run and populates the given vector. 83 | * Pipelines are then run in a thread pool. 84 | */ 85 | virtual void preparePipelines(std::vector>& pipelines) = 0; 86 | /** 87 | * Runs and post-processing code when pipelines are done executing. 88 | */ 89 | virtual void finalize(std::vector>& pipelines) { ( void )pipelines; }; 90 | }; 91 | 92 | bool runAlg(std::vector args, Alg &alg); 93 | 94 | void removeFiles(const std::vector &tileOutputFiles, bool removeParentDirIfEmpty = true); 95 | 96 | ////////////// 97 | 98 | 99 | struct Info : public Alg 100 | { 101 | Info() { needsSingleCrs = false; } 102 | 103 | // impl 104 | virtual void addArgs() override; 105 | virtual bool checkArgs() override; 106 | virtual void preparePipelines(std::vector>& pipelines) override; 107 | virtual void finalize(std::vector>& pipelines) override; 108 | }; 109 | 110 | struct Translate : public Alg 111 | { 112 | // parameters from the user 113 | std::string outputFile; 114 | std::string assignCrs; 115 | std::string transformCrs; 116 | std::string transformCoordOp; 117 | std::string outputFormat; // las / laz / copc 118 | 119 | // args - initialized in addArgs() 120 | pdal::Arg* argOutput = nullptr; 121 | pdal::Arg* argOutputFormat = nullptr; 122 | 123 | std::vector tileOutputFiles; 124 | 125 | // impl 126 | virtual void addArgs() override; 127 | virtual bool checkArgs() override; 128 | virtual void preparePipelines(std::vector>& pipelines) override; 129 | virtual void finalize(std::vector>& pipelines) override; 130 | }; 131 | 132 | struct Density : public Alg 133 | { 134 | // parameters from the user 135 | std::string outputFile; 136 | double resolution = 0; 137 | 138 | // tiling setup for parallel runs 139 | TileAlignment tileAlignment; 140 | 141 | // args - initialized in addArgs() 142 | pdal::Arg* argOutput = nullptr; 143 | pdal::Arg* argRes = nullptr; 144 | pdal::Arg* argTileSize = nullptr; 145 | pdal::Arg* argTileOriginX = nullptr; 146 | pdal::Arg* argTileOriginY = nullptr; 147 | 148 | std::vector tileOutputFiles; 149 | 150 | // impl 151 | virtual void addArgs() override; 152 | virtual bool checkArgs() override; 153 | virtual void preparePipelines(std::vector>& pipelines) override; 154 | virtual void finalize(std::vector>& pipelines) override; 155 | 156 | // new 157 | std::unique_ptr pipeline(ParallelJobInfo *tile = nullptr) const; 158 | }; 159 | 160 | 161 | struct Boundary : public Alg 162 | { 163 | // parameters from the user 164 | std::string outputFile; 165 | double resolution = 0; // cell size of the hexbin filter (if zero, it will be estimated by PDAL) 166 | int pointsThreshold = 0; // min. number of points in order to have a cell considered occupied 167 | 168 | // args - initialized in addArgs() 169 | pdal::Arg* argOutput = nullptr; 170 | pdal::Arg* argResolution = nullptr; 171 | pdal::Arg* argPointsThreshold = nullptr; 172 | 173 | // impl 174 | virtual void addArgs() override; 175 | virtual bool checkArgs() override; 176 | virtual void preparePipelines(std::vector>& pipelines) override; 177 | virtual void finalize(std::vector>& pipelines) override; 178 | 179 | }; 180 | 181 | 182 | struct Clip : public Alg 183 | { 184 | // parameters from the user 185 | std::string outputFile; 186 | std::string polygonFile; 187 | std::string outputFormat; // las / laz / copc 188 | 189 | // args - initialized in addArgs() 190 | pdal::Arg* argOutput = nullptr; 191 | pdal::Arg* argOutputFormat = nullptr; 192 | pdal::Arg* argPolygon = nullptr; 193 | 194 | std::vector tileOutputFiles; 195 | 196 | // impl 197 | virtual void addArgs() override; 198 | virtual bool checkArgs() override; 199 | virtual void preparePipelines(std::vector>& pipelines) override; 200 | virtual void finalize(std::vector>& pipelines) override; 201 | 202 | // new 203 | //std::unique_ptr pipeline(ParallelTileInfo *tile, const pdal::Options &crop_opts) const; 204 | }; 205 | 206 | 207 | struct Merge : public Alg 208 | { 209 | 210 | // parameters from the user 211 | std::string outputFile; 212 | std::vector inputFiles; 213 | std::string inputFileList; 214 | 215 | // args - initialized in addArgs() 216 | pdal::Arg* argOutput = nullptr; 217 | 218 | Merge() { hasSingleInput = false; } 219 | 220 | // impl 221 | virtual void addArgs() override; 222 | virtual bool checkArgs() override; 223 | virtual void preparePipelines(std::vector>& pipelines) override; 224 | 225 | }; 226 | 227 | 228 | struct Thin : public Alg 229 | { 230 | // parameters from the user 231 | std::string outputFile; 232 | std::string mode; // "every-nth" or "sample" 233 | int stepEveryN; // keep every N-th point 234 | double stepSample; // cell size for Poisson sampling 235 | std::string outputFormat; // las / laz / copc 236 | 237 | // args - initialized in addArgs() 238 | pdal::Arg* argOutput = nullptr; 239 | pdal::Arg* argMode = nullptr; 240 | pdal::Arg* argStepEveryN = nullptr; 241 | pdal::Arg* argStepSample = nullptr; 242 | pdal::Arg* argOutputFormat = nullptr; 243 | 244 | std::vector tileOutputFiles; 245 | 246 | // impl 247 | virtual void addArgs() override; 248 | virtual bool checkArgs() override; 249 | virtual void preparePipelines(std::vector>& pipelines) override; 250 | virtual void finalize(std::vector>& pipelines) override; 251 | }; 252 | 253 | 254 | 255 | struct ToRaster : public Alg 256 | { 257 | // parameters from the user 258 | std::string outputFile; 259 | double resolution = 0; 260 | std::string attribute; 261 | double collarSize = 0; 262 | 263 | // tiling setup for parallel runs 264 | TileAlignment tileAlignment; 265 | 266 | // args - initialized in addArgs() 267 | pdal::Arg* argOutput = nullptr; 268 | pdal::Arg* argRes = nullptr; 269 | pdal::Arg* argAttribute = nullptr; 270 | 271 | pdal::Arg* argTileSize = nullptr; 272 | pdal::Arg* argTileOriginX = nullptr; 273 | pdal::Arg* argTileOriginY = nullptr; 274 | 275 | std::vector tileOutputFiles; 276 | 277 | // impl 278 | virtual void addArgs() override; 279 | virtual bool checkArgs() override; 280 | virtual void preparePipelines(std::vector>& pipelines) override; 281 | virtual void finalize(std::vector>& pipelines) override; 282 | }; 283 | 284 | 285 | struct ToRasterTin : public Alg 286 | { 287 | // parameters from the user 288 | std::string outputFile; 289 | double resolution = 0; 290 | double collarSize = 0; 291 | 292 | // tiling setup for parallel runs 293 | TileAlignment tileAlignment; 294 | 295 | // args - initialized in addArgs() 296 | pdal::Arg* argOutput = nullptr; 297 | pdal::Arg* argRes = nullptr; 298 | pdal::Arg* argTileSize = nullptr; 299 | pdal::Arg* argTileOriginX = nullptr; 300 | pdal::Arg* argTileOriginY = nullptr; 301 | 302 | std::vector tileOutputFiles; 303 | 304 | ToRasterTin() { isStreaming = false; } 305 | 306 | // impl 307 | virtual void addArgs() override; 308 | virtual bool checkArgs() override; 309 | virtual void preparePipelines(std::vector>& pipelines) override; 310 | virtual void finalize(std::vector>& pipelines) override; 311 | }; 312 | 313 | 314 | struct ToVector : public Alg 315 | { 316 | // parameters from the user 317 | std::string outputFile; 318 | std::vector attributes; 319 | 320 | // args - initialized in addArgs() 321 | pdal::Arg* argOutput = nullptr; 322 | //pdal::Arg* argAttribute = nullptr; 323 | 324 | std::vector tileOutputFiles; 325 | 326 | // impl 327 | virtual void addArgs() override; 328 | virtual bool checkArgs() override; 329 | virtual void preparePipelines(std::vector>& pipelines) override; 330 | virtual void finalize(std::vector>& pipelines) override; 331 | }; 332 | -------------------------------------------------------------------------------- /src/boundary.cpp: -------------------------------------------------------------------------------- 1 | /***************************************************************************** 2 | * Copyright (c) 2023, Lutra Consulting Ltd. and Hobu, Inc. * 3 | * * 4 | * All rights reserved. * 5 | * * 6 | * This program is free software; you can redistribute it and/or modify * 7 | * it under the terms of the GNU General Public License as published by * 8 | * the Free Software Foundation; either version 3 of the License, or * 9 | * (at your option) any later version. * 10 | * * 11 | ****************************************************************************/ 12 | 13 | #include 14 | #include 15 | 16 | #include 17 | #include 18 | #include 19 | 20 | #include 21 | #include 22 | #include 23 | 24 | #include "utils.hpp" 25 | #include "alg.hpp" 26 | #include "vpc.hpp" 27 | 28 | using namespace pdal; 29 | 30 | 31 | void Boundary::addArgs() 32 | { 33 | argOutput = &programArgs.add("output,o", "Output vector file", outputFile); 34 | 35 | argResolution = &programArgs.add("resolution", "Resolution of cells used to calculate boundary. " 36 | "If not specified, it will be estimated from first 5000 points.", resolution); 37 | argPointsThreshold = &programArgs.add("threshold", "Minimal number of points in a cell to consider cell occupied.", pointsThreshold); 38 | } 39 | 40 | bool Boundary::checkArgs() 41 | { 42 | if (!argOutput->set()) 43 | { 44 | std::cerr << "missing output" << std::endl; 45 | return false; 46 | } 47 | 48 | if (!argResolution->set() && argPointsThreshold->set()) 49 | { 50 | std::cerr << "Resolution argument must be set when points threshold is set." << std::endl; 51 | return false; 52 | } 53 | 54 | if (!argPointsThreshold->set()) 55 | { 56 | pointsThreshold = 15; // the same default as in PDAL for HexBin filter 57 | } 58 | 59 | return true; 60 | } 61 | 62 | static std::unique_ptr pipeline(ParallelJobInfo *tile, double resolution, int pointsThreshold) 63 | { 64 | assert(tile); 65 | assert(tile->inputFilenames.size() == 1); 66 | 67 | std::unique_ptr manager( new PipelineManager ); 68 | 69 | Stage& r = manager->makeReader(tile->inputFilenames[0], ""); 70 | 71 | Stage *last = &r; 72 | 73 | // filtering 74 | if (!tile->filterBounds.empty()) 75 | { 76 | Options filter_opts; 77 | filter_opts.add(pdal::Option("bounds", tile->filterBounds)); 78 | 79 | if (readerSupportsBounds(r)) 80 | { 81 | // Reader of the format can do the filtering - use that whenever possible! 82 | r.addOptions(filter_opts); 83 | } 84 | else 85 | { 86 | // Reader can't do the filtering - do it with a filter 87 | last = &manager->makeFilter( "filters.crop", *last, filter_opts); 88 | } 89 | } 90 | if (!tile->filterExpression.empty()) 91 | { 92 | Options filter_opts; 93 | filter_opts.add(pdal::Option("expression", tile->filterExpression)); 94 | last = &manager->makeFilter( "filters.expression", *last, filter_opts); 95 | } 96 | 97 | // TODO: what edge size? (by default samples 5000 points if not specified 98 | // TODO: set threshold ? (default at least 16 points to keep the cell) 99 | // btw. if threshold=0, there are still missing points because of simplification (smooth=True) 100 | 101 | pdal::Options hexbin_opts; 102 | if (resolution != 0) 103 | { 104 | hexbin_opts.add(pdal::Option("edge_size", resolution)); 105 | } 106 | hexbin_opts.add(pdal::Option("threshold", pointsThreshold)); 107 | (void)manager->makeFilter( "filters.hexbin", *last, hexbin_opts ); 108 | 109 | return manager; 110 | } 111 | 112 | void Boundary::preparePipelines(std::vector>& pipelines) 113 | { 114 | if (ends_with(inputFile, ".vpc")) 115 | { 116 | // VPC handling 117 | VirtualPointCloud vpc; 118 | if (!vpc.read(inputFile)) 119 | return; 120 | 121 | for (const VirtualPointCloud::File& f : vpc.files) 122 | { 123 | ParallelJobInfo tile(ParallelJobInfo::FileBased, BOX2D(), filterExpression, filterBounds); 124 | tile.inputFilenames.push_back(f.filename); 125 | pipelines.push_back(pipeline(&tile, resolution, pointsThreshold)); 126 | } 127 | } 128 | else 129 | { 130 | ParallelJobInfo tile(ParallelJobInfo::Single, BOX2D(), filterExpression, filterBounds); 131 | tile.inputFilenames.push_back(inputFile); 132 | pipelines.push_back(pipeline(&tile, resolution, pointsThreshold)); 133 | } 134 | } 135 | 136 | static std::string extractPolygon(PipelineManager &pipeline) 137 | { 138 | pdal::MetadataNode mn = pipeline.getMetadata(); 139 | //Utils::toJSON(mn, std::cout); 140 | 141 | pdal::MetadataNode hb = mn.findChild("filters.hexbin").findChild("boundary"); 142 | //Utils::toJSON(hb, std::cout); 143 | return hb.value(); 144 | } 145 | 146 | void Boundary::finalize(std::vector>& pipelines) 147 | { 148 | if (pipelines.empty()) 149 | return; 150 | 151 | GDALAllRegister(); 152 | 153 | OGRSpatialReferenceH hSrs; 154 | hSrs = OSRNewSpatialReference(crs.getWKT().c_str()); 155 | assert(hSrs); 156 | 157 | OGRwkbGeometryType wkbType = /*wkt.find("MULTI") == std::string::npos ? wkbPolygon :*/ wkbMultiPolygon; 158 | 159 | OGRSFDriverH hDriver = OGRGetDriverByName("GPKG"); 160 | if (hDriver == nullptr) 161 | { 162 | std::cerr << "Failed to create GPKG driver" << std::endl; 163 | return; 164 | } 165 | 166 | GDALDatasetH hDS = GDALCreate( hDriver, outputFile.c_str(), 0, 0, 0, GDT_Unknown, nullptr ); 167 | if (hDS == nullptr) 168 | { 169 | std::cerr << "Failed to create output file: " << outputFile << std::endl; 170 | return; 171 | } 172 | 173 | OGRLayerH hLayer = GDALDatasetCreateLayer( hDS, "boundary", hSrs, wkbType, nullptr ); 174 | if (hLayer == nullptr) 175 | { 176 | std::cerr << "Failed to create layer in the output file: " << outputFile << std::endl; 177 | return; 178 | } 179 | 180 | // TODO: to a union of boundary polygons 181 | 182 | for (auto& pipePtr : pipelines) 183 | { 184 | // extract boundary polygon 185 | std::string wkt = extractPolygon(*pipePtr.get()); 186 | 187 | OGRGeometryH geom; 188 | char *wkt_ptr = wkt.data(); 189 | if (OGR_G_CreateFromWkt(&wkt_ptr, hSrs, &geom) != OGRERR_NONE) 190 | { 191 | std::cerr << "Failed to parse geometry: " << wkt << std::endl; 192 | } 193 | if ( wkbFlatten(OGR_G_GetGeometryType(geom)) == wkbPolygon) 194 | { 195 | // this function takes ownership of "geom" and then creates new instance 196 | geom = OGR_G_ForceToMultiPolygon(geom); 197 | } 198 | OGRFeatureH hFeature = OGR_F_Create(OGR_L_GetLayerDefn(hLayer)); 199 | if (OGR_F_SetGeometryDirectly(hFeature, geom) != OGRERR_NONE) 200 | { 201 | std::cerr << "Could not set geometry " << wkt << std::endl; 202 | } 203 | if (OGR_L_CreateFeature(hLayer, hFeature) != OGRERR_NONE) 204 | { 205 | std::cerr << "Failed to create a new feature in the output file!" << std::endl; 206 | } 207 | 208 | OGR_F_Destroy(hFeature); 209 | } 210 | 211 | OSRDestroySpatialReference(hSrs); 212 | GDALClose(hDS); 213 | } 214 | -------------------------------------------------------------------------------- /src/clip.cpp: -------------------------------------------------------------------------------- 1 | /***************************************************************************** 2 | * Copyright (c) 2023, Lutra Consulting Ltd. and Hobu, Inc. * 3 | * * 4 | * All rights reserved. * 5 | * * 6 | * This program is free software; you can redistribute it and/or modify * 7 | * it under the terms of the GNU General Public License as published by * 8 | * the Free Software Foundation; either version 3 of the License, or * 9 | * (at your option) any later version. * 10 | * * 11 | ****************************************************************************/ 12 | 13 | #include 14 | #include 15 | #include 16 | 17 | #include 18 | #include 19 | #include 20 | #include 21 | #include 22 | 23 | #include 24 | 25 | #include "utils.hpp" 26 | #include "alg.hpp" 27 | #include "vpc.hpp" 28 | 29 | using namespace pdal; 30 | 31 | namespace fs = std::filesystem; 32 | 33 | 34 | void Clip::addArgs() 35 | { 36 | argOutput = &programArgs.add("output,o", "Output point cloud file", outputFile); 37 | argOutputFormat = &programArgs.add("output-format", "Output format (las/laz/copc)", outputFormat); 38 | argPolygon = &programArgs.add("polygon,p", "Input polygon vector file", polygonFile); 39 | } 40 | 41 | bool Clip::checkArgs() 42 | { 43 | if (!argOutput->set()) 44 | { 45 | std::cerr << "missing output" << std::endl; 46 | return false; 47 | } 48 | if (!argPolygon->set()) 49 | { 50 | std::cerr << "missing polygon" << std::endl; 51 | return false; 52 | } 53 | 54 | if (argOutputFormat->set()) 55 | { 56 | if (outputFormat != "las" && outputFormat != "laz") 57 | { 58 | std::cerr << "unknown output format: " << outputFormat << std::endl; 59 | return false; 60 | } 61 | } 62 | else 63 | outputFormat = "las"; // uncompressed by default 64 | 65 | return true; 66 | } 67 | 68 | 69 | // populate polygons into filters.crop options 70 | bool loadPolygons(const std::string &polygonFile, pdal::Options& crop_opts, BOX2D& bbox) 71 | { 72 | GDALAllRegister(); 73 | 74 | GDALDatasetH hDS = GDALOpenEx( polygonFile.c_str(), GDAL_OF_VECTOR, NULL, NULL, NULL ); 75 | if( hDS == NULL ) 76 | { 77 | std::cerr << "Could not open input polygon file: " << polygonFile << std::endl; 78 | return false; 79 | } 80 | 81 | // TODO: reproject polygons to the CRS of the point cloud if they are not the same 82 | 83 | OGRLayerH hLayer = GDALDatasetGetLayer(hDS, 0); 84 | //hLayer = GDALDatasetGetLayerByName( hDS, "point" ); 85 | 86 | OGREnvelope fullEnvelope; 87 | 88 | OGR_L_ResetReading(hLayer); 89 | OGRFeatureH hFeature; 90 | while( (hFeature = OGR_L_GetNextFeature(hLayer)) != NULL ) 91 | { 92 | OGRGeometryH hGeometry = OGR_F_GetGeometryRef(hFeature); 93 | if ( hGeometry != NULL ) 94 | { 95 | OGREnvelope envelope; 96 | OGR_G_GetEnvelope(hGeometry, &envelope); 97 | if (!fullEnvelope.IsInit()) 98 | fullEnvelope = envelope; 99 | else 100 | fullEnvelope.Merge(envelope); 101 | crop_opts.add(pdal::Option("polygon", pdal::Polygon(hGeometry))); 102 | } 103 | OGR_F_Destroy( hFeature ); 104 | } 105 | GDALClose( hDS ); 106 | 107 | bbox = BOX2D(fullEnvelope.MinX, fullEnvelope.MinY, fullEnvelope.MaxX, fullEnvelope.MaxY); 108 | 109 | return true; 110 | } 111 | 112 | 113 | static std::unique_ptr pipeline(ParallelJobInfo *tile, const pdal::Options &crop_opts) 114 | { 115 | assert(tile); 116 | 117 | std::unique_ptr manager( new PipelineManager ); 118 | 119 | Stage& r = manager->makeReader( tile->inputFilenames[0], ""); 120 | 121 | Stage *last = &r; 122 | 123 | // filtering 124 | if (!tile->filterBounds.empty()) 125 | { 126 | Options filter_opts; 127 | filter_opts.add(pdal::Option("bounds", tile->filterBounds)); 128 | 129 | if (readerSupportsBounds(r)) 130 | { 131 | // Reader of the format can do the filtering - use that whenever possible! 132 | r.addOptions(filter_opts); 133 | } 134 | else 135 | { 136 | // Reader can't do the filtering - do it with a filter 137 | last = &manager->makeFilter( "filters.crop", *last, filter_opts); 138 | } 139 | } 140 | if (!tile->filterExpression.empty()) 141 | { 142 | Options filter_opts; 143 | filter_opts.add(pdal::Option("expression", tile->filterExpression)); 144 | last = &manager->makeFilter( "filters.expression", *last, filter_opts); 145 | } 146 | 147 | last = &manager->makeFilter( "filters.crop", *last, crop_opts ); 148 | 149 | pdal::Options writer_opts; 150 | writer_opts.add(pdal::Option("forward", "all")); 151 | manager->makeWriter( tile->outputFilename, "", *last, writer_opts); 152 | 153 | return manager; 154 | } 155 | 156 | 157 | void Clip::preparePipelines(std::vector>& pipelines) 158 | { 159 | pdal::Options crop_opts; 160 | BOX2D bbox; 161 | if (!loadPolygons(polygonFile, crop_opts, bbox)) 162 | return; 163 | 164 | if (ends_with(inputFile, ".vpc")) 165 | { 166 | // for /tmp/hello.vpc we will use /tmp/hello dir for all results 167 | fs::path outputParentDir = fs::path(outputFile).parent_path(); 168 | fs::path outputSubdir = outputParentDir / fs::path(outputFile).stem(); 169 | fs::create_directories(outputSubdir); 170 | 171 | // VPC handling 172 | VirtualPointCloud vpc; 173 | if (!vpc.read(inputFile)) 174 | return; 175 | 176 | for (const VirtualPointCloud::File& f : vpc.files) 177 | { 178 | if (!bbox.overlaps(f.bbox.to2d())) 179 | { 180 | totalPoints -= f.count; 181 | continue; // we can safely skip 182 | } 183 | 184 | if (verbose) 185 | { 186 | std::cout << "using " << f.filename << std::endl; 187 | } 188 | 189 | ParallelJobInfo tile(ParallelJobInfo::FileBased, BOX2D(), filterExpression, filterBounds); 190 | tile.inputFilenames.push_back(f.filename); 191 | 192 | // for input file /x/y/z.las that goes to /tmp/hello.vpc, 193 | // individual output file will be called /tmp/hello/z.las 194 | fs::path inputBasename = fs::path(f.filename).stem(); 195 | 196 | // if the output is not VPC las file format is forced to avoid time spent on compression, files will be later merged into single output and removed anyways 197 | if (!ends_with(outputFile, ".vpc")) 198 | tile.outputFilename = (outputSubdir / inputBasename).string() + ".las"; 199 | else 200 | tile.outputFilename = (outputSubdir / inputBasename).string() + "." + outputFormat; 201 | 202 | tileOutputFiles.push_back(tile.outputFilename); 203 | 204 | pipelines.push_back(pipeline(&tile, crop_opts)); 205 | } 206 | } 207 | else 208 | { 209 | if (ends_with(outputFile, ".copc.laz")) 210 | { 211 | isStreaming = false; 212 | } 213 | ParallelJobInfo tile(ParallelJobInfo::Single, BOX2D(), filterExpression, filterBounds); 214 | tile.inputFilenames.push_back(inputFile); 215 | tile.outputFilename = outputFile; 216 | pipelines.push_back(pipeline(&tile, crop_opts)); 217 | } 218 | } 219 | 220 | void Clip::finalize(std::vector>&) 221 | { 222 | if (tileOutputFiles.empty()) 223 | return; 224 | 225 | // now build a new output VPC 226 | std::vector args; 227 | args.push_back("--output=" + outputFile); 228 | for (std::string f : tileOutputFiles) 229 | args.push_back(f); 230 | 231 | if (ends_with(outputFile, ".vpc")) 232 | { 233 | // now build a new output VPC 234 | buildVpc(args); 235 | } 236 | else 237 | { 238 | // merge all the output files into a single file 239 | Merge merge; 240 | // for copc set isStreaming to false 241 | if (ends_with(outputFile, ".copc.laz")) 242 | { 243 | merge.isStreaming = false; 244 | } 245 | runAlg(args, merge); 246 | 247 | // remove files as they are not needed anymore - they are merged 248 | removeFiles(tileOutputFiles, true); 249 | } 250 | } 251 | -------------------------------------------------------------------------------- /src/density.cpp: -------------------------------------------------------------------------------- 1 | /***************************************************************************** 2 | * Copyright (c) 2023, Lutra Consulting Ltd. and Hobu, Inc. * 3 | * * 4 | * All rights reserved. * 5 | * * 6 | * This program is free software; you can redistribute it and/or modify * 7 | * it under the terms of the GNU General Public License as published by * 8 | * the Free Software Foundation; either version 3 of the License, or * 9 | * (at your option) any later version. * 10 | * * 11 | ****************************************************************************/ 12 | 13 | #include 14 | #include 15 | #include 16 | 17 | #include 18 | #include 19 | #include 20 | 21 | #include "utils.hpp" 22 | #include "alg.hpp" 23 | #include "vpc.hpp" 24 | 25 | using namespace pdal; 26 | 27 | namespace fs = std::filesystem; 28 | 29 | void Density::addArgs() 30 | { 31 | argOutput = &programArgs.add("output,o", "Output raster file", outputFile); 32 | argRes = &programArgs.add("resolution,r", "Resolution of the density grid", resolution); 33 | argTileSize = &programArgs.add("tile-size", "Size of a tile for parallel runs", tileAlignment.tileSize); 34 | argTileOriginX = &programArgs.add("tile-origin-x", "X origin of a tile for parallel runs", tileAlignment.originX); 35 | argTileOriginY = &programArgs.add("tile-origin-y", "Y origin of a tile for parallel runs", tileAlignment.originY); 36 | } 37 | 38 | bool Density::checkArgs() 39 | { 40 | if (!argOutput->set()) 41 | { 42 | std::cerr << "missing output" << std::endl; 43 | return false; 44 | } 45 | if (!argRes->set()) 46 | { 47 | std::cerr << "missing resolution" << std::endl; 48 | return false; 49 | } 50 | 51 | // TODO: not sure why ProgramArgs cleans the default value 52 | if (!argTileSize->set()) 53 | { 54 | tileAlignment.tileSize = 1000; 55 | } 56 | 57 | if (!argTileOriginX->set()) 58 | tileAlignment.originX = -1; 59 | if (!argTileOriginY->set()) 60 | tileAlignment.originY = -1; 61 | 62 | return true; 63 | } 64 | 65 | 66 | std::unique_ptr Density::pipeline(ParallelJobInfo *tile) const 67 | { 68 | std::unique_ptr manager( new PipelineManager ); 69 | 70 | std::vector readers; 71 | for (const std::string &f : tile->inputFilenames) 72 | { 73 | readers.push_back(&manager->makeReader(f, "")); 74 | } 75 | 76 | std::vector last = readers; 77 | 78 | // find out what will be the bounding box for this job 79 | // (there could be also no bbox if there's no "bounds" filter and no tiling) 80 | BOX2D filterBox = !tile->filterBounds.empty() ? parseBounds(tile->filterBounds).to2d() : BOX2D(); 81 | BOX2D box = intersectTileBoxWithFilterBox(tile->box, filterBox); 82 | 83 | if (box.valid()) 84 | { 85 | // We are going to do filtering of points based on 2D box. Ideally we want to do 86 | // the filtering in the reader (if the reader can do it efficiently like copc/ept), 87 | // otherwise we have to add filters.crop stage to filter points after they were read 88 | 89 | for (Stage* reader : readers) 90 | { 91 | if (readerSupportsBounds(*reader)) 92 | { 93 | // add "bounds" option to reader 94 | pdal::Options copc_opts; 95 | copc_opts.add(pdal::Option("threads", 1)); 96 | copc_opts.add(pdal::Option("bounds", box_to_pdal_bounds(box))); 97 | reader->addOptions(copc_opts); 98 | } 99 | } 100 | 101 | if (!allReadersSupportBounds(readers) && !tile->filterBounds.empty()) 102 | { 103 | // At least some readers can't do the filtering - do it with a filter 104 | Options filter_opts; 105 | filter_opts.add(pdal::Option("bounds", tile->filterBounds)); 106 | Stage *filterCrop = &manager->makeFilter( "filters.crop", filter_opts); 107 | for (Stage *s : last) 108 | filterCrop->setInput(*s); 109 | last.clear(); 110 | last.push_back(filterCrop); 111 | } 112 | } 113 | 114 | if (!tile->filterExpression.empty()) 115 | { 116 | Options filter_opts; 117 | filter_opts.add(pdal::Option("expression", tile->filterExpression)); 118 | Stage *filterExpr = &manager->makeFilter( "filters.expression", filter_opts); 119 | for (Stage *s : last) 120 | filterExpr->setInput(*s); 121 | last.clear(); 122 | last.push_back(filterExpr); 123 | } 124 | 125 | pdal::Options writer_opts; 126 | writer_opts.add(pdal::Option("binmode", true)); 127 | writer_opts.add(pdal::Option("output_type", "count")); 128 | writer_opts.add(pdal::Option("resolution", resolution)); 129 | 130 | writer_opts.add(pdal::Option("data_type", "int16")); // 16k points in a cell should be enough? :) 131 | writer_opts.add(pdal::Option("gdalopts", "TILED=YES")); 132 | writer_opts.add(pdal::Option("gdalopts", "COMPRESS=DEFLATE")); 133 | 134 | if (box.valid()) 135 | { 136 | BOX2D box2 = box; 137 | // fix tile size - PDAL's writers.gdal adds one pixel (see GDALWriter::createGrid()), 138 | // because it probably expects that that the bounds and resolution do not perfectly match 139 | box2.maxx -= resolution; 140 | box2.maxy -= resolution; 141 | 142 | writer_opts.add(pdal::Option("bounds", box_to_pdal_bounds(box2))); 143 | } 144 | 145 | // TODO: "writers.gdal: Requested driver 'COG' does not support file creation."" 146 | // writer_opts.add(pdal::Option("gdaldriver", "COG")); 147 | 148 | pdal::StageCreationOptions opts{ tile->outputFilename, "", nullptr, writer_opts, "" }; 149 | Stage& w = manager->makeWriter( opts ); 150 | for (Stage *stage : last) 151 | w.setInput(*stage); 152 | 153 | return manager; 154 | } 155 | 156 | 157 | void Density::preparePipelines(std::vector>& pipelines) 158 | { 159 | if (ends_with(inputFile, ".vpc")) 160 | { 161 | // using spatial processing 162 | 163 | VirtualPointCloud vpc; 164 | if (!vpc.read(inputFile)) 165 | return; 166 | 167 | // for /tmp/hello.tif we will use /tmp/hello dir for all results 168 | fs::path outputParentDir = fs::path(outputFile).parent_path(); 169 | fs::path outputSubdir = outputParentDir / fs::path(outputFile).stem(); 170 | fs::create_directories(outputSubdir); 171 | 172 | bool unalignedFiles = false; 173 | 174 | // TODO: optionally adjust origin to have nicer numbers for bounds? 175 | if (tileAlignment.originX == -1) 176 | tileAlignment.originX = bounds.minx; 177 | if (tileAlignment.originY == -1) 178 | tileAlignment.originY = bounds.miny; 179 | 180 | // align bounding box of data to the grid 181 | TileAlignment gridAlignment = tileAlignment; 182 | gridAlignment.tileSize = resolution; 183 | Tiling gridTiling = gridAlignment.coverBounds(bounds.to2d()); 184 | std::cout << "grid " << gridTiling.tileCountX << "x" << gridTiling.tileCountY << std::endl; 185 | BOX2D gridBounds = gridTiling.fullBox(); 186 | 187 | Tiling t = tileAlignment.coverBounds(gridBounds); 188 | std::cout << "tiles " << t.tileCountX << " " << t.tileCountY << std::endl; 189 | 190 | totalPoints = 0; // we need to recalculate as we may use some points multiple times 191 | for (int iy = 0; iy < t.tileCountY; ++iy) 192 | { 193 | for (int ix = 0; ix < t.tileCountX; ++ix) 194 | { 195 | BOX2D tileBox = t.boxAt(ix, iy); 196 | 197 | // for tiles that are smaller than full box - only use intersection 198 | // to avoid empty areas in resulting rasters 199 | tileBox.clip(gridBounds); 200 | 201 | if (!filterBounds.empty() && !intersectionBox2D(tileBox, parseBounds(filterBounds).to2d()).valid()) 202 | { 203 | if (verbose) 204 | std::cout << "skipping tile " << iy << " " << ix << " -- " << tileBox.toBox() << std::endl; 205 | continue; 206 | } 207 | 208 | ParallelJobInfo tile(ParallelJobInfo::Spatial, tileBox, filterExpression, filterBounds); 209 | for (const VirtualPointCloud::File & f: vpc.overlappingBox2D(tileBox)) 210 | { 211 | tile.inputFilenames.push_back(f.filename); 212 | totalPoints += f.count; 213 | } 214 | if (tile.inputFilenames.empty()) 215 | continue; // no input files for this tile 216 | 217 | if (tile.inputFilenames.size() > 1) 218 | unalignedFiles = true; 219 | 220 | // create temp output file names 221 | // for tile (x=2,y=3) that goes to /tmp/hello.tif, 222 | // individual output file will be called /tmp/hello/2_3.tif 223 | fs::path inputBasename = std::to_string(ix) + "_" + std::to_string(iy); 224 | tile.outputFilename = (outputSubdir / inputBasename).string() + ".tif"; 225 | 226 | tileOutputFiles.push_back(tile.outputFilename); 227 | 228 | pipelines.push_back(pipeline(&tile)); 229 | } 230 | } 231 | 232 | if (unalignedFiles) 233 | { 234 | std::cerr << std::endl; 235 | std::cerr << "Warning: input files not perfectly aligned with tile grid - processing may take longer." << std::endl; 236 | std::cerr << "Consider using --tile-size, --tile-origin-x, --tile-origin-y arguments" << std::endl; 237 | std::cerr << std::endl; 238 | } 239 | } 240 | else if (ends_with(inputFile, ".copc.laz")) 241 | { 242 | // using square tiles for single COPC 243 | 244 | // for /tmp/hello.tif we will use /tmp/hello dir for all results 245 | fs::path outputParentDir = fs::path(outputFile).parent_path(); 246 | fs::path outputSubdir = outputParentDir / fs::path(outputFile).stem(); 247 | fs::create_directories(outputSubdir); 248 | 249 | if (tileAlignment.originX == -1) 250 | tileAlignment.originX = bounds.minx; 251 | if (tileAlignment.originY == -1) 252 | tileAlignment.originY = bounds.miny; 253 | 254 | Tiling t = tileAlignment.coverBounds(bounds.to2d()); 255 | 256 | for (int iy = 0; iy < t.tileCountY; ++iy) 257 | { 258 | for (int ix = 0; ix < t.tileCountX; ++ix) 259 | { 260 | BOX2D tileBox = t.boxAt(ix, iy); 261 | 262 | if (!filterBounds.empty() && !intersectionBox2D(tileBox, parseBounds(filterBounds).to2d()).valid()) 263 | { 264 | if (verbose) 265 | std::cout << "skipping tile " << iy << " " << ix << " -- " << tileBox.toBox() << std::endl; 266 | continue; 267 | } 268 | 269 | ParallelJobInfo tile(ParallelJobInfo::Spatial, tileBox, filterExpression, filterBounds); 270 | tile.inputFilenames.push_back(inputFile); 271 | 272 | // create temp output file names 273 | // for tile (x=2,y=3) that goes to /tmp/hello.tif, 274 | // individual output file will be called /tmp/hello/2_3.tif 275 | fs::path inputBasename = std::to_string(ix) + "_" + std::to_string(iy); 276 | tile.outputFilename = (outputSubdir / inputBasename).string() + ".tif"; 277 | 278 | tileOutputFiles.push_back(tile.outputFilename); 279 | 280 | pipelines.push_back(pipeline(&tile)); 281 | } 282 | } 283 | } 284 | else 285 | { 286 | // single input LAS/LAZ - no parallelism 287 | 288 | ParallelJobInfo tile(ParallelJobInfo::Single, BOX2D(), filterExpression, filterBounds); 289 | tile.inputFilenames.push_back(inputFile); 290 | tile.outputFilename = outputFile; 291 | pipelines.push_back(pipeline(&tile)); 292 | } 293 | 294 | } 295 | 296 | 297 | void Density::finalize(std::vector>&) 298 | { 299 | if (!tileOutputFiles.empty()) 300 | { 301 | rasterTilesToCog(tileOutputFiles, outputFile); 302 | 303 | // clean up the temporary directory 304 | fs::path outputParentDir = fs::path(outputFile).parent_path(); 305 | fs::path outputSubdir = outputParentDir / fs::path(outputFile).stem(); 306 | fs::remove_all(outputSubdir); 307 | } 308 | } 309 | -------------------------------------------------------------------------------- /src/info.cpp: -------------------------------------------------------------------------------- 1 | /***************************************************************************** 2 | * Copyright (c) 2023, Lutra Consulting Ltd. and Hobu, Inc. * 3 | * * 4 | * All rights reserved. * 5 | * * 6 | * This program is free software; you can redistribute it and/or modify * 7 | * it under the terms of the GNU General Public License as published by * 8 | * the Free Software Foundation; either version 3 of the License, or * 9 | * (at your option) any later version. * 10 | * * 11 | ****************************************************************************/ 12 | 13 | #include 14 | #include 15 | #include 16 | 17 | #include 18 | #include 19 | #include 20 | #include 21 | #include 22 | 23 | #include 24 | 25 | #include "utils.hpp" 26 | #include "alg.hpp" 27 | #include "vpc.hpp" 28 | 29 | using namespace pdal; 30 | 31 | namespace fs = std::filesystem; 32 | 33 | 34 | void Info::addArgs() 35 | { 36 | } 37 | 38 | bool Info::checkArgs() 39 | { 40 | return true; 41 | } 42 | 43 | 44 | // extract information from WKT - write to "crs" and "units" strings 45 | static void formatCrsInfo(const std::string &crsWkt, std::string &crs, std::string &units) 46 | { 47 | if (crsWkt.empty()) 48 | { 49 | crs = "(unknown)"; 50 | units = "(unknown)"; 51 | } 52 | else if (crsWkt == "_mix_") 53 | { 54 | // this can only happen in case of VPC - it is our special marker 55 | // when there are two or more different CRS in the input files 56 | crs = "(multiple)"; 57 | units = "(unknown)"; 58 | } 59 | else 60 | { 61 | // this code is quite clumsy... most of what we need is handled by pdal::SpatialReference, but: 62 | // - there's no API to get human readable name of the CRS 63 | // - https://github.com/PDAL/PDAL/issues/3943 - SpatialReference::identifyVerticalEPSG() is broken 64 | // - https://github.com/PDAL/PDAL/issues/3946 - SpatialReference::getHorizontalUnits() may return incorrect units 65 | pdal::SpatialReference sr(crsWkt); 66 | std::string wktHoriz = sr.getHorizontal(); 67 | std::string wktVert = sr.getVertical(); 68 | CRS c(crsWkt); 69 | CRS ch(wktHoriz); 70 | CRS cv(wktVert); 71 | std::string crsEpsg = c.identifyEPSG(); 72 | std::string crsHorizEpsg = sr.identifyHorizontalEPSG(); 73 | std::string crsVertEpsg = cv.identifyEPSG(); 74 | 75 | // CRS description and EPSG codes if available 76 | crs = c.name(); 77 | if (!crsEpsg.empty()) 78 | crs += " (EPSG:" + crsEpsg + ")"; 79 | else if (!crsHorizEpsg.empty() && !crsVertEpsg.empty()) 80 | crs += " (EPSG:" + crsHorizEpsg + "+" + crsVertEpsg + ")"; // this syntax of compound EPSG codes is understood by PROJ too 81 | else if (!crsHorizEpsg.empty()) 82 | crs += " (EPSG:" + crsHorizEpsg + " + ?)"; 83 | 84 | if (wktVert.empty()) 85 | { 86 | crs += " (vertical CRS missing!)"; 87 | } 88 | 89 | // horizontal and vertical units 90 | std::string unitsHoriz = ch.units(); 91 | std::string unitsVert = cv.units(); 92 | if (unitsHoriz == unitsVert) 93 | units = unitsHoriz; 94 | else 95 | { 96 | units = "horizontal=" + unitsHoriz + " vertical=" + unitsVert; 97 | } 98 | // TODO: add a warning when horizontal and vertical units do not match (?) 99 | } 100 | 101 | } 102 | 103 | 104 | void Info::preparePipelines(std::vector>&) 105 | { 106 | if (ends_with(inputFile, ".vpc")) 107 | { 108 | VirtualPointCloud vpc; 109 | if (!vpc.read(inputFile)) 110 | return; 111 | 112 | // TODO: other global metadata 113 | 114 | point_count_t total = 0; 115 | BOX3D box = !vpc.files.empty() ? vpc.files[0].bbox : BOX3D(); 116 | for (const VirtualPointCloud::File& f : vpc.files) 117 | { 118 | total += f.count; 119 | box.grow(f.bbox); 120 | } 121 | 122 | std::string crsName; 123 | std::string units; 124 | formatCrsInfo(vpc.crsWkt, crsName, units); 125 | 126 | std::cout << "VPC " << vpc.files.size() << " files" << std::endl; 127 | std::cout << "count " << total << std::endl; 128 | std::cout << "extent " << box.minx << " " << box.miny << " " << box.minz << std::endl; 129 | std::cout << " " << box.maxx << " " << box.maxy << " " << box.maxz << std::endl; 130 | std::cout << "crs " << crsName << std::endl; 131 | std::cout << "units " << units << std::endl; 132 | 133 | // list individual files 134 | std::cout << std::endl << "Files:" << std::endl; 135 | for (const VirtualPointCloud::File& f : vpc.files) 136 | { 137 | // TODO: maybe add more info? 138 | std::cout << f.filename << std::endl; 139 | } 140 | 141 | // TODO: optionally run stats on the whole VPC 142 | } 143 | else 144 | { 145 | MetadataNode layout; 146 | MetadataNode meta = getReaderMetadata(inputFile, &layout); 147 | 148 | std::string crs; 149 | std::string units; 150 | std::string crsWkt = meta.findChild("srs").findChild("compoundwkt").value(); 151 | formatCrsInfo(crsWkt, crs, units); 152 | 153 | std::cout << "LAS " << meta.findChild("major_version").value() << "." << meta.findChild("minor_version").value() << std::endl; 154 | std::cout << "point format " << meta.findChild("dataformat_id").value() << std::endl; 155 | std::cout << "count " << meta.findChild("count").value() << std::endl; 156 | std::cout << "scale " << meta.findChild("scale_x").value() << " " << meta.findChild("scale_y").value() << " " << meta.findChild("scale_z").value() << std::endl; 157 | std::cout << "offset " << meta.findChild("offset_x").value() << " " << meta.findChild("offset_y").value() << " " << meta.findChild("offset_z").value() << std::endl; 158 | std::cout << "extent " << meta.findChild("minx").value() << " " << meta.findChild("miny").value() << " " << meta.findChild("minz").value() << std::endl; 159 | std::cout << " " << meta.findChild("maxx").value() << " " << meta.findChild("maxy").value() << " " << meta.findChild("maxz").value() << std::endl; 160 | std::cout << "crs " << crs << std::endl; 161 | std::cout << "units " << units << std::endl; 162 | // TODO: file size in MB ? 163 | 164 | // TODO: possibly show extra metadata: (probably --verbose mode) 165 | // - creation date + software ID + system ID 166 | // - filesource ID 167 | // - VLR info 168 | 169 | std::cout << std::endl << "Attributes:" << std::endl; 170 | MetadataNodeList dims = layout.children("dimensions"); 171 | for (auto &dim : dims) 172 | { 173 | std::string name = dim.findChild("name").value(); 174 | std::string type = dim.findChild("type").value(); 175 | int size = dim.findChild("size").value(); 176 | std::cout << " - " << name << " " << type << " " << size << std::endl; 177 | } 178 | 179 | // TODO: optionally run filters.stats to get basic stats + counts of classes, returns, ... 180 | } 181 | } 182 | 183 | void Info::finalize(std::vector>&) 184 | { 185 | } 186 | -------------------------------------------------------------------------------- /src/main.cpp: -------------------------------------------------------------------------------- 1 | /***************************************************************************** 2 | * Copyright (c) 2023, Lutra Consulting Ltd. and Hobu, Inc. * 3 | * * 4 | * All rights reserved. * 5 | * * 6 | * This program is free software; you can redistribute it and/or modify * 7 | * it under the terms of the GNU General Public License as published by * 8 | * the Free Software Foundation; either version 3 of the License, or * 9 | * (at your option) any later version. * 10 | * * 11 | ****************************************************************************/ 12 | 13 | /* 14 | TODO: 15 | - algs that output point cloud: support multi-copc or single-copc output - as a post-processing step? 16 | - VPC: overlapping files? do not allow - require tiling 17 | */ 18 | 19 | #include 20 | #include 21 | 22 | #include 23 | 24 | #include "alg.hpp" 25 | #include "vpc.hpp" 26 | 27 | extern int runTile(std::vector arglist); // tile/tile.cpp 28 | 29 | 30 | void printUsage() 31 | { 32 | std::cout << "usage: pdal_wrench []" << std::endl; 33 | std::cout << " pdal_wrench [--help]" << std::endl; 34 | std::cout << std::endl; 35 | std::cout << "Available commands:" << std::endl; 36 | std::cout << " boundary Exports a polygon file containing boundary" << std::endl; 37 | std::cout << " build_vpc Creates a virtual point cloud" << std::endl; 38 | std::cout << " clip Outputs only points that are inside of the clipping polygons" << std::endl; 39 | std::cout << " density Exports a raster where each cell contains number of points" << std::endl; 40 | std::cout << " info Prints basic metadata from the point cloud file" << std::endl; 41 | std::cout << " merge Merges multiple point cloud files to a single one" << std::endl; 42 | std::cout << " thin Creates a thinned version of the point cloud (with fewer points)" << std::endl; 43 | std::cout << " tile Creates square tiles from input data" << std::endl; 44 | std::cout << " to_raster Exports point cloud data to a 2D raster grid" << std::endl; 45 | std::cout << " to_raster_tin Exports point cloud data to a 2D raster grid using triangulation" << std::endl; 46 | std::cout << " to_vector Exports point cloud data to a vector layer with 3D points" << std::endl; 47 | std::cout << " translate Converts to a different file format, reproject, and more" << std::endl; 48 | } 49 | 50 | 51 | #if defined(_WIN32) && defined(_MSC_VER) 52 | int wmain( int argc, wchar_t *argv[ ], wchar_t *envp[ ] ) 53 | #else 54 | int main(int argc, char* argv[]) 55 | #endif 56 | { 57 | if (argc < 2) 58 | { 59 | printUsage(); 60 | return 1; 61 | } 62 | std::string cmd = pdal::FileUtils::fromNative(argv[1]); 63 | 64 | std::vector args; 65 | for ( int i = 2; i < argc; ++i ) 66 | args.push_back(pdal::FileUtils::fromNative(argv[i])); 67 | 68 | if (cmd == "--help" || cmd == "help") 69 | { 70 | printUsage(); 71 | } 72 | else if (cmd == "density") 73 | { 74 | Density density; 75 | runAlg(args, density); 76 | } 77 | else if (cmd == "boundary") 78 | { 79 | Boundary boundary; 80 | runAlg(args, boundary); 81 | } 82 | else if (cmd == "clip") 83 | { 84 | Clip clip; 85 | runAlg(args, clip); 86 | } 87 | else if (cmd == "build_vpc") 88 | { 89 | buildVpc(args); 90 | } 91 | else if (cmd == "merge") 92 | { 93 | Merge merge; 94 | runAlg(args, merge); 95 | } 96 | else if (cmd == "thin") 97 | { 98 | Thin thin; 99 | runAlg(args, thin); 100 | } 101 | else if (cmd == "to_raster") 102 | { 103 | ToRaster toRaster; 104 | runAlg(args, toRaster); 105 | } 106 | else if (cmd == "to_raster_tin") 107 | { 108 | ToRasterTin toRasterTin; 109 | runAlg(args, toRasterTin); 110 | } 111 | else if (cmd == "to_vector") 112 | { 113 | ToVector toVector; 114 | runAlg(args, toVector); 115 | } 116 | else if (cmd == "info") 117 | { 118 | Info info; 119 | runAlg(args, info); 120 | } 121 | else if (cmd == "translate") 122 | { 123 | Translate translate; 124 | runAlg(args, translate); 125 | } 126 | else if (cmd == "tile") 127 | { 128 | runTile(args); 129 | } 130 | else 131 | { 132 | std::cerr << "unknown command: " << cmd << std::endl; 133 | return 1; 134 | } 135 | 136 | return 0; 137 | } 138 | -------------------------------------------------------------------------------- /src/merge.cpp: -------------------------------------------------------------------------------- 1 | /***************************************************************************** 2 | * Copyright (c) 2023, Lutra Consulting Ltd. and Hobu, Inc. * 3 | * * 4 | * All rights reserved. * 5 | * * 6 | * This program is free software; you can redistribute it and/or modify * 7 | * it under the terms of the GNU General Public License as published by * 8 | * the Free Software Foundation; either version 3 of the License, or * 9 | * (at your option) any later version. * 10 | * * 11 | ****************************************************************************/ 12 | 13 | #include 14 | #include 15 | #include 16 | #include 17 | 18 | #include 19 | #include 20 | #include 21 | #include 22 | #include 23 | 24 | #include 25 | 26 | #include "utils.hpp" 27 | #include "alg.hpp" 28 | #include "vpc.hpp" 29 | 30 | using namespace pdal; 31 | 32 | namespace fs = std::filesystem; 33 | 34 | 35 | void Merge::addArgs() 36 | { 37 | argOutput = &programArgs.add("output,o", "Output virtual point cloud file", outputFile); 38 | // we set hasSingleInput=false so the default "input,i" argument is not added 39 | programArgs.add("files", "input files", inputFiles).setPositional(); 40 | programArgs.add("input-file-list", "Read input files from a text file", inputFileList); 41 | 42 | } 43 | 44 | bool Merge::checkArgs() 45 | { 46 | if (!argOutput->set()) 47 | { 48 | std::cerr << "missing output" << std::endl; 49 | return false; 50 | } 51 | 52 | return true; 53 | } 54 | 55 | static std::unique_ptr pipeline(ParallelJobInfo *tile) 56 | { 57 | assert(tile); 58 | 59 | std::unique_ptr manager( new PipelineManager ); 60 | 61 | std::vector readers; 62 | for (const std::string& f : tile->inputFilenames) 63 | { 64 | readers.push_back(&manager->makeReader(f, "")); 65 | } 66 | 67 | std::vector last = readers; 68 | 69 | // filtering 70 | if (!tile->filterBounds.empty()) 71 | { 72 | Options filter_opts; 73 | filter_opts.add(pdal::Option("bounds", tile->filterBounds)); 74 | 75 | if (allReadersSupportBounds(readers)) 76 | { 77 | // Reader of the format can do the filtering - use that whenever possible! 78 | for (Stage *r : readers) 79 | r->addOptions(filter_opts); 80 | } 81 | else 82 | { 83 | // Reader can't do the filtering - do it with a filter 84 | Stage *filterCrop = &manager->makeFilter( "filters.crop", filter_opts); 85 | for (Stage *s : last) 86 | filterCrop->setInput(*s); 87 | last.clear(); 88 | last.push_back(filterCrop); 89 | } 90 | } 91 | if (!tile->filterExpression.empty()) 92 | { 93 | Options filter_opts; 94 | filter_opts.add(pdal::Option("expression", tile->filterExpression)); 95 | Stage *filterExpr = &manager->makeFilter( "filters.expression", filter_opts); 96 | for (Stage *s : last) 97 | filterExpr->setInput(*s); 98 | last.clear(); 99 | last.push_back(filterExpr); 100 | } 101 | 102 | pdal::Options options; 103 | options.add(pdal::Option("forward", "all")); 104 | Stage* writer = &manager->makeWriter(tile->outputFilename, "", options); 105 | for (Stage *s : last) 106 | writer->setInput(*s); 107 | 108 | return manager; 109 | } 110 | 111 | void Merge::preparePipelines(std::vector>& pipelines) 112 | { 113 | ParallelJobInfo tile(ParallelJobInfo::Single, BOX2D(), filterExpression, filterBounds); 114 | if (!inputFileList.empty()) 115 | { 116 | std::ifstream inputFile(inputFileList); 117 | std::string line; 118 | if(!inputFile) 119 | { 120 | std::cerr << "failed to open input file list: " << inputFileList << std::endl; 121 | return; 122 | } 123 | 124 | while (std::getline(inputFile, line)) 125 | { 126 | inputFiles.push_back(line); 127 | } 128 | } 129 | 130 | std::vector vpcFilesToRemove; 131 | vpcFilesToRemove.reserve(inputFiles.size()); 132 | 133 | for (const std::string& inputFile : inputFiles) 134 | { 135 | if (ends_with(inputFile, ".vpc")) 136 | { 137 | vpcFilesToRemove.push_back(inputFile); 138 | 139 | VirtualPointCloud vpc; 140 | if (!vpc.read(inputFile)) 141 | std::cerr << "could not open input VPC: " << inputFile << std::endl; 142 | return; 143 | 144 | for (const VirtualPointCloud::File& vpcSingleFile : vpc.files) 145 | { 146 | inputFiles.push_back(vpcSingleFile.filename); 147 | } 148 | } 149 | } 150 | 151 | for (const std::string& f : vpcFilesToRemove) 152 | { 153 | inputFiles.erase(std::remove(inputFiles.begin(), inputFiles.end(), f), inputFiles.end()); 154 | } 155 | 156 | tile.inputFilenames = inputFiles; 157 | tile.outputFilename = outputFile; 158 | 159 | if (ends_with(outputFile, ".copc.laz")) 160 | { 161 | isStreaming = false; 162 | } 163 | 164 | pipelines.push_back(pipeline(&tile)); 165 | 166 | // only algs with single input have the number of points figured out already 167 | totalPoints = 0; 168 | for (const std::string& f : inputFiles) 169 | { 170 | QuickInfo qi = getQuickInfo(f); 171 | totalPoints += qi.m_pointCount; 172 | } 173 | } 174 | -------------------------------------------------------------------------------- /src/thin.cpp: -------------------------------------------------------------------------------- 1 | /***************************************************************************** 2 | * Copyright (c) 2023, Lutra Consulting Ltd. and Hobu, Inc. * 3 | * * 4 | * All rights reserved. * 5 | * * 6 | * This program is free software; you can redistribute it and/or modify * 7 | * it under the terms of the GNU General Public License as published by * 8 | * the Free Software Foundation; either version 3 of the License, or * 9 | * (at your option) any later version. * 10 | * * 11 | ****************************************************************************/ 12 | 13 | #include 14 | #include 15 | #include 16 | 17 | #include 18 | #include 19 | #include 20 | #include 21 | #include 22 | 23 | #include 24 | 25 | #include "utils.hpp" 26 | #include "alg.hpp" 27 | #include "vpc.hpp" 28 | 29 | using namespace pdal; 30 | 31 | namespace fs = std::filesystem; 32 | 33 | 34 | // TODO: add support for filters.sample and/or filters.voxeldownsize 35 | // (both in streaming mode but more memory intense - keeping occupation grid) 36 | 37 | void Thin::addArgs() 38 | { 39 | argOutput = &programArgs.add("output,o", "Output point cloud file", outputFile); 40 | argOutputFormat = &programArgs.add("output-format", "Output format (las/laz/copc)", outputFormat); 41 | argMode = &programArgs.add("mode", " 'every-nth' or 'sample' - either to keep every N-th point or to keep points based on their distance", mode); 42 | argStepEveryN = &programArgs.add("step-every-nth", "Keep every N-th point", stepEveryN); 43 | argStepSample = &programArgs.add("step-sample", "Minimum spacing between points", stepSample); 44 | } 45 | 46 | bool Thin::checkArgs() 47 | { 48 | if (!argOutput->set()) 49 | { 50 | std::cerr << "missing output" << std::endl; 51 | return false; 52 | } 53 | 54 | if (!argMode->set()) 55 | { 56 | std::cerr << "missing mode" << std::endl; 57 | return false; 58 | } 59 | else if (mode == "every-nth") 60 | { 61 | if (!argStepEveryN->set()) 62 | { 63 | std::cerr << "missing step for every N-th point mode" << std::endl; 64 | return false; 65 | } 66 | } 67 | else if (mode == "sample") 68 | { 69 | if (!argStepSample->set()) 70 | { 71 | std::cerr << "missing step for sampling mode" << std::endl; 72 | return false; 73 | } 74 | } 75 | else 76 | { 77 | std::cerr << "unknown mode: " << mode << std::endl; 78 | return false; 79 | } 80 | 81 | if (argOutputFormat->set()) 82 | { 83 | if (outputFormat != "las" && outputFormat != "laz") 84 | { 85 | std::cerr << "unknown output format: " << outputFormat << std::endl; 86 | return false; 87 | } 88 | } 89 | else 90 | outputFormat = "las"; // uncompressed by default 91 | 92 | return true; 93 | } 94 | 95 | 96 | static std::unique_ptr pipeline(ParallelJobInfo *tile, std::string mode, int stepEveryN, double stepSample) 97 | { 98 | std::unique_ptr manager( new PipelineManager ); 99 | 100 | Stage& r = manager->makeReader( tile->inputFilenames[0], ""); 101 | 102 | Stage *last = &r; 103 | 104 | // filtering 105 | if (!tile->filterBounds.empty()) 106 | { 107 | Options filter_opts; 108 | filter_opts.add(pdal::Option("bounds", tile->filterBounds)); 109 | 110 | if (readerSupportsBounds(r)) 111 | { 112 | // Reader of the format can do the filtering - use that whenever possible! 113 | r.addOptions(filter_opts); 114 | } 115 | else 116 | { 117 | // Reader can't do the filtering - do it with a filter 118 | last = &manager->makeFilter( "filters.crop", *last, filter_opts); 119 | } 120 | } 121 | if (!tile->filterExpression.empty()) 122 | { 123 | Options filter_opts; 124 | filter_opts.add(pdal::Option("expression", tile->filterExpression)); 125 | last = &manager->makeFilter( "filters.expression", *last, filter_opts); 126 | } 127 | 128 | if (mode == "every-nth") 129 | { 130 | pdal::Options decim_opts; 131 | decim_opts.add(pdal::Option("step", stepEveryN)); 132 | last = &manager->makeFilter( "filters.decimation", *last, decim_opts ); 133 | } 134 | else if (mode == "sample") 135 | { 136 | pdal::Options sample_opts; 137 | sample_opts.add(pdal::Option("cell", stepSample)); 138 | last = &manager->makeFilter( "filters.sample", *last, sample_opts ); 139 | } 140 | 141 | pdal::Options writer_opts; 142 | writer_opts.add(pdal::Option("forward", "all")); // TODO: maybe we could use lower scale than the original 143 | manager->makeWriter( tile->outputFilename, "", *last, writer_opts); 144 | 145 | return manager; 146 | } 147 | 148 | 149 | void Thin::preparePipelines(std::vector>& pipelines) 150 | { 151 | if (ends_with(inputFile, ".vpc")) 152 | { 153 | // for /tmp/hello.vpc we will use /tmp/hello dir for all results 154 | fs::path outputParentDir = fs::path(outputFile).parent_path(); 155 | fs::path outputSubdir = outputParentDir / fs::path(outputFile).stem(); 156 | fs::create_directories(outputSubdir); 157 | 158 | // VPC handling 159 | VirtualPointCloud vpc; 160 | if (!vpc.read(inputFile)) 161 | return; 162 | 163 | for (const VirtualPointCloud::File& f : vpc.files) 164 | { 165 | ParallelJobInfo tile(ParallelJobInfo::FileBased, BOX2D(), filterExpression, filterBounds); 166 | tile.inputFilenames.push_back(f.filename); 167 | 168 | // for input file /x/y/z.las that goes to /tmp/hello.vpc, 169 | // individual output file will be called /tmp/hello/z.las 170 | fs::path inputBasename = fs::path(f.filename).stem(); 171 | 172 | if (!ends_with(outputFile, ".vpc")) 173 | tile.outputFilename = (outputSubdir / inputBasename).string() + ".las"; 174 | else 175 | tile.outputFilename = (outputSubdir / inputBasename).string() + "." + outputFormat; 176 | 177 | tileOutputFiles.push_back(tile.outputFilename); 178 | 179 | pipelines.push_back(pipeline(&tile, mode, stepEveryN, stepSample)); 180 | } 181 | } 182 | else 183 | { 184 | if (ends_with(outputFile, ".copc.laz")) 185 | { 186 | isStreaming = false; 187 | } 188 | ParallelJobInfo tile(ParallelJobInfo::Single, BOX2D(), filterExpression, filterBounds); 189 | tile.inputFilenames.push_back(inputFile); 190 | tile.outputFilename = outputFile; 191 | pipelines.push_back(pipeline(&tile, mode, stepEveryN, stepSample)); 192 | } 193 | } 194 | 195 | void Thin::finalize(std::vector>&) 196 | { 197 | if (tileOutputFiles.empty()) 198 | return; 199 | 200 | // now build a new output VPC 201 | std::vector args; 202 | args.push_back("--output=" + outputFile); 203 | for (std::string f : tileOutputFiles) 204 | args.push_back(f); 205 | 206 | if (ends_with(outputFile, ".vpc")) 207 | { 208 | // now build a new output VPC 209 | buildVpc(args); 210 | } 211 | else 212 | { 213 | // merge all the output files into a single file 214 | Merge merge; 215 | // for copc set isStreaming to false 216 | if (ends_with(outputFile, ".copc.laz")) 217 | { 218 | merge.isStreaming = false; 219 | } 220 | runAlg(args, merge); 221 | 222 | // remove files as they are not needed anymore - they are merged 223 | removeFiles(tileOutputFiles, true); 224 | } 225 | } 226 | -------------------------------------------------------------------------------- /src/tile/BufferCache.cpp: -------------------------------------------------------------------------------- 1 | /***************************************************************************** 2 | * Copyright (c) 2020, Hobu, Inc. (info@hobu.co) * 3 | * * 4 | * All rights reserved. * 5 | * * 6 | * This program is free software; you can redistribute it and/or modify * 7 | * it under the terms of the GNU General Public License as published by * 8 | * the Free Software Foundation; either version 3 of the License, or * 9 | * (at your option) any later version. * 10 | * * 11 | ****************************************************************************/ 12 | 13 | 14 | #include 15 | 16 | #include "BufferCache.hpp" 17 | 18 | namespace untwine 19 | { 20 | namespace epf 21 | { 22 | 23 | // If we have a buffer in the cache, return it. Otherwise create a new one and return that. 24 | // If nonblock is true and there are no available buffers, return null. 25 | DataVecPtr BufferCache::fetch(std::unique_lock& lock, bool nonblock) 26 | { 27 | if (nonblock && m_buffers.empty() && m_count >= MaxBuffers) 28 | return nullptr; 29 | 30 | m_cv.wait(lock, [this](){ return m_buffers.size() || m_count < MaxBuffers; }); 31 | if (m_buffers.size()) 32 | { 33 | DataVecPtr buf(std::move(m_buffers.back())); 34 | m_buffers.pop_back(); 35 | return buf; 36 | } 37 | 38 | // m_count tracks the number of created buffers. We only create MaxBuffers buffers. 39 | // If we've created that many, we wait until one is available. 40 | m_count++; 41 | return DataVecPtr(new DataVec(BufSize)); 42 | } 43 | 44 | // Put a buffer back in the cache. 45 | void BufferCache::replace(DataVecPtr&& buf) 46 | { 47 | m_buffers.push_back(std::move(buf)); 48 | 49 | if (m_count == MaxBuffers) 50 | m_cv.notify_one(); 51 | } 52 | 53 | } // namespace epf 54 | } // namespace untwine 55 | -------------------------------------------------------------------------------- /src/tile/BufferCache.hpp: -------------------------------------------------------------------------------- 1 | /***************************************************************************** 2 | * Copyright (c) 2020, Hobu, Inc. (info@hobu.co) * 3 | * * 4 | * All rights reserved. * 5 | * * 6 | * This program is free software; you can redistribute it and/or modify * 7 | * it under the terms of the GNU General Public License as published by * 8 | * the Free Software Foundation; either version 3 of the License, or * 9 | * (at your option) any later version. * 10 | * * 11 | ****************************************************************************/ 12 | 13 | 14 | #pragma once 15 | 16 | #include 17 | #include 18 | #include 19 | 20 | #include "EpfTypes.hpp" 21 | 22 | namespace untwine 23 | { 24 | namespace epf 25 | { 26 | 27 | // This is simply a cache of data buffers to avoid continuous allocation and deallocation. 28 | class BufferCache 29 | { 30 | public: 31 | BufferCache() : m_count(0) 32 | {} 33 | 34 | DataVecPtr fetch(std::unique_lock& lock, bool nonblock); 35 | void replace(DataVecPtr&& buf); 36 | 37 | private: 38 | std::deque m_buffers; 39 | std::condition_variable m_cv; 40 | int m_count; 41 | }; 42 | 43 | } // namespace epf 44 | } // namespace untwine 45 | -------------------------------------------------------------------------------- /src/tile/Cell.cpp: -------------------------------------------------------------------------------- 1 | /***************************************************************************** 2 | * Copyright (c) 2020, Hobu, Inc. (info@hobu.co) * 3 | * * 4 | * All rights reserved. * 5 | * * 6 | * This program is free software; you can redistribute it and/or modify * 7 | * it under the terms of the GNU General Public License as published by * 8 | * the Free Software Foundation; either version 3 of the License, or * 9 | * (at your option) any later version. * 10 | * * 11 | ****************************************************************************/ 12 | 13 | 14 | #include "Cell.hpp" 15 | #include "Writer.hpp" 16 | 17 | namespace untwine 18 | { 19 | namespace epf 20 | { 21 | 22 | // NOTE - After write(), the cell is invalid and must be initialized or destroyed. 23 | void Cell::write() 24 | { 25 | size_t size = m_pos - m_buf->data(); 26 | if (size) 27 | m_writer->enqueue(m_key, std::move(m_buf), size); 28 | else 29 | m_writer->replace(std::move(m_buf)); 30 | } 31 | 32 | void Cell::initialize(const Cell *exclude) 33 | { 34 | m_buf = m_cellMgr->getBuffer(exclude); 35 | m_pos = m_buf->data(); 36 | m_endPos = m_pos + m_pointSize * (BufSize / m_pointSize); 37 | } 38 | 39 | void Cell::advance() 40 | { 41 | m_pos += m_pointSize; 42 | if (m_pos >= m_endPos) 43 | { 44 | write(); 45 | initialize(this); 46 | } 47 | } 48 | 49 | /// 50 | /// CellMgr 51 | /// 52 | 53 | CellMgr::CellMgr(int pointSize, Writer *writer) : m_pointSize(pointSize), m_writer(writer) 54 | {} 55 | 56 | Cell *CellMgr::get(const TileKey& key, const Cell *lastCell) 57 | { 58 | auto it = m_cells.find(key); 59 | if (it == m_cells.end()) 60 | { 61 | std::unique_ptr cell(new Cell(key, m_pointSize, m_writer, this, lastCell)); 62 | it = m_cells.insert( {key, std::move(cell)} ).first; 63 | } 64 | 65 | return it->second.get(); 66 | } 67 | 68 | DataVecPtr CellMgr::getBuffer(const Cell *exclude) 69 | { 70 | DataVecPtr b = m_writer->fetchBuffer(); 71 | 72 | // If we couldn't fetch a buffer, flush all the the buffers for this processor and 73 | // try again, but block. 74 | if (!b) 75 | { 76 | flush(exclude); 77 | b = m_writer->fetchBufferBlocking(); 78 | } 79 | return b; 80 | } 81 | 82 | // Eliminate all the cells and their associated data buffers except the `exclude` 83 | // cell. 84 | void CellMgr::flush(const Cell *exclude) 85 | { 86 | CellMap::iterator it = m_cells.end(); 87 | 88 | if (exclude) 89 | it = m_cells.find(exclude->key()); 90 | 91 | // If there was no exclude cell just clear the cells. 92 | // Otherwise, save the exclude cell, clear the list, and reinsert. 93 | // Cells are written when they are destroyed. 94 | if (it == m_cells.end()) 95 | m_cells.clear(); 96 | else 97 | { 98 | std::unique_ptr c = std::move(it->second); 99 | m_cells.clear(); 100 | m_cells.insert({c->key(), std::move(c)}); 101 | } 102 | } 103 | 104 | } // namespace epf 105 | } // namespace untwine 106 | -------------------------------------------------------------------------------- /src/tile/Cell.hpp: -------------------------------------------------------------------------------- 1 | /***************************************************************************** 2 | * Copyright (c) 2020, Hobu, Inc. (info@hobu.co) * 3 | * * 4 | * All rights reserved. * 5 | * * 6 | * This program is free software; you can redistribute it and/or modify * 7 | * it under the terms of the GNU General Public License as published by * 8 | * the Free Software Foundation; either version 3 of the License, or * 9 | * (at your option) any later version. * 10 | * * 11 | ****************************************************************************/ 12 | 13 | #pragma once 14 | 15 | #include 16 | #include 17 | #include 18 | #include 19 | #include 20 | 21 | #include "EpfTypes.hpp" 22 | #include "Point.hpp" 23 | #include "TileKey.hpp" 24 | 25 | namespace untwine 26 | { 27 | namespace epf 28 | { 29 | 30 | class Writer; 31 | class CellMgr; 32 | 33 | // A cell represents a voxel that contains points. All cells are the same size. A cell has 34 | // a buffer which is filled by points. When the buffer is filled, it's passed to the writer 35 | // and a new buffer is created. 36 | class Cell 37 | { 38 | public: 39 | using FlushFunc = std::function; 40 | 41 | Cell(const TileKey& key, int pointSize, Writer *writer, CellMgr *mgr, const Cell *lastCell) : 42 | m_key(key), m_pointSize(pointSize), m_writer(writer), m_cellMgr(mgr) 43 | { 44 | assert(pointSize < BufSize); 45 | initialize(lastCell); 46 | } 47 | ~Cell() 48 | { 49 | write(); 50 | } 51 | 52 | void initialize(const Cell *exclude); 53 | Point point() 54 | { return Point(m_pos); } 55 | TileKey key() const 56 | { return m_key; } 57 | void copyPoint(Point& b) 58 | { std::copy(b.data(), b.data() + m_pointSize, m_pos); } 59 | void advance(); 60 | 61 | private: 62 | DataVecPtr m_buf; 63 | TileKey m_key; 64 | uint8_t *m_pos; 65 | uint8_t *m_endPos; 66 | int m_pointSize; 67 | Writer *m_writer; 68 | CellMgr *m_cellMgr; 69 | 70 | void write(); 71 | }; 72 | 73 | class CellMgr 74 | { 75 | friend class Cell; 76 | public: 77 | CellMgr(int pointSize, Writer *writer); 78 | 79 | Cell *get(const TileKey& key, const Cell *lastCell = nullptr); 80 | 81 | private: 82 | using CellMap = std::unordered_map>; 83 | int m_pointSize; 84 | Writer *m_writer; 85 | CellMap m_cells; 86 | 87 | DataVecPtr getBuffer(const Cell* exclude); 88 | void flush(const Cell *exclude); 89 | }; 90 | 91 | 92 | } // namespace epf 93 | } // namespace untwine 94 | -------------------------------------------------------------------------------- /src/tile/Common.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #ifdef _WIN32 4 | #include 5 | #endif 6 | 7 | #include 8 | 9 | using PointCount = uint64_t; 10 | 11 | class FatalError : public std::runtime_error 12 | { 13 | public: 14 | inline FatalError(std::string const& msg) : std::runtime_error(msg) 15 | {} 16 | }; 17 | 18 | namespace untwine 19 | { 20 | 21 | // We check both _WIN32 and _MSC_VER to deal with MinGW, which doesn't support the special 22 | // Windows wide character interfaces for streams. 23 | #if defined(_WIN32) && defined(_MSC_VER) 24 | inline std::wstring toNative(const std::string& in) 25 | { 26 | if (in.empty()) 27 | return std::wstring(); 28 | 29 | int len = MultiByteToWideChar(CP_UTF8, 0, in.data(), in.length(), nullptr, 0); 30 | std::wstring out(len, 0); 31 | if (MultiByteToWideChar(CP_UTF8, 0, in.data(), in.length(), out.data(), len) == 0) 32 | { 33 | char buf[200] {}; 34 | len = FormatMessageA(0, 0, GetLastError(), 0, buf, 199, 0); 35 | throw FatalError("Can't convert UTF8 to UTF16: " + std::string(buf, len)); 36 | } 37 | return out; 38 | } 39 | 40 | inline std::string fromNative(const std::wstring& in) 41 | { 42 | if (in.empty()) 43 | return std::string(); 44 | 45 | int len = WideCharToMultiByte(CP_UTF8, 0, in.data(), in.length(), nullptr, 0, nullptr, nullptr); 46 | std::string out(len, 0); 47 | if (WideCharToMultiByte(CP_UTF8, 0, in.data(), in.length(), out.data(), len, nullptr, nullptr) == 0) 48 | { 49 | char buf[200] {}; 50 | len = FormatMessageA(0, 0, GetLastError(), 0, buf, 199, 0); 51 | throw FatalError("Can't convert UTF16 to UTF8: " + std::string(buf, len)); 52 | } 53 | return out; 54 | } 55 | #else 56 | inline std::string toNative(const std::string& in) 57 | { 58 | return in; 59 | } 60 | 61 | inline std::string fromNative(const std::string& in) 62 | { 63 | return in; 64 | } 65 | #endif 66 | 67 | } 68 | -------------------------------------------------------------------------------- /src/tile/EpfTypes.hpp: -------------------------------------------------------------------------------- 1 | /***************************************************************************** 2 | * Copyright (c) 2020, Hobu, Inc. (info@hobu.co) * 3 | * * 4 | * All rights reserved. * 5 | * * 6 | * This program is free software; you can redistribute it and/or modify * 7 | * it under the terms of the GNU General Public License as published by * 8 | * the Free Software Foundation; either version 3 of the License, or * 9 | * (at your option) any later version. * 10 | * * 11 | ****************************************************************************/ 12 | 13 | 14 | #pragma once 15 | 16 | #include 17 | #include 18 | #include 19 | 20 | #include 21 | #include 22 | #include 23 | #include 24 | 25 | #include "FileDimInfo.hpp" 26 | #include "TileKey.hpp" 27 | 28 | namespace untwine 29 | { 30 | namespace epf 31 | { 32 | 33 | using DataVec = std::vector; 34 | using DataVecPtr = std::unique_ptr; 35 | using Totals = std::unordered_map; 36 | constexpr int MaxPointsPerNode = 100000; 37 | constexpr int BufSize = 4096 * 10; 38 | constexpr int MaxBuffers = 1000; 39 | constexpr int NumWriters = 4; 40 | constexpr int NumFileProcessors = 8; 41 | 42 | struct FileInfo 43 | { 44 | FileInfo() : numPoints(0), start(0) 45 | {} 46 | 47 | std::string filename; 48 | std::string driver; 49 | DimInfoList dimInfo; 50 | uint64_t numPoints; 51 | uint64_t start; 52 | pdal::BOX3D bounds; 53 | pdal::SpatialReference srs; 54 | 55 | bool valid() const 56 | { return filename.size(); } 57 | }; 58 | 59 | } // namespace epf 60 | } // namespace untwine 61 | -------------------------------------------------------------------------------- /src/tile/FileDimInfo.hpp: -------------------------------------------------------------------------------- 1 | /***************************************************************************** 2 | * Copyright (c) 2020, Hobu, Inc. (info@hobu.co) * 3 | * * 4 | * All rights reserved. * 5 | * * 6 | * This program is free software; you can redistribute it and/or modify * 7 | * it under the terms of the GNU General Public License as published by * 8 | * the Free Software Foundation; either version 3 of the License, or * 9 | * (at your option) any later version. * 10 | * * 11 | ****************************************************************************/ 12 | 13 | #pragma once 14 | 15 | #include 16 | 17 | namespace untwine 18 | { 19 | 20 | struct FileDimInfo 21 | { 22 | FileDimInfo() 23 | {} 24 | 25 | FileDimInfo(const std::string& name) : name(name), extraDim(false) 26 | {} 27 | 28 | std::string name; 29 | pdal::Dimension::Type type; 30 | int offset; 31 | pdal::Dimension::Id dim; 32 | bool extraDim; 33 | }; 34 | 35 | using DimInfoList = std::vector; 36 | 37 | inline std::ostream& operator<<(std::ostream& out, const FileDimInfo& fdi) 38 | { 39 | out << fdi.name << " " << (int)fdi.type << " " << fdi.offset; 40 | return out; 41 | } 42 | 43 | inline std::istream& operator>>(std::istream& in, FileDimInfo& fdi) 44 | { 45 | int type; 46 | in >> fdi.name >> type >> fdi.offset; 47 | fdi.type = (pdal::Dimension::Type)type; 48 | return in; 49 | } 50 | 51 | } // namespace untwine 52 | -------------------------------------------------------------------------------- /src/tile/FileProcessor.cpp: -------------------------------------------------------------------------------- 1 | /***************************************************************************** 2 | * Copyright (c) 2020, Hobu, Inc. (info@hobu.co) * 3 | * * 4 | * All rights reserved. * 5 | * * 6 | * This program is free software; you can redistribute it and/or modify * 7 | * it under the terms of the GNU General Public License as published by * 8 | * the Free Software Foundation; either version 3 of the License, or * 9 | * (at your option) any later version. * 10 | * * 11 | ****************************************************************************/ 12 | 13 | 14 | #include "FileProcessor.hpp" 15 | #include "Common.hpp" 16 | #include "../utils.hpp" 17 | 18 | #include 19 | #include 20 | #include 21 | 22 | namespace untwine 23 | { 24 | namespace epf 25 | { 26 | 27 | FileProcessor::FileProcessor(const FileInfo& fi, size_t pointSize, const TileGrid& grid, 28 | untwine::epf::Writer *writer, ProgressBar& progressBar) : 29 | m_fi(fi), m_cellMgr(pointSize, writer), m_grid(grid), m_progressBar(progressBar) 30 | {} 31 | 32 | void FileProcessor::run() 33 | { 34 | pdal::Options opts; 35 | opts.add("filename", m_fi.filename); 36 | opts.add("count", m_fi.numPoints); 37 | #ifdef PDAL_LAS_START 38 | if (m_fi.driver == "readers.las") 39 | opts.add("start", m_fi.start); 40 | #endif 41 | 42 | pdal::StageFactory factory; 43 | pdal::Stage *s = factory.createStage(m_fi.driver); 44 | s->setOptions(opts); 45 | 46 | PointCount count = 0; 47 | PointCount countTotal = 0; 48 | 49 | // We need to move the data from the PointRef to some output buffer. We copy the data 50 | // to the end of the *last* output buffer we used in hopes that it's the right one. 51 | // If it's not we lose and we're forced to move that data to the another cell, 52 | // which then becomes the active cell. 53 | 54 | // This is some random cell that ultimately won't get used, but it contains a buffer 55 | // into which we can write data. 56 | Cell *cell = m_cellMgr.get(TileKey()); 57 | 58 | pdal::StreamCallbackFilter f; 59 | f.setCallback([this, &count, &countTotal, &cell](pdal::PointRef& point) 60 | { 61 | // Write the data into the point buffer in the cell. This is the *last* 62 | // cell buffer that we used. We're hoping that it's the right one. 63 | Point p = cell->point(); 64 | for (const FileDimInfo& fdi : m_fi.dimInfo) 65 | point.getField(reinterpret_cast(p.data() + fdi.offset), 66 | fdi.dim, fdi.type); 67 | 68 | // Find the actual cell that this point belongs in. If it's not the one 69 | // we chose, copy the data to the correct cell. 70 | TileKey cellIndex = m_grid.key(p.x(), p.y(), p.z()); 71 | if (cellIndex != cell->key()) 72 | { 73 | // Make sure that we exclude the current cell from any potential flush so 74 | // that no other thread can claim its data buffer and overwrite it before 75 | // we have a chance to copy from it in copyPoint(). 76 | cell = m_cellMgr.get(cellIndex, cell); 77 | cell->copyPoint(p); 78 | } 79 | // Advance the cell - move the buffer pointer so when we refer to the cell's 80 | // point, we're referring to the next location in the cell's buffer. 81 | cell->advance(); 82 | count++; 83 | countTotal++; 84 | 85 | if (count == 100'000) // ProgressWriter::ChunkSize) 86 | { 87 | m_progressBar.add(count); 88 | count = 0; 89 | } 90 | 91 | return true; 92 | } 93 | ); 94 | f.setInput(*s); 95 | 96 | pdal::FixedPointTable t(1000); 97 | 98 | try 99 | { 100 | f.prepare(t); 101 | f.execute(t); 102 | } 103 | catch (const pdal::pdal_error& err) 104 | { 105 | throw FatalError(err.what()); 106 | } 107 | 108 | // We normally call update for every CountIncrement points, but at the end, just 109 | // tell the progress writer the number that we've done since the last update. 110 | m_progressBar.add(count); 111 | } 112 | 113 | } // namespace epf 114 | } // namespace untwine 115 | -------------------------------------------------------------------------------- /src/tile/FileProcessor.hpp: -------------------------------------------------------------------------------- 1 | /***************************************************************************** 2 | * Copyright (c) 2020, Hobu, Inc. (info@hobu.co) * 3 | * * 4 | * All rights reserved. * 5 | * * 6 | * This program is free software; you can redistribute it and/or modify * 7 | * it under the terms of the GNU General Public License as published by * 8 | * the Free Software Foundation; either version 3 of the License, or * 9 | * (at your option) any later version. * 10 | * * 11 | ****************************************************************************/ 12 | 13 | 14 | #include "EpfTypes.hpp" 15 | #include "TileGrid.hpp" 16 | #include "Cell.hpp" 17 | 18 | struct ProgressBar; 19 | 20 | namespace untwine 21 | { 22 | 23 | namespace epf 24 | { 25 | 26 | class Writer; 27 | 28 | // Processes a single input file (FileInfo) and writes data to the Writer. 29 | class FileProcessor 30 | { 31 | public: 32 | FileProcessor(const FileInfo& fi, size_t pointSize, const TileGrid& grid, Writer *writer, 33 | ProgressBar& progressBar); 34 | 35 | Cell *getCell(const TileKey& key); 36 | void run(); 37 | 38 | private: 39 | FileInfo m_fi; 40 | CellMgr m_cellMgr; 41 | TileGrid m_grid; 42 | ProgressBar& m_progressBar; 43 | }; 44 | 45 | } // namespace epf 46 | } // namespace untwine 47 | -------------------------------------------------------------------------------- /src/tile/Las.cpp: -------------------------------------------------------------------------------- 1 | /***************************************************************************** 2 | * Copyright (c) 2021, Hobu, Inc. (info@hobu.co) * 3 | * * 4 | * All rights reserved. * 5 | * * 6 | * This program is free software; you can redistribute it and/or modify * 7 | * it under the terms of the GNU General Public License as published by * 8 | * the Free Software Foundation; either version 3 of the License, or * 9 | * (at your option) any later version. * 10 | * * 11 | ****************************************************************************/ 12 | 13 | #include "Las.hpp" 14 | 15 | namespace untwine 16 | { 17 | 18 | const pdal::Dimension::IdList& pdrfDims(int pdrf) 19 | { 20 | using namespace pdal; 21 | 22 | if (pdrf < 0 || pdrf > 10) 23 | pdrf = 10; 24 | 25 | using D = Dimension::Id; 26 | static const Dimension::IdList dims[11] 27 | { 28 | // 0 29 | { D::X, D::Y, D::Z, D::Intensity, D::ReturnNumber, D::NumberOfReturns, D::ScanDirectionFlag, 30 | D::EdgeOfFlightLine, D::Classification, D::ScanAngleRank, D::UserData, D::PointSourceId }, 31 | // 1 32 | { D::X, D::Y, D::Z, D::Intensity, D::ReturnNumber, D::NumberOfReturns, D::ScanDirectionFlag, 33 | D::EdgeOfFlightLine, D::Classification, D::ScanAngleRank, D::UserData, D::PointSourceId, 34 | D::GpsTime }, 35 | // 2 36 | { D::X, D::Y, D::Z, D::Intensity, D::ReturnNumber, D::NumberOfReturns, D::ScanDirectionFlag, 37 | D::EdgeOfFlightLine, D::Classification, D::ScanAngleRank, D::UserData, D::PointSourceId, 38 | D::Red, D::Green, D::Blue }, 39 | // 3 40 | { D::X, D::Y, D::Z, D::Intensity, D::ReturnNumber, D::NumberOfReturns, D::ScanDirectionFlag, 41 | D::EdgeOfFlightLine, D::Classification, D::ScanAngleRank, D::UserData, D::PointSourceId, 42 | D::GpsTime, D::Red, D::Green, D::Blue }, 43 | {}, 44 | {}, 45 | // 6 46 | { D::X, D::Y, D::Z, D::Intensity, D::ReturnNumber, D::NumberOfReturns, D::ScanDirectionFlag, 47 | D::EdgeOfFlightLine, D::Classification, D::ScanAngleRank, D::UserData, D::PointSourceId, 48 | D::GpsTime, D::ScanChannel, D::ClassFlags }, 49 | // 7 50 | { D::X, D::Y, D::Z, D::Intensity, D::ReturnNumber, D::NumberOfReturns, D::ScanDirectionFlag, 51 | D::EdgeOfFlightLine, D::Classification, D::ScanAngleRank, D::UserData, D::PointSourceId, 52 | D::GpsTime, D::ScanChannel, D::ClassFlags, D::Red, D::Green, D::Blue }, 53 | // 8 54 | { D::X, D::Y, D::Z, D::Intensity, D::ReturnNumber, D::NumberOfReturns, D::ScanDirectionFlag, 55 | D::EdgeOfFlightLine, D::Classification, D::ScanAngleRank, D::UserData, D::PointSourceId, 56 | D::GpsTime, D::ScanChannel, D::ClassFlags, D::Red, D::Green, D::Blue, D::Infrared }, 57 | {}, 58 | {} 59 | }; 60 | return dims[pdrf]; 61 | } 62 | 63 | const pdal::Dimension::IdList& extentDims(int pdrf) 64 | { 65 | using namespace pdal; 66 | 67 | if (pdrf < 0 || pdrf > 10) 68 | pdrf = 10; 69 | 70 | using D = Dimension::Id; 71 | static const Dimension::IdList dims[11] 72 | { 73 | {}, // 0 74 | {}, // 1 75 | {}, // 2 76 | {}, // 3 77 | {}, // 4 78 | {}, // 5 79 | // 6 80 | { D::X, D::Y, D::Z, D::Intensity, D::ReturnNumber, D::NumberOfReturns, D::ScanChannel, 81 | D::ScanDirectionFlag, D::EdgeOfFlightLine, D::Classification, D::UserData, 82 | D::ScanAngleRank, D::PointSourceId, D::GpsTime }, 83 | // 7 84 | { D::X, D::Y, D::Z, D::Intensity, D::ReturnNumber, D::NumberOfReturns, D::ScanChannel, 85 | D::ScanDirectionFlag, D::EdgeOfFlightLine, D::Classification, D::UserData, 86 | D::ScanAngleRank, D::PointSourceId, D::GpsTime, D::Red, D::Green, D::Blue }, 87 | // 8 88 | { D::X, D::Y, D::Z, D::Intensity, D::ReturnNumber, D::NumberOfReturns, D::ScanChannel, 89 | D::ScanDirectionFlag, D::EdgeOfFlightLine, D::Classification, D::UserData, 90 | D::ScanAngleRank, D::PointSourceId, D::GpsTime, D::Red, D::Green, D::Blue, D::Infrared }, 91 | {}, 92 | {} 93 | }; 94 | return dims[pdrf]; 95 | } 96 | 97 | } // namespace untwine 98 | -------------------------------------------------------------------------------- /src/tile/Las.hpp: -------------------------------------------------------------------------------- 1 | /***************************************************************************** 2 | * Copyright (c) 2021, Hobu, Inc. (info@hobu.co) * 3 | * * 4 | * All rights reserved. * 5 | * * 6 | * This program is free software; you can redistribute it and/or modify * 7 | * it under the terms of the GNU General Public License as published by * 8 | * the Free Software Foundation; either version 3 of the License, or * 9 | * (at your option) any later version. * 10 | * * 11 | ****************************************************************************/ 12 | 13 | #include 14 | 15 | namespace untwine 16 | { 17 | 18 | const pdal::Dimension::IdList& pdrfDims(int pdrf); 19 | const pdal::Dimension::IdList& extentDims(int pdrf); 20 | 21 | } // namespace untwine 22 | -------------------------------------------------------------------------------- /src/tile/Point.hpp: -------------------------------------------------------------------------------- 1 | /***************************************************************************** 2 | * Copyright (c) 2020, Hobu, Inc. (info@hobu.co) * 3 | * * 4 | * All rights reserved. * 5 | * * 6 | * This program is free software; you can redistribute it and/or modify * 7 | * it under the terms of the GNU General Public License as published by * 8 | * the Free Software Foundation; either version 3 of the License, or * 9 | * (at your option) any later version. * 10 | * * 11 | ****************************************************************************/ 12 | 13 | 14 | #pragma once 15 | 16 | namespace untwine 17 | { 18 | 19 | // Utterly trivial wrapper around a pointer. 20 | class Point 21 | { 22 | public: 23 | Point() : m_data(nullptr) 24 | {} 25 | 26 | Point(uint8_t *data) : m_data(data) 27 | {} 28 | Point(char *data) : m_data(reinterpret_cast(data)) 29 | {} 30 | 31 | uint8_t *data() 32 | { return m_data; } 33 | double x() const 34 | { 35 | double d; 36 | memcpy(&d, ddata(), sizeof(d)); 37 | return d; 38 | } 39 | double y() const 40 | { 41 | double d; 42 | memcpy(&d, ddata() + 1, sizeof(d)); 43 | return d; 44 | } 45 | double z() const 46 | { 47 | double d; 48 | memcpy(&d, ddata() + 2, sizeof(d)); 49 | return d; 50 | } 51 | 52 | char *cdata() const 53 | { return reinterpret_cast(m_data); } 54 | double *ddata() const 55 | { return reinterpret_cast(m_data); } 56 | 57 | private: 58 | uint8_t *m_data; 59 | }; 60 | 61 | } // namespace untwine 62 | -------------------------------------------------------------------------------- /src/tile/README.md: -------------------------------------------------------------------------------- 1 | 2 | Implements tiling of point clouds in two passes: 3 | 1. Read input files and write raw point data to files in a temporary directory 4 | 2. Write tiles as LAS/LAZ files from the temp point data files 5 | 6 | The first pass is entirely based on untwine's "epf" implementation, with only 7 | minor changes to grid/voxel structure to accommodate tiling requirements 8 | (fixed tile edge size, only using X/Y dimensions for tile keys, single level). 9 | Using commit `66cafb` as a base of the fork. 10 | 11 | Single pass tiling can be done with "pdal tile" kernel, but it can easily run out 12 | of open files (it keeps all output LAS/LAZ files open until it is finished). 13 | 14 | License: GPL3+ 15 | -------------------------------------------------------------------------------- /src/tile/ThreadPool.cpp: -------------------------------------------------------------------------------- 1 | /****************************************************************************** 2 | * Copyright (c) 2018, Connor Manning 3 | * 4 | * All rights reserved. 5 | * 6 | * Redistribution and use in source and binary forms, with or without 7 | * modification, are permitted provided that the following 8 | * conditions are met: 9 | * 10 | * * Redistributions of source code must retain the above copyright 11 | * notice, this list of conditions and the following disclaimer. 12 | * * Redistributions in binary form must reproduce the above copyright 13 | * notice, this list of conditions and the following disclaimer in 14 | * the documentation and/or other materials provided 15 | * with the distribution. 16 | * * Neither the name of the Martin Isenburg or Iowa Department 17 | * of Natural Resources nor the names of its contributors may be 18 | * used to endorse or promote products derived from this software 19 | * without specific prior written permission. 20 | * 21 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 | * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 | * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 24 | * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 25 | * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 26 | * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 27 | * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS 28 | * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED 29 | * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 30 | * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT 31 | * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY 32 | * OF SUCH DAMAGE. 33 | ****************************************************************************/ 34 | 35 | #include "ThreadPool.hpp" 36 | 37 | #include 38 | 39 | namespace untwine 40 | { 41 | 42 | void ThreadPool::go() 43 | { 44 | std::lock_guard lock(m_mutex); 45 | if (m_running) 46 | return; 47 | 48 | m_running = true; 49 | 50 | for (std::size_t i(0); i < m_numThreads; ++i) 51 | { 52 | m_threads.emplace_back([this]() { work(); }); 53 | } 54 | } 55 | 56 | void ThreadPool::work() 57 | { 58 | while (true) 59 | { 60 | std::unique_lock lock(m_mutex); 61 | m_consumeCv.wait(lock, [this]() 62 | { 63 | return m_tasks.size() || !m_running; 64 | }); 65 | 66 | if (m_tasks.size()) 67 | { 68 | ++m_outstanding; 69 | auto task(std::move(m_tasks.front())); 70 | m_tasks.pop(); 71 | 72 | lock.unlock(); 73 | 74 | // Notify add(), which may be waiting for a spot in the queue. 75 | m_produceCv.notify_all(); 76 | 77 | std::string err; 78 | 79 | if (m_trap) 80 | { 81 | try 82 | { 83 | task(); 84 | } 85 | catch (std::exception& e) 86 | { 87 | err = e.what(); 88 | } 89 | catch (...) 90 | { 91 | err = m_catchall; 92 | } 93 | } 94 | else 95 | task(); 96 | 97 | lock.lock(); 98 | --m_outstanding; 99 | if (err.size()) 100 | { 101 | if (m_verbose) 102 | std::cout << "Exception in pool task: " << err << std::endl; 103 | m_errors.push_back(err); 104 | } 105 | lock.unlock(); 106 | 107 | // Notify await(), which may be waiting for a running task. 108 | m_produceCv.notify_all(); 109 | } 110 | else if (!m_running) 111 | { 112 | return; 113 | } 114 | } 115 | } 116 | 117 | } // namespace untwine 118 | 119 | -------------------------------------------------------------------------------- /src/tile/ThreadPool.hpp: -------------------------------------------------------------------------------- 1 | /****************************************************************************** 2 | * Copyright (c) 2018, Connor Manning 3 | * 4 | * All rights reserved. 5 | * 6 | * Redistribution and use in source and binary forms, with or without 7 | * modification, are permitted provided that the following 8 | * conditions are met: 9 | * 10 | * * Redistributions of source code must retain the above copyright 11 | * notice, this list of conditions and the following disclaimer. 12 | * * Redistributions in binary form must reproduce the above copyright 13 | * notice, this list of conditions and the following disclaimer in 14 | * the documentation and/or other materials provided 15 | * with the distribution. 16 | * * Neither the name of the Martin Isenburg or Iowa Department 17 | * of Natural Resources nor the names of its contributors may be 18 | * used to endorse or promote products derived from this software 19 | * without specific prior written permission. 20 | * 21 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 | * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 | * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 24 | * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 25 | * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 26 | * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 27 | * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS 28 | * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED 29 | * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 30 | * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT 31 | * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY 32 | * OF SUCH DAMAGE. 33 | ****************************************************************************/ 34 | 35 | #pragma once 36 | 37 | #include 38 | #include 39 | #include 40 | #include 41 | #include 42 | #include 43 | 44 | //#include "Common.hpp" 45 | 46 | namespace untwine 47 | { 48 | 49 | class ThreadPool 50 | { 51 | public: 52 | // After numThreads tasks are actively running, and queueSize tasks have 53 | // been enqueued to wait for an available worker thread, subsequent calls 54 | // to Pool::add will block until an enqueued task has been popped from the 55 | // queue. 56 | ThreadPool(std::size_t numThreads, int64_t queueSize = -1, 57 | bool verbose = false) : 58 | m_queueSize(queueSize), 59 | m_numThreads(std::max(numThreads, 1)), m_verbose(verbose) 60 | { 61 | assert(m_queueSize != 0); 62 | go(); 63 | } 64 | 65 | ~ThreadPool() 66 | { join(); } 67 | 68 | ThreadPool(const ThreadPool& other) = delete; 69 | ThreadPool& operator=(const ThreadPool& other) = delete; 70 | 71 | // Start worker threads. 72 | void go(); 73 | 74 | // Disallow the addition of new tasks and wait for all currently running 75 | // tasks to complete. 76 | void join() 77 | { 78 | std::unique_lock lock(m_mutex); 79 | if (!m_running) return; 80 | m_running = false; 81 | lock.unlock(); 82 | 83 | m_consumeCv.notify_all(); 84 | for (auto& t : m_threads) t.join(); 85 | m_threads.clear(); 86 | } 87 | 88 | // join() and empty the queue of tasks that may have been waiting to run. 89 | void stop() 90 | { 91 | join(); 92 | 93 | // Effectively clear the queue. 94 | std::queue> q; 95 | m_tasks.swap(q); 96 | } 97 | 98 | // Wait for all current tasks to complete. As opposed to join, tasks may 99 | // continue to be added while a thread is await()-ing the queue to empty. 100 | void await() 101 | { 102 | std::unique_lock lock(m_mutex); 103 | m_produceCv.wait(lock, [this]() 104 | { 105 | return !m_outstanding && m_tasks.empty(); 106 | }); 107 | } 108 | 109 | // Join and restart. 110 | void cycle() 111 | { join(); go(); } 112 | 113 | // Change the number of threads. Current threads will be joined. 114 | void resize(const std::size_t numThreads) 115 | { 116 | join(); 117 | m_numThreads = numThreads; 118 | go(); 119 | } 120 | 121 | // Determine if worker threads had errors. 122 | bool hasErrors() const 123 | { 124 | std::unique_lock lock(m_mutex); 125 | 126 | return m_errors.size(); 127 | } 128 | 129 | // Clear worker thread errors, returning the list of current errors in the process. 130 | std::vector clearErrors() 131 | { 132 | std::unique_lock lock(m_mutex); 133 | 134 | std::vector out = m_errors; 135 | m_errors.clear(); 136 | return out; 137 | } 138 | 139 | 140 | // Add a threaded task, blocking until a thread is available. If join() is 141 | // called, add() may not be called again until go() is called and completes. 142 | bool add(std::function task) 143 | { 144 | std::unique_lock lock(m_mutex); 145 | if (!m_running) 146 | return false; 147 | 148 | m_produceCv.wait(lock, [this]() 149 | { 150 | return m_queueSize < 0 || m_tasks.size() < (size_t)m_queueSize; 151 | }); 152 | 153 | m_tasks.emplace(task); 154 | 155 | // Notify worker that a task is available. 156 | lock.unlock(); 157 | m_consumeCv.notify_all(); 158 | return true; 159 | } 160 | 161 | std::size_t size() const 162 | { return m_numThreads; } 163 | 164 | std::size_t numThreads() const 165 | { return m_numThreads; } 166 | 167 | // Turn on or off exception trapping of worker threads. Optionally set a catchall string 168 | // to be used when a exception is caught that doesn't derive from std::exception. 169 | void trap(bool trapExceptions, const std::string& catchall = "Unknown error") 170 | { 171 | std::unique_lock l(m_mutex); 172 | 173 | m_trap = trapExceptions; 174 | m_catchall = catchall; 175 | m_errors.clear(); 176 | } 177 | 178 | private: 179 | // Worker thread function. Wait for a task and run it. 180 | void work(); 181 | 182 | int64_t m_queueSize; 183 | std::size_t m_numThreads; 184 | bool m_verbose; 185 | std::vector m_threads; 186 | std::queue> m_tasks; 187 | std::vector m_errors; 188 | 189 | std::size_t m_outstanding = 0; 190 | bool m_running = false; 191 | bool m_trap = false; 192 | std::string m_catchall = "Unknown error."; 193 | 194 | mutable std::mutex m_mutex; 195 | std::condition_variable m_produceCv; 196 | std::condition_variable m_consumeCv; 197 | }; 198 | 199 | } // namespace untwine 200 | 201 | -------------------------------------------------------------------------------- /src/tile/TileGrid.cpp: -------------------------------------------------------------------------------- 1 | /***************************************************************************** 2 | * Copyright (c) 2020, Hobu, Inc. (info@hobu.co) * 3 | * * 4 | * All rights reserved. * 5 | * * 6 | * This program is free software; you can redistribute it and/or modify * 7 | * it under the terms of the GNU General Public License as published by * 8 | * the Free Software Foundation; either version 3 of the License, or * 9 | * (at your option) any later version. * 10 | * * 11 | ****************************************************************************/ 12 | 13 | 14 | #include 15 | #include 16 | 17 | #include "TileGrid.hpp" 18 | 19 | using namespace pdal; 20 | 21 | namespace untwine 22 | { 23 | namespace epf 24 | { 25 | 26 | void TileGrid::expand(const BOX3D& bounds, size_t points) 27 | { 28 | m_bounds.grow(bounds); 29 | double xside = m_bounds.maxx - m_bounds.minx; 30 | double yside = m_bounds.maxy - m_bounds.miny; 31 | m_millionPoints += size_t(points / 1000000.0); 32 | 33 | m_gridSizeX = std::ceil(xside / m_tileLength); 34 | m_gridSizeY = std::ceil(yside / m_tileLength); 35 | } 36 | 37 | 38 | TileKey TileGrid::key(double x, double y, double z) const 39 | { 40 | (void)z; 41 | int xi = (int)std::floor((x - m_bounds.minx) / m_tileLength); 42 | int yi = (int)std::floor((y - m_bounds.miny) / m_tileLength); 43 | xi = (std::min)((std::max)(0, xi), m_gridSizeX - 1); 44 | yi = (std::min)((std::max)(0, yi), m_gridSizeY - 1); 45 | return TileKey(xi, yi, 0); 46 | } 47 | 48 | } // namespace epf 49 | } // namespace untwine 50 | -------------------------------------------------------------------------------- /src/tile/TileGrid.hpp: -------------------------------------------------------------------------------- 1 | /***************************************************************************** 2 | * Copyright (c) 2020, Hobu, Inc. (info@hobu.co) * 3 | * * 4 | * All rights reserved. * 5 | * * 6 | * This program is free software; you can redistribute it and/or modify * 7 | * it under the terms of the GNU General Public License as published by * 8 | * the Free Software Foundation; either version 3 of the License, or * 9 | * (at your option) any later version. * 10 | * * 11 | ****************************************************************************/ 12 | 13 | 14 | #pragma once 15 | 16 | #include 17 | 18 | #include "TileKey.hpp" 19 | 20 | 21 | namespace untwine 22 | { 23 | namespace epf 24 | { 25 | 26 | class TileGrid 27 | { 28 | public: 29 | 30 | void setTileLength( double len ) { m_tileLength = len; } 31 | 32 | void expand(const pdal::BOX3D& bounds, size_t points); 33 | TileKey key(double x, double y, double z) const; 34 | pdal::BOX3D conformingBounds() const 35 | { return m_bounds; } 36 | 37 | private: 38 | double m_tileLength = 100; 39 | int m_gridSizeX = 0; 40 | int m_gridSizeY = 0; 41 | pdal::BOX3D m_bounds; 42 | size_t m_millionPoints = 0; 43 | }; 44 | 45 | } // namespace epf 46 | } // namespace untwine 47 | -------------------------------------------------------------------------------- /src/tile/TileKey.hpp: -------------------------------------------------------------------------------- 1 | /***************************************************************************** 2 | * Copyright (c) 2020, Hobu, Inc. (info@hobu.co) * 3 | * * 4 | * All rights reserved. * 5 | * * 6 | * This program is free software; you can redistribute it and/or modify * 7 | * it under the terms of the GNU General Public License as published by * 8 | * the Free Software Foundation; either version 3 of the License, or * 9 | * (at your option) any later version. * 10 | * * 11 | ****************************************************************************/ 12 | 13 | #include 14 | #include 15 | 16 | #pragma once 17 | 18 | namespace untwine 19 | { 20 | 21 | // This key supports large levels, but requires a larger key. 22 | class TileKey 23 | { 24 | public: 25 | TileKey() : m_x(0), m_y(0), m_z(0) 26 | {} 27 | 28 | TileKey(int x, int y, int z) : m_x(x), m_y(y), m_z(z) 29 | {} 30 | 31 | int x() const 32 | { return m_x; } 33 | int y() const 34 | { return m_y; } 35 | int z() const 36 | { return m_z; } 37 | 38 | std::string toString() const 39 | { return (std::string)(*this); } 40 | 41 | operator std::string() const 42 | { 43 | return std::to_string(m_x) + '-' + std::to_string(m_y); 44 | } 45 | 46 | private: 47 | int m_x; 48 | int m_y; 49 | int m_z; // MD: unused but kept for now 50 | }; 51 | 52 | inline bool operator==(const TileKey& k1, const TileKey& k2) 53 | { 54 | return k1.x() == k2.x() && k1.y() == k2.y() && k1.z() == k2.z(); 55 | } 56 | 57 | inline bool operator!=(const TileKey& k1, const TileKey& k2) 58 | { 59 | return k1.x() != k2.x() || k1.y() != k2.y() || k1.z() != k2.z(); 60 | } 61 | 62 | inline std::ostream& operator<<(std::ostream& out, const TileKey& k) 63 | { 64 | out << k.toString(); 65 | return out; 66 | } 67 | 68 | inline bool operator<(const TileKey& k1, const TileKey& k2) 69 | { 70 | if (k1.x() != k2.x()) 71 | return k1.x() < k2.x(); 72 | if (k1.y() != k2.y()) 73 | return k1.y() < k2.y(); 74 | return k1.z() < k2.z(); 75 | } 76 | 77 | } // namespace untwine 78 | 79 | namespace std 80 | { 81 | template<> struct hash 82 | { 83 | size_t operator()(const untwine::TileKey& k) const noexcept 84 | { 85 | size_t t; 86 | 87 | static_assert(sizeof(size_t) == sizeof(uint64_t) || 88 | sizeof(size_t) == sizeof(uint32_t), 89 | "Only systems with 32 and 64 bit size_t are currently supported."); 90 | 91 | // Counting on the compiler to optimize away the wrong branch. 92 | if (sizeof(size_t) == sizeof(uint64_t)) 93 | { 94 | // For this to work well we just assume that the values are no more than 16 bits. 95 | t = size_t(k.x()) << 48; 96 | t |= size_t(k.y()) << 32; 97 | t |= size_t(k.z()) << 16; 98 | } 99 | else if (sizeof(size_t) == sizeof(uint32_t)) 100 | { 101 | t = size_t((k.x() << 24) | 0xFF000000); 102 | t |= size_t((k.y() << 16) | 0xFF0000); 103 | t |= size_t((k.z() << 8) | 0xFF00); 104 | } 105 | return t; 106 | } 107 | }; 108 | } 109 | -------------------------------------------------------------------------------- /src/tile/Writer.cpp: -------------------------------------------------------------------------------- 1 | /***************************************************************************** 2 | * Copyright (c) 2020, Hobu, Inc. (info@hobu.co) * 3 | * * 4 | * All rights reserved. * 5 | * * 6 | * This program is free software; you can redistribute it and/or modify * 7 | * it under the terms of the GNU General Public License as published by * 8 | * the Free Software Foundation; either version 3 of the License, or * 9 | * (at your option) any later version. * 10 | * * 11 | ****************************************************************************/ 12 | 13 | 14 | #include 15 | 16 | #include 17 | 18 | #include "Writer.hpp" 19 | #include "Common.hpp" 20 | #include "TileKey.hpp" 21 | 22 | using namespace pdal; 23 | 24 | namespace untwine 25 | { 26 | namespace epf 27 | { 28 | 29 | Writer::Writer(const std::string& directory, int numThreads, size_t pointSize) : 30 | m_directory(directory), m_pool(numThreads), m_stop(false), m_pointSize(pointSize) 31 | { 32 | std::function f = std::bind(&Writer::run, this); 33 | while (numThreads--) 34 | m_pool.add(f); 35 | } 36 | 37 | std::string Writer::path(const TileKey& key) 38 | { 39 | return m_directory + "/" + key.toString() + ".bin"; 40 | } 41 | 42 | Totals Writer::totals(size_t minSize) 43 | { 44 | Totals t; 45 | 46 | for (auto ti = m_totals.begin(); ti != m_totals.end(); ++ti) 47 | if (ti->second >= minSize) 48 | t.insert(*ti); 49 | return t; 50 | } 51 | 52 | DataVecPtr Writer::fetchBuffer() 53 | { 54 | std::unique_lock lock(m_mutex); 55 | 56 | // If there are fewer items in the queue than we have FileProcessors, we may choose not 57 | // to block and return a nullptr, expecting that the caller will flush outstanding cells. 58 | return m_bufferCache.fetch(lock, m_queue.size() < NumFileProcessors); 59 | } 60 | 61 | 62 | DataVecPtr Writer::fetchBufferBlocking() 63 | { 64 | std::unique_lock lock(m_mutex); 65 | 66 | return m_bufferCache.fetch(lock, false); 67 | } 68 | 69 | 70 | void Writer::enqueue(const TileKey& key, DataVecPtr data, size_t dataSize) 71 | { 72 | { 73 | std::lock_guard lock(m_mutex); 74 | m_totals[key] += (dataSize / m_pointSize); 75 | m_queue.push_back({key, std::move(data), dataSize}); 76 | } 77 | m_available.notify_one(); 78 | } 79 | 80 | void Writer::replace(DataVecPtr data) 81 | { 82 | std::lock_guard lock(m_mutex); 83 | m_bufferCache.replace(std::move(data)); 84 | } 85 | 86 | void Writer::stop() 87 | { 88 | { 89 | std::lock_guard lock(m_mutex); 90 | m_stop = true; 91 | } 92 | m_available.notify_all(); 93 | m_pool.join(); 94 | std::vector errors = m_pool.clearErrors(); 95 | if (errors.size()) 96 | throw FatalError(errors.front()); 97 | } 98 | 99 | void Writer::run() 100 | { 101 | while (true) 102 | { 103 | WriteData wd; 104 | 105 | // Loop waiting for data. 106 | while (true) 107 | { 108 | std::unique_lock lock(m_mutex); 109 | 110 | // Look for a queue entry that represents a key that we aren't already 111 | // actively processing. 112 | //ABELL - Perhaps a writer should grab and write *all* the entries in the queue 113 | // that match the key we found. 114 | auto li = m_queue.begin(); 115 | for (; li != m_queue.end(); ++li) 116 | if (std::find(m_active.begin(), m_active.end(), li->key) == m_active.end()) 117 | break; 118 | 119 | // If there is no data to process, exit if we're stopping. Wait otherwise. 120 | // If there is data to process, stick the key on the active list and 121 | // remove the item from the queue and break to do the actual write. 122 | if (li == m_queue.end()) 123 | { 124 | if (m_stop) 125 | return; 126 | m_available.wait(lock); 127 | } 128 | else 129 | { 130 | m_active.push_back(li->key); 131 | wd = std::move(*li); 132 | m_queue.erase(li); 133 | break; 134 | } 135 | } 136 | 137 | // Open the file. Write the data. Stick the buffer back on the cache. 138 | // Remove the key from the active key list. 139 | std::ofstream out(untwine::toNative(path(wd.key)), std::ios::app | std::ios::binary); 140 | out.write(reinterpret_cast(wd.data->data()), wd.dataSize); 141 | out.close(); 142 | if (!out) 143 | throw FatalError("Failure writing to '" + path(wd.key) + "'."); 144 | 145 | std::lock_guard lock(m_mutex); 146 | m_bufferCache.replace(std::move(wd.data)); 147 | m_active.remove(wd.key); 148 | } 149 | } 150 | 151 | } // namespace epf 152 | } // namespace untwine 153 | -------------------------------------------------------------------------------- /src/tile/Writer.hpp: -------------------------------------------------------------------------------- 1 | /***************************************************************************** 2 | * Copyright (c) 2020, Hobu, Inc. (info@hobu.co) * 3 | * * 4 | * All rights reserved. * 5 | * * 6 | * This program is free software; you can redistribute it and/or modify * 7 | * it under the terms of the GNU General Public License as published by * 8 | * the Free Software Foundation; either version 3 of the License, or * 9 | * (at your option) any later version. * 10 | * * 11 | ****************************************************************************/ 12 | 13 | 14 | #pragma once 15 | 16 | #include 17 | #include 18 | #include 19 | #include 20 | 21 | #include "EpfTypes.hpp" 22 | #include "BufferCache.hpp" 23 | #include "ThreadPool.hpp" 24 | #include "TileKey.hpp" 25 | 26 | namespace untwine 27 | { 28 | namespace epf 29 | { 30 | 31 | // The writer has some number of threads that actually write data to the files for tiles. When 32 | // a processor has a full tile (or a partial that it needs to discard/finish), it sticks it 33 | // on the queue for one of the writer threads to pick up and process. 34 | // 35 | // We can't have multiple writer threads write to the same file simultaneously, so rather than 36 | // lock (which might stall threads that could otherwise be working), we make sure that only 37 | // one writer thread is working on a file at a time by sticking 38 | // the key of the thread in an "active" list. A writer thread looking for work will ignore 39 | // any buffer on the queue that's for a file currently being handled by another writer thread. 40 | // 41 | // The writer owns a buffer cache. The cache manages the actual data buffers that are filled 42 | // by the file processors and written by a writer thread. The buffers are created as needed 43 | // until some predefined number of buffers is hit in order to limit memory use. 44 | // Once a writer is done with a buffer, it sticks it back on the cache 45 | // and then notifes the some processor that a buffer is available in case the processor 46 | // is waiting for a free buffer. 47 | // 48 | // Since processors try to hold onto buffers until they are full, there can be times at 49 | // which the buffers are exhaused and no more are available, but none are ready to be 50 | // written. In this case, the buffers for the processor needing a new buffer flushed to 51 | // the queue even if they aren't full so that they can be reused. The active buffer for a 52 | // flushing processor is reserved, so there need to be at least one more buffer than the 53 | // number of file processors, though typically there are many more buffers than file processors. 54 | // 55 | // Buffers containing no points are never queued, but if a processor flush occurs, they are 56 | // replaced on the buffer cache for reuse. Empty buffers can happen because if a cell has had 57 | // its buffer written, it immediately grabs a new buffer even if it hasn't seen a point 58 | // destined for that cell - we don't want to tear down the cell just to recreate it. 59 | // The thinking is that if we've filled a buffer for a cell, there's 60 | // probably at least one more point going to that cell from the source. 61 | class Writer 62 | { 63 | struct WriteData 64 | { 65 | TileKey key; 66 | DataVecPtr data; 67 | size_t dataSize; 68 | }; 69 | 70 | public: 71 | Writer(const std::string& directory, int numThreads, size_t pointSize); 72 | 73 | void replace(DataVecPtr data); 74 | void enqueue(const TileKey& key, DataVecPtr data, size_t dataSize); 75 | void stop(); 76 | const Totals& totals() 77 | { return m_totals; } 78 | Totals totals(size_t minSize); 79 | DataVecPtr fetchBuffer(); 80 | DataVecPtr fetchBufferBlocking(); 81 | 82 | private: 83 | std::string path(const TileKey& key); 84 | void run(); 85 | 86 | std::string m_directory; 87 | ThreadPool m_pool; 88 | BufferCache m_bufferCache; 89 | bool m_stop; 90 | size_t m_pointSize; 91 | std::list m_queue; 92 | std::list m_active; 93 | Totals m_totals; 94 | std::mutex m_mutex; 95 | std::condition_variable m_available; 96 | }; 97 | 98 | } // namespace epf 99 | } // namespace untwine 100 | -------------------------------------------------------------------------------- /src/to_vector.cpp: -------------------------------------------------------------------------------- 1 | /***************************************************************************** 2 | * Copyright (c) 2023, Lutra Consulting Ltd. and Hobu, Inc. * 3 | * * 4 | * All rights reserved. * 5 | * * 6 | * This program is free software; you can redistribute it and/or modify * 7 | * it under the terms of the GNU General Public License as published by * 8 | * the Free Software Foundation; either version 3 of the License, or * 9 | * (at your option) any later version. * 10 | * * 11 | ****************************************************************************/ 12 | 13 | #include 14 | #include 15 | #include 16 | 17 | #include 18 | #include 19 | #include 20 | #include 21 | #include 22 | 23 | #include 24 | 25 | #include "utils.hpp" 26 | #include "alg.hpp" 27 | #include "vpc.hpp" 28 | 29 | using namespace pdal; 30 | 31 | namespace fs = std::filesystem; 32 | 33 | 34 | void ToVector::addArgs() 35 | { 36 | argOutput = &programArgs.add("output,o", "Output vector file", outputFile); 37 | programArgs.add("attribute", "Attributes to include", attributes); 38 | } 39 | 40 | bool ToVector::checkArgs() 41 | { 42 | if (!argOutput->set()) 43 | { 44 | std::cerr << "missing output" << std::endl; 45 | return false; 46 | } 47 | 48 | return true; 49 | } 50 | 51 | 52 | static std::unique_ptr pipeline(ParallelJobInfo *tile, const std::vector &attributes) 53 | { 54 | std::unique_ptr manager( new PipelineManager ); 55 | 56 | Stage& r = manager->makeReader( tile->inputFilenames[0], ""); 57 | 58 | Stage *last = &r; 59 | 60 | // filtering 61 | if (!tile->filterBounds.empty()) 62 | { 63 | Options filter_opts; 64 | filter_opts.add(pdal::Option("bounds", tile->filterBounds)); 65 | 66 | if (readerSupportsBounds(r)) 67 | { 68 | // Reader of the format can do the filtering - use that whenever possible! 69 | r.addOptions(filter_opts); 70 | } 71 | else 72 | { 73 | // Reader can't do the filtering - do it with a filter 74 | last = &manager->makeFilter( "filters.crop", *last, filter_opts); 75 | } 76 | } 77 | if (!tile->filterExpression.empty()) 78 | { 79 | Options filter_opts; 80 | filter_opts.add(pdal::Option("expression", tile->filterExpression)); 81 | last = &manager->makeFilter( "filters.expression", *last, filter_opts); 82 | } 83 | 84 | pdal::Options writer_opts; 85 | writer_opts.add(pdal::Option("ogrdriver", "GPKG")); 86 | if (!attributes.empty()) 87 | writer_opts.add(pdal::Option("attr_dims", join_strings(attributes, ','))); 88 | (void)manager->makeWriter( tile->outputFilename, "writers.ogr", *last, writer_opts); 89 | 90 | return manager; 91 | } 92 | 93 | 94 | void ToVector::preparePipelines(std::vector>& pipelines) 95 | { 96 | if (ends_with(inputFile, ".vpc")) 97 | { 98 | // for /tmp/hello.vpc we will use /tmp/hello dir for all results 99 | fs::path outputParentDir = fs::path(outputFile).parent_path(); 100 | fs::path outputSubdir = outputParentDir / fs::path(outputFile).stem(); 101 | fs::create_directories(outputSubdir); 102 | 103 | // VPC handling 104 | VirtualPointCloud vpc; 105 | if (!vpc.read(inputFile)) 106 | return; 107 | 108 | for (const VirtualPointCloud::File& f : vpc.files) 109 | { 110 | ParallelJobInfo tile(ParallelJobInfo::FileBased, BOX2D(), filterExpression, filterBounds); 111 | tile.inputFilenames.push_back(f.filename); 112 | 113 | // for input file /x/y/z.las that goes to /tmp/hello.vpc, 114 | // individual output file will be called /tmp/hello/z.las 115 | fs::path inputBasename = fs::path(f.filename).stem(); 116 | tile.outputFilename = (outputSubdir / inputBasename).string() + ".gpkg"; 117 | 118 | tileOutputFiles.push_back(tile.outputFilename); 119 | 120 | pipelines.push_back(pipeline(&tile, attributes)); 121 | } 122 | } 123 | else 124 | { 125 | ParallelJobInfo tile(ParallelJobInfo::Single, BOX2D(), filterExpression, filterBounds); 126 | tile.inputFilenames.push_back(inputFile); 127 | tile.outputFilename = outputFile; 128 | pipelines.push_back(pipeline(&tile, attributes)); 129 | } 130 | } 131 | 132 | void ToVector::finalize(std::vector>&) 133 | { 134 | if (tileOutputFiles.empty()) 135 | return; 136 | 137 | if (ends_with(inputFile, ".vpc")) 138 | { 139 | // for /tmp/hello.vpc we will use /tmp/hello dir for all results 140 | fs::path outputParentDir = fs::path(outputFile).parent_path(); 141 | fs::path outputSubdir = outputParentDir / fs::path(outputFile).stem(); 142 | fs::path vrtFile = outputSubdir / "all.vrt"; 143 | 144 | // remove vrt if it exist 145 | if (fs::exists(vrtFile)) 146 | { 147 | fs::remove(vrtFile); 148 | } 149 | 150 | std::ofstream outputStreamFile(vrtFile); 151 | 152 | // check if file opened successfully and write VRT content to it 153 | if (outputStreamFile.is_open()) { 154 | 155 | outputStreamFile << "" << std::endl; 156 | outputStreamFile << "" << std::endl; 157 | 158 | for (const std::string& f : tileOutputFiles) 159 | { 160 | outputStreamFile << "" << std::endl; 161 | outputStreamFile << "" << f << "" << std::endl; 162 | outputStreamFile << "points" << std::endl; 163 | outputStreamFile << "" << std::endl; 164 | } 165 | 166 | outputStreamFile << "" << std::endl; 167 | outputStreamFile << "" << std::endl; 168 | 169 | // close file 170 | outputStreamFile.close(); 171 | } 172 | else 173 | { 174 | std::cerr << "Failed to open VRT file for writing: " << vrtFile.string() << std::endl; 175 | return; 176 | } 177 | 178 | // options for translate - empty 179 | GDALVectorTranslateOptions *options = GDALVectorTranslateOptionsNew(nullptr, NULL); 180 | 181 | // open VRT file and check it 182 | GDALDatasetH vrtDs = GDALOpenEx(vrtFile.string().c_str(), GDAL_OF_VECTOR | GDAL_OF_READONLY, NULL, NULL, NULL); 183 | if (!vrtDs) 184 | { 185 | std::cerr << "Failed to open composite VRT file" << std::endl; 186 | fs::remove(vrtFile); 187 | return; 188 | } 189 | 190 | // translate to resulting file and check it 191 | GDALDatasetH resultDS = GDALVectorTranslate(outputFile.c_str(), nullptr, 1, (GDALDatasetH *)&vrtDs, options, nullptr); 192 | if (!resultDS) 193 | { 194 | std::cerr << "Failed to create output file" << std::endl; 195 | GDALClose(vrtDs); 196 | fs::remove(vrtFile); 197 | return; 198 | } 199 | 200 | // close datasets 201 | GDALClose(vrtDs); 202 | GDALClose(resultDS); 203 | 204 | // delete temporary files 205 | for (const std::string& f : tileOutputFiles) 206 | { 207 | fs::remove(f); 208 | } 209 | 210 | // delete vrt file 211 | fs::remove(vrtFile); 212 | 213 | // delete dir if empty 214 | if (fs::is_empty(outputSubdir)) 215 | { 216 | fs::remove(outputSubdir); 217 | } 218 | } 219 | } 220 | -------------------------------------------------------------------------------- /src/translate.cpp: -------------------------------------------------------------------------------- 1 | /***************************************************************************** 2 | * Copyright (c) 2023, Lutra Consulting Ltd. and Hobu, Inc. * 3 | * * 4 | * All rights reserved. * 5 | * * 6 | * This program is free software; you can redistribute it and/or modify * 7 | * it under the terms of the GNU General Public License as published by * 8 | * the Free Software Foundation; either version 3 of the License, or * 9 | * (at your option) any later version. * 10 | * * 11 | ****************************************************************************/ 12 | 13 | #include 14 | #include 15 | #include 16 | 17 | #include 18 | #include 19 | #include 20 | #include 21 | #include 22 | 23 | #include 24 | 25 | #include "utils.hpp" 26 | #include "alg.hpp" 27 | #include "vpc.hpp" 28 | 29 | using namespace pdal; 30 | 31 | namespace fs = std::filesystem; 32 | 33 | 34 | void Translate::addArgs() 35 | { 36 | argOutput = &programArgs.add("output,o", "Output point cloud file", outputFile); 37 | argOutputFormat = &programArgs.add("output-format", "Output format (las/laz/copc)", outputFormat); 38 | programArgs.add("assign-crs", "Assigns CRS to data (no reprojection)", assignCrs); 39 | programArgs.add("transform-crs", "Transforms (reprojects) data to another CRS", transformCrs); 40 | programArgs.add("transform-coord-op", "Details on how to do the transform of coordinates when --transform-crs is used. " 41 | "It can be a PROJ pipeline or a WKT2 CoordinateOperation. " 42 | "When not specified, PROJ will pick the default transform.", transformCoordOp); 43 | } 44 | 45 | bool Translate::checkArgs() 46 | { 47 | if (!argOutput->set()) 48 | { 49 | std::cerr << "missing output" << std::endl; 50 | return false; 51 | } 52 | 53 | // TODO: or use the same format as the reader? 54 | if (argOutputFormat->set()) 55 | { 56 | if (outputFormat != "las" && outputFormat != "laz") 57 | { 58 | std::cerr << "unknown output format: " << outputFormat << std::endl; 59 | return false; 60 | } 61 | } 62 | else 63 | outputFormat = "las"; // uncompressed by default 64 | 65 | if (!transformCoordOp.empty() && transformCrs.empty()) 66 | { 67 | std::cerr << "Need to specify also --transform-crs when --transform-coord-op is used." << std::endl; 68 | return false; 69 | } 70 | 71 | return true; 72 | } 73 | 74 | 75 | static std::unique_ptr pipeline(ParallelJobInfo *tile, std::string assignCrs, std::string transformCrs, std::string transformCoordOp) 76 | { 77 | std::unique_ptr manager( new PipelineManager ); 78 | 79 | Options reader_opts; 80 | if (!assignCrs.empty()) 81 | reader_opts.add(pdal::Option("override_srs", assignCrs)); 82 | 83 | Stage& r = manager->makeReader( tile->inputFilenames[0], "", reader_opts); 84 | 85 | Stage *last = &r; 86 | 87 | // filtering 88 | if (!tile->filterBounds.empty()) 89 | { 90 | Options filter_opts; 91 | filter_opts.add(pdal::Option("bounds", tile->filterBounds)); 92 | 93 | if (readerSupportsBounds(r)) 94 | { 95 | // Reader of the format can do the filtering - use that whenever possible! 96 | r.addOptions(filter_opts); 97 | } 98 | else 99 | { 100 | // Reader can't do the filtering - do it with a filter 101 | last = &manager->makeFilter( "filters.crop", *last, filter_opts); 102 | } 103 | } 104 | if (!tile->filterExpression.empty()) 105 | { 106 | Options filter_opts; 107 | filter_opts.add(pdal::Option("expression", tile->filterExpression)); 108 | last = &manager->makeFilter( "filters.expression", *last, filter_opts); 109 | } 110 | 111 | // optional reprojection 112 | Stage* reproject = nullptr; 113 | if (!transformCrs.empty()) 114 | { 115 | Options transform_opts; 116 | transform_opts.add(pdal::Option("out_srs", transformCrs)); 117 | if (!transformCoordOp.empty()) 118 | { 119 | transform_opts.add(pdal::Option("coord_op", transformCoordOp)); 120 | reproject = &manager->makeFilter( "filters.projpipeline", *last, transform_opts); 121 | } 122 | else 123 | { 124 | reproject = &manager->makeFilter( "filters.reprojection", *last, transform_opts); 125 | } 126 | last = reproject; 127 | } 128 | 129 | pdal::Options writer_opts; 130 | if (!reproject) 131 | { 132 | // let's use the same offset & scale & header & vlrs as the input 133 | writer_opts.add(pdal::Option("forward", "all")); 134 | } 135 | else 136 | { 137 | // avoid adding offset as it probably wouldn't work 138 | // TODO: maybe adjust scale as well - depending on the CRS 139 | writer_opts.add(pdal::Option("forward", "header,scale,vlr")); 140 | writer_opts.add(pdal::Option("offset_x", "auto")); 141 | writer_opts.add(pdal::Option("offset_y", "auto")); 142 | writer_opts.add(pdal::Option("offset_z", "auto")); 143 | } 144 | 145 | (void)manager->makeWriter( tile->outputFilename, "", *last, writer_opts); 146 | 147 | return manager; 148 | } 149 | 150 | 151 | void Translate::preparePipelines(std::vector>& pipelines) 152 | { 153 | if (ends_with(inputFile, ".vpc")) 154 | { 155 | // for /tmp/hello.vpc we will use /tmp/hello dir for all results 156 | fs::path outputParentDir = fs::path(outputFile).parent_path(); 157 | fs::path outputSubdir = outputParentDir / fs::path(outputFile).stem(); 158 | fs::create_directories(outputSubdir); 159 | 160 | // VPC handling 161 | VirtualPointCloud vpc; 162 | if (!vpc.read(inputFile)) 163 | return; 164 | 165 | for (const VirtualPointCloud::File& f : vpc.files) 166 | { 167 | ParallelJobInfo tile(ParallelJobInfo::FileBased, BOX2D(), filterExpression, filterBounds); 168 | tile.inputFilenames.push_back(f.filename); 169 | 170 | // for input file /x/y/z.las that goes to /tmp/hello.vpc, 171 | // individual output file will be called /tmp/hello/z.las 172 | fs::path inputBasename = fs::path(f.filename).stem(); 173 | 174 | if (!ends_with(outputFile, ".vpc")) 175 | tile.outputFilename = (outputSubdir / inputBasename).string() + ".las"; 176 | else 177 | tile.outputFilename = (outputSubdir / inputBasename).string() + outputFormat; 178 | 179 | tileOutputFiles.push_back(tile.outputFilename); 180 | 181 | pipelines.push_back(pipeline(&tile, assignCrs, transformCrs, transformCoordOp)); 182 | } 183 | } 184 | else 185 | { 186 | if (ends_with(outputFile, ".copc.laz")) 187 | { 188 | isStreaming = false; 189 | } 190 | ParallelJobInfo tile(ParallelJobInfo::Single, BOX2D(), filterExpression, filterBounds); 191 | tile.inputFilenames.push_back(inputFile); 192 | tile.outputFilename = outputFile; 193 | pipelines.push_back(pipeline(&tile, assignCrs, transformCrs, transformCoordOp)); 194 | } 195 | } 196 | 197 | void Translate::finalize(std::vector>&) 198 | { 199 | if (tileOutputFiles.empty()) 200 | return; 201 | 202 | std::vector args; 203 | args.push_back("--output=" + outputFile); 204 | for (std::string f : tileOutputFiles) 205 | args.push_back(f); 206 | 207 | if (ends_with(outputFile, ".vpc")) 208 | { 209 | // now build a new output VPC 210 | buildVpc(args); 211 | } 212 | else 213 | { 214 | // merge all the output files into a single file 215 | Merge merge; 216 | // for copc set isStreaming to false 217 | if (ends_with(outputFile, ".copc.laz")) 218 | { 219 | merge.isStreaming = false; 220 | } 221 | runAlg(args, merge); 222 | 223 | // remove files as they are not needed anymore - they are merged 224 | removeFiles(tileOutputFiles, true); 225 | } 226 | } 227 | -------------------------------------------------------------------------------- /src/utils.cpp: -------------------------------------------------------------------------------- 1 | /***************************************************************************** 2 | * Copyright (c) 2023, Lutra Consulting Ltd. and Hobu, Inc. * 3 | * * 4 | * All rights reserved. * 5 | * * 6 | * This program is free software; you can redistribute it and/or modify * 7 | * it under the terms of the GNU General Public License as published by * 8 | * the Free Software Foundation; either version 3 of the License, or * 9 | * (at your option) any later version. * 10 | * * 11 | ****************************************************************************/ 12 | 13 | #include "utils.hpp" 14 | 15 | #include 16 | #include 17 | #include 18 | 19 | #include 20 | #include 21 | #include 22 | #include 23 | 24 | #include 25 | 26 | using namespace pdal; 27 | 28 | 29 | static ProgressBar sProgressBar; 30 | 31 | 32 | // Table subclass that also takes care of updating progress in streaming pipelines 33 | class MyTable : public FixedPointTable 34 | { 35 | public: 36 | MyTable(point_count_t capacity) : FixedPointTable(capacity) {} 37 | 38 | protected: 39 | virtual void reset() 40 | { 41 | sProgressBar.add(); 42 | FixedPointTable::reset(); 43 | } 44 | }; 45 | 46 | 47 | std::string box_to_pdal_bounds(const BOX2D &box) 48 | { 49 | std::ostringstream oss; 50 | oss << std::fixed << "([" << box.minx << "," << box.maxx << "],[" << box.miny << "," << box.maxy << "])"; 51 | return oss.str(); // "([xmin, xmax], [ymin, ymax])" 52 | } 53 | 54 | 55 | QuickInfo getQuickInfo(std::string inputFile) 56 | { 57 | // TODO: handle the case when driver is not inferred 58 | std::string driver = StageFactory::inferReaderDriver(inputFile); 59 | if (driver.empty()) 60 | { 61 | std::cerr << "Could not infer driver for input file: " << inputFile << std::endl; 62 | return QuickInfo(); 63 | } 64 | 65 | StageFactory factory; 66 | Stage *reader = factory.createStage(driver); // reader is owned by the factory 67 | pdal::Options opts; 68 | opts.add("filename", inputFile); 69 | reader->setOptions(opts); 70 | return reader->preview(); 71 | 72 | // PipelineManager m; 73 | // Stage &r = m.makeReader(inputFile, ""); 74 | // return r.preview().m_pointCount; 75 | } 76 | 77 | MetadataNode getReaderMetadata(std::string inputFile, MetadataNode *pointLayoutMeta) 78 | { 79 | // compared to quickinfo / preview, this provides more info... 80 | 81 | PipelineManager m; 82 | Stage &r = m.makeReader(inputFile, ""); 83 | FixedPointTable table(10000); 84 | r.prepare(table); 85 | if (pointLayoutMeta) 86 | { 87 | *pointLayoutMeta = table.layout()->toMetadata(); 88 | } 89 | return r.getMetadata(); 90 | } 91 | 92 | #define CHUNK_SIZE 100000 93 | 94 | void runPipelineParallel(point_count_t totalPoints, bool isStreaming, std::vector>& pipelines, int max_threads, bool verbose) 95 | { 96 | int num_chunks = totalPoints / CHUNK_SIZE; 97 | 98 | if (verbose) 99 | { 100 | std::cout << "total points: " << (float)totalPoints / 1'000'000 << "M" << std::endl; 101 | 102 | std::cout << "jobs " << pipelines.size() << std::endl; 103 | std::cout << "max threads " << max_threads << std::endl; 104 | if (!isStreaming) 105 | std::cout << "running in non-streaming mode!" << std::endl; 106 | } 107 | 108 | auto start = std::chrono::high_resolution_clock::now(); 109 | 110 | sProgressBar.init(isStreaming ? num_chunks : pipelines.size()); 111 | 112 | int nThreads = (std::min)( (int)pipelines.size(), max_threads ); 113 | ThreadPool p(nThreads); 114 | for (size_t i = 0; i < pipelines.size(); ++i) 115 | { 116 | PipelineManager* pipeline = pipelines[i].get(); 117 | if (isStreaming) 118 | { 119 | p.add([pipeline]() { 120 | 121 | MyTable table(CHUNK_SIZE); 122 | pipeline->executeStream(table); 123 | 124 | }); 125 | } 126 | else 127 | { 128 | p.add([pipeline, &pipelines, i]() { 129 | pipeline->execute(); 130 | pipelines[i].reset(); // to free the point table and views (meshes, rasters) 131 | sProgressBar.add(); 132 | }); 133 | } 134 | } 135 | 136 | //std::cout << "starting to wait" << std::endl; 137 | 138 | // while (p.tasksInQueue() + p.tasksInProgress()) 139 | // { 140 | // //std::cout << "progress: " << p.tasksInQueue() << " " << p.tasksInProgress() << " cnt " << cntPnt/1'000 << std::endl; 141 | // std::this_thread::sleep_for(500ms); 142 | // } 143 | 144 | p.join(); 145 | 146 | sProgressBar.done(); 147 | 148 | auto stop = std::chrono::high_resolution_clock::now(); 149 | auto duration = std::chrono::duration_cast(stop - start); 150 | if (verbose) 151 | { 152 | std::cout << "time " << duration.count()/1000. << " s" << std::endl; 153 | } 154 | } 155 | 156 | 157 | static GDALDatasetH rasterTilesToVrt(const std::vector &inputFiles, const std::string &outputVrtFile) 158 | { 159 | // build a VRT so that all tiles can be handled as a single data source 160 | 161 | std::vector dsNames; 162 | for ( const std::string &t : inputFiles ) 163 | { 164 | dsNames.push_back(t.c_str()); 165 | } 166 | 167 | // https://gdal.org/api/gdal_utils.html 168 | GDALDatasetH ds = GDALBuildVRT(outputVrtFile.c_str(), (int)dsNames.size(), nullptr, dsNames.data(), nullptr, nullptr); 169 | return ds; 170 | } 171 | 172 | static bool rasterVrtToCog(GDALDatasetH ds, const std::string &outputFile) 173 | { 174 | const char* args[] = { "-of", "COG", "-co", "COMPRESS=DEFLATE", NULL }; 175 | GDALTranslateOptions* psOptions = GDALTranslateOptionsNew((char**)args, NULL); 176 | 177 | GDALDatasetH dsFinal = GDALTranslate(outputFile.c_str(), ds, psOptions, nullptr); 178 | GDALTranslateOptionsFree(psOptions); 179 | if (!dsFinal) 180 | return false; 181 | GDALClose(dsFinal); 182 | return true; 183 | } 184 | 185 | bool rasterTilesToCog(const std::vector &inputFiles, const std::string &outputFile) 186 | { 187 | std::string outputVrt = outputFile; 188 | assert(ends_with(outputVrt, ".tif")); 189 | outputVrt.erase(outputVrt.rfind(".tif"), 4); 190 | outputVrt += ".vrt"; 191 | 192 | GDALDatasetH ds = rasterTilesToVrt(inputFiles, outputVrt); 193 | 194 | if (!ds) 195 | return false; 196 | 197 | rasterVrtToCog(ds, outputFile); 198 | GDALClose(ds); 199 | 200 | std::filesystem::remove(outputVrt); 201 | 202 | return true; 203 | } 204 | 205 | bool readerSupportsBounds(Stage &reader) 206 | { 207 | // these readers support "bounds" option with a 2D/3D bounding box, and based 208 | // on it, they will do very efficient reading of data and only return what's 209 | // in the given bounding box 210 | return reader.getName() == "readers.copc" || reader.getName() == "readers.ept"; 211 | } 212 | 213 | bool allReadersSupportBounds(const std::vector &readers) 214 | { 215 | for (Stage *r : readers) 216 | { 217 | if (!readerSupportsBounds(*r)) 218 | return false; 219 | } 220 | return true; 221 | } 222 | 223 | pdal::Bounds parseBounds(const std::string &boundsStr) 224 | { 225 | // if the input string is not a correct 2D/3D PDAL bounds then parse() 226 | // will throw an exception 227 | pdal::Bounds b; 228 | std::string::size_type pos(0); 229 | b.parse(boundsStr, pos); 230 | return b; 231 | } 232 | 233 | BOX2D intersectionBox2D(const BOX2D &b1, const BOX2D &b2) 234 | { 235 | BOX2D b; 236 | b.minx = b1.minx > b2.minx ? b1.minx : b2.minx; 237 | b.miny = b1.miny > b2.miny ? b1.miny : b2.miny; 238 | b.maxx = b1.maxx < b2.maxx ? b1.maxx : b2.maxx; 239 | b.maxy = b1.maxy < b2.maxy ? b1.maxy : b2.maxy; 240 | if (b.minx > b.maxx || b.miny > b.maxy) 241 | return BOX2D(); 242 | return b; 243 | } 244 | 245 | 246 | BOX2D intersectTileBoxWithFilterBox(const BOX2D &tileBox, const BOX2D &filterBox) 247 | { 248 | if (tileBox.valid() && filterBox.valid()) 249 | { 250 | return intersectionBox2D(tileBox, filterBox); 251 | } 252 | else if (tileBox.valid()) 253 | { 254 | return tileBox; 255 | } 256 | else if (filterBox.valid()) 257 | { 258 | return filterBox; 259 | } 260 | else 261 | { 262 | return BOX2D(); // invalid box 263 | } 264 | } 265 | -------------------------------------------------------------------------------- /src/utils.hpp: -------------------------------------------------------------------------------- 1 | /***************************************************************************** 2 | * Copyright (c) 2023, Lutra Consulting Ltd. and Hobu, Inc. * 3 | * * 4 | * All rights reserved. * 5 | * * 6 | * This program is free software; you can redistribute it and/or modify * 7 | * it under the terms of the GNU General Public License as published by * 8 | * the Free Software Foundation; either version 3 of the License, or * 9 | * (at your option) any later version. * 10 | * * 11 | ****************************************************************************/ 12 | 13 | #pragma once 14 | 15 | #include 16 | #include 17 | 18 | using namespace pdal; 19 | 20 | // tiling scheme containing tileCountX x tileCountY square tiles of tileSize x tileSize, 21 | // with lower-left corner of the tiling being at [tileStartX,tileStartY] 22 | struct Tiling 23 | { 24 | int tileCountX; 25 | int tileCountY; 26 | double tileStartX; 27 | double tileStartY; 28 | double tileSize; 29 | 30 | BOX2D fullBox() const 31 | { 32 | return BOX2D(tileStartX, 33 | tileStartY, 34 | tileStartX + tileSize * tileCountX, 35 | tileStartY + tileSize * tileCountY); 36 | } 37 | 38 | BOX2D boxAt(int ix, int iy) const 39 | { 40 | return BOX2D(tileStartX + tileSize*ix, 41 | tileStartY + tileSize*iy, 42 | tileStartX + tileSize*(ix+1), 43 | tileStartY + tileSize*(iy+1)); 44 | } 45 | }; 46 | 47 | // specification that square tiles of tileSize x tileSize should be aligned: 48 | // so that all corners have coordinates [originX + N*tileSize, originY + M*tileSize] 49 | // where N,M are some integer values 50 | struct TileAlignment 51 | { 52 | double originX; 53 | double originY; 54 | double tileSize; 55 | 56 | // returns tiling that fully covers given bounding box, using this tile alignment 57 | Tiling coverBounds(const BOX2D &box) const 58 | { 59 | Tiling t; 60 | t.tileSize = tileSize; 61 | double offsetX = fmod(originX, tileSize); 62 | double offsetY = fmod(originY, tileSize); 63 | t.tileStartX = floor((box.minx - offsetX)/tileSize)*tileSize + offsetX; 64 | t.tileStartY = floor((box.miny - offsetY)/tileSize)*tileSize + offsetY; 65 | t.tileCountX = ceil((box.maxx - t.tileStartX)/tileSize); 66 | t.tileCountY = ceil((box.maxy - t.tileStartY)/tileSize); 67 | return t; 68 | } 69 | }; 70 | 71 | struct ParallelJobInfo 72 | { 73 | enum ParallelMode { 74 | Single, //!< no parallelism 75 | FileBased, //!< each input file processed separately 76 | Spatial, //!< using tiles - "box" should be used 77 | } mode; 78 | 79 | ParallelJobInfo(ParallelMode m = Single): mode(m) {} 80 | ParallelJobInfo(ParallelMode m, const BOX2D &b, const std::string fe, const std::string fb) 81 | : mode(m), box(b), filterExpression(fe), filterBounds(fb) {} 82 | 83 | // what input point cloud files to read for a job 84 | std::vector inputFilenames; 85 | 86 | // what is the output file name of this job 87 | std::string outputFilename; 88 | 89 | // bounding box for this job (for input/output) 90 | BOX2D box; 91 | 92 | // PDAL filter expression to apply on all pipelines 93 | std::string filterExpression; 94 | 95 | // PDAL filter on 2D or 3D bounds to apply on all pipelines 96 | // Format is "([xmin, xmax], [ymin, ymax])" or "([xmin, xmax], [ymin, ymax], [zmin, zmax])" 97 | std::string filterBounds; 98 | 99 | // modes of operation: 100 | // A. multi input without box (LAS/LAZ) -- per file strategy 101 | // - all input files are processed, no filtering on bounding box 102 | // B. multi input with box (anything) -- tile strategy 103 | // - all input files are processed, but with filtering applied 104 | // - COPC: filtering inside readers.copc with "bounds" argument 105 | // - LAS/LAZ: filter either using CropFilter after reader -or- "where" 106 | 107 | // streaming algs: 108 | // - multi-las: if not overlapping: mode A 109 | // if overlapping: mode A - with a warning it is inefficient? 110 | // - multi-copc: mode B 111 | // - single-copc: mode B or just single pipeline 112 | }; 113 | 114 | 115 | 116 | #include 117 | #include 118 | 119 | // few CRS-related functions that cover in addition to what pdal::SpatialReference doess not provide 120 | struct CRS 121 | { 122 | public: 123 | // construct CRS using a well-known text definition (WKT) 124 | CRS(std::string s = "") 125 | { 126 | ptr.reset( OGRSpatialReference::FromHandle(OSRNewSpatialReference(s.size() ? s.c_str() : nullptr)) ); 127 | } 128 | 129 | std::string name() { return ptr ? ptr->GetName() : ""; } 130 | 131 | // workaround for https://github.com/PDAL/PDAL/issues/3943 132 | std::string identifyEPSG() 133 | { 134 | if (!ptr) 135 | return ""; 136 | 137 | if (const char* c = ptr->GetAuthorityCode(nullptr)) 138 | return std::string(c); 139 | 140 | if (ptr->AutoIdentifyEPSG() == OGRERR_NONE) 141 | { 142 | if (const char* c = ptr->GetAuthorityCode(nullptr)) 143 | return std::string(c); 144 | } 145 | 146 | return ""; 147 | } 148 | 149 | // workaround for https://github.com/PDAL/PDAL/issues/3946 150 | std::string units() 151 | { 152 | if (!ptr) 153 | return std::string(); 154 | 155 | const char* units(nullptr); 156 | 157 | // The returned value remains internal to the OGRSpatialReference 158 | // and should not be freed, or modified. It may be invalidated on 159 | // the next OGRSpatialReference call. 160 | (void)ptr->GetLinearUnits(&units); 161 | std::string tmp(units); 162 | Utils::trim(tmp); 163 | return tmp; 164 | } 165 | 166 | private: 167 | struct OGRDeleter 168 | { 169 | void operator()(OGRSpatialReference* o) 170 | { 171 | OSRDestroySpatialReference(OGRSpatialReference::ToHandle(o)); 172 | }; 173 | }; 174 | 175 | std::unique_ptr ptr; 176 | }; 177 | 178 | 179 | // GDAL-style progress bar: 180 | // 0...10...20...30...40...50...60...70...80...90...100 - done. 181 | struct ProgressBar 182 | { 183 | private: 184 | uint64_t total = 0; 185 | uint64_t current = 0; 186 | int last_percent = -1; // -1 = not started, 0-50 means 0-100 percent 187 | std::mutex mutex; 188 | 189 | public: 190 | void init(uint64_t tot) 191 | { 192 | total = tot; 193 | current = 0; 194 | last_percent = -1; 195 | add(0); 196 | } 197 | 198 | void add(uint64_t count = 1) 199 | { 200 | mutex.lock(); 201 | current += count; 202 | 203 | int new_percent = (int)std::round((std::min)(1.0,((double)current/(double)total))*100)/2; 204 | while (new_percent > last_percent) 205 | { 206 | ++last_percent; 207 | if (last_percent % 5 == 0) 208 | std::cout << last_percent*2 << std::flush; 209 | else 210 | std::cout << "." << std::flush; 211 | } 212 | mutex.unlock(); 213 | } 214 | 215 | void done() 216 | { 217 | std::cout << " - done." << std::endl; 218 | } 219 | }; 220 | 221 | 222 | QuickInfo getQuickInfo(std::string inputFile); 223 | 224 | MetadataNode getReaderMetadata(std::string inputFile, MetadataNode *pointLayoutMeta = nullptr); 225 | 226 | void runPipelineParallel(point_count_t totalPoints, bool isStreaming, std::vector>& pipelines, int max_threads, bool verbose); 227 | 228 | std::string box_to_pdal_bounds(const BOX2D &box); 229 | 230 | pdal::Bounds parseBounds(const std::string &boundsStr); 231 | 232 | bool readerSupportsBounds(Stage &reader); 233 | 234 | bool allReadersSupportBounds(const std::vector &readers); 235 | 236 | BOX2D intersectionBox2D(const BOX2D &b1, const BOX2D &b2); 237 | 238 | BOX2D intersectTileBoxWithFilterBox(const BOX2D &tileBox, const BOX2D &filterBox); 239 | 240 | 241 | inline bool ends_with(std::string const & value, std::string const & ending) 242 | { 243 | if (ending.size() > value.size()) return false; 244 | return std::equal(ending.rbegin(), ending.rend(), value.rbegin()); 245 | } 246 | 247 | 248 | inline std::string join_strings(const std::vector& list, char delimiter) 249 | { 250 | std::string output; 251 | for (std::vector::const_iterator it = list.begin(); it != list.end(); ++it) 252 | { 253 | output += *it; 254 | if (it != list.end() - 1) 255 | output += delimiter; 256 | } 257 | return output; 258 | } 259 | 260 | 261 | bool rasterTilesToCog(const std::vector &inputFiles, const std::string &outputFile); 262 | -------------------------------------------------------------------------------- /src/vpc.hpp: -------------------------------------------------------------------------------- 1 | /***************************************************************************** 2 | * Copyright (c) 2023, Lutra Consulting Ltd. and Hobu, Inc. * 3 | * * 4 | * All rights reserved. * 5 | * * 6 | * This program is free software; you can redistribute it and/or modify * 7 | * it under the terms of the GNU General Public License as published by * 8 | * the Free Software Foundation; either version 3 of the License, or * 9 | * (at your option) any later version. * 10 | * * 11 | ****************************************************************************/ 12 | 13 | #pragma once 14 | 15 | #include 16 | #include 17 | 18 | #include 19 | #include 20 | #include 21 | 22 | using namespace pdal; 23 | 24 | 25 | void buildVpc(std::vector args); 26 | 27 | 28 | struct VirtualPointCloud 29 | { 30 | //! Schema for a single attribute 31 | struct SchemaItem 32 | { 33 | SchemaItem(const std::string &n, const std::string &t, int s): name(n), type(t), size(s) {} 34 | 35 | std::string name; 36 | std::string type; 37 | int size; 38 | }; 39 | 40 | //! Stats for a single attribute 41 | struct StatsItem 42 | { 43 | StatsItem(const std::string &n, uint32_t p, double a, point_count_t c, double max, double min, double st, double vr) 44 | : name(n), position(p), average(a), count(c), maximum(max), minimum(min), stddev(st), variance(vr) {} 45 | 46 | std::string name; 47 | uint32_t position; 48 | double average; 49 | point_count_t count; 50 | double maximum; 51 | double minimum; 52 | double stddev; 53 | double variance; 54 | }; 55 | 56 | struct File 57 | { 58 | std::string filename; 59 | point_count_t count; 60 | std::string boundaryWkt; // not pdal::Geometry because of https://github.com/PDAL/PDAL/issues/4016 61 | BOX3D bbox; 62 | std::string crsWkt; 63 | std::string datetime; // RFC 3339 encoded date/time - e.g. 2023-01-01T12:00:00Z 64 | std::vector schema; // we're not using it, just for STAC export 65 | std::vector stats; 66 | 67 | // support for overview point clouds - currently we assume a file refers to at most a single overview file 68 | // (when building VPC with overviews, we create one overview file for all source data) 69 | std::string overviewFilename; 70 | }; 71 | 72 | std::vector files; 73 | std::string crsWkt; // valid WKT for CRS of all files (or empty string if undefined, or "_mix_" if a mixture of CRS was seen) 74 | 75 | void clear(); 76 | void dump(); 77 | bool read(std::string filename); 78 | bool write(std::string filename); 79 | 80 | point_count_t totalPoints() const; 81 | BOX3D box3d() const; 82 | 83 | //! returns files that have bounding box overlapping the given bounding box 84 | std::vector overlappingBox2D(const BOX2D &box); 85 | }; 86 | -------------------------------------------------------------------------------- /tests/conftest.py: -------------------------------------------------------------------------------- 1 | import subprocess 2 | import typing 3 | 4 | import pdal 5 | import pytest 6 | import requests 7 | import utils 8 | 9 | 10 | @pytest.fixture(autouse=True, scope="session") 11 | def _prepare_data(): 12 | 13 | test_data_folder = utils.test_data_folder() 14 | 15 | if not test_data_folder.exists(): 16 | test_data_folder.mkdir(parents=True) 17 | 18 | base_data = utils.test_data_filepath("stadium-utm.laz") 19 | 20 | if not base_data.exists(): 21 | # PDAL autzen stadium dataset 22 | url = "https://media.githubusercontent.com/media/PDAL/data/refs/heads/main/autzen/stadium-utm.laz" 23 | 24 | r = requests.get(url, timeout=10 * 60) 25 | 26 | with open(base_data, "wb") as f: 27 | f.write(r.content) 28 | # Run the pdal_wrench command 29 | 30 | laz_file = pdal.Reader(base_data.as_posix()).pipeline() 31 | number_points = laz_file.execute() 32 | 33 | assert number_points == 693895 34 | 35 | files_for_vpc = [] 36 | for i in range(1, 5): 37 | 38 | clip_gpkg_file = utils.test_data_filepath(f"rectangle{i}.gpkg") 39 | clipped_laz_file = utils.test_data_filepath(f"data_clipped{i}.laz") 40 | 41 | files_for_vpc.append(clipped_laz_file) 42 | 43 | if not clipped_laz_file.exists(): 44 | 45 | input_file = base_data 46 | 47 | res = subprocess.run( 48 | [ 49 | utils.pdal_wrench_path(), 50 | "clip", 51 | "--polygon", 52 | str(clip_gpkg_file), 53 | "--output", 54 | str(clipped_laz_file), 55 | "--input", 56 | str(input_file), 57 | ], 58 | check=True, 59 | ) 60 | 61 | assert res.returncode == 0 62 | 63 | clipped_laz = pdal.Reader(clipped_laz_file.as_posix()).pipeline() 64 | number_points = clipped_laz.execute() 65 | 66 | assert number_points > 0 67 | 68 | assert clipped_laz_file.exists() 69 | 70 | vpc_file = utils.test_data_filepath("data.vpc") 71 | 72 | if not vpc_file.exists(): 73 | res = subprocess.run( 74 | [ 75 | utils.pdal_wrench_path(), 76 | "build_vpc", 77 | "--output", 78 | vpc_file.as_posix(), 79 | *[f.as_posix() for f in files_for_vpc], 80 | ], 81 | check=True, 82 | ) 83 | 84 | assert res.returncode == 0 85 | 86 | assert vpc_file.exists() 87 | 88 | vpc = pdal.Reader(vpc_file.as_posix()).pipeline() 89 | number_points = vpc.execute() 90 | 91 | assert number_points == 338163 92 | 93 | base_copc_data = utils.test_data_filepath("stadium-utm.copc.laz") 94 | 95 | if not base_copc_data.exists(): 96 | res = subprocess.run( 97 | [ 98 | utils.pdal_wrench_path(), 99 | "translate", 100 | f"--input={base_data.as_posix()}", 101 | f"--output={base_copc_data.as_posix()}", 102 | ], 103 | check=True, 104 | ) 105 | 106 | assert res.returncode == 0 107 | 108 | assert base_copc_data.exists() 109 | 110 | vpc = pdal.Reader.copc(base_copc_data.as_posix()).pipeline() 111 | number_points = vpc.execute() 112 | 113 | assert number_points == 693895 114 | 115 | 116 | @pytest.fixture 117 | def laz_files() -> typing.List[str]: 118 | """Return a list of laz files""" 119 | files = [] 120 | for i in range(1, 5): 121 | files.append(utils.test_data_filepath(f"data_clipped{i}.laz").as_posix()) 122 | return files 123 | 124 | 125 | @pytest.fixture 126 | def main_laz_file() -> str: 127 | "Return path to the main laz file" 128 | return utils.test_data_filepath("stadium-utm.laz").as_posix() 129 | 130 | 131 | @pytest.fixture 132 | def vpc_file() -> str: 133 | "Return path to the vpc file" 134 | return utils.test_data_filepath("data.vpc").as_posix() 135 | 136 | 137 | @pytest.fixture 138 | def main_copc_file() -> str: 139 | "Return path to the main copc file" 140 | return utils.test_data_filepath("stadium-utm.copc.laz").as_posix() 141 | -------------------------------------------------------------------------------- /tests/data/rectangle.gpkg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PDAL/wrench/7214679e5b9537bc49b2fc2343ae778b5adb66de/tests/data/rectangle.gpkg -------------------------------------------------------------------------------- /tests/data/rectangle1.gpkg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PDAL/wrench/7214679e5b9537bc49b2fc2343ae778b5adb66de/tests/data/rectangle1.gpkg -------------------------------------------------------------------------------- /tests/data/rectangle2.gpkg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PDAL/wrench/7214679e5b9537bc49b2fc2343ae778b5adb66de/tests/data/rectangle2.gpkg -------------------------------------------------------------------------------- /tests/data/rectangle3.gpkg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PDAL/wrench/7214679e5b9537bc49b2fc2343ae778b5adb66de/tests/data/rectangle3.gpkg -------------------------------------------------------------------------------- /tests/data/rectangle4.gpkg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PDAL/wrench/7214679e5b9537bc49b2fc2343ae778b5adb66de/tests/data/rectangle4.gpkg -------------------------------------------------------------------------------- /tests/test_boundary.py: -------------------------------------------------------------------------------- 1 | import subprocess 2 | 3 | import utils 4 | 5 | 6 | def test_boundary_laz(main_laz_file: str): 7 | """Test boundary on las function""" 8 | 9 | output = utils.test_data_filepath("boundary-laz.gpkg") 10 | 11 | res = subprocess.run( 12 | [ 13 | utils.pdal_wrench_path(), 14 | "boundary", 15 | f"--output={output.as_posix()}", 16 | f"--input={main_laz_file}", 17 | ], 18 | check=True, 19 | ) 20 | 21 | assert res.returncode == 0 22 | 23 | assert output.exists() 24 | 25 | 26 | def test_boundary_copc(main_copc_file: str): 27 | """Test boundary on copc function""" 28 | 29 | output = utils.test_data_filepath("boundary_copc.gpkg") 30 | 31 | res = subprocess.run( 32 | [ 33 | utils.pdal_wrench_path(), 34 | "boundary", 35 | f"--output={output.as_posix()}", 36 | f"--input={main_copc_file}", 37 | ], 38 | check=True, 39 | ) 40 | 41 | assert res.returncode == 0 42 | 43 | assert output.exists() 44 | 45 | 46 | def test_boundary_on_vpc(vpc_file: str): 47 | """Test boundary on vpc function""" 48 | 49 | output = utils.test_data_filepath("boundary-vpc.gpkg") 50 | 51 | res = subprocess.run( 52 | [ 53 | utils.pdal_wrench_path(), 54 | "boundary", 55 | f"--output={output.as_posix()}", 56 | f"--input={vpc_file}", 57 | ], 58 | check=True, 59 | ) 60 | 61 | assert res.returncode == 0 62 | 63 | assert output.exists() 64 | -------------------------------------------------------------------------------- /tests/test_clip.py: -------------------------------------------------------------------------------- 1 | import subprocess 2 | 3 | import numpy as np 4 | import pdal 5 | import utils 6 | 7 | 8 | def test_clip_laz_to_las(main_laz_file: str): 9 | """Test clip las function""" 10 | 11 | clipped_las_file = utils.test_data_filepath("clipped.las") 12 | 13 | res = subprocess.run( 14 | [ 15 | utils.pdal_wrench_path(), 16 | "clip", 17 | f"--output={clipped_las_file.as_posix()}", 18 | f"--input={main_laz_file}", 19 | f"--polygon={utils.test_data_filepath('rectangle1.gpkg')}", 20 | ], 21 | check=True, 22 | ) 23 | 24 | assert res.returncode == 0 25 | 26 | assert clipped_las_file.exists() 27 | 28 | pipeline = pdal.Reader.las(filename=clipped_las_file.as_posix()).pipeline() 29 | 30 | number_of_points = pipeline.execute() 31 | 32 | assert number_of_points == 66832 33 | 34 | # points as numpy array 35 | array = pipeline.arrays[0] 36 | 37 | # all points 38 | assert isinstance(array, np.ndarray) 39 | 40 | # first point 41 | assert isinstance(array[0], np.void) 42 | assert len(array[0]) == 20 43 | assert isinstance(array[0]["X"], np.float64) 44 | assert isinstance(array[0]["Y"], np.float64) 45 | assert isinstance(array[0]["Z"], np.float64) 46 | assert isinstance(array[0]["Intensity"], np.uint16) 47 | 48 | 49 | def test_clip_laz_to_copc(main_laz_file: str): 50 | """Test clip las function""" 51 | 52 | clipped_las_file = utils.test_data_filepath("clipped.copc.laz") 53 | 54 | res = subprocess.run( 55 | [ 56 | utils.pdal_wrench_path(), 57 | "clip", 58 | f"--output={clipped_las_file.as_posix()}", 59 | f"--input={main_laz_file}", 60 | f"--polygon={utils.test_data_filepath('rectangle1.gpkg')}", 61 | ], 62 | check=True, 63 | ) 64 | 65 | assert res.returncode == 0 66 | 67 | assert clipped_las_file.exists() 68 | 69 | pipeline = pdal.Reader.las(filename=clipped_las_file.as_posix()).pipeline() 70 | 71 | number_of_points = pipeline.execute() 72 | 73 | assert number_of_points == 66832 74 | 75 | 76 | def test_clip_vpc_to_copc(vpc_file: str): 77 | """Test clip vpc to copc function""" 78 | 79 | clipped_file = utils.test_data_filepath("clipped-vpc.copc.laz") 80 | 81 | res = subprocess.run( 82 | [ 83 | utils.pdal_wrench_path(), 84 | "clip", 85 | f"--output={clipped_file.as_posix()}", 86 | f"--input={vpc_file}", 87 | f"--polygon={utils.test_data_filepath('rectangle.gpkg')}", 88 | ], 89 | check=True, 90 | ) 91 | 92 | assert res.returncode == 0 93 | 94 | assert clipped_file.exists() 95 | 96 | pipeline = pdal.Reader.copc(filename=clipped_file.as_posix()).pipeline() 97 | 98 | number_of_points = pipeline.execute() 99 | 100 | assert number_of_points == 19983 101 | 102 | 103 | def test_clip_copc_to_laz(main_copc_file: str): 104 | """Test clip las function""" 105 | 106 | clipped_laz_file = utils.test_data_filepath("clipped-copc-input.laz") 107 | 108 | res = subprocess.run( 109 | [ 110 | utils.pdal_wrench_path(), 111 | "clip", 112 | f"--output={clipped_laz_file.as_posix()}", 113 | f"--input={main_copc_file}", 114 | f"--polygon={utils.test_data_filepath('rectangle1.gpkg')}", 115 | ], 116 | check=True, 117 | ) 118 | 119 | assert res.returncode == 0 120 | 121 | assert clipped_laz_file.exists() 122 | 123 | pipeline = pdal.Reader(filename=clipped_laz_file.as_posix()).pipeline() 124 | 125 | number_of_points = pipeline.execute() 126 | 127 | assert number_of_points == 66832 128 | -------------------------------------------------------------------------------- /tests/test_merge.py: -------------------------------------------------------------------------------- 1 | import subprocess 2 | import typing 3 | 4 | import numpy as np 5 | import pdal 6 | import utils 7 | 8 | 9 | def test_merge_las(laz_files: typing.List[str]): 10 | """Test merge las function""" 11 | 12 | merged_las_file = utils.test_data_filepath("data_merged.las") 13 | 14 | res = subprocess.run( 15 | [ 16 | utils.pdal_wrench_path(), 17 | "merge", 18 | f"--output={merged_las_file.as_posix()}", 19 | *laz_files, 20 | ], 21 | check=True, 22 | ) 23 | 24 | assert res.returncode == 0 25 | 26 | pipeline = pdal.Reader.las(filename=merged_las_file.as_posix()).pipeline() 27 | 28 | number_of_points = pipeline.execute() 29 | 30 | assert number_of_points == 338163 31 | 32 | # points as numpy array 33 | array = pipeline.arrays[0] 34 | 35 | # all points 36 | assert isinstance(array, np.ndarray) 37 | 38 | # first point 39 | assert isinstance(array[0], np.void) 40 | assert len(array[0]) == 20 41 | assert isinstance(array[0]["X"], np.float64) 42 | assert isinstance(array[0]["Y"], np.float64) 43 | assert isinstance(array[0]["Z"], np.float64) 44 | assert isinstance(array[0]["Intensity"], np.uint16) 45 | 46 | 47 | def test_merge_vpc(vpc_file: str): 48 | """Test merge of vpc file""" 49 | 50 | merged_las_file = utils.test_data_filepath("data_merged.las") 51 | 52 | res = subprocess.run( 53 | [ 54 | utils.pdal_wrench_path(), 55 | "merge", 56 | f"--output={merged_las_file.as_posix()}", 57 | vpc_file, 58 | ], 59 | check=True, 60 | ) 61 | 62 | assert res.returncode == 0 63 | 64 | pipeline = pdal.Reader.las(filename=merged_las_file.as_posix()).pipeline() 65 | 66 | number_of_points = pipeline.execute() 67 | 68 | assert number_of_points == 338163 69 | 70 | # points as numpy array 71 | array = pipeline.arrays[0] 72 | 73 | # all points 74 | assert isinstance(array, np.ndarray) 75 | 76 | # first point 77 | assert isinstance(array[0], np.void) 78 | assert len(array[0]) == 20 79 | assert isinstance(array[0]["X"], np.float64) 80 | assert isinstance(array[0]["Y"], np.float64) 81 | assert isinstance(array[0]["Z"], np.float64) 82 | assert isinstance(array[0]["Intensity"], np.uint16) 83 | 84 | 85 | def test_merge_to_copc(laz_files: typing.List[str]): 86 | """Test merge to copc function""" 87 | 88 | merged_copc_file = utils.test_data_filepath("data_merged.copc.laz") 89 | 90 | res = subprocess.run( 91 | [ 92 | utils.pdal_wrench_path(), 93 | "merge", 94 | f"--output={merged_copc_file.as_posix()}", 95 | *laz_files, 96 | ], 97 | check=True, 98 | ) 99 | 100 | assert res.returncode == 0 101 | 102 | pipeline = pdal.Reader(filename=merged_copc_file.as_posix()).pipeline() 103 | 104 | number_of_points = pipeline.execute() 105 | 106 | assert number_of_points == 97965 107 | -------------------------------------------------------------------------------- /tests/test_thin.py: -------------------------------------------------------------------------------- 1 | import subprocess 2 | import typing 3 | 4 | import pdal 5 | import utils 6 | 7 | 8 | def test_thin_laz_to_las(main_laz_file: str): 9 | """Test thin to las function""" 10 | 11 | output = utils.test_data_filepath("thin.las") 12 | 13 | res = subprocess.run( 14 | [ 15 | utils.pdal_wrench_path(), 16 | "thin", 17 | "--mode=every-nth", 18 | "--step-every-nth=5", 19 | f"--input={main_laz_file}", 20 | f"--output={output.as_posix()}", 21 | ], 22 | check=True, 23 | ) 24 | 25 | assert res.returncode == 0 26 | 27 | assert output.exists() 28 | 29 | pipeline = pdal.Reader.las(filename=output.as_posix()).pipeline() 30 | 31 | number_of_points = pipeline.execute() 32 | 33 | assert number_of_points == 138779 34 | 35 | 36 | def test_thin_laz_to_copc(main_laz_file: str): 37 | """Test thin to copc function""" 38 | 39 | output = utils.test_data_filepath("thin.copc.laz") 40 | 41 | res = subprocess.run( 42 | [ 43 | utils.pdal_wrench_path(), 44 | "thin", 45 | "--mode=every-nth", 46 | "--step-every-nth=5", 47 | f"--input={main_laz_file}", 48 | f"--output={output.as_posix()}", 49 | ], 50 | check=True, 51 | ) 52 | 53 | assert res.returncode == 0 54 | 55 | assert output.exists() 56 | 57 | pipeline = pdal.Reader.copc(filename=output.as_posix()).pipeline() 58 | 59 | number_of_points = pipeline.execute() 60 | 61 | assert number_of_points == 138779 62 | 63 | 64 | def test_thin_vpc_to_copc(vpc_file: str): 65 | """Test thin vpc to copc function""" 66 | 67 | output = utils.test_data_filepath("thin-vpc.copc.laz") 68 | 69 | res = subprocess.run( 70 | [ 71 | utils.pdal_wrench_path(), 72 | "thin", 73 | "--mode=every-nth", 74 | "--step-every-nth=5", 75 | f"--input={vpc_file}", 76 | f"--output={output.as_posix()}", 77 | ], 78 | check=True, 79 | ) 80 | 81 | assert res.returncode == 0 82 | 83 | assert output.exists() 84 | 85 | pipeline = pdal.Reader.copc(filename=output.as_posix()).pipeline() 86 | 87 | number_of_points = pipeline.execute() 88 | 89 | assert number_of_points == 19593 90 | 91 | 92 | def test_thin_copc_to_laz(main_copc_file: str): 93 | """Test thin copc to laz function""" 94 | 95 | output = utils.test_data_filepath("thin-copc-input.laz") 96 | 97 | res = subprocess.run( 98 | [ 99 | utils.pdal_wrench_path(), 100 | "thin", 101 | "--mode=every-nth", 102 | "--step-every-nth=5", 103 | f"--input={main_copc_file}", 104 | f"--output={output.as_posix()}", 105 | ], 106 | check=True, 107 | ) 108 | 109 | assert res.returncode == 0 110 | 111 | assert output.exists() 112 | 113 | pipeline = pdal.Reader(filename=output.as_posix()).pipeline() 114 | 115 | number_of_points = pipeline.execute() 116 | 117 | assert number_of_points == 138779 118 | -------------------------------------------------------------------------------- /tests/test_to_vector.py: -------------------------------------------------------------------------------- 1 | import subprocess 2 | 3 | import utils 4 | from osgeo import ogr 5 | 6 | ogr.UseExceptions() 7 | 8 | 9 | def test_to_vector_las_file(main_laz_file: str): 10 | """Test to_vector las function""" 11 | 12 | gpkg_file = utils.test_data_filepath("points.gpkg") 13 | if gpkg_file.exists(): 14 | gpkg_file.unlink() 15 | 16 | res = subprocess.run( 17 | [ 18 | utils.pdal_wrench_path(), 19 | "to_vector", 20 | f"--output={gpkg_file.as_posix()}", 21 | f"--input={main_laz_file}", 22 | ], 23 | check=True, 24 | ) 25 | 26 | assert res.returncode == 0 27 | 28 | assert gpkg_file.exists() 29 | 30 | ds: ogr.DataSource = ogr.Open(gpkg_file.as_posix()) 31 | 32 | assert ds 33 | assert ds.GetLayerCount() == 1 34 | 35 | layer: ogr.Layer = ds.GetLayer(0) 36 | 37 | assert layer 38 | assert layer.GetName() == "points" 39 | assert layer.GetFeatureCount() == 693895 40 | 41 | 42 | def test_to_vector_vpc_file(vpc_file: str): 43 | """Test to_vector las function""" 44 | 45 | gpkg_file = utils.test_data_filepath("points.gpkg") 46 | if gpkg_file.exists(): 47 | gpkg_file.unlink() 48 | 49 | res = subprocess.run( 50 | [ 51 | utils.pdal_wrench_path(), 52 | "to_vector", 53 | f"--output={gpkg_file.as_posix()}", 54 | f"--input={vpc_file}", 55 | ], 56 | check=True, 57 | ) 58 | 59 | assert res.returncode == 0 60 | 61 | assert gpkg_file.exists() 62 | 63 | temp_folder = gpkg_file.parent / gpkg_file.stem 64 | 65 | assert not temp_folder.exists() 66 | 67 | ds: ogr.DataSource = ogr.Open(gpkg_file.as_posix()) 68 | 69 | assert ds 70 | assert ds.GetLayerCount() == 1 71 | 72 | layer: ogr.Layer = ds.GetLayer(0) 73 | 74 | assert layer 75 | assert layer.GetName() == "points" 76 | assert layer.GetFeatureCount() == 338163 77 | -------------------------------------------------------------------------------- /tests/test_translate.py: -------------------------------------------------------------------------------- 1 | import subprocess 2 | import typing 3 | 4 | import pdal 5 | import utils 6 | 7 | 8 | def test_translate_laz_to_las(main_laz_file: str): 9 | """Test translate las function""" 10 | 11 | output = utils.test_data_filepath("translate.las") 12 | 13 | res = subprocess.run( 14 | [ 15 | utils.pdal_wrench_path(), 16 | "translate", 17 | f"--input={main_laz_file}", 18 | f"--output={output.as_posix()}", 19 | ], 20 | check=True, 21 | ) 22 | 23 | assert res.returncode == 0 24 | 25 | assert output.exists() 26 | 27 | pipeline = pdal.Reader.las(filename=output.as_posix()).pipeline() 28 | 29 | number_of_points = pipeline.execute() 30 | 31 | assert number_of_points == 693895 32 | 33 | 34 | def test_translate_laz_to_copc(main_laz_file: str): 35 | """Test translate las function""" 36 | 37 | output = utils.test_data_filepath("translate.copc.laz") 38 | 39 | res = subprocess.run( 40 | [ 41 | utils.pdal_wrench_path(), 42 | "translate", 43 | f"--input={main_laz_file}", 44 | f"--output={output.as_posix()}", 45 | ], 46 | check=True, 47 | ) 48 | 49 | assert res.returncode == 0 50 | 51 | assert output.exists() 52 | 53 | pipeline = pdal.Reader.copc(filename=output.as_posix()).pipeline() 54 | 55 | number_of_points = pipeline.execute() 56 | 57 | assert number_of_points == 693895 58 | 59 | 60 | def test_translate_vpc_to_copc(vpc_file: str): 61 | """Test translate vpc to copc function""" 62 | 63 | output = utils.test_data_filepath("translate-vpc.copc.laz") 64 | 65 | res = subprocess.run( 66 | [ 67 | utils.pdal_wrench_path(), 68 | "translate", 69 | f"--input={vpc_file}", 70 | f"--output={output.as_posix()}", 71 | ], 72 | check=True, 73 | ) 74 | 75 | assert res.returncode == 0 76 | 77 | assert output.exists() 78 | 79 | pipeline = pdal.Reader.copc(filename=output.as_posix()).pipeline() 80 | 81 | number_of_points = pipeline.execute() 82 | 83 | assert number_of_points == 97965 84 | 85 | 86 | def test_translate_copc_to_laz(main_copc_file: str): 87 | """Test translate copc to copc function""" 88 | 89 | output = utils.test_data_filepath("translate-copc-input.laz") 90 | 91 | res = subprocess.run( 92 | [ 93 | utils.pdal_wrench_path(), 94 | "translate", 95 | f"--input={main_copc_file}", 96 | f"--output={output.as_posix()}", 97 | ], 98 | check=True, 99 | ) 100 | 101 | assert res.returncode == 0 102 | 103 | assert output.exists() 104 | 105 | pipeline = pdal.Reader(filename=output.as_posix()).pipeline() 106 | 107 | number_of_points = pipeline.execute() 108 | 109 | assert number_of_points == 693895 110 | -------------------------------------------------------------------------------- /tests/utils.py: -------------------------------------------------------------------------------- 1 | import shutil 2 | from pathlib import Path 3 | 4 | 5 | def test_data_folder() -> Path: 6 | """Return data folder for tests""" 7 | return Path(__file__).parent / "data" 8 | 9 | 10 | def test_data_filepath(file_name: str) -> Path: 11 | """Return path to file in data folder""" 12 | return test_data_folder() / file_name 13 | 14 | 15 | def pdal_wrench_path() -> str: 16 | """Return path to pdal_wrench executable""" 17 | executable_name = "pdal_wrench" 18 | 19 | # check system executable exists 20 | if shutil.which(executable_name): 21 | return executable_name 22 | 23 | # try to look in build directory 24 | path_pdal_wrench = Path(__file__).parent.parent / "build" / executable_name 25 | 26 | if not path_pdal_wrench.exists(): 27 | raise FileNotFoundError(path_pdal_wrench) 28 | 29 | return path_pdal_wrench.as_posix() 30 | -------------------------------------------------------------------------------- /vpc-spec.md: -------------------------------------------------------------------------------- 1 | # Virtual Point Cloud (VPC) 2 | 3 | ## Purpose 4 | 5 | Draft specification for a file format that groups multiple point cloud files to be treated as a single dataset. 6 | 7 | Inspired by GDAL's VRTs and PDAL's tindex. 8 | 9 | Goals: 10 | - load VPC in QGIS as a single point cloud layer (rather than each file as a separate map layer) 11 | - run processing tools on a VPC as input 12 | - simple format that's easy to read/write 13 | - allow referencing both local and remote datasets 14 | - allow working with both indexed (e.g. COPC or EPT) and non-indexed (e.g. LAS/LAZ) datasets 15 | 16 | Non-goals: 17 | - support other kinds of data than (georeferenced) point clouds 18 | 19 | ## File format 20 | 21 | We are using [STAC API ItemCollection](https://github.com/radiantearth/stac-api-spec/blob/main/fragments/itemcollection/README.md) as the file format and expecting these STAC extensions: 22 | - [pointcloud](https://github.com/stac-extensions/pointcloud/) 23 | - [projection](https://github.com/stac-extensions/projection/) 24 | 25 | Note: ItemCollection not the same thing as a [STAC Collection](https://github.com/radiantearth/stac-spec/blob/master/collection-spec/README.md). An ItemCollection is essentially a single JSON file representing a GeoJSON FeatureCollection containing STAC Items. A STAC Collection is also a JSON file, 26 | but with a different structure and extra metadata, but more importantly, it only links to other standalone JSON files (STAC items) which 27 | is impractical for our use case as we strongly prefer to have the whole virtual point cloud definition in a single file (for easy manipulation). 28 | 29 | We use `.vpc` extension (to allow easy format recognition based on the extension). 30 | 31 | Normally a virtual point cloud file will not provide any links to other STAC entities (e.g. to a parent, to a root or to self) because often it will be created ad-hoc. 32 | 33 | Why STAC: 34 | - it is a good fit into the larger ecosystem of data catalogs, avoiding creation of a new format 35 | - supported natively by PDAL as well (readers.stac) 36 | - search endpoint on STAC API servers returns the same ItemCollection that we use, so a search result can be fed directly as input 37 | - the ItemCollection file is an ordinary GeoJSON and other clients can consume it (to at least show boundaries of individual files) 38 | 39 | ### Coordinate Reference Systems (CRS) 40 | 41 | Each referenced file can have its own CRS and it is defined through the "projection" STAC extension - either using `proj:epsg` or `proj:wkt2` or `proj:projjson`. It is recommended that a single virtual point cloud only references files in the same CRS. 42 | 43 | Please note that `proj:epsg` only allows a single EPSG code to be specified. When working with a compound CRS that has an EPSG code for both horizontal and vertical CRS (often written as `EPSG:1234+5678` where `EPSG:1234` is horizontal and `EPSG:5678` vertical), but there is no EPSG code for the compound CRS, this can't be encoded in `proj:epsg` and it is preferrable to use an alternative encoding (e.g. `proj:wkt2`). 44 | 45 | ### Statistics 46 | 47 | The STAC pointcloud extension defines optional `pc:statistics` entry with statistics for each attribute. There is a limitation that currently it does not define how to store distinct values where it makes sense and their counts (e.g. Classification attribute) - see https://github.com/stac-extensions/pointcloud/issues/5. 48 | 49 | ### Boundaries 50 | 51 | The format requires that boundaries of referenced files are defined in WGS 84 (since GeoJSON requires that) - either as a simple 2D or 3D bounding box or as a boundary geometry (polygon / multi-polygon). In addition to that, the "projection" STAC extension allows that the bounding box or boundary geometry can be specified in native coordinate reference system - this is strongly recommended and if `proj:bbox` or `proj:geometry` are present, they will be used instead of their WGS 84 equivalents. 52 | 53 | It is also strongly recommended to provide bounding box in 3D rather than just 2D bounding box (3D viewers can correctly place the expected box in the 3D space rather than guessing the vertical range). 54 | 55 | ### Overviews 56 | 57 | Overviews are useful for client software to show preview of the point cloud when zoomed out, without having to open all individual files and only rely on overviews 58 | (the same idea as with overviews of raster layers). 59 | Overviews are an optional feature, they do not need to be provided. 60 | 61 | In a simple case, there would be a single overview point cloud for a whole VPC - using thinned original data (e.g. every 1000-th point) and merged from individual data files to a single file. 62 | 63 | In case of very large point clouds, using a single overview point cloud may not be enough, and it may be useful to have a hierarchy of overviews with a tiling scheme (e.g. 1 overview at level 0, 4 overviews at level 1, 16 overviews at level 2 - each level having different density). 64 | 65 | Overviews are defined as extra assets of STAC items in addition to the actual data file - it is expected that they have `overview` role defined. An overview point cloud may be referenced from multiple STAC items. 66 | 67 | In case of a hierarchy of overviews, each STAC item may have multiple overview assets linked (e.g. one for level 0, one for level 1, one for level 2). STAC protocol does not provide a way to distinguish which overview is at what level (no place to write spacing between points or density), it is up to the client software to collect all referenced overviews and query their properties. 68 | 69 | A sample of encoding overview point cloud in addition to the actual data: 70 | ```json 71 | "assets": { 72 | "data": { 73 | "href": "./mydata_54_1.copc.laz", 74 | "roles": [ 75 | "data" 76 | ] 77 | }, 78 | "overview": { 79 | "href": "./mydata-overview.copc.laz", 80 | "roles": [ 81 | "overview" 82 | ] 83 | } 84 | } 85 | ``` 86 | --------------------------------------------------------------------------------