├── .circleci └── config.yml ├── .gitignore ├── CHANGELOG.md ├── Dockerfile ├── LICENSE.txt ├── README.md ├── VERSION ├── bin └── package.sh ├── build-and-test.sh └── python ├── Dockerfile ├── PYVERSION ├── README.md ├── bin └── package-python.sh ├── lambda └── lambda_function.py ├── requirements-pre.txt └── requirements.txt /.circleci/config.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | 3 | references: 4 | restore_repo: &restore_repo 5 | restore_cache: 6 | keys: 7 | - v1-repo-{{ .Branch }}-{{ .Revision }} 8 | - v1-repo-{{ .Branch }} 9 | - v1-repo 10 | save_repo: &save_repo 11 | save_cache: 12 | key: v1-repo-{{ .Branch }}-{{ .Revision }} 13 | paths: 14 | - ~/project 15 | 16 | jobs: 17 | build_and_deploy: 18 | machine: 19 | docker_layer_caching: true 20 | steps: 21 | - *restore_repo 22 | - checkout 23 | - *save_repo 24 | - run: 25 | name: Install dependencies 26 | command: | 27 | pip install awscli 28 | sudo apt-get install jq 29 | - run: 30 | name: Build application Docker image 31 | command: | 32 | # update known hosts 33 | mkdir -p ~/.ssh 34 | ssh-keyscan github.com >> ~/.ssh/known_hosts 35 | # build and publish geolambda 36 | VERSION=$(cat VERSION) 37 | docker build . -t developmentseed/geolambda:${VERSION} 38 | docker run --rm -v ${PWD}:/home/geolambda --entrypoint package.sh -it developmentseed/geolambda:${VERSION} 39 | docker login -u $DOCKER_USER -p $DOCKER_PASS 40 | docker tag developmentseed/geolambda:${VERSION} developmentseed/geolambda:latest 41 | docker push developmentseed/geolambda:latest 42 | docker push developmentseed/geolambda:${VERSION} 43 | # build and publish geolambda-python docker image 44 | cd python 45 | docker build --build-arg VERSION=${VERSION} . -t developmentseed/geolambda:${VERSION}-python 46 | docker run --rm -v ${PWD}:/home/geolambda --entrypoint package-python.sh -it developmentseed/geolambda:${VERSION}-python 47 | docker push developmentseed/geolambda:${VERSION}-python 48 | cd .. 49 | # deploy public Lambda layers 50 | pip install awscli 51 | for region in us-east-1 us-west-2 eu-central-1 eu-west-2 eu-north-1 52 | do 53 | LVERSION="$(aws lambda publish-layer-version --region ${region} \ 54 | --layer-name geolambda --license-info 'MIT' \ 55 | --description 'Native geospatial libaries for all runtimes' \ 56 | --zip-file fileb://lambda-deploy.zip | jq '.Version')" 57 | aws lambda add-layer-version-permission --region ${region} \ 58 | --layer-name geolambda --action lambda:GetLayerVersion \ 59 | --statement-id public --version-number ${LVERSION} --principal '*' 60 | LVERSION="$(aws lambda publish-layer-version --region ${region} \ 61 | --layer-name geolambda-python --license-info 'MIT' \ 62 | --description 'Geospatial Python libraries' \ 63 | --zip-file fileb://python/lambda-deploy.zip | jq '.Version')" 64 | aws lambda add-layer-version-permission --region ${region} \ 65 | --layer-name geolambda-python --action lambda:GetLayerVersion \ 66 | --statement-id public --version-number ${LVERSION} --principal '*' 67 | done 68 | # deploy Python Lambda layers 69 | 70 | 71 | workflows: 72 | version: 2 73 | build_and_deploy: 74 | jobs: 75 | - build_and_deploy: 76 | filters: 77 | branches: 78 | only: master 79 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .env 2 | lambda-deploy.zip 3 | lambda-layer-deploy.zip 4 | .DS_Store -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | All notable changes to this project will be documented in this file. 3 | 4 | The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) 5 | and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html). 6 | 7 | ## [Unreleased] 8 | 9 | ### Added 10 | - Added deployments to `eu-west-2` and `eu-north-1` 11 | 12 | ### Fixed 13 | - README updates regarding versions 14 | 15 | ### Changed 16 | - Keep symlinks when zipping, resulting in smaller deploy package 17 | - Python Lambda Layers no longer need the base GeoLambda layer in addition to the python layer 18 | - Manually compile sqlite3 used in Proj, GDAL, nghttp2 19 | - GDAL 3.2.1 20 | - Proj 7.2.1 21 | - GEOS 3.8.1 22 | - GeoTIFF 1.5.1 23 | - HDF4 4.2.15 24 | - HDF5 1.10.7 25 | - NetCDF 4.7.4 26 | - Nghttp2 1.41.0 27 | - OpenJPEG 2.4.0 28 | - libJPEG Turbo 2.4.0 29 | - Curl 7.73.0 30 | - Webp 1.1.0 31 | - Zstd 1.4.5 32 | - OpenSSL 1.1.1 33 | - In python layer: 34 | - Python 3.7.4 -> 3.7.9 35 | - rasterio 1.1.0 -> 1.2.0 36 | - shapely 1.6.4.post2 -> 1.7.1 37 | - pyproj 2.4.0 -> 3.0.0.post1 38 | 39 | ### Removed 40 | - GDAL Elasticsearch driver 41 | 42 | 43 | ## [v2.0.0] - 2019-10-25 44 | 45 | ### Added 46 | - Added OpenSSL, which is used to compile versions of Python 3.7+, it is not packaged in GeoLambda Layer 47 | - OPENSSL_VERSION=1.0.2 48 | 49 | ### Changed 50 | - Updated GDAL, PROJ, and Geotiff libraries. 51 | - GDAL_VERSION=3.0.1 52 | - PROJ_VERSION=6.2.0 53 | - GEOTIFF_VERSION=1.5.1 54 | 55 | 56 | ## [v1.2.0] - 2019-10-24 57 | 58 | ### Added 59 | - A Python GeoLambda Layer is now published, along with the base Lambda Layer. The base layer *must* be included in any Lambda that uses the Python GeoLambda layer. It includes the Python libraries GDAL, rasterio, pyproj, and shapely. 60 | - Base geolambda includes OpenSSL 1.0.2, as this is required for compiling Python 3.7+. It is not included in the Lambda layer, just the base layer for ease of creating different versions of Python child images. 61 | 62 | 63 | ### Changed 64 | - The python diectory, and the new Lambda layer, now uses Python 3.7.4 65 | - GDAL_VERSION=2.4.2 66 | - GEOS_VERSION=3.8.0 67 | - NETCDF_VERSION=4.7.1 68 | - NGHTTP2_VERSION=1.39.2 69 | - OPENJPEG_VERSION=2.3.1 70 | - CURL_VERSION=7.66.0 71 | - LIBJPEG_TURBO_VERSION=2.0.3 72 | - WEBP_VERSION=1.0.3 73 | - ZSTD_VERSION=1.4.3 74 | 75 | 76 | ## [v1.1.0] - 2018-03-22 77 | 78 | ### Added 79 | - Make compatible with Lambda Layers 80 | - Python example 81 | - Improve documentation 82 | - More libraries (CURL with http2, webp, zstd, libjpegturbo) 83 | - GeoTIFF now compiled from scratch rather than GGDAL builtin 84 | - Published public lambda layers - see README for ARNs 85 | - GEOTIFF_VERSION=1.4.3 86 | - OPENJPEG_VERSION=2.3.0 87 | - LIBJPEG_TURBO_VERSION=2.0.1 88 | - CURL_VERSION=7.59.0 89 | - NGHTTP2_VERSION=1.35.1 90 | - WEBP_VERSION=1.0.1 91 | - ZSTD_VERSION=1.3.8 92 | 93 | ### Changed 94 | - Major refactor, GeoLambda base now runtime agnostic contains system libraries only 95 | - GDAL_VERSION=2.4.1 96 | - GEOS_VERSION=3.7.1 97 | - HDF4_VERSION=4.2.14 98 | - HDF5_VERSION=1.10.5 99 | - NETCDF_VERSION=4.6.2 100 | 101 | ### Removed 102 | - Removed Python codes to make geolambda system libraries only 103 | 104 | 105 | ## [v1.0.0] - 2018-07-27 106 | 107 | Initial release 108 | 109 | Package Versions 110 | - PROJ_VERSION=5.2.0 111 | - GEOS_VERSION=3.6.2 112 | - HDF4_VERSION=4.2.12 113 | - SZIP_VERSION=2.1.1 114 | - HDF5_VERSION=1.10.1 115 | - NETCDF_VERSION=4.6.1 116 | - OPENJPEG_VERSION=2.3.0 117 | - PKGCONFIG_VERSION=0.29.2 118 | - GDAL_VERSION=2.3.1 119 | 120 | [Unreleased]: https://github.com/sat-utils/sat-stac/compare/master...develop 121 | [v2.1.0]: https://github.com/developmentseed/geolambda/compare/2.0.0...2.1.0 122 | [v2.0.0]: https://github.com/developmentseed/geolambda/compare/1.2.0...2.0.0 123 | [v1.2.0]: https://github.com/developmentseed/geolambda/compare/1.1.0...1.2.0 124 | [v1.1.0]: https://github.com/developmentseed/geolambda/compare/1.0.0...1.1.0 125 | [v1.0.0]: https://github.com/developmentseed/geolambda/tree/1.0.0 126 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM lambci/lambda:build-provided 2 | 3 | LABEL maintainer="Development Seed " 4 | LABEL authors="Matthew Hanson " 5 | 6 | # install system libraries 7 | RUN \ 8 | yum makecache fast; \ 9 | yum install -y wget libpng-devel nasm rsync; \ 10 | yum install -y bash-completion --enablerepo=epel; \ 11 | yum clean all; \ 12 | yum autoremove 13 | 14 | # versions of packages 15 | ENV \ 16 | GDAL_VERSION=3.2.1 \ 17 | PROJ_VERSION=7.2.1 \ 18 | GEOS_VERSION=3.8.1 \ 19 | GEOTIFF_VERSION=1.6.0 \ 20 | HDF4_VERSION=4.2.15 \ 21 | HDF5_VERSION=1.10.7 \ 22 | NETCDF_VERSION=4.7.4 \ 23 | NGHTTP2_VERSION=1.41.0 \ 24 | OPENJPEG_VERSION=2.4.0 \ 25 | LIBJPEG_TURBO_VERSION=2.0.6 \ 26 | CURL_VERSION=7.73.0 \ 27 | PKGCONFIG_VERSION=0.29.2 \ 28 | SZIP_VERSION=2.1.1 \ 29 | WEBP_VERSION=1.1.0 \ 30 | ZSTD_VERSION=1.4.5 \ 31 | OPENSSL_VERSION=1.1.1 32 | 33 | # Paths to things 34 | ENV \ 35 | BUILD=/build \ 36 | NPROC=4 \ 37 | PREFIX=/usr/local \ 38 | GDAL_CONFIG=/usr/local/bin/gdal-config \ 39 | LD_LIBRARY_PATH=/usr/local/lib:/usr/local/lib64 \ 40 | PKG_CONFIG_PATH=/usr/local/lib/pkgconfig:/usr/lib64/pkgconfig \ 41 | GDAL_DATA=/usr/local/share/gdal \ 42 | PROJ_LIB=/usr/local/share/proj 43 | 44 | # switch to a build directory 45 | WORKDIR /build 46 | 47 | # pkg-config - version > 2.5 required for GDAL 2.3+ 48 | RUN \ 49 | mkdir pkg-config; \ 50 | wget -qO- https://pkg-config.freedesktop.org/releases/pkg-config-$PKGCONFIG_VERSION.tar.gz \ 51 | | tar xvz -C pkg-config --strip-components=1; cd pkg-config; \ 52 | ./configure --prefix=$PREFIX CFLAGS="-O2 -Os"; \ 53 | make -j ${NPROC} install; \ 54 | cd ../; rm -rf pkg-config 55 | 56 | # sqlite3 (required by proj) 57 | RUN \ 58 | mkdir sqlite3; \ 59 | wget -qO- https://www.sqlite.org/2020/sqlite-autoconf-3330000.tar.gz \ 60 | | tar xvz -C sqlite3 --strip-components=1; cd sqlite3; \ 61 | ./configure --prefix=$PREFIX; \ 62 | make; make install; \ 63 | cd ../; rm -rf sqlite3; 64 | 65 | # proj 66 | RUN \ 67 | mkdir proj; \ 68 | wget -qO- http://download.osgeo.org/proj/proj-$PROJ_VERSION.tar.gz \ 69 | | tar xvz -C proj --strip-components=1; cd proj; \ 70 | SQLITE3_LIBS="=L$PREFIX/lib -lsqlite3" SQLITE3_INCLUDE_DIRS=$PREFIX/include/proj ./configure --prefix=$PREFIX; \ 71 | make -j ${NPROC} install; \ 72 | cd ..; rm -rf proj 73 | 74 | 75 | # nghttp2 76 | RUN \ 77 | mkdir nghttp2; \ 78 | wget -qO- https://github.com/nghttp2/nghttp2/releases/download/v${NGHTTP2_VERSION}/nghttp2-${NGHTTP2_VERSION}.tar.gz \ 79 | | tar xvz -C nghttp2 --strip-components=1; cd nghttp2; \ 80 | ./configure --enable-lib-only --prefix=${PREFIX}; \ 81 | make -j ${NPROC} install; \ 82 | cd ..; rm -rf nghttp2 83 | 84 | # curl 85 | RUN \ 86 | mkdir curl; \ 87 | wget -qO- https://curl.haxx.se/download/curl-${CURL_VERSION}.tar.gz \ 88 | | tar xvz -C curl --strip-components=1; cd curl; \ 89 | ./configure --prefix=${PREFIX} --disable-manual --disable-cookies --with-nghttp2=${PREFIX}; \ 90 | make -j ${NPROC} install; \ 91 | cd ..; rm -rf curl 92 | 93 | # GEOS 94 | RUN \ 95 | mkdir geos; \ 96 | wget -qO- http://download.osgeo.org/geos/geos-$GEOS_VERSION.tar.bz2 \ 97 | | tar xvj -C geos --strip-components=1; cd geos; \ 98 | ./configure --enable-python --prefix=$PREFIX CFLAGS="-O2 -Os"; \ 99 | make -j ${NPROC} install; \ 100 | cd ..; rm -rf geos 101 | 102 | # szip (for hdf) 103 | RUN \ 104 | mkdir szip; \ 105 | wget -qO- https://support.hdfgroup.org/ftp/lib-external/szip/$SZIP_VERSION/src/szip-$SZIP_VERSION.tar.gz \ 106 | | tar xvz -C szip --strip-components=1; cd szip; \ 107 | ./configure --prefix=$PREFIX; \ 108 | make -j ${NPROC} install; \ 109 | cd ..; rm -rf szip 110 | 111 | # libhdf4 112 | RUN \ 113 | mkdir hdf4; \ 114 | wget -qO- https://support.hdfgroup.org/ftp/HDF/releases/HDF$HDF4_VERSION/src/hdf-$HDF4_VERSION.tar \ 115 | | tar xv -C hdf4 --strip-components=1; cd hdf4; \ 116 | ./configure \ 117 | --prefix=$PREFIX \ 118 | --with-szlib=$PREFIX \ 119 | --enable-shared \ 120 | --disable-netcdf \ 121 | --disable-fortran; \ 122 | make -j ${NPROC} install; \ 123 | cd ..; rm -rf hdf4 124 | 125 | # libhdf5 126 | RUN \ 127 | mkdir hdf5; \ 128 | wget -qO- https://support.hdfgroup.org/ftp/HDF5/releases/hdf5-${HDF5_VERSION%.*}/hdf5-${HDF5_VERSION}/src/hdf5-$HDF5_VERSION.tar.gz \ 129 | | tar xvz -C hdf5 --strip-components=1; cd hdf5; \ 130 | ./configure \ 131 | --prefix=$PREFIX \ 132 | --with-szlib=$PREFIX; \ 133 | make -j ${NPROC} install; \ 134 | cd ..; rm -rf hdf5 135 | 136 | # NetCDF 137 | RUN \ 138 | mkdir netcdf; \ 139 | wget -qO- https://github.com/Unidata/netcdf-c/archive/v$NETCDF_VERSION.tar.gz \ 140 | | tar xvz -C netcdf --strip-components=1; cd netcdf; \ 141 | ./configure --prefix=$PREFIX --enable-hdf4; \ 142 | make -j ${NPROC} install; \ 143 | cd ..; rm -rf netcdf 144 | 145 | # WEBP 146 | RUN \ 147 | mkdir webp; \ 148 | wget -qO- https://storage.googleapis.com/downloads.webmproject.org/releases/webp/libwebp-${WEBP_VERSION}.tar.gz \ 149 | | tar xvz -C webp --strip-components=1; cd webp; \ 150 | CFLAGS="-O2 -Wl,-S" PKG_CONFIG_PATH="/usr/lib64/pkgconfig" ./configure --prefix=$PREFIX; \ 151 | make -j ${NPROC} install; \ 152 | cd ..; rm -rf webp 153 | 154 | # ZSTD 155 | RUN \ 156 | mkdir zstd; \ 157 | wget -qO- https://github.com/facebook/zstd/archive/v${ZSTD_VERSION}.tar.gz \ 158 | | tar -xvz -C zstd --strip-components=1; cd zstd; \ 159 | make -j ${NPROC} install PREFIX=$PREFIX ZSTD_LEGACY_SUPPORT=0 CFLAGS=-O1 --silent; \ 160 | cd ..; rm -rf zstd 161 | 162 | # openjpeg 163 | RUN \ 164 | mkdir openjpeg; \ 165 | wget -qO- https://github.com/uclouvain/openjpeg/archive/v$OPENJPEG_VERSION.tar.gz \ 166 | | tar xvz -C openjpeg --strip-components=1; cd openjpeg; mkdir build; cd build; \ 167 | cmake .. -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=$PREFIX; \ 168 | make -j ${NPROC} install; \ 169 | cd ../..; rm -rf openjpeg 170 | 171 | # jpeg_turbo 172 | RUN \ 173 | mkdir jpeg; \ 174 | wget -qO- https://github.com/libjpeg-turbo/libjpeg-turbo/archive/${LIBJPEG_TURBO_VERSION}.tar.gz \ 175 | | tar xvz -C jpeg --strip-components=1; cd jpeg; \ 176 | cmake -G"Unix Makefiles" -DCMAKE_INSTALL_PREFIX=$PREFIX .; \ 177 | make -j $(nproc) install; \ 178 | cd ..; rm -rf jpeg 179 | 180 | # geotiff 181 | RUN \ 182 | mkdir geotiff; \ 183 | wget -qO- https://download.osgeo.org/geotiff/libgeotiff/libgeotiff-$GEOTIFF_VERSION.tar.gz \ 184 | | tar xvz -C geotiff --strip-components=1; cd geotiff; \ 185 | ./configure --prefix=${PREFIX} \ 186 | --with-proj=${PREFIX} --with-jpeg=${PREFIX} --with-zip=yes;\ 187 | make -j ${NPROC} install; \ 188 | cd ${BUILD}; rm -rf geotiff 189 | 190 | # GDAL 191 | RUN \ 192 | mkdir gdal; \ 193 | wget -qO- http://download.osgeo.org/gdal/$GDAL_VERSION/gdal-$GDAL_VERSION.tar.gz \ 194 | | tar xvz -C gdal --strip-components=1; cd gdal; \ 195 | ./configure \ 196 | --disable-debug \ 197 | --disable-static \ 198 | --prefix=${PREFIX} \ 199 | --with-openjpeg \ 200 | --with-geotiff=${PREFIX} \ 201 | --with-hdf4=${PREFIX} \ 202 | --with-hdf5=${PREFIX} \ 203 | --with-netcdf=${PREFIX} \ 204 | --with-webp=${PREFIX} \ 205 | --with-zstd=${PREFIX} \ 206 | --with-jpeg=${PREFIX} \ 207 | --with-threads=yes \ 208 | --with-sqlite3=$PREFIX \ 209 | --with-curl=${PREFIX}/bin/curl-config \ 210 | --without-python \ 211 | --without-libtool \ 212 | --disable-driver-elastic \ 213 | --with-geos=$PREFIX/bin/geos-config \ 214 | --with-hide-internal-symbols=yes \ 215 | CFLAGS="-O2 -Os" CXXFLAGS="-O2 -Os" \ 216 | LDFLAGS="-Wl,-rpath,'\$\$ORIGIN'"; \ 217 | make -j ${NPROC} install; \ 218 | cd ${BUILD}; rm -rf gdal 219 | 220 | # Open SSL is needed for building Python so it's included here for ease 221 | RUN \ 222 | mkdir openssl; \ 223 | wget -qO- https://www.openssl.org/source/openssl-${OPENSSL_VERSION}.tar.gz \ 224 | | tar xvz -C openssl --strip-components=1; cd openssl; \ 225 | ./config shared --prefix=${PREFIX}/openssl --openssldir=${PREFIX}/openssl; \ 226 | make depend; make install; cd ..; rm -rf openssl 227 | 228 | 229 | # Copy shell scripts and config files over 230 | COPY bin/* /usr/local/bin/ 231 | 232 | WORKDIR /home/geolambda 233 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | Copyright (c) 2016 Development Seed 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy 4 | of this software and associated documentation files (the "Software"), to deal 5 | in the Software without restriction, including without limitation the rights 6 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | copies of the Software, and to permit persons to whom the Software is 8 | furnished to do so, subject to the following conditions: 9 | 10 | The above copyright notice and this permission notice shall be included in all 11 | copies or substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 19 | SOFTWARE. 20 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # GeoLambda: geospatial AWS Lambda Layer 2 | 3 | The GeoLambda project provides public Docker images and AWS Lambda Layers containing common geospatial native libraries. GeoLambda contains the libraries for GDAL, Proj, GEOS, GeoTIFF, HDF4/5, SZIP, NetCDF, OpenJPEG, WEBP, ZSTD, and others. For some applications you may wish to minimize the size of the libraries by excluding unused libraries, or you may wish to add other libraries. In this case this repository can be used as a template to create your own Docker image or Lambda Layer following the instructions in this README. 4 | 5 | This repository also contains additional images and layers for specific runtimes. Using them as a Layer assumes the use of the base GeoLambda layer. 6 | 7 | - [Python](python/README.md) 8 | 9 | ## Usage 10 | 11 | While GeoLambda was initially intended for AWS Lambda it is also useful as a base geospatial Docker image. For detailed info on what is included in each image, see the Dockerfile for that version or the [CHANGELOG](CHANGELOG.md). A version summary is provided here: 12 | 13 | | geolambda | GDAL | Notes | 14 | | --------- | ---- | ----- | 15 | | 1.0.0 | 2.3.1 | | 16 | | 1.1.0 | 2.4.1 | | 17 | | 1.2.0 | 2.4.2 | Separate Python (3.7.4) image and Lambda Layer added | 18 | | 2.0.0 | 3.0.1 | libgeotiff 1.5.1, proj 6.2.0 | 19 | | 2.1.0 | 3.2.1 | libgeotiff 1.6.0, proj 7.2.1, openjpeg 2.4.0, Python layer 3.7.9 | 20 | 21 | #### Environment variables 22 | 23 | When using GeoLambda some environment variables need to be set. These are set in the Docker image, but if using the Lambda Layer they will need to be set: 24 | 25 | - GDAL_DATA=/opt/share/gdal 26 | - PROJ_LIB=/opt/share/proj (only needed for GeoLambda 2.0.0+) 27 | 28 | ### Lambda Layers 29 | 30 | If you just wish to use the publicly available Lambda layers you will need the ARN for the layer in the same region as your Lambda function. Currently, the latest GeoLambda layers are deployed in `us-east-1`, `us-west-2`, `eu-central-1`, `eu-west-2`, and `eu-north-1`. If you want to use it in another region please file an issue or you can also create your own layer using this repository (see instructions below on 'Create a new version'). 31 | 32 | #### v2.1.0 33 | 34 | | Region | ARN | 35 | | ------ | --- | 36 | | us-east-1 | arn:aws:lambda:us-east-1:552188055668:layer:geolambda:4 | 37 | | us-west-2 | arn:aws:lambda:us-west-2:552188055668:layer:geolambda:4 | 38 | | eu-central-1 | arn:aws:lambda:eu-central-1:552188055668:layer:geolambda:4 | 39 | | eu-west-2 | | 40 | | eu-north-1 | | 41 | 42 | #### v2.1.0-python 43 | 44 | See the [GeoLambda Python README](python/README.md). The Python Lambda Layer includes the libraries `numpy`, `rasterio`, `GDAL`, `pyproj`, and `shapely`, plus everything in the standard GeoLambda layer. Note this is a change from v2.0.0 where both Layers needed to be included in a Lambda. 45 | 46 | | Region | ARN | 47 | | ------ | --- | 48 | | us-east-1 | arn:aws:lambda:us-east-1:552188055668:layer:geolambda-python:3 | 49 | | us-west-2 | arn:aws:lambda:us-west-2:552188055668:layer:geolambda-python:3 | 50 | | eu-central-1 | arn:aws:lambda:eu-central-1:552188055668:layer:geolambda-python:3 | 51 | | eu-west-2 | | 52 | | eu-north-1 | | 53 | 54 | 55 | #### v2.0.0 56 | 57 | | Region | ARN | 58 | | ------ | --- | 59 | | us-east-1 | arn:aws:lambda:us-east-1:552188055668:layer:geolambda:4 | 60 | | us-west-2 | arn:aws:lambda:us-west-2:552188055668:layer:geolambda:4 | 61 | | eu-central-1 | arn:aws:lambda:eu-central-1:552188055668:layer:geolambda:4 | 62 | 63 | #### v2.0.0-python 64 | 65 | See the [GeoLambda Python README](python/README.md). The Python Lambda Layer includes the libraries `numpy`, `rasterio`, `GDAL`, `pyproj`, and `shapely`. This is an addition to the standard GeoLambda layer; if you wish to use GeoLambda-Python, both layers must be included. 66 | 67 | | Region | ARN | 68 | | ------ | --- | 69 | | us-east-1 | arn:aws:lambda:us-east-1:552188055668:layer:geolambda-python:3 | 70 | | us-west-2 | arn:aws:lambda:us-west-2:552188055668:layer:geolambda-python:3 | 71 | | eu-central-1 | arn:aws:lambda:eu-central-1:552188055668:layer:geolambda-python:3 | 72 | 73 | #### v1.2.0 74 | 75 | | Region | ARN | 76 | | ------ | --- | 77 | | us-east-1 | arn:aws:lambda:us-east-1:552188055668:layer:geolambda:2 | 78 | | us-west-2 | arn:aws:lambda:us-west-2:552188055668:layer:geolambda:2 | 79 | | eu-central-1 | arn:aws:lambda:eu-central-1:552188055668:layer:geolambda:2 | 80 | 81 | #### v1.2.0-python 82 | 83 | See the [GeoLambda Python README](python/README.md). The Python Lambda Layer includes the libraries `numpy`, `rasterio`, `GDAL`, `pyproj`, and `shapely`. This is an addition to the standard GeoLambda layer; if you wish to use GeoLambda-Python, both layers must be included. 84 | 85 | | Region | ARN | 86 | | ------ | --- | 87 | | us-east-1 | arn:aws:lambda:us-east-1:552188055668:layer:geolambda-python:1 | 88 | | us-west-2 | arn:aws:lambda:us-west-2:552188055668:layer:geolambda-python:1 | 89 | | eu-central-1 | arn:aws:lambda:eu-central-1:552188055668:layer:geolambda-python:1 | 90 | 91 | #### v1.1.0 92 | 93 | | Region | ARN | 94 | | ------ | --- | 95 | | us-east-1 | arn:aws:lambda:us-east-1:552188055668:layer:geolambda:1 | 96 | | us-west-2 | arn:aws:lambda:us-west-2:552188055668:layer:geolambda:1 | 97 | | eu-central-1 | arn:aws:lambda:eu-central-1:552188055668:layer:geolambda:1 | 98 | 99 | 100 | ### Docker images 101 | 102 | The Docker images used to create the Lambda layer are also published to Docker Hub, and thus are also suitable for general use as a base image for geospatial applications. 103 | 104 | The developmentseed/geolambda image in Docker Hub is tagged by version. 105 | 106 | $ docker pull developmentseed/geolambda: 107 | 108 | Or just include it in your own Dockerfile as the base image. 109 | 110 | ``` 111 | FROM developmentseed/geolambda: 112 | ``` 113 | 114 | The GeoLambda image does not have an entrypoint defined, so a command must be provided when you run it. This example will mount the current directory to /work and run the container interactively. 115 | 116 | $ docker run --rm -v $PWD:/home/geolambda -it developmentseed/geolambda:latest /bin/bash 117 | 118 | All of the GDAL CLI tools are installed so could be run on images in the current directory. 119 | 120 | 121 | ## Development 122 | 123 | Contributions to the geolambda project are encouraged. The goal is to provide a turnkey method for developing and deploying geospatial applications to AWS. The 'master' branch in this repository contains the current state as deployed to the Docker Hub images `developmentseed/geolambda:latest` and `devlopmentseed/geolambda-python:latest`, along with a tag of the version. The 'develop' branch is the development version and is not deployed to Docker Hub. 124 | 125 | When making a merge to the `master` branch be sure to increment the `VERSION` file. Circle will push the new version as a tag to GitHub and build and push the image to Docker Hub. If a GitHub tag already exists with that version the process will fail. 126 | 127 | ### Create a new version 128 | 129 | Use the Dockerfile here as a template for your own version of GeoLambda. Simply edit it to remove or add additional libraries, then build and tag with your own name. The steps below are used to create a new official version of GeoLambda, replace `developmentseed/geolambda` with your own name. 130 | 131 | To create a new version of GeoLambda follow these steps. Note that this is the manual process of what is currently done in CircleCI, so it is not necessary to perform them but they are useful as an example for deploying your own versions. 132 | 133 | 1. update the version in the `VERSION` file 134 | 135 | The version in the VERSION file will be used to tag the Docker images and create a GitHub tag. 136 | 137 | 2. build the image: 138 | 139 | ``` 140 | $ VERSION=$(cat VERSION) 141 | $ docker build . -t developmentseed/geolambda:${VERSION} 142 | ``` 143 | 144 | 3. Push the image to Docker Hub: 145 | 146 | ``` 147 | $ docker push developmentseed/geolambda:${VERSION} 148 | ``` 149 | 150 | 4. Create deployment package using the built-in [packaging script](bin/package.sh) 151 | 152 | ``` 153 | $ docker run --rm -v $PWD:/home/geolambda \ 154 | -it developmentseed/geolambda:${VERSION} package.sh 155 | ``` 156 | 157 | This will create a lambda/ directory containing the native libraries and related files, along with a `lambda-deploy.zip` file that can be deployed as a Lambda layer. 158 | 159 | 5. Push as Lambda layer (if layer already exists a new version will be created) 160 | 161 | ``` 162 | $ aws lambda publish-layer-version \ 163 | --layer-name geolambda \ 164 | --license-info "MIT" \ 165 | --description "Native geospatial libaries for all runtimes" \ 166 | --zip-file fileb://lambda-deploy.zip 167 | ``` 168 | 169 | 6. Make layer public (needs to be done each time a new version is published) 170 | 171 | ``` 172 | $ aws lambda add-layer-version-permission --layer-name geolambda \ 173 | --statement-id public --version-number 1 --principal '*' \ 174 | --action lambda:GetLayerVersion 175 | ``` 176 | -------------------------------------------------------------------------------- /VERSION: -------------------------------------------------------------------------------- 1 | 2.1.0 2 | -------------------------------------------------------------------------------- /bin/package.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # directory used for deployment 4 | export DEPLOY_DIR=lambda 5 | 6 | # make deployment directory and add lambda handler 7 | mkdir -p $DEPLOY_DIR/lib 8 | 9 | # copy libs 10 | cp -P ${PREFIX}/lib/*.so* $DEPLOY_DIR/lib/ 11 | cp -P ${PREFIX}/lib64/libjpeg*.so* $DEPLOY_DIR/lib/ 12 | 13 | strip $DEPLOY_DIR/lib/* || true 14 | 15 | # copy GDAL_DATA files over 16 | mkdir -p $DEPLOY_DIR/share 17 | rsync -ax $PREFIX/share/gdal $DEPLOY_DIR/share/ 18 | rsync -ax $PREFIX/share/proj $DEPLOY_DIR/share/ 19 | 20 | # zip up deploy package 21 | cd $DEPLOY_DIR 22 | zip --symlinks -ruq ../lambda-deploy.zip ./ 23 | -------------------------------------------------------------------------------- /build-and-test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | VERSION=$(cat VERSION) 4 | PYVERSION=$(cat python/PYVERSION) 5 | 6 | docker build . -t developmentseed/geolambda:${VERSION} 7 | docker run --rm -v $PWD:/home/geolambda -it developmentseed/geolambda:${VERSION} package.sh 8 | 9 | cd python 10 | docker build . --build-arg VERSION=${VERSION} -t developmentseed/geolambda:${VERSION}-python 11 | docker run -v ${PWD}:/home/geolambda -t developmentseed/geolambda:${VERSION}-python package-python.sh 12 | 13 | docker run -e GDAL_DATA=/opt/share/gdal -e PROJ_LIB=/opt/share/proj \ 14 | --rm -v ${PWD}/lambda:/var/task lambci/lambda:python3.7 lambda_function.lambda_handler '{}' 15 | -------------------------------------------------------------------------------- /python/Dockerfile: -------------------------------------------------------------------------------- 1 | ARG VERSION=2.1.0 2 | FROM developmentseed/geolambda:${VERSION} 3 | 4 | LABEL maintainer="Development Seed " 5 | LABEL authors="Matthew Hanson " 6 | 7 | ARG PYVERSION=3.7.9 8 | 9 | # install Python 10 | ENV \ 11 | PYENV_ROOT=/root/.pyenv \ 12 | PATH=/root/.pyenv/shims:/root/.pyenv/bin:$PATH 13 | 14 | RUN \ 15 | curl https://pyenv.run | bash; \ 16 | CONFIGURE_OPTS="--with-openssl=${PREFIX}/openssl --enable-loadable-sqlite-extensions" \ 17 | LD_RUN_PATH="${PREFIX}/openssl/lib" \ 18 | pyenv install ${PYVERSION}; \ 19 | pyenv global ${PYVERSION}; \ 20 | pip install --upgrade pip 21 | 22 | COPY requirements*.txt ./ 23 | 24 | RUN \ 25 | pip install -r requirements-pre.txt; \ 26 | pip install -r requirements.txt 27 | 28 | COPY bin/* /usr/local/bin/ 29 | -------------------------------------------------------------------------------- /python/PYVERSION: -------------------------------------------------------------------------------- 1 | 3.7.9 2 | -------------------------------------------------------------------------------- /python/README.md: -------------------------------------------------------------------------------- 1 | # Python GeoLambda 2 | 3 | The Python GeoLambda image is meant to be used as a template for your own Python based geospatial project. Follow the usage instructions to package, test, and deploy your own Lambda. 4 | 5 | ## Usage 6 | 7 | These instructions show how to use the files in this directory to create your own Python based geospatial Lambda function and get it deployed. 8 | 9 | **1. Update requirements** 10 | 11 | First a Docker image will need to be created based on GeoLambda that includes the dependencies needed. The first thing is to is update the `requirements-pre.txt` and `requirement.txt` files with the dependencies needed. The `requirements-pre.txt` is a way to specify build dependencies to packages in `requirements.txt`. For instance, rasterio requires numpy is installed before rasterio. This is a well known pip problem that is fixed in [PEP 518](https://www.python.org/dev/peps/pep-0518/) but is not used by all packages so this work-around is needed. 12 | 13 | **2. Create handler** 14 | 15 | An example Lambda handler is located at [lambda/lambda_function.py](lambda/lambda_function.py). Ideally most of the logic in your Lambda should be in packages and the handler should only be responsible for taking in an event, calling relevant functions, and returning some output. Keep the handler code as small as possible. Move more complex code into packages or make them other functions in the [lambda/lambda_function.py](lambda/lambda_function.py) file. 16 | 17 | **3. Build image** 18 | 19 | Now, use the [Dockerfile](Dockerfile) can be used to create a new Docker image based on any version of GeoLambda with any version of Python by providing the versions as build arguments to `docker run`. This will install the specified version of Python along with any Python packages provided in [requirements.txt](requirements.txt). 20 | 21 | ``` 22 | $ VERSION=2.1.0 23 | $ docker build . --build-arg VERSION=${VERSION} --build-arg PYVERSION=3.7.9 -t :latest 24 | ``` 25 | 26 | If not provided, `VERSION` (the version of GeoLambda to use) will default to `latest` and `PYVERSION` (Python version) will default to `3.7.9`. 27 | 28 | **4. Set up development environment and lambda layer zip file** 29 | 30 | First you'll need to build a `geolambda` base layer, and then you'll have to build `geolambda-python` layer located in `python/` directory. 31 | 32 | All that's needed to create a development package is to install the Python packages into the `python/lambda/` directory using the `package-python.sh` script. This copies the installed site-packages over the lambda directory, but excluding some libraries since they won't be needed. This includes packages that are already pre-installed in Lambda like boto3, as well as files and libraries that wouldn't be used operationally (e.g., testing files). 33 | 34 | This will create geolambda base layer (`lambda-deploy.zip`) file and `/geolambda/lambda` directory needed for development. 35 | 36 | ``` 37 | # current dir: geolambda (up one level) 38 | 39 | $ docker run --rm -v $PWD:/home/geolambda -it developmentseed/geolambda:${VERSION} package.sh 40 | ``` 41 | 42 | This will copy site-packages in /geolambda/python/lambda directory needed for development. Also, it will create a `lambda-deploy.zip` file in the current directory. 43 | Contents of the zip file are following AWS conventions: https://docs.aws.amazon.com/lambda/latest/dg/configuration-layers.html#configuration-layers-path 44 | 45 | ``` 46 | # current dir: geolambda/python 47 | 48 | $ docker run -v ${PWD}:/home/geolambda -t :latest package-python.sh 49 | ``` 50 | 51 | **5. Testing deployment package** 52 | 53 | You can use the [LambCI Docker images](https://github.com/lambci/docker-lambda) to test out your handler in the Lambda environment by mounting the base GeoLambda Lambda layer (`/geolambda/lambda`) and the lambda directory (`/geolambda/python/lambda`) created above. 54 | 55 | ``` 56 | # current dir: geolambda/python 57 | 58 | $ docker run -e GDAL_DATA=/opt/share/gdal -e PROJ_LIB=/opt/share/proj \ 59 | --rm -v ${PWD}/lambda:/var/task lambci/lambda:python3.7 lambda_function.lambda_handler '{}' 60 | ``` 61 | 62 | The last argument is a JSON string that will be passed as the event payload to the handler function. 63 | 64 | **6. Deploy as Lambda layer (if layer already exists a new version will be created)** 65 | 66 | Deploy the Lambda layer by using AWS CLI. 67 | 68 | ``` 69 | $ aws lambda publish-layer-version \ 70 | --layer-name geolambda-python \ 71 | --description "Python bindings for GDAL" \ 72 | --zip-file fileb://lambda-deploy.zip 73 | ``` 74 | 75 | ### Pre-built images 76 | 77 | Builds of this default Python image are also available on Docker Hub as tags in the `developmentseed/geolambda` repository. 78 | 79 | $ docker pull developmentseed/geolambda:${VERSION}-python 80 | 81 | To run the image interactively: 82 | 83 | $ docker run -v ${PWD}:/home/geolambda --rm -it developmentseed/geolambda:${VERSION}-python 84 | 85 | This image includes the dependencies specified in the default [requirements.txt](requirements.txt) file. 86 | 87 | 88 | ## Development 89 | 90 | To create a new official version of a Python GeoLambda Docker image for publication to Docker Hub follow these steps *after* following the [steps for publishing a new base GeoLambda](../README.md). These steps are performed automatically in [CircleCI](../.circleci/config.yml) 91 | 92 | From this directory: 93 | 94 | ``` 95 | $ VERSION=$(cat ../VERSION) 96 | $ docker build . --build-arg VERSION=${VERSION} -t developmentseed/geolambda:${VERSION}-python 97 | ``` 98 | 99 | Provide `--build-arg PYVERSION=X.Y.Z` to build other Python versions and change the tag to be `${VERSION}-python` 100 | 101 | 102 | 103 | -------------------------------------------------------------------------------- /python/bin/package-python.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # directory used for development 4 | export DEPLOY_DIR=lambda 5 | 6 | # make deployment directory and add lambda handler 7 | mkdir -p $DEPLOY_DIR/lib 8 | 9 | # copy libs 10 | cp -P ${PREFIX}/lib/*.so* $DEPLOY_DIR/lib/ 11 | cp -P ${PREFIX}/lib64/libjpeg*.so* $DEPLOY_DIR/lib/ 12 | 13 | strip $DEPLOY_DIR/lib/* || true 14 | 15 | # copy GDAL_DATA files over 16 | mkdir -p $DEPLOY_DIR/share 17 | rsync -ax $PREFIX/share/gdal $DEPLOY_DIR/share/ 18 | rsync -ax $PREFIX/share/proj $DEPLOY_DIR/share/ 19 | 20 | # Get Python version 21 | PYVERSION=$(cat /root/.pyenv/version) 22 | MAJOR=${PYVERSION%%.*} 23 | MINOR=${PYVERSION#*.} 24 | PYVER=${PYVERSION%%.*}.${MINOR%%.*} 25 | PYPATH=/root/.pyenv/versions/$PYVERSION/lib/python${PYVER}/site-packages/ 26 | 27 | echo Creating deploy package for Python $PYVERSION 28 | 29 | EXCLUDE="boto3* botocore* pip* docutils* *.pyc setuptools* wheel* coverage* testfixtures* mock* *.egg-info *.dist-info __pycache__ easy_install.py" 30 | 31 | EXCLUDES=() 32 | for E in ${EXCLUDE} 33 | do 34 | EXCLUDES+=("--exclude ${E} ") 35 | done 36 | 37 | rsync -ax $PYPATH/ $DEPLOY_DIR/ ${EXCLUDES[@]} 38 | 39 | # prepare dir for lambda layer deployment - https://docs.aws.amazon.com/lambda/latest/dg/configuration-layers.html#configuration-layers-path 40 | cp -r $DEPLOY_DIR ./python 41 | rm ./python/lambda_function.py 42 | 43 | # zip up deploy package 44 | zip --symlinks -ruq lambda-deploy.zip ./python 45 | 46 | # cleanup 47 | rm -rf ./python -------------------------------------------------------------------------------- /python/lambda/lambda_function.py: -------------------------------------------------------------------------------- 1 | import os 2 | import subprocess 3 | import sys 4 | import logging 5 | 6 | # set up logger 7 | logger = logging.getLogger(__file__) 8 | logger.setLevel(logging.DEBUG) 9 | # commented out to avoid duplicate logs in lambda 10 | # logger.addHandler(logging.StreamHandler()) 11 | 12 | # imports used for the example code below 13 | from osgeo import gdal 14 | 15 | # not used in code below, but imported to test loading of libraries 16 | import rasterio 17 | import pyproj 18 | import shapely 19 | 20 | 21 | test_filename = 'https://landsat-pds.s3.amazonaws.com/c1/L8/086/240/LC08_L1GT_086240_20180827_20180827_01_RT/LC08_L1GT_086240_20180827_20180827_01_RT_B1.TIF' 22 | 23 | 24 | def lambda_handler(event, context=None): 25 | """ Lambda handler """ 26 | logger.debug(event) 27 | 28 | # this try block is for testing and info only, 29 | # it prints out info on the the libgdal binary and paths to linked libraries 30 | #try: 31 | # output = subprocess.check_output('readelf -d /opt/lib/libgdal.so'.split(' ')) 32 | # logger.info(output.decode()) 33 | # output = subprocess.check_output('ldd /opt/lib/libgdal.so'.split(' ')) 34 | # logger.info(output.decode()) 35 | #except Exception as e: 36 | # pass 37 | 38 | # process event payload and do something like this 39 | fname = event.get('filename', test_filename) 40 | fname = fname.replace('s3://', '/vsis3/') 41 | # open and return metadata 42 | ds = gdal.Open(fname) 43 | band = ds.GetRasterBand(1) 44 | stats = band.GetStatistics(0, 1) 45 | 46 | return stats 47 | 48 | 49 | if __name__ == "__main__": 50 | """ Test lambda_handler """ 51 | event = {'filename': test_filename} 52 | stats = lambda_handler(event) 53 | print(stats) -------------------------------------------------------------------------------- /python/requirements-pre.txt: -------------------------------------------------------------------------------- 1 | numpy 2 | -------------------------------------------------------------------------------- /python/requirements.txt: -------------------------------------------------------------------------------- 1 | GDAL==3.2.1 2 | rasterio==1.2.0 --no-binary rasterio 3 | shapely==1.7.1 4 | pyproj==3.0.0.post1 5 | --------------------------------------------------------------------------------