├── .gitignore ├── etc ├── mapserver │ └── mapserver.conf ├── supervisor │ └── conf.d │ │ └── supervisord.conf └── nginx │ └── sites-available │ └── mapserver_proxy.conf ├── LICENSE ├── Dockerfile ├── mapfiles └── mapfile.map └── README.md /.gitignore: -------------------------------------------------------------------------------- 1 | *~ -------------------------------------------------------------------------------- /etc/mapserver/mapserver.conf: -------------------------------------------------------------------------------- 1 | CONFIG 2 | ENV 3 | # Available variables (https://mapserver.org/environment_variables.html): 4 | # MS_MAP_PATTERN - regex used to validate map variable values 5 | # MS_MAPFILE - default mapfile if no value is passed in the map variable 6 | # MS_MAP_NO_PATH - limits access to a curated set of maps (by environment variable) 7 | MS_MAP_PATTERN="^\/usr\/src\/mapfiles\/([^\.][_A-Za-z0-9\-\.]+\/{1})*([_A-Za-z0-9\-\.]+\.(map))$" 8 | MS_MAPFILE "/usr/src/mapfiles/mapfile.map" 9 | MS_MAP_NO_PATH "true" 10 | MS_ERRORFILE "stderr" 11 | END 12 | MAPS 13 | # FOO "/usr/src/mapfiles/foo.map" 14 | END 15 | END 16 | -------------------------------------------------------------------------------- /etc/supervisor/conf.d/supervisord.conf: -------------------------------------------------------------------------------- 1 | [supervisord] 2 | nodaemon=true 3 | 4 | [fcgi-program:mapserver] 5 | ;socket=tcp://0.0.0.0:9090 6 | socket=unix:///var/run/mapserver.sock 7 | socket_owner=www-data 8 | socket_mode=0644 9 | command=/usr/local/bin/mapserv 10 | 11 | ; Hint: Try a higher number for increased parallelism! 12 | numprocs=1 13 | process_name=%(program_name)s_%(process_num)02d 14 | autorestart=true 15 | autostart=true 16 | 17 | ; Log everything to supervisor stdout so Docker will see it. 18 | ; http://veithen.github.io/2015/01/08/supervisord-redirecting-stdout.html 19 | stdout_logfile=/dev/stdout 20 | stdout_logfile_maxbytes=0 21 | 22 | stderr_logfile=/dev/stderr 23 | stderr_logfile_maxbytes=0 24 | 25 | [program:nginx] 26 | command = /usr/sbin/nginx -g "daemon off;" 27 | ; Log everything to supervisor stdout so Docker will see it. 28 | ; http://veithen.github.io/2015/01/08/supervisord-redirecting-stdout.html 29 | stdout_logfile=/dev/stdout 30 | stdout_logfile_maxbytes=0 31 | 32 | stderr_logfile=/dev/stderr 33 | stderr_logfile_maxbytes=0 34 | -------------------------------------------------------------------------------- /etc/nginx/sites-available/mapserver_proxy.conf: -------------------------------------------------------------------------------- 1 | # supervisord fcgi-program does not appear to work directly without a 2 | # proxy. This nginx conf proxies from port 80 to the mapserver fcgi 3 | # listening on TCP port 9090. 4 | server { 5 | access_log on; 6 | listen 80; 7 | server_name localhost; 8 | location / { 9 | # allow cross-origin requests to allow http clients (ie. OpenLayers running somewhere) to use GetCapabilities 10 | add_header 'Access-Control-Allow-Origin' '*'; 11 | 12 | #fastcgi_pass localhost:9090; 13 | fastcgi_pass unix:/var/run/mapserver.sock; 14 | fastcgi_param QUERY_STRING $query_string; 15 | fastcgi_param REQUEST_METHOD $request_method; 16 | fastcgi_param SCRIPT_FILENAME $document_root$fastcgi_script_name; 17 | 18 | # Use keepalive connections 19 | fastcgi_keep_conn on; 20 | 21 | # Redirect MapServer logs to stderr at an appropriate level 22 | # (debug, info, notice, warn, error (default), crit, alert, 23 | # and emerg). 24 | error_log stderr error; 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2016 Pete Schmitt 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM ghcr.io/osgeo/gdal:ubuntu-full-3.10.3 2 | 3 | ARG MAPSERVER_VERSION=8.4.0 4 | 5 | RUN \ 6 | # AWS CLI can be helpful with debugging 7 | curl "https://awscli.amazonaws.com/awscli-exe-linux-$(uname -m).zip" -o "/tmp/awscliv2.zip" && \ 8 | unzip -q -d /tmp /tmp/awscliv2.zip && \ 9 | /tmp/aws/install && \ 10 | rm -rf /tmp/awscliv2.zip /tmp/aws && \ 11 | # Use package manager to install dependencies 12 | apt-get update && apt-get upgrade -y && \ 13 | # don't allow tzdata to prompt the user for a setting. 14 | env DEBIAN_FRONTEND=noninteractive \ 15 | apt-get install -y --no-install-recommends \ 16 | nginx \ 17 | supervisor \ 18 | # Might be helpful for debugging 19 | emacs-nox \ 20 | less \ 21 | postgresql-client \ 22 | procps \ 23 | strace \ 24 | # MapServer dependencies 25 | build-essential \ 26 | cmake \ 27 | libcurl4-gnutls-dev \ 28 | libfcgi0ldbl \ 29 | libfcgi-dev \ 30 | libgeos-dev \ 31 | libpq-dev \ 32 | libxml2 \ 33 | libxml2-dev \ 34 | libpng-dev \ 35 | zlib1g \ 36 | zlib1g-dev \ 37 | libjpeg-turbo8 \ 38 | libjpeg-turbo8-dev \ 39 | libgif-dev \ 40 | libcairo2 \ 41 | libcairo2-dev \ 42 | librsvg2-2 \ 43 | librsvg2-dev \ 44 | libfribidi0 \ 45 | libfribidi-dev \ 46 | libfreetype6 \ 47 | libfreetype6-dev \ 48 | libharfbuzz0b \ 49 | libharfbuzz-dev \ 50 | protobuf-c-compiler \ 51 | libprotobuf-c-dev && \ 52 | # Build MapServer using libproj from the GDAL Docker 53 | curl https://download.osgeo.org/mapserver/mapserver-${MAPSERVER_VERSION}.tar.gz | tar zx -C /tmp && \ 54 | mkdir /tmp/mapserver-${MAPSERVER_VERSION}/build && \ 55 | cd /tmp/mapserver-${MAPSERVER_VERSION}/build && \ 56 | cmake .. \ 57 | -DWITH_CURL=1 \ 58 | -DWITH_CAIRO=1 \ 59 | -DWITH_RSVG=1 \ 60 | -DWITH_CLIENT_WMS=1 \ 61 | -DWITH_CLIENT_WFS=1 \ 62 | -DPROJ_LIBRARY=/usr/local/lib/libinternalproj.so \ 63 | -DCMAKE_C_FLAGS=-DPROJ_RENAME_SYMBOLS && \ 64 | make -j $(grep --count ^processor /proc/cpuinfo) && \ 65 | make install && \ 66 | # Cleanup 67 | apt-get autoremove -y && \ 68 | apt-get clean && \ 69 | rm -rf /var/lib/apt/lists/* /tmp/* 70 | 71 | ADD etc /etc 72 | RUN ln -sf /etc/nginx/sites-available/mapserver_proxy.conf /etc/nginx/sites-enabled/default 73 | COPY mapfiles /usr/src/mapfiles 74 | 75 | EXPOSE 80 76 | 77 | ENV MAPSERVER_CONFIG_FILE=/etc/mapserver/mapserver.conf 78 | 79 | CMD /usr/bin/supervisord 80 | -------------------------------------------------------------------------------- /mapfiles/mapfile.map: -------------------------------------------------------------------------------- 1 | MAP 2 | OUTPUTFORMAT 3 | NAME png24 4 | DRIVER "AGG/PNG" 5 | MIMETYPE "image/png" 6 | IMAGEMODE RGB 7 | EXTENSION "png" 8 | END 9 | 10 | OUTPUTFORMAT 11 | NAME jpeg 12 | DRIVER "AGG/JPEG" 13 | MIMETYPE "image/jpeg" 14 | IMAGEMODE RGB 15 | EXTENSION "jpg" 16 | END 17 | 18 | # How to return GeoJSON for Mapserver vector layers? 19 | # http://mapserver.org/output/ogr_output.html 20 | # https://github.com/mapserver/msautotest/blob/master/wxs/wfs_ogr.map 21 | OUTPUTFORMAT 22 | NAME "geojson" 23 | DRIVER "OGR/GEOJSON" 24 | MIMETYPE "application/json; subtype=geojson" 25 | FORMATOPTION "STORAGE=memory" 26 | FORMATOPTION "FORM=SIMPLE" 27 | FORMATOPTION "LCO:WRITE_BBOX=YES" 28 | END 29 | 30 | IMAGETYPE PNG24 31 | IMAGECOLOR 255 255 255 32 | 33 | # Write log to /var/log/mapserver.stderr (make sure the file exists) to see error log. 34 | CONFIG "MS_ERRORFILE" "stderr" 35 | CONFIG "CPL_VSIL_CURL_ALLOWED_EXTENSIONS" ".tif" # .shp .shx .dbf .qix .cpg .prj 36 | CONFIG "VSI_CACHE" "TRUE" 37 | # cache size in bytes 38 | CONFIG "VSI_CACHE_SIZE" "50000000" 39 | 40 | EXTENT -180 -90 180 90 41 | 42 | PROJECTION 43 | # Performance optimization: Instead of specifying 4326, set all the Proj4 parameters directly. 44 | # http://mapserver.org/optimization/mapfile.html#projections 45 | # http://spatialreference.org/ref/epsg/4326/proj4/ 46 | "init=epsg:4326" 47 | #"proj=longlat" 48 | #"ellps=WGS84" 49 | #"datum=WGS84" 50 | #"no_defs" 51 | END 52 | 53 | WEB 54 | # Uncomment the following and run `touch /var/log/mapserver_output.log` to see access log. 55 | #LOG /var/log/mapserver_output.log 56 | 57 | # Note: You should configure your webserver (ie. NGINX) to use 58 | # proxy_pass to forward all Mapserver traffic to 59 | # localhost:8080. Otherwise, edit ows_onlineresource 60 | # accordingly. 61 | METADATA 62 | "ows_title" "raster_layer" 63 | "ows_onlineresource" "http://localhost:8000?" 64 | "ows_srs" "EPSG:4326" 65 | "ows_enable_request" "*" 66 | "wms_srs" "EPSG:4326" 67 | "wms_feature_info_mime_type" "text/html" 68 | "wms_allow_getmap_without_styles" "true" 69 | "wfs_getfeature_formatlist" "ogrgml,geojson" 70 | END 71 | END 72 | 73 | MAXSIZE 4096 74 | 75 | LAYER 76 | NAME raster_layer 77 | METADATA 78 | "ows_title" "raster_layer_lowres" 79 | "ows_srs" "epsg:4326" 80 | END 81 | DATA "/vsicurl/https://github.com/OSGeo/gdal/raw/86193bf5942fb63b91b06a18df78efc73a2d869b/autotest/gdrivers/data/gtiff/int8.tif" 82 | STATUS OFF 83 | TYPE RASTER 84 | END 85 | 86 | END # mapfile 87 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Abstract 2 | ======== 3 | 4 | Mapserver >= 7.0 can render imagery stored in an AWS S3 bucket using a 5 | file handler provided by GDAL >= 2.1. Two file handlers are available 6 | `/vsicurl/` and `/vsis3/`. These handlers make use of [HTTP GET range 7 | requests](https://tools.ietf.org/html/rfc7233) to transfer the minimum 8 | data required. When images are properly prepared, access via the vsi 9 | drivers can be highly performant. 10 | 11 | VSI file handlers 12 | ================= 13 | Before configuring Mapserver to render imagery stored in an S3 bucket, 14 | ensure that `gdalinfo` can access the files on the command line. 15 | 16 | - [/vsicurl/](http://gdal.org/cpl__vsi_8h.html#a4f791960f2d86713d16e99e9c0c36258) 17 | can read from a static website, for example one hosted on S3. For 18 | example, [this file from the GDAL test suite](https://github.com/OSGeo/gdal/blob/master/autotest/gdrivers/data/gtiff/int8.tif) 19 | can be accessed via its /vsicurl/ driver. 20 | 21 | gdalinfo /vsicurl/https://github.com/OSGeo/gdal/raw/86193bf5942fb63b91b06a18df78efc73a2d869b/autotest/gdrivers/data/gtiff/int8.tif 22 | 23 | - [/vsis3/](https://gdal.org/en/stable/user/virtual_file_systems.html#vsis3) 24 | can be used to read from buckets which require AWS credentials. The 25 | vsis3 driver should fetch properly configured 26 | credentials. Credential management is out of scope for this 27 | document. 28 | 29 | Preparing imagery 30 | ================= 31 | 32 | The format & layout of your data have a critical impact on Mapserver 33 | performance. This is especially important when using the vsicurl 34 | drivers. To achieve high performance, you need to minimize the amount 35 | of data that needs to be transferred. The [GDAL COG 36 | driver](https://gdal.org/en/stable/drivers/raster/cog.html) puts 37 | GeoTIFF data in the optimal format for random access over /vsicurl/. 38 | Details of this format are outside the scope of this document. 39 | 40 | Layer configuration 41 | =================== 42 | 43 | This section documents how to configure a 44 | [mapfile](http://mapserver.org/mapfile/map.html) for /vsicurl/ data. 45 | 46 | Single image 47 | ------------ 48 | 49 | To have Mapserver render from a single source image, set the `DATA` to 50 | the `/vsicurl/` or `/vsis3/` path in the LAYER object of your mapfile: 51 | 52 | LAYER 53 | NAME landsat_tile 54 | DATA "/vsicurl/https://github.com/OSGeo/gdal/raw/86193bf5942fb63b91b06a18df78efc73a2d869b/autotest/gdrivers/data/gtiff/int8.tif" 55 | TYPE RASTER 56 | END 57 | 58 | Many images 59 | ----------- 60 | 61 | [gdaltindex](https://gdal.org/en/stable/programs/gdaltindex.html) can 62 | point to files having `/vsicurl/` paths. For example: 63 | 64 | gdaltindex tindex.gpkg /vsis3/landsat-pds/L8/021/036/LC80210362016114LGN00/LC80210362016114LGN00_B2.TIF /vsis3/landsat-pds/L8/021/036/LC80210362016114LGN00/LC80210362016114LGN00_B3.TIF 65 | 66 | Once you have the tile index, set your LAYER like so: 67 | 68 | LAYER 69 | NAME landsat_tiles 70 | TILEINDEX "/usr/src/mapfiles/tile_index.gpkg" 71 | TYPE RASTER 72 | END 73 | 74 | Performance Improvement: VSI Curl options 75 | ----------------------------------------- 76 | 77 | Some GDAL config options have an outsized impact on performance. 78 | These are well summarized at [TiTiler Performance 79 | Tuning](https://developmentseed.org/titiler/advanced/performance_tuning/). 80 | 81 | 82 | For example, consider: 83 | - [CPL_VSIL_CURL_ALLOWED_EXTENSIONS](https://trac.osgeo.org/gdal/wiki/ConfigOptions#CPL_VSIL_CURL_ALLOWED_EXTENSIONS). Use this to restrict only the file extensions you expect to actually need. 84 | - [VSI_CACHE](https://trac.osgeo.org/gdal/wiki/ConfigOptions#VSI_CACHE) results in less network traffic (even for simple things like `gdalinfo`)! 85 | - [VSI_CACHE_SIZE](https://trac.osgeo.org/gdal/wiki/ConfigOptions#VSI_CACHE) size of cache in bytes. 86 | - [GDAL_DISABLE_READDIR_ON_OPEN](https://trac.osgeo.org/gdal/wiki/ConfigOptions#GDAL_DISABLE_READDIR_ON_OPEN) might result in fewer HTTP GET requests when opening the GeoTIFF header. 87 | 88 | Example configuration for the MAP object of your MAPFILE: 89 | 90 | CONFIG "CPL_VSIL_CURL_ALLOWED_EXTENSIONS" ".tif" 91 | CONFIG "VSI_CACHE" "TRUE" 92 | # cache size in bytes 93 | CONFIG "VSI_CACHE_SIZE" "50000000" 94 | CONFIG "GDAL_DISABLE_READDIR_ON_OPEN" "TRUE" 95 | 96 | Docker Image 97 | ============ 98 | 99 | This repo includes a Docker image that can be used to render GeoTIFFs stored in AWS S3 via Mapserver-7.0.2 and gdal-2.1.1. 100 | 101 | docker build -t mapserver-docker:latest 102 | docker run --rm -it -p 8000:80 -v /Users/pschmitt/src/mapserver-docker/mapfiles:/usr/src/mapfiles mapserver-docker:latest 103 | 104 | A sample mapfile is available at `mapfiles/mapfile.map`. 105 | 106 | Once the container is running 107 | 108 | Render imagery with a WMS client like QGIS, OpenLayers or manually issue a request for a single tile: 109 | 110 | http://localhost:8000/mapserv?LAYERS=raster_layer&FORMAT=image%2Fpng&MAP=/usr/src/mapfiles/mapfile.map&TRANSPARENT=true&SERVICE=WMS&VERSION=1.1.1&REQUEST=GetMap&STYLES=&SRS=EPSG%3A4326&BBOX=57.00,27.05,57.01,27.06&WIDTH=256&HEIGHT=256 111 | 112 | Assorted Docs & Links 113 | ===================== 114 | 115 | - [GDAL Virtual File Systems](https://gdal.org/en/stable/user/virtual_file_systems.html) 116 | - [GDAL Cloud Optimized GeoTIFF driver](https://gdal.org/en/stable/drivers/raster/cog.html) 117 | - [TiTiler GDAL Performance Tuning](https://developmentseed.org/titiler/advanced/performance_tuning/) 118 | - [Mapserver Virtual File System](http://mapserver.org/input/virtual-file.html) 119 | 120 | Thanks to [Even Rouault](http://erouault.blogspot.com/) for his work 121 | on /vsis3/ support, the [Mapserver](http://www.mapserver.org/) team 122 | for an excellent tool and the 123 | [mapserver-users](https://lists.osgeo.org/listinfo/mapserver-users) 124 | mailing list! 125 | --------------------------------------------------------------------------------