├── .dockerignore ├── .envrc ├── .gitignore ├── .pre-commit-config.yaml ├── Dockerfile ├── LICENSE ├── Makefile ├── README.md ├── docker-compose.yml ├── docs ├── greyscale.png ├── greyscale_stretched.png └── rgb.png ├── pylama.ini ├── requirements-server.txt ├── requirements.txt ├── sample.env ├── server.py ├── template.yaml ├── templates ├── index.html └── preview.html └── virtual ├── __init__.py ├── catalogs.py ├── lambda.py └── web.py /.dockerignore: -------------------------------------------------------------------------------- 1 | .git 2 | deps 3 | functions 4 | node_modules 5 | tmp 6 | venv 7 | *.tif 8 | *.TIF 9 | *.env 10 | *.yaml 11 | *.yml 12 | *.vrt 13 | *.json 14 | *.png 15 | *.pyc 16 | .dockerignore 17 | .gitignore 18 | *.md 19 | .DS_Store 20 | .envrc 21 | !deps/prune.sh 22 | !deps/required.txt 23 | -------------------------------------------------------------------------------- /.envrc: -------------------------------------------------------------------------------- 1 | test -f .env && dotenv 2 | PATH_add bin 3 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules/ 2 | environment.json 3 | lib/ 4 | share/ 5 | venv/ 6 | .env 7 | .pypath/ 8 | project.json 9 | up.json 10 | .cache/ 11 | marblecutter.egg-info/ 12 | *.pyc 13 | *.tif 14 | staging.env 15 | production.env 16 | out.zip 17 | package-lock.json 18 | .venv/ 19 | .vscode/ 20 | .aws-sam/ 21 | packaged.yaml 22 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | repos: 2 | - repo: https://github.com/ambv/black 3 | rev: stable 4 | hooks: 5 | - id: black 6 | # args: [--fast] 7 | python_version: python3 8 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM quay.io/mojodna/gdal 2 | LABEL maintainer="Seth Fitzsimmons " 3 | 4 | ARG http_proxy 5 | 6 | ENV DEBIAN_FRONTEND noninteractive 7 | ENV LC_ALL C.UTF-8 8 | ENV GDAL_CACHEMAX 512 9 | ENV GDAL_DISABLE_READDIR_ON_OPEN TRUE 10 | ENV GDAL_HTTP_MERGE_CONSECUTIVE_RANGES YES 11 | ENV VSI_CACHE TRUE 12 | # tune this according to how much memory is available 13 | ENV VSI_CACHE_SIZE 536870912 14 | # override this accordingly; should be 2-4x $(nproc) 15 | ENV WEB_CONCURRENCY 4 16 | 17 | EXPOSE 8000 18 | 19 | RUN apt-get update \ 20 | && apt-get upgrade -y \ 21 | && apt-get install -y --no-install-recommends \ 22 | build-essential \ 23 | ca-certificates \ 24 | cython \ 25 | git \ 26 | python-pip \ 27 | python-wheel \ 28 | python-setuptools \ 29 | && apt-get clean \ 30 | && rm -rf /var/lib/apt/lists/* 31 | 32 | WORKDIR /opt/marblecutter 33 | 34 | COPY requirements-server.txt /opt/marblecutter/ 35 | COPY requirements.txt /opt/marblecutter/ 36 | 37 | RUN pip install -U numpy && \ 38 | pip install -r requirements-server.txt && \ 39 | rm -rf /root/.cache 40 | 41 | COPY virtual /opt/marblecutter/virtual 42 | 43 | USER nobody 44 | 45 | ENTRYPOINT ["gunicorn", "-k", "gevent", "-b", "0.0.0.0", "--access-logfile", "-", "virtual.web:app"] 46 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Original work Copyright 2016 Stamen Design 2 | Modified work Copyright 2016-2018 Seth Fitzsimmons 3 | Modified work Copyright 2016 American Red Cross 4 | Modified work Copyright 2016-2017 Humanitarian OpenStreetMap Team 5 | Modified work Copyright 2017 Mapzen 6 | Modified work Copyright 2018 Radiant.Earth 7 | 8 | Redistribution and use in source and binary forms, with or without 9 | modification, are permitted provided that the following conditions are met: 10 | 11 | 1. Redistributions of source code must retain the above copyright notice, this 12 | list of conditions and the following disclaimer. 13 | 14 | 2. Redistributions in binary form must reproduce the above copyright notice, 15 | this list of conditions and the following disclaimer in the documentation 16 | and/or other materials provided with the distribution. 17 | 18 | 3. Neither the name of the copyright holder nor the names of its contributors 19 | may be used to endorse or promote products derived from this software without 20 | specific prior written permission. 21 | 22 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 23 | ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 24 | WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 25 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 26 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 27 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 28 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 29 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 30 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 31 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | PATH := node_modules/.bin:$(PATH) 2 | STACK_NAME ?= "marblecutter-virtual" 3 | 4 | deploy: packaged.yaml 5 | sam deploy \ 6 | --template-file $< \ 7 | --stack-name $(STACK_NAME) \ 8 | --capabilities CAPABILITY_IAM \ 9 | --parameter-overrides DomainName=$(DOMAIN_NAME) 10 | 11 | packaged.yaml: .aws-sam/build/template.yaml 12 | sam package --s3-bucket $(S3_BUCKET) --output-template-file $@ 13 | 14 | .aws-sam/build/template.yaml: template.yaml requirements.txt virtual/*.py 15 | sam build --use-container 16 | 17 | clean: 18 | rm -rf .aws-sam/ packaged.yaml 19 | 20 | server: 21 | docker build --build-arg http_proxy=$(http_proxy) -t quay.io/mojodna/marblecutter-virtual . 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # marblecutter-virtual 2 | 3 | I am a tile server for HTTP(S)-accessible [Cloud Optimized GeoTIFFs 4 | (COGs)](http://www.cogeo.org/). 5 | 6 | I can also be seen as an example of a virtual `Catalog` implementation, drawing 7 | necessary metadata from URL parameters. For more information, check out [`VirtualCatalog`](virtual/catalogs.py) and [`web.py`](virtual/web.py). 8 | 9 | ## Running Locally 10 | 11 | The easiest way to get a working instance running locally is to use [Docker 12 | Compose](https://docs.docker.com/compose/): 13 | 14 | ```bash 15 | docker-compose up 16 | ``` 17 | 18 | A tile server will then be accessible on `localhost:8000`. To browse a map 19 | preview, visit 20 | `http://localhost:8000/preview?url=https%3A%2F%2Fs3-us-west-2.amazonaws.com%2Fplanet-disaster-data%2Fhurricane-harvey%2FSkySat_Freeport_s03_20170831T162740Z3.tif`. 21 | 22 | URLs (`url` in the query string) must be URL-encoded. From a browser's 23 | JavaScript console (or Node.js REPL), run: 24 | 25 | ```javascript 26 | encodeURIComponent("https://s3-us-west-2.amazonaws.com/planet-disaster-data/hurricane-harvey/SkySat_Freeport_s03_20170831T162740Z3.tif") 27 | ``` 28 | 29 | If you need to access non-public files on S3, set your environment accordingly 30 | (see `sample.env`), either by creating `.env` and uncommenting `env_file` in 31 | `docker-compose.yml` or by adding appropriate `environment` entries. 32 | 33 | ## Endpoints 34 | 35 | ### `/bounds` - Source image bounds (in geographic coordinates) 36 | 37 | #### Parameters 38 | 39 | * `url` - a URL to a valid COG. Required. 40 | 41 | #### Example 42 | 43 | ```bash 44 | $ curl "http://localhost:8000/bounds?url=https%3A%2F%2Fs3-us-west-2.amazonaws.com%2Fplanet-disaster-data%2Fhurricane-harvey%2FSkySat_Freeport_s03_20170831T162740Z3.tif" 45 | { 46 | "bounds": [ 47 | -95.46993599071261, 48 | 28.86905396361014, 49 | -95.2386152334213, 50 | 29.068190805522605 51 | ], 52 | "url": "https://s3-us-west-2.amazonaws.com/planet-disaster-data/hurricane-harvey/SkySat_Freeport_s03_20170831T162740Z3.tif" 53 | } 54 | ``` 55 | 56 | ### `/tiles/{z}/{x}/{y}` - Tiles 57 | 58 | #### Parameters 59 | 60 | * `url` - a URL to a valid COG. Required. 61 | * `rgb` - Source bands to map to RGB channels. Defaults to `1,2,3`. 62 | * `nodata` - a custom NODATA value. 63 | * `linearStretch` - whether to stretch output to match min/max values present in 64 | the source. Useful for raw sensor output, e.g. earth observation (EO) data. 65 | * `resample` - Specify a custom resampling method (e.g. for discrete values). 66 | Valid values (from `rasterio.enums.Resampling`): `nearest`, `bilinear`, 67 | `cubic`, `cubic_spline`, `lanczos`, `average`, `mode`, `gauss`, `max`, `min`, 68 | `med`, `q1`, `q3`. Defaults to `bilinear`. 69 | 70 | `@2x` can be added to the filename (after the `{y}` coordinate) to request 71 | retina tiles. The map preview will detect support for retina displays and 72 | request tiles accordingly. 73 | 74 | PNGs or JPEGs will be rendered depending on the presence of NODATA values in the 75 | source image (surfaced as transparency in the output). 76 | 77 | #### Examples 78 | 79 | ```bash 80 | $ curl "http://localhost:8000/tiles/14/3851/6812@2x?url=https%3A%2F%2Fs3-us-west-2.amazonaws.com%2Fplanet-disaster-data%2Fhurricane-harvey%2FSkySat_Freeport_s03_20170831T162740Z3.tif" | imgcat 81 | ``` 82 | 83 | ![RGB](docs/rgb.png) 84 | 85 | ```bash 86 | $ curl "http://localhost:8000/tiles/14/3851/6812@2x?url=https%3A%2F%2Fs3-us-west-2.amazonaws.com%2Fplanet-disaster-data%2Fhurricane-harvey%2FSkySat_Freeport_s03_20170831T162740Z3.tif&rgb=1,1,1" | imgcat 87 | ``` 88 | 89 | ![greyscale](docs/greyscale.png) 90 | 91 | ```bash 92 | $ curl "http://localhost:8000/tiles/14/3851/6812@2x?url=https%3A%2F%2Fs3-us-west-2.amazonaws.com%2Fplanet-disaster-data%2Fhurricane-harvey%2FSkySat_Freeport_s03_20170831T162740Z3.tif&rgb=1,1,1&linearStretch=true" | imgcat 93 | ``` 94 | 95 | ![greyscale stretched](docs/greyscale_stretched.png) 96 | 97 | ### `/tiles` - TileJSON 98 | 99 | #### Parameters 100 | 101 | See tile parameters. 102 | 103 | #### Example 104 | 105 | ```bash 106 | $ curl "http://localhost:8000/tiles?url=https%3A%2F%2Fs3-us-west-2.amazonaws.com%2Fplanet-disaster-data%2Fhurricane-harvey%2FSkySat_Freeport_s03_20170831T162740Z3.tif" 107 | { 108 | "bounds": [ 109 | -95.46993599071261, 110 | 28.86905396361014, 111 | -95.2386152334213, 112 | 29.068190805522605 113 | ], 114 | "center": [ 115 | -95.35427561206696, 116 | 28.968622384566373, 117 | 15 118 | ], 119 | "maxzoom": 21, 120 | "minzoom": 8, 121 | "name": "Untitled", 122 | "tilejson": "2.1.0", 123 | "tiles": [ 124 | "//localhost:8000/tiles/{z}/{x}/{y}?url=https%3A%2F%2Fs3-us-west-2.amazonaws.com%2Fplanet-disaster-data%2Fhurricane-harvey%2FSkySat_Freeport_s03_20170831T162740Z3.tif" 125 | ] 126 | } 127 | ``` 128 | 129 | ### `/preview` - Preview 130 | 131 | #### Parameters 132 | 133 | See tile parameters. 134 | 135 | #### Example 136 | 137 | `http://localhost:8000/preview?url=https%3A%2F%2Fs3-us-west-2.amazonaws.com%2Fplanet-disaster-data%2Fhurricane-harvey%2FSkySat_Freeport_s03_20170831T162740Z3.tif` 138 | 139 | ## Deploying to AWS 140 | 141 | marblecutter-virtual is deployed using the [AWS Serverless Application Model 142 | (SAM)](https://github.com/awslabs/serverless-application-model). 143 | 144 | Once you have the [SAM CLI](https://github.com/awslabs/aws-sam-cli) installed, you can build with: 145 | 146 | ```bash 147 | sam build --use-container 148 | ``` 149 | 150 | You can then test it locally as though it's running on Lambda + API Gateway 151 | (it will be _really_ slow, as function invocations are not re-used in the 152 | same way as on Lambda proper): 153 | 154 | ```bash 155 | sam local start-api 156 | ``` 157 | 158 | To deploy, first package the application: 159 | 160 | ```bash 161 | sam package --s3-bucket --output-template-file packaged.yaml 162 | ``` 163 | 164 | Once staged, it can be deployed: 165 | 166 | ```bash 167 | sam deploy \ 168 | --template-file packaged.yaml \ 169 | --stack-name marblecutter-virtual \ 170 | --capabilities CAPABILITY_IAM \ 171 | --parameter-overrides DomainName= 172 | ``` 173 | 174 | These commands are wrapped as a `deploy` target, so this can be done more 175 | simply with: 176 | 177 | ```bash 178 | S3_BUCKET= DOMAIN_NAME= make deploy 179 | ``` 180 | 181 | `` must be in the target AWS region (`AWS_DEFAULT_REGION`). 182 | 183 | NOTE: when setting up a Cloudfront distribution in front of a regional API 184 | Gateway endpoint (which is what this process does), an `Origin Custom Header` 185 | will be added: `X-Forwarded-Host` should be the hostname used for your 186 | Cloudfront distribution (otherwise auto-generated tile URLs will use the API 187 | Gateway domain; CF sends a `Host` header corresponding to the origin, not the 188 | CDN endpoint). -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '3' 2 | services: 3 | marblecutter: 4 | build: . 5 | environment: 6 | - PYTHONPATH=. 7 | volumes: 8 | - .:/opt/marblecutter/ 9 | ports: 10 | - "8000:8000" 11 | entrypoint: python 12 | command: server.py 13 | -------------------------------------------------------------------------------- /docs/greyscale.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mojodna/marblecutter-virtual/6bb9b7deceae401b8d529b7cc079427a43829c0e/docs/greyscale.png -------------------------------------------------------------------------------- /docs/greyscale_stretched.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mojodna/marblecutter-virtual/6bb9b7deceae401b8d529b7cc079427a43829c0e/docs/greyscale_stretched.png -------------------------------------------------------------------------------- /docs/rgb.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mojodna/marblecutter-virtual/6bb9b7deceae401b8d529b7cc079427a43829c0e/docs/rgb.png -------------------------------------------------------------------------------- /pylama.ini: -------------------------------------------------------------------------------- 1 | [pylama:pep8] 2 | max_line_length = 88 3 | 4 | [pylama:pycodestyle] 5 | max_line_length = 88 6 | -------------------------------------------------------------------------------- /requirements-server.txt: -------------------------------------------------------------------------------- 1 | -r requirements.txt 2 | 3 | gevent 4 | gunicorn 5 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | cachetools ~= 2.0.0 2 | flask-cors 3 | # marblecutter[web] ~= 0.3.1 4 | https://github.com/mojodna/marblecutter/archive/5b9040b.tar.gz#egg=marblecutter[web] 5 | rasterio[s3] ~= 1.0 6 | numpy 7 | serverless-wsgi 8 | # temporary workaround for https://github.com/mapbox/rasterio/issues/1651 9 | mock -------------------------------------------------------------------------------- /sample.env: -------------------------------------------------------------------------------- 1 | AWS_ACCESS_KEY_ID= 2 | AWS_SECRET_ACCESS_KEY= 3 | AWS_REGION=us-east-1 4 | S3_BUCKET= 5 | STACK_NAME=marblecutter-virtual 6 | DOMAIN_NAME= -------------------------------------------------------------------------------- /server.py: -------------------------------------------------------------------------------- 1 | # coding=utf-8 2 | from __future__ import division, print_function 3 | 4 | import logging 5 | import os 6 | 7 | from virtual.web import app 8 | 9 | logging.basicConfig(level=logging.INFO) 10 | logging.getLogger("rasterio._base").setLevel(logging.WARNING) 11 | LOG = logging.getLogger(__name__) 12 | 13 | 14 | if __name__ == "__main__": 15 | app.run(host="0.0.0.0", port=int(os.getenv("PORT", 8000)), debug=True) 16 | -------------------------------------------------------------------------------- /template.yaml: -------------------------------------------------------------------------------- 1 | AWSTemplateFormatVersion: '2010-09-09' 2 | Transform: AWS::Serverless-2016-10-31 3 | Description: > 4 | marblecutter-virtual 5 | 6 | SAM Template for marblecutter-virtual 7 | 8 | Parameters: 9 | DomainName: 10 | Type: String 11 | Description: Endpoint name 12 | AcmCertificateArn: 13 | Type: String 14 | Description: ACM Certificate ARN (must have been created in us-east-1) 15 | Default: "" 16 | 17 | Globals: 18 | Api: 19 | # API Gateway regional endpoints 20 | EndpointConfiguration: REGIONAL 21 | 22 | # enable CORS; to make more specific, change the origin wildcard 23 | # to a particular domain name, e.g. "'www.example.com'" 24 | Cors: 25 | AllowMethods: "'*'" 26 | AllowHeaders: "'*'" 27 | AllowOrigin: "'*'" 28 | 29 | # Send/receive binary data through the APIs 30 | BinaryMediaTypes: 31 | # This is equivalent to */* when deployed 32 | - "*~1*" 33 | 34 | Resources: 35 | # Lambda function 36 | MarblecutterVirtualFunction: 37 | Type: AWS::Serverless::Function 38 | Properties: 39 | CodeUri: . 40 | Policies: AmazonS3ReadOnlyAccess 41 | Handler: virtual.lambda.handle 42 | Runtime: python3.6 43 | Environment: 44 | Variables: 45 | AWS_REQUEST_PAYER: requester 46 | CPL_TMPDIR: /tmp 47 | GDAL_CACHEMAX: 75% 48 | GDAL_DATA: /var/task/rasterio/gdal_data 49 | GDAL_DISABLE_READDIR_ON_OPEN: TRUE 50 | GDAL_HTTP_MERGE_CONSECUTIVE_RANGES: YES 51 | # requires nghttp2 support 52 | GDAL_HTTP_VERSION: 2 53 | VSI_CACHE: TRUE 54 | VSI_CACHE_SIZE: 500000000 55 | Timeout: 15 56 | MemorySize: 1536 57 | Events: 58 | # API Gateway routes 59 | ProxyApiRoot: 60 | Type: Api 61 | Properties: 62 | Path: / 63 | Method: ANY 64 | ProxyApiGreedy: 65 | Type: Api 66 | Properties: 67 | Path: /{proxy+} 68 | Method: ANY 69 | 70 | # CloudFront Distribution 71 | CFDistribution: 72 | Type: AWS::CloudFront::Distribution 73 | Properties: 74 | DistributionConfig: 75 | Aliases: 76 | - !Ref DomainName 77 | Enabled: true 78 | Comment: marblecutter-virtual 79 | IPV6Enabled: true 80 | Origins: 81 | - 82 | Id: MarblecutterVirtualApi 83 | DomainName: !Sub "${ServerlessRestApi}.execute-api.${AWS::Region}.amazonaws.com" 84 | CustomOriginConfig: 85 | OriginProtocolPolicy: https-only 86 | OriginCustomHeaders: 87 | - HeaderName: X-Forwarded-Host 88 | HeaderValue: !Ref DomainName 89 | OriginPath: !Sub "/${ServerlessRestApi.Stage}" 90 | DefaultCacheBehavior: 91 | TargetOriginId: MarblecutterVirtualApi 92 | ForwardedValues: 93 | QueryString: true 94 | Cookies: 95 | Forward: none 96 | ViewerProtocolPolicy: allow-all 97 | # ViewerCertificate: 98 | # AcmCertificateArn: !Ref AcmCertificateArn 99 | # SslSupportMethod: sni-only 100 | 101 | Outputs: 102 | MarblecutterVirtualApi: 103 | Description: "API Gateway endpoint URL for for marblecutter-virtual" 104 | Value: !Sub "https://${ServerlessRestApi}.execute-api.${AWS::Region}.amazonaws.com/${ServerlessRestApi.Stage}" 105 | 106 | MarblecutterVirtualFunction: 107 | Description: "marblecutter-virtual Lambda Function ARN" 108 | Value: !GetAtt MarblecutterVirtualFunction.Arn 109 | 110 | MarblecutterVirtualFunctionIamRole: 111 | Description: "Implicit IAM Role created for marblecutter-virtual" 112 | Value: !GetAtt MarblecutterVirtualFunctionRole.Arn 113 | 114 | CFDistribution: 115 | Description: Cloudfront Distribution Domain Name 116 | Value: !GetAtt CFDistribution.DomainName -------------------------------------------------------------------------------- /templates/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 7 | 54 | 55 | 56 | 57 |
58 |
59 | Preview 60 | 61 | 62 | 63 | 64 | 65 | 83 | 84 | 85 |
86 | 87 |
88 |
89 | 90 | 91 | 92 | -------------------------------------------------------------------------------- /templates/preview.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 7 | 11 | 12 | 13 | 14 | 18 | 19 | {% if request.host == "tiles.rdnt.io" %} 20 | 21 | 22 | 25 | {% endif %} 26 | 35 | 36 | 37 | 38 |
39 | 121 | 122 | 123 | -------------------------------------------------------------------------------- /virtual/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mojodna/marblecutter-virtual/6bb9b7deceae401b8d529b7cc079427a43829c0e/virtual/__init__.py -------------------------------------------------------------------------------- /virtual/catalogs.py: -------------------------------------------------------------------------------- 1 | # coding=utf-8 2 | 3 | import logging 4 | import math 5 | 6 | from marblecutter import Bounds, get_resolution_in_meters, get_source, get_zoom 7 | from marblecutter.catalogs import WGS84_CRS, Catalog 8 | from marblecutter.utils import Source 9 | from rasterio import warp 10 | from rasterio.enums import Resampling 11 | 12 | LOG = logging.getLogger(__name__) 13 | 14 | 15 | class VirtualCatalog(Catalog): 16 | _rgb = None 17 | _nodata = None 18 | _linear_stretch = None 19 | _resample = None 20 | _expr = None 21 | 22 | def __init__(self, uri, rgb=None, nodata=None, linear_stretch=None, resample=None, expr=None): 23 | self._uri = uri 24 | 25 | if rgb: 26 | self._rgb = rgb 27 | 28 | if nodata: 29 | self._nodata = nodata 30 | 31 | if linear_stretch: 32 | self._linear_stretch = linear_stretch 33 | 34 | if expr: 35 | self._expr = expr 36 | 37 | try: 38 | # test whether provided resampling method is valid 39 | Resampling[resample] 40 | self._resample = resample 41 | except KeyError: 42 | self._resample = None 43 | 44 | self._meta = {} 45 | 46 | with get_source(self._uri) as src: 47 | self._bounds = warp.transform_bounds(src.crs, WGS84_CRS, *src.bounds) 48 | self._resolution = get_resolution_in_meters( 49 | Bounds(src.bounds, src.crs), (src.height, src.width) 50 | ) 51 | approximate_zoom = get_zoom(max(self._resolution), op=math.ceil) 52 | 53 | global_min = src.get_tag_item("TIFFTAG_MINSAMPLEVALUE") 54 | global_max = src.get_tag_item("TIFFTAG_MAXSAMPLEVALUE") 55 | 56 | for band in range(0, src.count): 57 | self._meta["values"] = self._meta.get("values", {}) 58 | self._meta["values"][band] = {} 59 | min_val = src.get_tag_item("STATISTICS_MINIMUM", bidx=band + 1) 60 | max_val = src.get_tag_item("STATISTICS_MAXIMUM", bidx=band + 1) 61 | mean_val = src.get_tag_item("STATISTICS_MEAN", bidx=band + 1) 62 | 63 | if min_val is not None: 64 | self._meta["values"][band]["min"] = float(min_val) 65 | elif global_min is not None: 66 | self._meta["values"][band]["min"] = float(global_min) 67 | 68 | if max_val is not None: 69 | self._meta["values"][band]["max"] = float(max_val) 70 | elif global_max is not None: 71 | self._meta["values"][band]["max"] = float(global_max) 72 | 73 | if mean_val is not None: 74 | self._meta["values"][band]["mean"] = float(mean_val) 75 | 76 | self._center = [ 77 | (self._bounds[0] + self.bounds[2]) / 2, 78 | (self._bounds[1] + self.bounds[3]) / 2, 79 | approximate_zoom - 3, 80 | ] 81 | self._maxzoom = approximate_zoom + 3 82 | self._minzoom = approximate_zoom - 10 83 | 84 | @property 85 | def uri(self): 86 | return self._uri 87 | 88 | def get_sources(self, bounds, resolution): 89 | recipes = {"imagery": True} 90 | 91 | if self._rgb is not None: 92 | recipes["rgb_bands"] = map(int, self._rgb.split(",")) 93 | 94 | if self._nodata is not None: 95 | recipes["nodata"] = self._nodata 96 | 97 | if self._linear_stretch is not None: 98 | recipes["linear_stretch"] = "per_band" 99 | 100 | if self._resample is not None: 101 | recipes["resample"] = self._resample 102 | 103 | if self._expr is not None: 104 | recipes["expr"] = self._expr 105 | 106 | yield Source( 107 | url=self._uri, 108 | name=self._name, 109 | resolution=self._resolution, 110 | band_info={}, 111 | meta=self._meta, 112 | recipes=recipes, 113 | ) 114 | -------------------------------------------------------------------------------- /virtual/lambda.py: -------------------------------------------------------------------------------- 1 | # coding=utf-8 2 | import logging 3 | import os 4 | import signal 5 | 6 | from virtual.web import app 7 | import serverless_wsgi 8 | 9 | logging.getLogger("rasterio._base").setLevel(logging.WARNING) 10 | logger = logging.getLogger(__name__) 11 | 12 | 13 | def handler(signum, frame): 14 | logger.error("Request timed out; shutting down to clean up.") 15 | os._exit(0) 16 | 17 | 18 | # Register the signal function handler 19 | signal.signal(signal.SIGALRM, handler) 20 | 21 | 22 | class TimeoutMiddleware: 23 | 24 | def __init__(self, app, timeout): 25 | self.timeout = timeout 26 | self.wrapped_app = app 27 | 28 | def __call__(self, environ, start_response): 29 | # set an interval timer in float seconds 30 | signal.setitimer(signal.ITIMER_REAL, self.timeout / 1000) 31 | try: 32 | return self.wrapped_app(environ, start_response) 33 | finally: 34 | # clear the interval timer 35 | signal.setitimer(signal.ITIMER_REAL, 0) 36 | 37 | 38 | def handle(event, context): 39 | context.get_remaining_time_in_millis() 40 | 41 | # transfer stage from event["requestContext"] to an X-Stage header 42 | event["headers"]["X-Stage"] = event.get("requestContext", {}).pop("stage", None) 43 | event["headers"]["Host"] = event["headers"].get( 44 | "X-Forwarded-Host", event["headers"].get("Host") 45 | ) 46 | 47 | app.wsgi_app = TimeoutMiddleware( 48 | app.wsgi_app, context.get_remaining_time_in_millis() - 50 49 | ) 50 | 51 | return serverless_wsgi.handle_request(app, event, context) 52 | -------------------------------------------------------------------------------- /virtual/web.py: -------------------------------------------------------------------------------- 1 | # coding=utf-8 2 | from __future__ import absolute_import 3 | 4 | import logging 5 | 6 | from cachetools.func import lru_cache 7 | from flask import Flask, Markup, jsonify, redirect, render_template, request 8 | from flask_cors import CORS 9 | from marblecutter import NoCatalogAvailable, tiling 10 | from marblecutter.formats.optimal import Optimal 11 | from marblecutter.transformations import Image 12 | from marblecutter.web import bp, url_for 13 | from mercantile import Tile 14 | 15 | try: 16 | from urllib.parse import urlparse, urlencode 17 | from urllib.request import urlopen, Request 18 | from urllib.error import HTTPError 19 | except ImportError: 20 | from urlparse import urlparse 21 | from urllib import urlencode 22 | from urllib2 import urlopen, Request, HTTPError 23 | 24 | from .catalogs import VirtualCatalog 25 | 26 | LOG = logging.getLogger(__name__) 27 | 28 | IMAGE_TRANSFORMATION = Image() 29 | IMAGE_FORMAT = Optimal() 30 | 31 | app = Flask("marblecutter-virtual") 32 | app.register_blueprint(bp) 33 | app.url_map.strict_slashes = False 34 | CORS(app, send_wildcard=True) 35 | 36 | 37 | @lru_cache() 38 | def make_catalog(args): 39 | if args.get("url", "") == "": 40 | raise NoCatalogAvailable() 41 | 42 | try: 43 | return VirtualCatalog( 44 | args["url"], 45 | rgb=args.get("rgb"), 46 | nodata=args.get("nodata"), 47 | linear_stretch=args.get("linearStretch"), 48 | resample=args.get("resample"), 49 | expr=args.get("expr", None) 50 | ) 51 | except Exception as e: 52 | LOG.exception(e) 53 | raise NoCatalogAvailable() 54 | 55 | 56 | @app.route("/") 57 | def index(): 58 | return (render_template("index.html"), 200, {"Content-Type": "text/html"}) 59 | 60 | 61 | @app.route("/tiles/") 62 | def meta(): 63 | catalog = make_catalog(request.args) 64 | 65 | meta = { 66 | "bounds": catalog.bounds, 67 | "center": catalog.center, 68 | "maxzoom": catalog.maxzoom, 69 | "minzoom": catalog.minzoom, 70 | "name": catalog.name, 71 | "tilejson": "2.1.0", 72 | "tiles": [ 73 | "{}{{z}}/{{x}}/{{y}}?{}".format( 74 | url_for("meta", _external=True, _scheme=""), urlencode(request.args) 75 | ) 76 | ], 77 | } 78 | 79 | return jsonify(meta) 80 | 81 | 82 | @app.route("/bounds/") 83 | def bounds(): 84 | catalog = make_catalog(request.args) 85 | 86 | return jsonify({"url": catalog.uri, "bounds": catalog.bounds}) 87 | 88 | 89 | @app.route("/preview") 90 | def preview(): 91 | try: 92 | # initialize the catalog so this route will fail if the source doesn't exist 93 | make_catalog(request.args) 94 | except Exception: 95 | return redirect(url_for("index"), code=303) 96 | 97 | return ( 98 | render_template( 99 | "preview.html", 100 | tilejson_url=Markup( 101 | url_for("meta", _external=True, _scheme="", **request.args) 102 | ), 103 | source_url=request.args["url"], 104 | ), 105 | 200, 106 | {"Content-Type": "text/html"}, 107 | ) 108 | 109 | 110 | @app.route("/tiles///") 111 | @app.route("/tiles///@x") 112 | def render_png(z, x, y, scale=1): 113 | catalog = make_catalog(request.args) 114 | tile = Tile(x, y, z) 115 | 116 | headers, data = tiling.render_tile( 117 | tile, 118 | catalog, 119 | format=IMAGE_FORMAT, 120 | transformation=IMAGE_TRANSFORMATION, 121 | scale=scale, 122 | ) 123 | 124 | headers.update(catalog.headers) 125 | 126 | return data, 200, headers 127 | --------------------------------------------------------------------------------