├── .editorconfig ├── .gitignore ├── .travis.yml ├── LICENSE ├── README.md ├── assets ├── co_overview.jpg ├── grca.jpg └── rhode_island_footprint.png ├── code ├── fill_mosaic_holes.py ├── naip.py └── overviews.py ├── data ├── naip_2011_2013_mosaic.json.gz ├── naip_2014_2015_mosaic.json.gz ├── naip_2015_2017_mosaic.json.gz └── naip_2016_2018_mosaic.json.gz ├── environment.yml ├── filled ├── naip_2011_2013_mosaic.json.gz ├── naip_2014_2015_mosaic.json.gz ├── naip_2015_2017_mosaic.json.gz ├── naip_2016_2018_mosaic.json.gz ├── naip_overview_2011_2013.json.gz ├── naip_overview_2014_2015.json.gz ├── naip_overview_2015_2017.json.gz └── naip_overview_2016_2018.json.gz ├── package.json ├── serverless.kyle.yml ├── serverless.yml └── site ├── .gitignore ├── README.md ├── package.json ├── public ├── favicon.ico ├── index.html ├── logo192.png ├── logo512.png ├── manifest.json └── share_preview_grca.jpg ├── src ├── App.css ├── App.js ├── App.test.js ├── constants.js ├── index.css ├── index.js ├── info-box.js ├── logo.svg ├── seo.js ├── style.json └── util.js └── yarn.lock /.editorconfig: -------------------------------------------------------------------------------- 1 | # EditorConfig is awesome: https://EditorConfig.org 2 | 3 | # top-most EditorConfig file 4 | root = true 5 | 6 | # Unix-style newlines with a newline ending every file 7 | [*] 8 | end_of_line = lf 9 | insert_final_newline = true 10 | charset = utf-8 11 | indent_style = space 12 | indent_size = 4 13 | trim_trailing_whitespace = true 14 | 15 | [Makefile] 16 | indent_style = tab 17 | 18 | [*.{js,yml,yaml,json}] 19 | indent_style = space 20 | indent_size = 2 21 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | cogeo-mosaic-tiler/ 2 | *.geojson 3 | *.json 4 | tmp 5 | data 6 | *.txt 7 | *.zip 8 | .serverless 9 | 10 | # Byte-compiled / optimized / DLL files 11 | __pycache__/ 12 | *.py[cod] 13 | *$py.class 14 | 15 | # C extensions 16 | *.so 17 | 18 | # Distribution / packaging 19 | .Python 20 | build/ 21 | develop-eggs/ 22 | dist/ 23 | downloads/ 24 | eggs/ 25 | .eggs/ 26 | lib/ 27 | lib64/ 28 | parts/ 29 | sdist/ 30 | var/ 31 | wheels/ 32 | pip-wheel-metadata/ 33 | share/python-wheels/ 34 | *.egg-info/ 35 | .installed.cfg 36 | *.egg 37 | MANIFEST 38 | 39 | # PyInstaller 40 | # Usually these files are written by a python script from a template 41 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 42 | *.manifest 43 | *.spec 44 | 45 | # Installer logs 46 | pip-log.txt 47 | pip-delete-this-directory.txt 48 | 49 | # Unit test / coverage reports 50 | htmlcov/ 51 | .tox/ 52 | .nox/ 53 | .coverage 54 | .coverage.* 55 | .cache 56 | nosetests.xml 57 | coverage.xml 58 | *.cover 59 | *.py,cover 60 | .hypothesis/ 61 | .pytest_cache/ 62 | cover/ 63 | 64 | # Translations 65 | *.mo 66 | *.pot 67 | 68 | # Django stuff: 69 | *.log 70 | local_settings.py 71 | db.sqlite3 72 | db.sqlite3-journal 73 | 74 | # Flask stuff: 75 | instance/ 76 | .webassets-cache 77 | 78 | # Scrapy stuff: 79 | .scrapy 80 | 81 | # Sphinx documentation 82 | docs/_build/ 83 | 84 | # PyBuilder 85 | target/ 86 | 87 | # Jupyter Notebook 88 | .ipynb_checkpoints 89 | 90 | # IPython 91 | profile_default/ 92 | ipython_config.py 93 | 94 | # pyenv 95 | # For a library or package, you might want to ignore these files since the code is 96 | # intended to run in multiple environments; otherwise, check them in: 97 | # .python-version 98 | 99 | # pipenv 100 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 101 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 102 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 103 | # install all needed dependencies. 104 | #Pipfile.lock 105 | 106 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow 107 | __pypackages__/ 108 | 109 | # Celery stuff 110 | celerybeat-schedule 111 | celerybeat.pid 112 | 113 | # SageMath parsed files 114 | *.sage.py 115 | 116 | # Environments 117 | .env 118 | .venv 119 | env/ 120 | venv/ 121 | ENV/ 122 | env.bak/ 123 | venv.bak/ 124 | 125 | # Spyder project settings 126 | .spyderproject 127 | .spyproject 128 | 129 | # Rope project settings 130 | .ropeproject 131 | 132 | # mypy 133 | .mypy_cache/ 134 | .dmypy.json 135 | dmypy.json 136 | 137 | # Pyre type checker 138 | .pyre/ 139 | 140 | # pytype static type analyzer 141 | .pytype/ 142 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: node_js 2 | node_js: 3 | - "10" 4 | branches: 5 | except: 6 | # Built website 7 | - gh_pages 8 | deploy: 9 | provider: script 10 | script: cd site && yarn install && yarn run deploy 11 | skip_cleanup: true 12 | on: 13 | branch: master 14 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2020 Kyle Barron 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # naip-cogeo-mosaic 2 | 3 | Serverless high-resolution NAIP map tiles from Cloud-Optimized GeoTIFFs for the 4 | lower 48 U.S. states. 5 | 6 | [Interactive example][example-website] 7 | 8 | [![](assets/grca.jpg)][example-website] 9 | 10 | 60cm-resolution NAIP imagery of the Grand Canyon from 2017. 11 | 12 | [example-website]: https://kylebarron.dev/naip-cogeo-mosaic 13 | 14 | ## Overview 15 | 16 | The [National Agriculture Imagery Program (NAIP)][naip-info] acquires aerial 17 | imagery during the agricultural growing seasons in the continental U.S. All NAIP 18 | imagery between 2011 and 2018 is stored in an AWS S3 [public dataset][naip-aws], 19 | and crucially the `naip-visualization` bucket stores images in [Cloud-Optimized 20 | GeoTIFF (COG) format][cog-format]. Because this data format supports streaming 21 | portions of the image at a time, it [enables serving a basemap of 22 | imagery][dynamic-map-tiling-blog] on demand without needing to preprocess and 23 | store any imagery. 24 | 25 | This repository is designed to create [MosaicJSON][mosaicjson] files 26 | representing collections of NAIP imagery that can be used with 27 | [`titiler`][titiler] to serve map tiles on demand. 28 | 29 | [naip-info]: https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/ 30 | [naip-aws]: https://registry.opendata.aws/naip/ 31 | [cog-format]: https://www.cogeo.org/ 32 | [dynamic-map-tiling-blog]: https://kylebarron.dev/blog/cog-mosaic/overview 33 | [titiler]: https://github.com/developmentseed/titiler 34 | [mosaicjson]: https://github.com/developmentseed/mosaicjson-spec 35 | 36 | ## Using 37 | 38 | If you'd like to get running quickly, you can use a built mosaicJSON 39 | file in the [`filled/` folder][filled/] and skip down to ["Deploy"](#deploy). 40 | 41 | Otherwise, the following describes how to create a custom mosaicJSON file from 42 | specified years of NAIP imagery available on AWS. 43 | 44 | [filled/]: https://github.com/kylebarron/naip-cogeo-mosaic/tree/master/filled 45 | 46 | ### Install 47 | 48 | Clone the repository and install Python dependencies. 49 | 50 | ``` 51 | git clone https://github.com/kylebarron/naip-cogeo-mosaic 52 | cd naip-cogeo-mosaic 53 | conda env create -f environment.yml 54 | source activate naip-cogeo-mosaic 55 | ``` 56 | 57 | If you prefer using pip, you can run 58 | 59 | ``` 60 | pip install awscli click pygeos 'rio-cogeo>=2.0' 'cogeo-mosaic>=3.0a5' 61 | ``` 62 | 63 | ### Select TIF URLs 64 | 65 | This section outlines methods for selecting files that represent a country-wide 66 | mosaic of NAIP imagery, which can then be put into a MosaicJSON file for 67 | serving. 68 | 69 | Download `manifest.txt`. This file has a listing of all files stored on the 70 | `naip-visualization` bucket. 71 | 72 | ```bash 73 | aws s3 cp s3://naip-visualization/manifest.txt ./manifest.txt --request-payer 74 | ``` 75 | 76 | In the NAIP program, different states are photographed in different years, with 77 | a target of imaging all states within every 3 years. [Here's an interactive 78 | map][naip-years] of when each state was photographed, (though it doesn't appear 79 | to include 2018 yet; [this graphic][naip_coverage_2018] shows which states were 80 | photographed in 2018). 81 | 82 | [naip-years]: https://www.arcgis.com/home/webmap/viewer.html?webmap=17944d45bbef42afb05a5652d7c28aa5 83 | [naip_coverage_2018]: https://www.fsa.usda.gov/Assets/USDA-FSA-Public/usdafiles/APFO/status-maps/pdfs/NAIP_Coverage_2018.pdf 84 | 85 | All (lower 48) states were photographed between 2011-2013, and again in 86 | 2014-2015. All states except Maine were photographed in 2016-2017. All states 87 | except Oregon were photographed in 2017-2018. 88 | 89 | Therefore, I'll generate four MosaicJSONs, with each spanning a range of 90 | 2011-2013, 2014-2015, 2015-2017, and 2016-2018. For the last two, I include an 91 | extra start year just for Maine/Oregon, but set each to use the latest available 92 | imagery, so only Maine takes imagery from 2015 and only Oregon takes imagery 93 | from 2016, respectively. 94 | 95 | The following code block selects imagery for each time span. You can run `python code/naip.py --help` for a full description of available options. 96 | 97 | ```bash 98 | python code/naip.py manifest \ 99 | -s 2011 \ 100 | -e 2013 \ 101 | --select-method last \ 102 | manifest.txt \ 103 | | sed -e 's|^|s3://naip-visualization/|' \ 104 | > urls_2011_2013.txt 105 | python code/naip.py manifest \ 106 | -s 2014 \ 107 | -e 2015 \ 108 | --select-method last \ 109 | manifest.txt \ 110 | | sed -e 's|^|s3://naip-visualization/|' \ 111 | > urls_2014_2015.txt 112 | python code/naip.py manifest \ 113 | -s 2015 \ 114 | -e 2017 \ 115 | --select-method last \ 116 | manifest.txt \ 117 | | sed -e 's|^|s3://naip-visualization/|' \ 118 | > urls_2015_2017.txt 119 | python code/naip.py manifest \ 120 | -s 2016 \ 121 | -e 2018 \ 122 | --select-method last \ 123 | manifest.txt \ 124 | | sed -e 's|^|s3://naip-visualization/|' \ 125 | > urls_2016_2018.txt 126 | ``` 127 | 128 | Each output file includes one filename for each quad identifier, deduplicated 129 | across years. 130 | 131 | ``` 132 | > head -n 5 urls_2016_2018.txt 133 | s3://naip-visualization/al/2017/100cm/rgb/30085/m_3008501_ne_16_1_20171018.tif 134 | s3://naip-visualization/al/2017/100cm/rgb/30085/m_3008501_nw_16_1_20171006.tif 135 | s3://naip-visualization/al/2017/100cm/rgb/30085/m_3008502_ne_16_1_20170909.tif 136 | s3://naip-visualization/al/2017/100cm/rgb/30085/m_3008502_nw_16_1_20170909.tif 137 | s3://naip-visualization/al/2017/100cm/rgb/30085/m_3008503_ne_16_1_20171017.tif 138 | ``` 139 | 140 | Additionally, files along state borders are deduplicated. Often, 141 | cells on state borders are duplicated across years. For example, this image is 142 | duplicated in both Texas's and Louisiana's datasets: 143 | 144 | ``` 145 | tx/2012/100cm/rgb/29093/m_2909302_ne_15_1_20120522.tif 146 | la/2013/100cm/rgb/29093/m_2909302_ne_15_1_20130702.tif 147 | ``` 148 | 149 | As you can tell by the cell and name, these are the same position across 150 | different years. I deduplicate these to reduce load on the lambda function 151 | parsing the mosaicJSON. 152 | 153 | To visualize the quads covered by a list of urls, you can visualize its 154 | footprint. For example, to get the mosaic footprint of Rhode Island from the 155 | 2011-2013 mosaic: 156 | 157 | ```bash 158 | export AWS_REQUEST_PAYER="requester" 159 | cat urls_2011_2013.txt \ 160 | | grep "^s3://naip-visualization/ri/" \ 161 | | cogeo-mosaic footprint - > footprint.geojson 162 | ``` 163 | 164 | And inspect it with [kepler.gl](https://github.com/kylebarron/keplergl_cli): 165 | 166 | ```bash 167 | kepler footprint.geojson 168 | ``` 169 | 170 | 171 | 172 | Here you can see that the tiles to be used in the mosaic of Rhode Island don't 173 | include the state's border. That's because the Python script to parse the 174 | manifest deduplicates tiles on the border when they're include in both states. 175 | If you looked at the footprint of Connecticut, you'd see the missing tiles on 176 | the border. 177 | 178 | Total number of files 179 | 180 | ```bash 181 | > wc -l urls_2011_2013.txt 182 | 213197 urls_2011_2013.txt 183 | ``` 184 | 185 | ### Create MosaicJSON 186 | 187 | NAIP imagery tiffs are in a requester pays bucket. In order to access them, you 188 | need to set the `AWS_REQUEST_PAYER` environment variable: 189 | 190 | ```bash 191 | export AWS_REQUEST_PAYER="requester" 192 | ``` 193 | 194 | I also found that on an AWS EC2 instance; `cogeo-mosaic create` was failing 195 | while it was working on my local computer. In general, if `cogeo-mosaic create` 196 | isn't working for some URL; you should run `rio info ` and see what the 197 | error is, since `cogeo-mosaic` uses `rasterio` internally, but doesn't currently 198 | print `rasterio` errors to stdout. In my case, I had to set the certificates 199 | path (see 200 | [cogeotiff/rio-tiler#19](https://github.com/cogeotiff/rio-tiler/issues/19), 201 | [mapbox/rasterio#942](https://github.com/mapbox/rasterio/issues/942)). 202 | 203 | ```bash 204 | export CURL_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt 205 | ``` 206 | 207 | I don't know how much data `cogeo-mosaic create` downloads (it only requests the 208 | GeoTIFF headers of each file), but it might be wise to run the mosaic creation 209 | on an AWS EC2 instance in the `us-west-2` region (the same region where the NAIP 210 | imagery is located), so that you don't have to pay for egress bandwidth on the 211 | requests. I found that creating the mosaic took about 1.5GB of memory; it 212 | finished in about 7 hours per mosaic on a `t2.small` instance. 213 | 214 | Then create the MosaicJSON file. GET requests are priced at `$0.0004` per 1000 215 | requests, so creating the MosaicJSON should cost `0.0004 * (200000 / 1000) = 0.08`. 8 cents! 216 | 217 | ```bash 218 | cat urls_2011_2013.txt \ 219 | | cogeo-mosaic create - \ 220 | > naip_2011_2013_mosaic.json 221 | cat urls_2014_2015.txt \ 222 | | cogeo-mosaic create - \ 223 | > naip_2014_2015_mosaic.json 224 | cat urls_2015_2017.txt \ 225 | | cogeo-mosaic create - \ 226 | > naip_2015_2017_mosaic.json 227 | cat urls_2016_2018.txt \ 228 | | cogeo-mosaic create - \ 229 | > naip_2016_2018_mosaic.json 230 | ``` 231 | 232 | #### Fill in missing quadkeys 233 | 234 | Some of these years have small missing areas. For example, in some years parts 235 | of Montana weren't photographed. `fill_mosaic_holes.py` is a simple script to fill mosaic quadkeys across years. 236 | 237 | This following tells the script to look at all the mosaics `data/*.json` and 238 | create new filled scripts output to the `filled/` folder. 239 | 240 | ```py 241 | python code/fill_mosaic_holes.py -o filled data/naip_201*.json 242 | ``` 243 | 244 | Note that this fills in entire quadkeys that are missing in one year but that 245 | exist in another. _However_, if a year is missing some areas, there will be 246 | quadkeys that _exist_ but only have _partial_ data. So without more effort there 247 | can still be some small holes in the data. See [issue #8][issue-8]. 248 | 249 | [issue-8]: https://github.com/kylebarron/naip-cogeo-mosaic/issues/8 250 | 251 | ## Deploy 252 | 253 | The older [`cogeo-mosaic-tiler`][cogeo-mosaic-tiler] is being deprecated in 254 | favor of the newer, more stable [`titiler`][titiler]. Refer to [`titiler`'s 255 | documentation][titiler-docs] for deployment instructions. **Note that since NAIP 256 | images are in a requester-pays bucket, you'll need to set 257 | `AWS_REQUEST_PAYER="requester"` in the environment.** 258 | 259 | [cogeo-mosaic-tiler]: https://github.com/developmentseed/cogeo-mosaic-tiler 260 | [titiler-docs]: https://developmentseed.org/titiler/ 261 | 262 | ### Upload MosaicJSON files 263 | 264 | In order for `titiler` to create your tiles on demand, it needs to access a 265 | MosaicJSON file, which you need to host somewhere accessible by `titiler` 266 | (preferably in the same AWS region). 267 | 268 | Generally the simplest method is uploading the JSON file to S3. However since 269 | these files are so large (~64MB uncompressed), I found that it was taking 2.5s 270 | to load and parse the JSON. 271 | 272 | As of v3, `cogeo-mosaic` (and thus also `titiler`) support alternate _backends_, 273 | such as [DynamoDB][dynamodb]. DynamoDB is a serverless database that makes 274 | loading the MosaicJSON fast, because the tiler only needs one or two reads, 275 | which each take around 10ms (as long as the DynamoDB table is in the same region 276 | as the tiler). 277 | 278 | [dynamodb]: https://aws.amazon.com/dynamodb/ 279 | 280 | For full backend docs, see [`cogeo-mosaic`'s documentation][cogeo-mosaic-docs]. 281 | 282 | [cogeo-mosaic-docs]: https://developmentseed.org/cogeo-mosaic/ 283 | 284 | #### DynamoDB 285 | 286 | If you wish to connect the tiler to one or more DynamoDB tables, you need to deploy with 287 | 288 | ```bash 289 | sls deploy --bucket your-mosaic-bucket --aws-account-id your-aws-account-id 290 | ``` 291 | 292 | You can find your AWS account ID with 293 | 294 | ```bash 295 | aws sts get-caller-identity 296 | ``` 297 | 298 | To upload a MosaicJSON to DynamoDB, run: 299 | 300 | ```bash 301 | pip install -U cogeo-mosaic>=3.0a5 302 | cogeo-mosaic upload --url 'dynamodb://{aws_region}/{table_name}' mosaic.json 303 | ``` 304 | 305 | That uploads the MosaicJSON to the given table in the specified region, creating 306 | the table if necessary. 307 | 308 | ### Proxy your endpoint with Cloudflare 309 | 310 | I like to proxy through Cloudflare to take advantage of their free caching. You can read my blog post [here][cloudflare_caching] to see how to do that. 311 | 312 | [cloudflare_caching]: https://kylebarron.dev/blog/caching-lambda-functions-cloudflare 313 | 314 | ## Low Zoom Overviews 315 | 316 | The NAIP imagery hosted on AWS in the `naip-visualization` bucket has 317 | full-resolution imagery plus 5 levels of internal overviews within each GeoTIFF. 318 | That means that it's fast to read image data for 6 or 7 zoom levels, but will 319 | slow down as you zoom out, since you'll necessarily need to read and combine 320 | many images, and perform downsampling on the fly. 321 | 322 | The native zoom range for source NAIP COG imagery is roughly 12-18. That means 323 | that for one zoom 6 tile, you'd have to combine imagery for 4^6 = 4,096 COGs on 324 | the fly. 325 | 326 | A way to solve this issue is to pregenerate lower zoom overviews given a mosaic. 327 | This removes some flexibility, as you have to choose upfront how to combine the 328 | higher-resolution images, but enables performant serving of lower-resolution 329 | imagery. 330 | 331 | **If you don't want to generate your own overviews, pregenerated overviews are 332 | available in a requester pays bucket of mine, and overview mosaics are 333 | available in the [`filled/`](filled/) directory.** My overview COGs are 334 | available at `s3://rp.kylebarron.dev/cog/naip/deflate/`. 335 | 336 | This section outlines how to create these overviews, which are themselves COGs. 337 | After creating the overviews, you'll have fast low-zoom imagery, as you can see 338 | in this screenshot of Colorado. 339 | 340 | [![](./assets/co_overview.jpg)](https://kylebarron.dev/naip-cogeo-mosaic/#6.75/39.02/-105.323) 341 | 342 | **Note, this code is experimental.** 343 | 344 | I want to have a seamless downsampled map. To do this I'll create overview COGs, 345 | and then create _another_ mosaic from these downsampled images. For zooms 12 and 346 | up, the map will fetch images using the full-resolution mosaic; for zooms 6-12, 347 | the map will use the lower-resolution mosaic. 348 | 349 | To make this mosaic, I'll create a new overview COG for each zoom 6 mercator 350 | tile. Then within a zoom 6 tile, only one source COG will need to be read. 351 | 352 | First, split the large, U.S.-wide mosaic into mosaics for each zoom 6 quadkey, 353 | using a script in `code/overviews.py`. 354 | 355 | ```bash 356 | python code/overviews.py split-mosaic \ 357 | -z 6 \ 358 | -o overview-mosaics \ 359 | --prefix naip_2011_2013_ \ 360 | filled/naip_2011_2013_mosaic.json 361 | python code/overviews.py split-mosaic \ 362 | -z 6 \ 363 | -o overview-mosaics \ 364 | --prefix naip_2014_2015_ \ 365 | filled/naip_2014_2015_mosaic.json 366 | python code/overviews.py split-mosaic \ 367 | -z 6 \ 368 | -o overview-mosaics \ 369 | --prefix naip_2015_2017_ \ 370 | filled/naip_2015_2017_mosaic.json 371 | python code/overviews.py split-mosaic \ 372 | -z 6 \ 373 | -o overview-mosaics \ 374 | --prefix naip_2016_2018_ \ 375 | filled/naip_2016_2018_mosaic.json 376 | ``` 377 | 378 | This creates about 50 temporary mosaics for each country-wide mosaic, and just 379 | over 200 total. Each of these mosaics defines the source input imagery necessary 380 | to create _one_ overview COG. 381 | 382 | Then we need a loop that will create an overview image for each of the above 383 | temporary mosaics. This step uses the `overview` code in 384 | [`cogeo-mosaic`][cogeo-mosaic]. 385 | 386 | [cogeo-mosaic]: https://github.com/developmentseed/cogeo-mosaic 387 | 388 | **I highly recommend to do this step on an EC2 instance in the same region as 389 | the NAIP data (`us-west-2`).** Since NAIP data is in a requester pays bucket, 390 | if you do this step elsewhere, you'll have to pay egress fees for intermediate 391 | imagery taken out of the region. Additionally, this code is not yet memory- 392 | optimized, so you may need an instance with a good amount of memory. 393 | 394 | ```bash 395 | # NAIP imagery is in a requester pays bucket 396 | export AWS_REQUEST_PAYER="requester" 397 | # Necessary on an EC2 instance 398 | export CURL_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt 399 | # Necessary to prevent expensive, unnecessary S3 LIST requests! 400 | export GDAL_DISABLE_READDIR_ON_OPEN=EMPTY_DIR 401 | mkdir out && cd out 402 | python ../code/overviews.py create-overviews \ 403 | `# Number of processes to use in parallel` \ 404 | -j 2 \ 405 | `# Input directory of temporary mosaics created above` \ 406 | `# Output dir is current directory` \ 407 | ../overview-mosaics 408 | ``` 409 | 410 | The output images _should_ be in COG already, but to make sure I'll use 411 | [`rio-cogeo`][rio-cogeo] to convert them before final storage on S3. You should 412 | use a `blocksize` and `overview-blocksize` that matches the size of imagery you 413 | use in your website. 414 | 415 | [rio-cogeo]: https://github.com/cogeotiff/rio-cogeo 416 | 417 | ```bash 418 | mkdir ../cog_deflate/ 419 | for file in $(ls *.tif); do 420 | rio cogeo create \ 421 | --blocksize 256 \ 422 | --overview-blocksize 256 \ 423 | $file ../cog_deflate/$file; 424 | done 425 | ``` 426 | 427 | Then upload your output images to an S3 bucket of yours, and finally, create new 428 | overview mosaics from these images. For full information, see its documentation 429 | (`code/overviews.py create-overview-mosaic --help`). As input you should pass a 430 | list of S3 urls to your overview COG files you just created. Don't rename the 431 | filenames before passing to the script. 432 | -------------------------------------------------------------------------------- /assets/co_overview.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kylebarron/naip-cogeo-mosaic/7298ed3797fe609d4c95f5adcb989b4ca8f9ffb7/assets/co_overview.jpg -------------------------------------------------------------------------------- /assets/grca.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kylebarron/naip-cogeo-mosaic/7298ed3797fe609d4c95f5adcb989b4ca8f9ffb7/assets/grca.jpg -------------------------------------------------------------------------------- /assets/rhode_island_footprint.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kylebarron/naip-cogeo-mosaic/7298ed3797fe609d4c95f5adcb989b4ca8f9ffb7/assets/rhode_island_footprint.png -------------------------------------------------------------------------------- /code/fill_mosaic_holes.py: -------------------------------------------------------------------------------- 1 | """ 2 | fill_mosaic_holes.py: Make sure all mosaics have the same keys 3 | """ 4 | 5 | import json 6 | from pathlib import Path 7 | 8 | import click 9 | 10 | 11 | @click.command() 12 | @click.option( 13 | '-o', 14 | '--out-dir', 15 | type=click.Path(dir_okay=True, file_okay=False, writable=True), 16 | help='Output directory for filled mosaics.') 17 | @click.argument('input', nargs=-1, type=click.Path(exists=True)) 18 | def main(input, out_dir): 19 | """Fill mosaic holes using other mosaics. 20 | 21 | If a quadkey exists in one MosaicJSON input but not in another, the JSON 22 | with missing data will be filled by another mosaic where the key exists. If 23 | only one mosaic is passed as input, it will be unchanged since there are no 24 | other mosaics to fill from. 25 | 26 | The order of input paths is the same order used for filling values of 27 | missing keys from other mosaics. 28 | """ 29 | mosaics = [] 30 | for path in input: 31 | with open(path) as f: 32 | mosaics.append(json.load(f)) 33 | 34 | mosaics = handle_mosaics(mosaics) 35 | for path, mosaic in zip(input, mosaics): 36 | Path(out_dir).mkdir(exist_ok=True, parents=True) 37 | with open(Path(out_dir) / Path(path).name, 'w') as f: 38 | json.dump(mosaic, f, separators=(',', ':')) 39 | 40 | 41 | def handle_mosaics(mosaics): 42 | """Fill gaps in mosaics 43 | 44 | If a quadkey is missing in one mosaic but exists in others, fill it in. 45 | 46 | Args: 47 | - mosaics: should be ordered from newest to oldest, so that the newest 48 | imagery is filled in. 49 | """ 50 | 51 | # Find all quadkeys 52 | quadkeys = set() 53 | for mosaic in mosaics: 54 | quadkeys.update(mosaic['tiles'].keys()) 55 | 56 | for mosaic in mosaics: 57 | for quadkey in quadkeys: 58 | if quadkey in mosaic['tiles'].keys(): 59 | continue 60 | 61 | # Find value in some other mosaic 62 | for m in mosaics: 63 | if quadkey in m['tiles'].keys(): 64 | mosaic['tiles'][quadkey] = m['tiles'][quadkey] 65 | 66 | # Make sure all mosaics have the same number of keys 67 | n_keys = [] 68 | for mosaic in mosaics: 69 | n_keys.append(len(mosaic['tiles'].keys())) 70 | 71 | msg = 'mosaics have different numbers of quadkeys' 72 | assert all(x == n_keys[0] for x in n_keys), msg 73 | 74 | return mosaics 75 | 76 | 77 | if __name__ == '__main__': 78 | main() 79 | -------------------------------------------------------------------------------- /code/naip.py: -------------------------------------------------------------------------------- 1 | """ 2 | Work with NAIP imagery and metadata 3 | """ 4 | import click 5 | 6 | 7 | @click.group() 8 | def main(): 9 | pass 10 | 11 | 12 | @main.command() 13 | @click.option( 14 | '-s', 15 | '--start-year', 16 | type=int, 17 | required=True, 18 | help='Start imagery year, from 2011-2018') 19 | @click.option( 20 | '-e', 21 | '--end-year', 22 | type=int, 23 | required=True, 24 | help='End imagery year, from 2011-2018') 25 | @click.option( 26 | '--select-method', 27 | type=click.Choice(['first', 'last'], case_sensitive=False), 28 | required=False, 29 | default='last', 30 | show_default=True, 31 | help='image selection method') 32 | @click.argument('file', type=click.File()) 33 | def manifest(start_year, end_year, select_method, file): 34 | """Select TIF URLs from manifest 35 | 36 | All states were photographed between 2011-2013, and again in 2014-2015. All 37 | states except Maine were photographed in 2016-2017. All states except Oregon 38 | were photographed in 2017-2018. 39 | """ 40 | if not 2011 <= start_year <= 2018: 41 | raise ValueError('start_year must be between 2011-2018') 42 | if not 2011 <= end_year <= 2018: 43 | raise ValueError('end_year must be between 2011-2018') 44 | 45 | skip_lines = ['manifest.txt', 'readme.html', 'readme.txt'] 46 | lines = [] 47 | for line in file: 48 | line = line.strip() 49 | 50 | if line in skip_lines: 51 | continue 52 | 53 | lines.append(line) 54 | 55 | state_years = {} 56 | for line in lines: 57 | state, year = line.split('/')[:2] 58 | state_years[state] = state_years.get(state, set()) 59 | state_years[state].add(int(year)) 60 | 61 | # (state_abbr, year) 62 | combos = [] 63 | for state, years in state_years.items(): 64 | match_years = [y for y in years if start_year <= y <= end_year] 65 | if match_years: 66 | if select_method == 'first': 67 | combos.append((state, str(min(match_years)))) 68 | elif select_method == 'last': 69 | combos.append((state, str(max(match_years)))) 70 | else: 71 | raise ValueError('invalid select_method') 72 | 73 | # (al, 2011) -> al/2011 74 | match_strs = ['/'.join(c) for c in combos] 75 | matched_lines = [ 76 | l for l in lines 77 | if l.endswith('.tif') and any(l.startswith(s) for s in match_strs)] 78 | 79 | # Deduplicate 80 | for l in deduplicate_urls(lines=matched_lines, select_method=select_method): 81 | print(l) 82 | 83 | 84 | def deduplicate_urls(lines, select_method): 85 | """Deduplicate urls by cell 86 | 87 | Often, cells on state borders are duplicated across years. For example, this tile is duplicated in both Texas's and Louisiana's datasets: 88 | 89 | tx/2012/100cm/rgb/29093/m_2909302_ne_15_1_20120522.tif 90 | la/2013/100cm/rgb/29093/m_2909302_ne_15_1_20130702.tif 91 | 92 | As you can tell by the cell and name, these are the same position across 93 | different years. I deduplicate these to reduce load on the lambda function 94 | parsing the mosaicJSON. 95 | """ 96 | # block: (year, url) 97 | data = {} 98 | for line in lines: 99 | block_id = name_to_id(line) 100 | year = name_to_year(line) 101 | 102 | existing = data.get(block_id) 103 | if existing is None: 104 | # Not a duplicate; add and continue 105 | data[block_id] = (year, line) 106 | continue 107 | 108 | # A duplicate; check whether to replace existing 109 | existing_year = existing[0] 110 | if select_method == 'last': 111 | if year > existing_year: 112 | data[block_id] = (year, line) 113 | continue 114 | 115 | elif select_method == 'first': 116 | if year < existing_year: 117 | data[block_id] = (year, line) 118 | continue 119 | 120 | else: 121 | raise ValueError('invalid select_method') 122 | 123 | return [t[1] for t in data.values()] 124 | 125 | 126 | def name_to_year(s): 127 | return int(s.split('/')[1]) 128 | 129 | 130 | def name_to_id(s): 131 | """Generate block identifier from url string 132 | 133 | Example input: 134 | 135 | al/2013/100cm/rgb/30085/m_3008501_ne_16_1_20130928.tif 136 | """ 137 | # cell: 30085 138 | # stem: m_3008501_ne_16_1_20130928.tif 139 | cell, stem = s.split('/')[4:] 140 | 141 | # Keep text before date 142 | # m_3008501_ne_16_1_ 143 | stem = stem[:18] 144 | 145 | # Concatenate 146 | return f'{cell}/{stem}' 147 | 148 | 149 | if __name__ == '__main__': 150 | main() 151 | -------------------------------------------------------------------------------- /code/overviews.py: -------------------------------------------------------------------------------- 1 | import json 2 | import os 3 | import re 4 | from copy import deepcopy 5 | from functools import partial 6 | from multiprocessing import Pool 7 | from pathlib import Path 8 | 9 | import click 10 | import mercantile 11 | from rasterio.rio import options 12 | from rio_cogeo.profiles import cog_profiles 13 | 14 | from cogeo_mosaic.mosaic import MosaicJSON 15 | from cogeo_mosaic.overviews import create_low_level_cogs 16 | 17 | 18 | @click.group() 19 | def main(): 20 | pass 21 | 22 | 23 | @click.command() 24 | @click.option( 25 | '-o', 26 | '--outdir', 27 | type=click.Path(dir_okay=True, file_okay=False, writable=True), 28 | help='Output directory for mosaics') 29 | @click.option( 30 | '-z', 31 | '--overview-zoom', 32 | type=int, 33 | help='Overview zoom level', 34 | default=6, 35 | show_default=True) 36 | @click.option( 37 | '--prefix', default='', required=False, help='Prefix for output Mosaics') 38 | @click.argument('mosaic', type=click.File()) 39 | def split_mosaic(outdir, overview_zoom, mosaic, prefix): 40 | """Split full mosaic into overview mercator-aligned mosaics 41 | """ 42 | # Input mosaic is an opened file 43 | mosaic = json.load(mosaic) 44 | 45 | outdir = Path(outdir) 46 | outdir.mkdir(exist_ok=True, parents=True) 47 | 48 | quadkeys = list(mosaic['tiles'].keys()) 49 | 50 | overview_quadkeys = {qk[:overview_zoom] for qk in quadkeys} 51 | 52 | for overview_qk in overview_quadkeys: 53 | overview_mosaic = subset_mosaic(mosaic, overview_qk, overview_zoom) 54 | out_path = outdir / f'{prefix}{overview_qk}.json' 55 | 56 | with open(out_path, 'w') as f: 57 | json.dump(overview_mosaic.dict(exclude_none=True), f) 58 | 59 | 60 | def subset_mosaic(mosaic, overview_qk, overview_zoom): 61 | """Create subset of mosaic within a single overview quadkey 62 | 63 | Args: 64 | - overview_qk: zoom 6 quadkey 65 | """ 66 | qk_tiles = { 67 | k: v 68 | for k, v in mosaic['tiles'].items() 69 | if k[:overview_zoom] == overview_qk} 70 | bounds = mercantile.bounds(mercantile.quadkey_to_tile(overview_qk)) 71 | 72 | # The new mosaic needs to be the same minzoom, quadkey zoom as 73 | new_mosaic = deepcopy(mosaic) 74 | new_mosaic['tiles'] = qk_tiles 75 | new_mosaic['bounds'] = bounds 76 | return MosaicJSON(**new_mosaic) 77 | 78 | 79 | @click.command() 80 | @click.argument("input_dir", type=click.Path()) 81 | @click.option( 82 | '-j', '--n-proc', type=int, default=4, help='# of processes in pool') 83 | @click.option( 84 | "--cog-profile", 85 | "-p", 86 | type=click.Choice(cog_profiles.keys()), 87 | default="deflate", 88 | help="Cloud Optimized GeoTIFF profile (default: deflate).", 89 | ) 90 | @click.option( 91 | "--overview-level", 92 | type=int, 93 | default=6, 94 | help="Max internal overivew level for the COG. " 95 | f"Will be used to get the size of each COG. Default is {256 * 2 **6}", 96 | ) 97 | @options.creation_options 98 | def create_overviews( 99 | input_dir, n_proc, cog_profile, overview_level, creation_options): 100 | 101 | files = [x for x in Path(input_dir).iterdir() if x.suffix == '.json'] 102 | 103 | output_profile = cog_profiles.get(cog_profile) 104 | output_profile.update(dict(BIGTIFF=os.getenv("BIGTIFF", "IF_SAFER"))) 105 | if creation_options: 106 | output_profile.update(creation_options) 107 | 108 | config = dict( 109 | GDAL_NUM_THREADS="ALL_CPU", 110 | GDAL_TIFF_INTERNAL_MASK=os.getenv("GDAL_TIFF_INTERNAL_MASK", True), 111 | GDAL_TIFF_OVR_BLOCKSIZE="128", 112 | ) 113 | 114 | _create_overview = partial( 115 | create_overview, 116 | output_profile=output_profile, 117 | config=config, 118 | overview_level=overview_level) 119 | 120 | with Pool(n_proc) as p: 121 | p.map(_create_overview, files) 122 | 123 | 124 | def create_overview(file, output_profile, config, overview_level): 125 | create_low_level_cogs( 126 | str(file), 127 | output_profile=output_profile, 128 | prefix=Path(file).stem, 129 | max_overview_level=overview_level, 130 | config=config, 131 | threads=1, 132 | ) 133 | 134 | 135 | @click.command() 136 | @click.argument("urls", type=click.File()) 137 | @click.option( 138 | '--quadkey-zoom', 139 | type=int, 140 | default=6, 141 | show_default=True, 142 | help='Quadkey zoom level for overview') 143 | @click.option( 144 | '--min-zoom', 145 | type=int, 146 | default=6, 147 | show_default=True, 148 | help='Min zoom level for overview') 149 | @click.option( 150 | '--max-zoom', 151 | type=int, 152 | default=11, 153 | show_default=True, 154 | help='Max zoom level for overview') 155 | def create_overview_mosaic(urls, quadkey_zoom, min_zoom, max_zoom): 156 | """Create mosaic representing overview 157 | """ 158 | # Input is file object 159 | urls = [l.strip() for l in urls.readlines()] 160 | 161 | quadkeys = [parse_url(url, quadkey_zoom) for url in urls] 162 | 163 | # Find bounds of quadkeys 164 | bboxes = [ 165 | mercantile.bounds(mercantile.quadkey_to_tile(qk)) for qk in quadkeys] 166 | minx = min(bboxes, key=lambda bbox: bbox[0])[0] 167 | miny = min(bboxes, key=lambda bbox: bbox[1])[1] 168 | maxx = max(bboxes, key=lambda bbox: bbox[2])[2] 169 | maxy = max(bboxes, key=lambda bbox: bbox[3])[3] 170 | bounds = [minx, miny, maxx, maxy] 171 | 172 | # Find center 173 | center = [(minx + maxx) / 2, (miny + maxy) / 2, min_zoom] 174 | 175 | tiles = {} 176 | for qk, url in zip(quadkeys, urls): 177 | tiles[qk] = [url] 178 | 179 | mosaic = { 180 | "mosaicjson": "0.0.2", 181 | "minzoom": min_zoom, 182 | "maxzoom": max_zoom, 183 | "quadkey_zoom": 6, 184 | "bounds": bounds, 185 | "center": center, 186 | "tiles": tiles} 187 | 188 | # Validation 189 | mosaic = MosaicJSON(**mosaic).dict(exclude_none=True) 190 | print(json.dumps(mosaic, separators=(',', ':'))) 191 | 192 | 193 | def parse_url(url, quadkey_zoom): 194 | """Parse quadkey from url 195 | """ 196 | name = url.split('/')[-1] 197 | match = re.findall(r'([0-3]{{{0}}})'.format(quadkey_zoom), name) 198 | qk = match[-1] 199 | return qk 200 | 201 | 202 | main.add_command(split_mosaic) 203 | main.add_command(create_overviews) 204 | main.add_command(create_overview_mosaic) 205 | 206 | if __name__ == '__main__': 207 | main() 208 | -------------------------------------------------------------------------------- /data/naip_2011_2013_mosaic.json.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kylebarron/naip-cogeo-mosaic/7298ed3797fe609d4c95f5adcb989b4ca8f9ffb7/data/naip_2011_2013_mosaic.json.gz -------------------------------------------------------------------------------- /data/naip_2014_2015_mosaic.json.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kylebarron/naip-cogeo-mosaic/7298ed3797fe609d4c95f5adcb989b4ca8f9ffb7/data/naip_2014_2015_mosaic.json.gz -------------------------------------------------------------------------------- /data/naip_2015_2017_mosaic.json.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kylebarron/naip-cogeo-mosaic/7298ed3797fe609d4c95f5adcb989b4ca8f9ffb7/data/naip_2015_2017_mosaic.json.gz -------------------------------------------------------------------------------- /data/naip_2016_2018_mosaic.json.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kylebarron/naip-cogeo-mosaic/7298ed3797fe609d4c95f5adcb989b4ca8f9ffb7/data/naip_2016_2018_mosaic.json.gz -------------------------------------------------------------------------------- /environment.yml: -------------------------------------------------------------------------------- 1 | name: naip-cogeo-mosaic 2 | channels: 3 | - conda-forge 4 | dependencies: 5 | - awscli 6 | - click 7 | - pygeos 8 | - pip 9 | - pip: 10 | - cogeo-mosaic>=3.0a5 11 | - rio-cogeo>=2.0a4 12 | -------------------------------------------------------------------------------- /filled/naip_2011_2013_mosaic.json.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kylebarron/naip-cogeo-mosaic/7298ed3797fe609d4c95f5adcb989b4ca8f9ffb7/filled/naip_2011_2013_mosaic.json.gz -------------------------------------------------------------------------------- /filled/naip_2014_2015_mosaic.json.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kylebarron/naip-cogeo-mosaic/7298ed3797fe609d4c95f5adcb989b4ca8f9ffb7/filled/naip_2014_2015_mosaic.json.gz -------------------------------------------------------------------------------- /filled/naip_2015_2017_mosaic.json.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kylebarron/naip-cogeo-mosaic/7298ed3797fe609d4c95f5adcb989b4ca8f9ffb7/filled/naip_2015_2017_mosaic.json.gz -------------------------------------------------------------------------------- /filled/naip_2016_2018_mosaic.json.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kylebarron/naip-cogeo-mosaic/7298ed3797fe609d4c95f5adcb989b4ca8f9ffb7/filled/naip_2016_2018_mosaic.json.gz -------------------------------------------------------------------------------- /filled/naip_overview_2011_2013.json.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kylebarron/naip-cogeo-mosaic/7298ed3797fe609d4c95f5adcb989b4ca8f9ffb7/filled/naip_overview_2011_2013.json.gz -------------------------------------------------------------------------------- /filled/naip_overview_2014_2015.json.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kylebarron/naip-cogeo-mosaic/7298ed3797fe609d4c95f5adcb989b4ca8f9ffb7/filled/naip_overview_2014_2015.json.gz -------------------------------------------------------------------------------- /filled/naip_overview_2015_2017.json.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kylebarron/naip-cogeo-mosaic/7298ed3797fe609d4c95f5adcb989b4ca8f9ffb7/filled/naip_overview_2015_2017.json.gz -------------------------------------------------------------------------------- /filled/naip_overview_2016_2018.json.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kylebarron/naip-cogeo-mosaic/7298ed3797fe609d4c95f5adcb989b4ca8f9ffb7/filled/naip_overview_2016_2018.json.gz -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "serverless-aerial-imagery", 3 | "version": "0.1.0", 4 | "description": ".", 5 | "main": "index.js", 6 | "scripts": { 7 | "test": "echo \"no test specified\"" 8 | }, 9 | "repository": { 10 | "type": "git", 11 | "url": "git+https://github.com/kylebarron/serverless-aerial-imagery.git" 12 | }, 13 | "author": "Kyle Barron ", 14 | "license": "MIT", 15 | "bugs": { 16 | "url": "https://github.com/kylebarron/serverless-aerial-imagery/issues" 17 | }, 18 | "homepage": "https://github.com/kylebarron/serverless-aerial-imagery#readme" 19 | } -------------------------------------------------------------------------------- /serverless.kyle.yml: -------------------------------------------------------------------------------- 1 | service: naip-cogeo-mosaic 2 | 3 | provider: 4 | name: aws 5 | runtime: python3.7 6 | stage: ${opt:stage, 'production'} 7 | region: "us-west-2" 8 | 9 | deploymentBucket: ${opt:bucket, 'mosaics-us-west-2.kylebarron.dev'} 10 | httpApi: 11 | cors: 12 | allowedOrigins: 13 | # You can't do http://localhost:* and http://localhost doesn't work 14 | - http://localhost:3000 15 | - http://localhost:3001 16 | - http://localhost:3002 17 | - http://localhost:3003 18 | - http://localhost:3004 19 | - http://localhost:3005 20 | - http://localhost:3006 21 | - http://localhost:8000 22 | - http://localhost:8001 23 | - http://localhost:8002 24 | - http://localhost:8003 25 | - http://localhost:8004 26 | - http://localhost:8005 27 | - http://localhost:8006 28 | - http://localhost:8080 29 | - http://localhost:8081 30 | - http://localhost:8082 31 | - http://localhost:8083 32 | - http://localhost:8084 33 | - https://all-transit.com 34 | - https://kylebarron.dev 35 | - https://kylebarron.github.io 36 | - https://landsat3d.com 37 | - https://landsat8.earth 38 | - https://nst.guide 39 | - https://nstguide.com 40 | - https://sentinel2.earth 41 | - https://trails3d.com 42 | allowedHeaders: 43 | - Authorization 44 | - Content-Type 45 | - X-Amz-Date 46 | - X-Amz-Security-Token 47 | - X-Amz-User-Agent 48 | - X-Api-Key 49 | allowedMethods: 50 | - GET 51 | - OPTIONS 52 | exposedResponseHeaders: 53 | # Source asset strings per tile 54 | - X-ASSETS 55 | maxAge: 6000 # In seconds 56 | 57 | # Add Tags to resources 58 | stackTags: 59 | Project: naip-cogeo-mosaic 60 | 61 | apiGateway: 62 | binaryMediaTypes: 63 | - "*/*" 64 | minimumCompressionSize: 1 65 | 66 | # Add other buckets here if needed 67 | iamRoleStatements: 68 | - Effect: "Allow" 69 | Action: 70 | - "s3:GetObject" 71 | - "s3:HeadObject" 72 | - "s3:PutObject" 73 | Resource: 74 | - "arn:aws:s3:::${self:provider.deploymentBucket}*" 75 | 76 | - Effect: "Allow" 77 | Action: 78 | - "s3:GetObject" 79 | - "s3:HeadObject" 80 | Resource: 81 | - "arn:aws:s3:::*" 82 | 83 | - Effect: "Allow" 84 | Action: 85 | - "dynamodb:GetItem" 86 | Resource: 87 | # Allow access to all dynamodb tables in region 88 | - "arn:aws:dynamodb:${self:provider.region}:961053664803:table/*" 89 | 90 | package: 91 | artifact: cogeo-mosaic-tiler/package.zip 92 | 93 | functions: 94 | app: 95 | handler: cogeo_mosaic_tiler.handlers.app.app 96 | memorySize: 1536 97 | timeout: 8 98 | layers: 99 | - arn:aws:lambda:${self:provider.region}:524387336408:layer:gdal24-py37-geolayer:1 100 | environment: 101 | # Necessary since NAIP bucket is requester-pays 102 | AWS_REQUEST_PAYER: requester 103 | # Default: One week cache control, one week stale while revalidate 104 | CACHE_CONTROL: ${opt:cache-control, 'public,max-age=604800,stale-while-revalidate=604800'} 105 | CPL_TMPDIR: /tmp 106 | CPL_VSIL_CURL_ALLOWED_EXTENSIONS: .tif,.TIF 107 | GDAL_CACHEMAX: 25% 108 | GDAL_DATA: /opt/share/gdal 109 | # https://github.com/OSGeo/gdal/issues/909#issuecomment-420036545 110 | GDAL_DISABLE_READDIR_ON_OPEN: EMPTY_DIR 111 | GDAL_HTTP_MERGE_CONSECUTIVE_RANGES: YES 112 | GDAL_HTTP_MULTIPLEX: YES 113 | GDAL_HTTP_VERSION: 2 114 | MAX_THREADS: 1 115 | MOSAIC_DEF_BUCKET: ${self:provider.deploymentBucket} 116 | PROJ_LIB: /opt/share/proj 117 | PYTHONWARNINGS: ignore 118 | VSI_CACHE: TRUE 119 | VSI_CACHE_SIZE: 536870912 120 | events: 121 | - httpApi: 122 | path: /{proxy+} 123 | method: "*" 124 | -------------------------------------------------------------------------------- /serverless.yml: -------------------------------------------------------------------------------- 1 | service: naip-cogeo-mosaic 2 | 3 | provider: 4 | name: aws 5 | runtime: python3.7 6 | stage: ${opt:stage, 'production'} 7 | region: "us-west-2" 8 | 9 | deploymentBucket: ${opt:bucket} 10 | httpApi: 11 | cors: true 12 | 13 | # Add Tags to resources 14 | stackTags: 15 | Project: cogeo 16 | 17 | apiGateway: 18 | binaryMediaTypes: 19 | - "*/*" 20 | minimumCompressionSize: 1 21 | 22 | # Add other buckets here if needed 23 | iamRoleStatements: 24 | - Effect: "Allow" 25 | Action: 26 | - "s3:GetObject" 27 | - "s3:HeadObject" 28 | - "s3:PutObject" 29 | Resource: 30 | - "arn:aws:s3:::${self:provider.deploymentBucket}*" 31 | 32 | - Effect: "Allow" 33 | Action: 34 | - "s3:GetObject" 35 | - "s3:HeadObject" 36 | Resource: 37 | - "arn:aws:s3:::*" 38 | 39 | - Effect: "Allow" 40 | Action: 41 | - "dynamodb:GetItem" 42 | Resource: 43 | # Allow access to all dynamodb tables in region 44 | - "arn:aws:dynamodb:${self:provider.region}:${opt:aws-account-id, '961053664803'}:table/*" 45 | 46 | package: 47 | artifact: cogeo-mosaic-tiler/package.zip 48 | 49 | functions: 50 | app: 51 | handler: cogeo_mosaic_tiler.handlers.app.app 52 | memorySize: 1536 53 | timeout: 8 54 | layers: 55 | - arn:aws:lambda:${self:provider.region}:524387336408:layer:gdal24-py37-geolayer:1 56 | environment: 57 | # Necessary since NAIP bucket is requester-pays 58 | AWS_REQUEST_PAYER: requester 59 | CACHE_CONTROL: ${opt:cache-control, 'max-age=3600'} 60 | CPL_TMPDIR: /tmp 61 | CPL_VSIL_CURL_ALLOWED_EXTENSIONS: .tif,.TIF 62 | GDAL_CACHEMAX: 25% 63 | GDAL_DATA: /opt/share/gdal 64 | # https://github.com/OSGeo/gdal/issues/909#issuecomment-420036545 65 | GDAL_DISABLE_READDIR_ON_OPEN: EMPTY_DIR 66 | GDAL_HTTP_MERGE_CONSECUTIVE_RANGES: YES 67 | GDAL_HTTP_MULTIPLEX: YES 68 | GDAL_HTTP_VERSION: 2 69 | MAX_THREADS: 1 70 | MOSAIC_DEF_BUCKET: ${self:provider.deploymentBucket} 71 | PROJ_LIB: /opt/share/proj 72 | PYTHONWARNINGS: ignore 73 | VSI_CACHE: TRUE 74 | VSI_CACHE_SIZE: 536870912 75 | events: 76 | - httpApi: 77 | path: /{proxy+} 78 | method: "*" 79 | -------------------------------------------------------------------------------- /site/.gitignore: -------------------------------------------------------------------------------- 1 | # See https://help.github.com/articles/ignoring-files/ for more about ignoring files. 2 | 3 | # dependencies 4 | /node_modules 5 | /.pnp 6 | .pnp.js 7 | 8 | # testing 9 | /coverage 10 | 11 | # production 12 | /build 13 | 14 | # misc 15 | .DS_Store 16 | .env.local 17 | .env.development.local 18 | .env.test.local 19 | .env.production.local 20 | 21 | npm-debug.log* 22 | yarn-debug.log* 23 | yarn-error.log* 24 | -------------------------------------------------------------------------------- /site/README.md: -------------------------------------------------------------------------------- 1 | This project was bootstrapped with [Create React App](https://github.com/facebook/create-react-app). 2 | 3 | ## Available Scripts 4 | 5 | In the project directory, you can run: 6 | 7 | ### `yarn start` 8 | 9 | Runs the app in the development mode.
10 | Open [http://localhost:3000](http://localhost:3000) to view it in the browser. 11 | 12 | The page will reload if you make edits.
13 | You will also see any lint errors in the console. 14 | 15 | ### `yarn test` 16 | 17 | Launches the test runner in the interactive watch mode.
18 | See the section about [running tests](https://facebook.github.io/create-react-app/docs/running-tests) for more information. 19 | 20 | ### `yarn build` 21 | 22 | Builds the app for production to the `build` folder.
23 | It correctly bundles React in production mode and optimizes the build for the best performance. 24 | 25 | The build is minified and the filenames include the hashes.
26 | Your app is ready to be deployed! 27 | 28 | See the section about [deployment](https://facebook.github.io/create-react-app/docs/deployment) for more information. 29 | 30 | ### `yarn eject` 31 | 32 | **Note: this is a one-way operation. Once you `eject`, you can’t go back!** 33 | 34 | If you aren’t satisfied with the build tool and configuration choices, you can `eject` at any time. This command will remove the single build dependency from your project. 35 | 36 | Instead, it will copy all the configuration files and the transitive dependencies (webpack, Babel, ESLint, etc) right into your project so you have full control over them. All of the commands except `eject` will still work, but they will point to the copied scripts so you can tweak them. At this point you’re on your own. 37 | 38 | You don’t have to ever use `eject`. The curated feature set is suitable for small and middle deployments, and you shouldn’t feel obligated to use this feature. However we understand that this tool wouldn’t be useful if you couldn’t customize it when you are ready for it. 39 | 40 | ## Learn More 41 | 42 | You can learn more in the [Create React App documentation](https://facebook.github.io/create-react-app/docs/getting-started). 43 | 44 | To learn React, check out the [React documentation](https://reactjs.org/). 45 | 46 | ### Code Splitting 47 | 48 | This section has moved here: https://facebook.github.io/create-react-app/docs/code-splitting 49 | 50 | ### Analyzing the Bundle Size 51 | 52 | This section has moved here: https://facebook.github.io/create-react-app/docs/analyzing-the-bundle-size 53 | 54 | ### Making a Progressive Web App 55 | 56 | This section has moved here: https://facebook.github.io/create-react-app/docs/making-a-progressive-web-app 57 | 58 | ### Advanced Configuration 59 | 60 | This section has moved here: https://facebook.github.io/create-react-app/docs/advanced-configuration 61 | 62 | ### Deployment 63 | 64 | This section has moved here: https://facebook.github.io/create-react-app/docs/deployment 65 | 66 | ### `yarn build` fails to minify 67 | 68 | This section has moved here: https://facebook.github.io/create-react-app/docs/troubleshooting#npm-run-build-fails-to-minify 69 | -------------------------------------------------------------------------------- /site/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "site", 3 | "version": "0.1.0", 4 | "homepage": "https://kylebarron.dev/naip-cogeo-mosaic", 5 | "license": "MIT", 6 | "author": "Kyle Barron ", 7 | "dependencies": { 8 | "@testing-library/jest-dom": "^4.2.4", 9 | "@testing-library/react": "^9.3.2", 10 | "@testing-library/user-event": "^7.1.2", 11 | "immutable": "^4.0.0-rc.12", 12 | "react": "^16.13.0", 13 | "react-dom": "^16.13.0", 14 | "react-helmet": "^6.1.0", 15 | "react-map-gl": "^5.2.3", 16 | "react-scripts": "3.4.0", 17 | "semantic-ui-react": "^0.88.2" 18 | }, 19 | "scripts": { 20 | "start": "react-scripts start", 21 | "build": "react-scripts build", 22 | "test": "react-scripts test", 23 | "eject": "react-scripts eject", 24 | "deploy": "react-scripts build && gh-pages -d build -b gh-pages -r https://$GH_TOKEN@github.com/kylebarron/naip-cogeo-mosaic.git" 25 | }, 26 | "eslintConfig": { 27 | "extends": "react-app" 28 | }, 29 | "browserslist": { 30 | "production": [ 31 | ">0.2%", 32 | "not dead", 33 | "not op_mini all" 34 | ], 35 | "development": [ 36 | "last 1 chrome version", 37 | "last 1 firefox version", 38 | "last 1 safari version" 39 | ] 40 | }, 41 | "devDependencies": { 42 | "gh-pages": "^2.2.0" 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /site/public/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kylebarron/naip-cogeo-mosaic/7298ed3797fe609d4c95f5adcb989b4ca8f9ffb7/site/public/favicon.ico -------------------------------------------------------------------------------- /site/public/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | naip-cogeo-mosaic 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 30 | 31 | 32 | 33 |
34 | 44 | 45 | 46 | -------------------------------------------------------------------------------- /site/public/logo192.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kylebarron/naip-cogeo-mosaic/7298ed3797fe609d4c95f5adcb989b4ca8f9ffb7/site/public/logo192.png -------------------------------------------------------------------------------- /site/public/logo512.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kylebarron/naip-cogeo-mosaic/7298ed3797fe609d4c95f5adcb989b4ca8f9ffb7/site/public/logo512.png -------------------------------------------------------------------------------- /site/public/manifest.json: -------------------------------------------------------------------------------- 1 | { 2 | "short_name": "React App", 3 | "name": "Create React App Sample", 4 | "icons": [ 5 | { 6 | "src": "favicon.ico", 7 | "sizes": "64x64 32x32 24x24 16x16", 8 | "type": "image/x-icon" 9 | }, 10 | { 11 | "src": "logo192.png", 12 | "type": "image/png", 13 | "sizes": "192x192" 14 | }, 15 | { 16 | "src": "logo512.png", 17 | "type": "image/png", 18 | "sizes": "512x512" 19 | } 20 | ], 21 | "start_url": ".", 22 | "display": "standalone", 23 | "theme_color": "#000000", 24 | "background_color": "#ffffff" 25 | } 26 | -------------------------------------------------------------------------------- /site/public/share_preview_grca.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kylebarron/naip-cogeo-mosaic/7298ed3797fe609d4c95f5adcb989b4ca8f9ffb7/site/public/share_preview_grca.jpg -------------------------------------------------------------------------------- /site/src/App.css: -------------------------------------------------------------------------------- 1 | @import url('https://api.mapbox.com/mapbox-gl-js/v1.10.0/mapbox-gl.css'); 2 | @import url('https://cdn.jsdelivr.net/npm/semantic-ui@2.4.2/dist/semantic.min.css'); 3 | 4 | .App { 5 | text-align: center; 6 | } 7 | 8 | .App-logo { 9 | height: 40vmin; 10 | pointer-events: none; 11 | } 12 | 13 | @media (prefers-reduced-motion: no-preference) { 14 | .App-logo { 15 | animation: App-logo-spin infinite 20s linear; 16 | } 17 | } 18 | 19 | .App-header { 20 | background-color: #282c34; 21 | min-height: 100vh; 22 | display: flex; 23 | flex-direction: column; 24 | align-items: center; 25 | justify-content: center; 26 | font-size: calc(10px + 2vmin); 27 | color: white; 28 | } 29 | 30 | .App-link { 31 | color: #61dafb; 32 | } 33 | 34 | @keyframes App-logo-spin { 35 | from { 36 | transform: rotate(0deg); 37 | } 38 | to { 39 | transform: rotate(360deg); 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /site/src/App.js: -------------------------------------------------------------------------------- 1 | import React from "react"; 2 | import "./App.css"; 3 | import ReactMapGL, { 4 | NavigationControl, 5 | ScaleControl, 6 | Layer, 7 | } from "react-map-gl"; 8 | import { Map } from "immutable"; 9 | import InfoBox from "./info-box"; 10 | import { 11 | getViewStateFromHash, 12 | setQueryParams, 13 | getMosaicFromQueryParams, 14 | } from "./util"; 15 | import { fullResMosaics, overviewMosaics } from "./constants"; 16 | const DEFAULT_MAP_STYLE = require("./style.json"); 17 | 18 | const INITIAL_MOSAIC_YEAR_RANGE = "2016-2018"; 19 | const INITIAL_VIEWPORT = { 20 | latitude: 36.07832, 21 | longitude: -111.8695, 22 | zoom: 13, 23 | bearing: 0, 24 | pitch: 0, 25 | }; 26 | 27 | function naipUrl(mosaicUrl) { 28 | // Do saturation client side for speed 29 | // const color_ops = "sigmoidal RGB 4 0.5, saturation 1.25"; 30 | const params = { 31 | url: mosaicUrl, 32 | }; 33 | const searchParams = new URLSearchParams(params); 34 | let baseUrl = "https://us-west-2-lambda.kylebarron.dev/naip/{z}/{x}/{y}.jpg?"; 35 | return baseUrl + searchParams.toString(); 36 | } 37 | 38 | function constructMapStyle(mosaicYearRange) { 39 | const fullResMosaicUrl = fullResMosaics[mosaicYearRange]; 40 | const overviewMosaicUrl = overviewMosaics[mosaicYearRange]; 41 | 42 | DEFAULT_MAP_STYLE.sources["naip"] = { 43 | type: "raster", 44 | tiles: [naipUrl(fullResMosaicUrl)], 45 | tileSize: 256, 46 | minzoom: 12, 47 | maxzoom: 18, 48 | attribution: 49 | '© USDA', 50 | }; 51 | DEFAULT_MAP_STYLE.sources["naip-overview"] = { 52 | type: "raster", 53 | tiles: [naipUrl(overviewMosaicUrl)], 54 | tileSize: 256, 55 | minzoom: 6, 56 | maxzoom: 11, 57 | attribution: 58 | '© USDA', 59 | }; 60 | return Map(DEFAULT_MAP_STYLE); 61 | } 62 | 63 | class NAIPMap extends React.Component { 64 | render() { 65 | const { mapStyle, viewport, onViewportChange } = this.props; 66 | 67 | return ( 68 | 77 | 92 | 101 | 102 |
103 | 104 |
105 | 106 |
107 | 108 |
109 | 110 | ); 111 | } 112 | } 113 | 114 | class App extends React.Component { 115 | state = { 116 | viewport: { 117 | ...INITIAL_VIEWPORT, 118 | ...getViewStateFromHash(window.location.hash), 119 | }, 120 | mosaicYearRange: getMosaicFromQueryParams() || INITIAL_MOSAIC_YEAR_RANGE, 121 | mapStyle: constructMapStyle( 122 | getMosaicFromQueryParams() || INITIAL_MOSAIC_YEAR_RANGE 123 | ), 124 | }; 125 | 126 | render() { 127 | const { mosaicYearRange, mapStyle, viewport } = this.state; 128 | return ( 129 |
130 | this.setState({ viewport })} 134 | /> 135 | { 139 | setQueryParams({ mosaic: selected }); 140 | this.setState({ 141 | mosaicYearRange: selected, 142 | mapStyle: constructMapStyle(selected), 143 | }); 144 | }} 145 | /> 146 |
147 | ); 148 | } 149 | } 150 | 151 | export default App; 152 | 153 | document.body.style.margin = 0; 154 | -------------------------------------------------------------------------------- /site/src/App.test.js: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | import { render } from '@testing-library/react'; 3 | import App from './App'; 4 | 5 | test('renders learn react link', () => { 6 | const { getByText } = render(); 7 | const linkElement = getByText(/learn react/i); 8 | expect(linkElement).toBeInTheDocument(); 9 | }); 10 | -------------------------------------------------------------------------------- /site/src/constants.js: -------------------------------------------------------------------------------- 1 | // Mapping from mosaic common identifier to URL 2 | export const fullResMosaics = { 3 | "2011-2013": 4 | "dynamodb://us-west-2/74f48044f38db32666078e75f3439d8e62cf9e25820afc79ea6ce19f", 5 | "2014-2015": 6 | "dynamodb://us-west-2/5395d9e7bba4eeaa6af4842e1a7b9d3ea9dfc2a74373ae24698809e9", 7 | "2015-2017": 8 | "dynamodb://us-west-2/7610d6d77fca346802fb21b89668cb12ef3162a31eb71734a8aaf5de", 9 | "2016-2018": 10 | "dynamodb://us-west-2/94c61bd217e1211db47cf7f8b95bbc8e5e7d68a26cd9099319cf15f9", 11 | }; 12 | 13 | // Mapping from mosaic common identifier to URL 14 | export const overviewMosaics = { 15 | "2011-2013": 16 | "s3://mosaics-us-west-2.kylebarron.dev/mosaics/naip/naip_overview_2011_2013.json.gz", 17 | "2014-2015": 18 | "s3://mosaics-us-west-2.kylebarron.dev/mosaics/naip/naip_overview_2014_2015.json.gz", 19 | "2015-2017": 20 | "s3://mosaics-us-west-2.kylebarron.dev/mosaics/naip/naip_overview_2015_2017.json.gz", 21 | "2016-2018": 22 | "s3://mosaics-us-west-2.kylebarron.dev/mosaics/naip/naip_overview_2016_2018.json.gz", 23 | }; 24 | 25 | export const mosaicOptions = [ 26 | { 27 | key: "2011-2013", 28 | value: "2011-2013", 29 | text: "Imagery Range: 2011-2013", 30 | }, 31 | { 32 | key: "2014-2015", 33 | value: "2014-2015", 34 | text: "Imagery Range: 2014-2015", 35 | }, 36 | { 37 | key: "2015-2017", 38 | value: "2015-2017", 39 | text: "Imagery Range: 2015-2017", 40 | }, 41 | { 42 | key: "2016-2018", 43 | value: "2016-2018", 44 | text: "Imagery Range: 2016-2018", 45 | }, 46 | ]; 47 | -------------------------------------------------------------------------------- /site/src/index.css: -------------------------------------------------------------------------------- 1 | body { 2 | margin: 0; 3 | font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', 'Roboto', 'Oxygen', 4 | 'Ubuntu', 'Cantarell', 'Fira Sans', 'Droid Sans', 'Helvetica Neue', 5 | sans-serif; 6 | -webkit-font-smoothing: antialiased; 7 | -moz-osx-font-smoothing: grayscale; 8 | } 9 | 10 | code { 11 | font-family: source-code-pro, Menlo, Monaco, Consolas, 'Courier New', 12 | monospace; 13 | } 14 | -------------------------------------------------------------------------------- /site/src/index.js: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | import ReactDOM from 'react-dom'; 3 | import './index.css'; 4 | import App from './App'; 5 | 6 | ReactDOM.render(, document.getElementById('root')); 7 | -------------------------------------------------------------------------------- /site/src/info-box.js: -------------------------------------------------------------------------------- 1 | import React from "react"; 2 | import { Accordion, Select, Icon } from "semantic-ui-react"; 3 | import { mosaicOptions } from "./constants"; 4 | 5 | export default function InfoBox(props) { 6 | const { mosaicYearRange, onChange, zoomIn } = props; 7 | 8 | const panels = [ 9 | { 10 | key: "header", 11 | title: "Serverless High-Resolution Imagery", 12 | content: { 13 | content: ( 14 |

15 | Serverless high-resolution (up to 0.6m){" "} 16 | 21 | NAIP 22 | {" "} 23 | map tiles, generated on demand from an{" "} 24 | 29 | AWS public dataset 30 | {" "} 31 | of Cloud-Optimized GeoTIFFs. 32 |
33 | 38 | 39 | Github 40 | 41 |
42 | 47 | 48 | Blog post 49 | 50 |

51 | ), 52 | }, 53 | }, 54 | ]; 55 | 56 | return ( 57 |
73 | 74 | {/*
Serverless High-Res Imagery
*/} 75 | 76 | {zoomIn && ( 77 |

78 | Zoom in to see imagery. 79 |

80 | )} 81 |