├── .gitignore ├── CHANGELOG.md ├── LICENSE ├── README.md ├── handler.js ├── package.json ├── serverless.yml └── utils.js /.gitignore: -------------------------------------------------------------------------------- 1 | # Logs 2 | logs 3 | *.log 4 | npm-debug.log* 5 | yarn-debug.log* 6 | yarn-error.log* 7 | 8 | # Runtime data 9 | pids 10 | *.pid 11 | *.seed 12 | *.pid.lock 13 | 14 | # Directory for instrumented libs generated by jscoverage/JSCover 15 | lib-cov 16 | 17 | # Coverage directory used by tools like istanbul 18 | coverage 19 | 20 | # nyc test coverage 21 | .nyc_output 22 | 23 | # Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files) 24 | .grunt 25 | 26 | # Bower dependency directory (https://bower.io/) 27 | bower_components 28 | 29 | # node-waf configuration 30 | .lock-wscript 31 | 32 | # Compiled binary addons (http://nodejs.org/api/addons.html) 33 | build/Release 34 | 35 | # Dependency directories 36 | node_modules/ 37 | jspm_packages/ 38 | 39 | # Typescript v1 declaration files 40 | typings/ 41 | 42 | # Optional npm cache directory 43 | .npm 44 | 45 | # Optional eslint cache 46 | .eslintcache 47 | 48 | # Optional REPL history 49 | .node_repl_history 50 | 51 | # Output of 'npm pack' 52 | *.tgz 53 | 54 | # Yarn Integrity file 55 | .yarn-integrity 56 | 57 | # dotenv environment variables file 58 | .env 59 | 60 | .serverless/ 61 | .eslintrc.js 62 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | Changelog 2 | ========= 3 | 4 | 0.2.0 5 | ----- 6 | - Change CBERS bucket 7 | 8 | 9 | 0.1.2 10 | ----- 11 | - sentinel sceneid UTMzome bug fix 12 | 13 | 14 | 0.1.1 15 | ----- 16 | - add cbers support 17 | 18 | 19 | 0.1.0 20 | ----- 21 | **almost breaking change** 22 | - add `full` option to force reading S2/L8 metadata 23 | 24 | 0.0.1 25 | ----- 26 | - initial release 27 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | BSD 3-Clause License 2 | 3 | Copyright (c) 2017, RemotePixel.ca 4 | All rights reserved. 5 | 6 | Redistribution and use in source and binary forms, with or without 7 | modification, are permitted provided that the following conditions are met: 8 | 9 | * Redistributions of source code must retain the above copyright notice, this 10 | list of conditions and the following disclaimer. 11 | 12 | * Redistributions in binary form must reproduce the above copyright notice, 13 | this list of conditions and the following disclaimer in the documentation 14 | and/or other materials provided with the distribution. 15 | 16 | * Neither the name of the copyright holder nor the names of its 17 | contributors may be used to endorse or promote products derived from 18 | this software without specific prior written permission. 19 | 20 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 23 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 24 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 25 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 26 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 27 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 28 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # aws-sat-api 2 | 3 | :warning: this lib is not actively maintained and has been replaced by https://github.com/RemotePixel/aws-sat-api-py 4 | 5 | Simple Serverless API for Sentinel-2 and Landsat-8 data hosted on AWS 6 | 7 | A really simple non-spatial API to get Landsat-8 and Sentinel-2(A and B) images hosed on AWS S3 8 | 9 | # Installation 10 | 11 | ##### Requirement 12 | - AWS Account 13 | - node + npm 14 | 15 | ```bash 16 | git clone https://github.com/remotepixel/aws-sat-api.git 17 | cd aws-sat-api/ 18 | 19 | npm install serverless 20 | #configure serverless (https://serverless.com/framework/docs/providers/aws/guide/credentials/) 21 | 22 | sls deploy 23 | ``` 24 | 25 | :tada: You should be all set there. 26 | 27 | 28 | #### Live Demo: https://viewer.remotepixel.ca 29 | -------------------------------------------------------------------------------- /handler.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | const logger = require('fastlog')('sat-api'); 4 | const utils = require('./utils.js'); 5 | 6 | /** 7 | * landsat handler function. 8 | * 9 | * @param {object} event - input 10 | * @param {object} context - 11 | * @param {function} callback - 12 | */ 13 | 14 | module.exports.landsat = (event, context, callback) => { 15 | logger.info('Received event: ' + JSON.stringify(event)); 16 | 17 | if (event.row === '') return callback(new Error('ROW param missing!')); 18 | if (event.path === '') return callback(new Error('PATH param missing!')); 19 | 20 | utils.get_landsat(event.path, event.row, event.full) 21 | .then(data => { 22 | return callback(null, { 23 | request: { path: event.path, row: event.row }, 24 | meta: { found: data.length }, 25 | results: data 26 | }); 27 | }) 28 | .catch(err => { 29 | logger.error(err); 30 | return callback(new Error('API Error')); 31 | }); 32 | }; 33 | 34 | 35 | module.exports.cbers = (event, context, callback) => { 36 | logger.info('Received event: ' + JSON.stringify(event)); 37 | 38 | if (event.row === '') return callback(new Error('ROW param missing!')); 39 | if (event.path === '') return callback(new Error('PATH param missing!')); 40 | 41 | utils.get_cbers(event.path, event.row) 42 | .then(data => { 43 | return callback(null, { 44 | request: { path: event.path, row: event.row }, 45 | meta: { found: data.length }, 46 | results: data 47 | }); 48 | }) 49 | .catch(err => { 50 | logger.error(err); 51 | return callback(new Error('API Error')); 52 | }); 53 | }; 54 | 55 | 56 | /** 57 | * sentinel handler function. 58 | * 59 | * @param {object} event - input 60 | * @param {object} context - 61 | * @param {function} callback - 62 | */ 63 | 64 | module.exports.sentinel = (event, context, callback) => { 65 | logger.info('Received event: ' + JSON.stringify(event)); 66 | 67 | if (event.utm === '') return callback(new Error('UTM param missing!')); 68 | if (event.lat === '') return callback(new Error('LAT param missing!')); 69 | if (event.grid === '') return callback(new Error('GRID param missing!')); 70 | 71 | utils.get_sentinel(event.utm, event.lat, event.grid, event.full) 72 | .then(data => { 73 | return callback(null, { 74 | request: { utm: event.utm, lat: event.lat, grid: event.grid}, 75 | meta: { found: data.length }, 76 | results: data 77 | }); 78 | }) 79 | .catch(err => { 80 | logger.error(err); 81 | return callback(new Error('API Error')); 82 | }); 83 | }; 84 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "aws-sat-api", 3 | "version": "0.1.2", 4 | "engines": { 5 | "node": "6.10.3" 6 | }, 7 | "repository": { 8 | "type": "git", 9 | "url": "git://github.com/remotepixel/aws-sat-api.git" 10 | }, 11 | "dependencies": { 12 | "aws-sdk": "^2.6.9", 13 | "moment": "^2.19.3", 14 | "fastlog": "^1.1.0" 15 | }, 16 | "devDependencies": { 17 | "@mapbox/mock-aws-sdk-js": "0.0.4", 18 | "codecov": "^2.2.0", 19 | "eslint": ">=3.8.1", 20 | "eslint-config-google": "^0.7.1", 21 | "eslint-config-standard": "^6.2.1", 22 | "eslint-plugin-promise": "^3.4.0", 23 | "eslint-plugin-standard": "^2.0.1", 24 | "nyc": "^10.0.0", 25 | "sinon": "^1.17.7", 26 | "tape": "^3.5.0" 27 | }, 28 | "scripts": { 29 | "test": "nyc tape test/*.test.js" 30 | }, 31 | "author": "Vincent Sarago for RemotePixel", 32 | "license": "BDS-3" 33 | } 34 | -------------------------------------------------------------------------------- /serverless.yml: -------------------------------------------------------------------------------- 1 | service: sat-api 2 | 3 | provider: 4 | name: aws 5 | 6 | runtime: nodejs6.10 7 | 8 | stage: ${opt:stage, 'production'} 9 | 10 | region: us-east-1 11 | 12 | iamRoleStatements: 13 | - Effect: "Allow" 14 | Action: 15 | - "s3:ListBucket" 16 | - "s3:GetObject" 17 | Resource: 18 | - "arn:aws:s3:::cbers-meta-pds*" 19 | - "arn:aws:s3:::landsat-pds*" 20 | - "arn:aws:s3:::sentinel-s2*" 21 | 22 | # deploymentBucket: remotepixel-${self:provider.region} 23 | 24 | functions: 25 | landsat: 26 | handler: handler.landsat 27 | memorySize: 512 28 | timeout: 10 29 | events: 30 | - http: 31 | integration: lambda 32 | path: landsat 33 | method: get 34 | cors: true 35 | request: 36 | template: 37 | application/json: '{ "path" : "$input.params(''path'')", "row" : "$input.params(''row'')", "full" : "$input.params(''full'')" }' 38 | 39 | cbers: 40 | handler: handler.cbers 41 | memorySize: 512 42 | timeout: 10 43 | events: 44 | - http: 45 | integration: lambda 46 | path: cbers 47 | method: get 48 | cors: true 49 | request: 50 | template: 51 | application/json: '{ "path" : "$input.params(''path'')", "row" : "$input.params(''row'')" }' 52 | 53 | 54 | sentinel: 55 | handler: handler.sentinel 56 | memorySize: 512 57 | timeout: 10 58 | events: 59 | - http: 60 | integration: lambda 61 | path: sentinel 62 | method: get 63 | cors: true 64 | request: 65 | template: 66 | application/json: '{ "utm" : "$input.params(''utm'')", "grid" : "$input.params(''grid'')", "lat" : "$input.params(''lat'')", "full" : "$input.params(''full'')" }' 67 | -------------------------------------------------------------------------------- /utils.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | const moment = require('moment'); 4 | const AWS = require('aws-sdk'); 5 | 6 | 7 | const zeroPad = (n, c) => { 8 | let s = String(n); 9 | if (s.length < c) s = zeroPad('0' + n, c); 10 | return s; 11 | }; 12 | 13 | 14 | const generate_year_range = (start, end) => { 15 | const years = []; 16 | for(var year = start; year <= end; year++){ 17 | years.push(year); 18 | } 19 | return years; 20 | }; 21 | 22 | 23 | const aws_list_directory = (bucket, prefix, s3) => { 24 | const params = { 25 | Bucket: bucket, 26 | Delimiter: '/', 27 | Prefix: prefix}; 28 | 29 | return s3.listObjectsV2(params).promise() 30 | .then(data => { 31 | return data.CommonPrefixes.map(e => { 32 | return e.Prefix; 33 | }); 34 | }) 35 | .catch(() => { 36 | return []; 37 | }); 38 | }; 39 | 40 | 41 | const parseSceneid_c1 = (sceneid) => { 42 | const sceneid_info = sceneid.split('_'); 43 | return { 44 | scene_id: sceneid, 45 | satellite: sceneid_info[0].slice(0,1) + sceneid_info[0].slice(3), 46 | sensor: sceneid_info[0].slice(1,2), 47 | correction_level: sceneid_info[1], 48 | path: sceneid_info[2].slice(0,3), 49 | row: sceneid_info[2].slice(3), 50 | acquisition_date: sceneid_info[3], 51 | ingestion_date: sceneid_info[4], 52 | collection: sceneid_info[5], 53 | category: sceneid_info[6] 54 | }; 55 | }; 56 | 57 | 58 | const parseSceneid_pre = (sceneid) => { 59 | return { 60 | scene_id: sceneid, 61 | satellite: sceneid.slice(2,3), 62 | sensor: sceneid.slice(1,2), 63 | path: sceneid.slice(3,6), 64 | row: sceneid.slice(6,9), 65 | acquisitionYear: sceneid.slice(9,13), 66 | acquisitionJulianDay: sceneid.slice(13,16), 67 | acquisition_date: moment().utc().year(sceneid.slice(9,13)).dayOfYear(sceneid.slice(13,16)).format('YYYYMMDD'), 68 | groundStationIdentifier: sceneid.slice(16,19), 69 | archiveVersion: sceneid.slice(19,21) 70 | }; 71 | }; 72 | 73 | 74 | const parseCBERSid = (sceneid) => { 75 | return { 76 | scene_id: sceneid, 77 | satellite: sceneid.split('_')[0], 78 | version: sceneid.split('_')[1], 79 | sensor: sceneid.split('_')[2], 80 | path: sceneid.split('_')[4], 81 | row: sceneid.split('_')[5], 82 | acquisition_date: sceneid.split('_')[3], 83 | processing_level: sceneid.split('_')[6] 84 | }; 85 | }; 86 | 87 | 88 | const get_l8_info = (bucket, key, s3) => { 89 | const params = { 90 | Bucket: bucket, 91 | Key: key}; 92 | 93 | return s3.getObject(params).promise() 94 | .then(data => { 95 | data = JSON.parse(data.Body.toString()); 96 | let metadata = data.L1_METADATA_FILE; 97 | return { 98 | cloud_coverage: metadata.IMAGE_ATTRIBUTES.CLOUD_COVER, 99 | cloud_coverage_land: metadata.IMAGE_ATTRIBUTES.CLOUD_COVER_LAND, 100 | sun_azimuth: metadata.IMAGE_ATTRIBUTES.SUN_AZIMUTH, 101 | sun_elevation: metadata.IMAGE_ATTRIBUTES.SUN_ELEVATION, 102 | geometry: { 103 | type: 'Polygon', 104 | coordinates: [[ 105 | [metadata.PRODUCT_METADATA.CORNER_UR_LON_PRODUCT, metadata.PRODUCT_METADATA.CORNER_UR_LAT_PRODUCT], 106 | [metadata.PRODUCT_METADATA.CORNER_UL_LON_PRODUCT, metadata.PRODUCT_METADATA.CORNER_UL_LAT_PRODUCT], 107 | [metadata.PRODUCT_METADATA.CORNER_LL_LON_PRODUCT, metadata.PRODUCT_METADATA.CORNER_LL_LAT_PRODUCT], 108 | [metadata.PRODUCT_METADATA.CORNER_LR_LON_PRODUCT, metadata.PRODUCT_METADATA.CORNER_LR_LAT_PRODUCT], 109 | [metadata.PRODUCT_METADATA.CORNER_UR_LON_PRODUCT, metadata.PRODUCT_METADATA.CORNER_UR_LAT_PRODUCT] 110 | ]] 111 | }}; 112 | }) 113 | .catch(() => { 114 | return {}; 115 | }); 116 | }; 117 | 118 | 119 | const get_s2_info = (bucket, key, s3) => { 120 | const params = { 121 | Bucket: bucket, 122 | Key: key}; 123 | 124 | return s3.getObject(params).promise() 125 | .then(data => { 126 | data = JSON.parse(data.Body.toString()); 127 | return { 128 | cloud_coverage: data.cloudyPixelPercentage, 129 | coverage: data.dataCoveragePercentage, 130 | geometry : data.tileGeometry, 131 | sat: data.productName.slice(0,3)}; 132 | }) 133 | .catch(() => { 134 | return {}; 135 | }); 136 | }; 137 | 138 | 139 | const get_landsat = (path, row, full=false) => { 140 | const s3 = new AWS.S3({region: 'us-west-2'}); 141 | const landsat_bucket = 'landsat-pds'; 142 | 143 | row = utils.zeroPad(row, 3); 144 | path = utils.zeroPad(path, 3); 145 | 146 | const level = ['L8', 'c1/L8']; 147 | 148 | // get list sceneid 149 | return Promise.all(level.map(e => { 150 | let prefix = `${e}/${path}/${row}/`; 151 | return utils.aws_list_directory(landsat_bucket, prefix, s3); 152 | })) 153 | .then(dirs => { 154 | dirs = [].concat.apply([], dirs); 155 | return Promise.all(dirs.map(e => { 156 | let landsat_id = e.split('/').slice(-2,-1)[0]; 157 | let info, aws_url; 158 | 159 | if (/L[COTEM]08_L\d{1}[A-Z]{2}_\d{6}_\d{8}_\d{8}_\d{2}_(T1|RT)/.exec(landsat_id)) { 160 | info = utils.parseSceneid_c1(landsat_id); 161 | info.type = info.category; 162 | aws_url = 'https://landsat-pds.s3.amazonaws.com/c1'; 163 | } else { 164 | info = utils.parseSceneid_pre(landsat_id); 165 | info.type = 'pre'; 166 | aws_url = 'https://landsat-pds.s3.amazonaws.com'; 167 | } 168 | info.browseURL = `${aws_url}/L8/${info.path}/${info.row}/${info.scene_id}/${info.scene_id}_thumb_large.jpg`; 169 | info.thumbURL = `${aws_url}/L8/${info.path}/${info.row}/${info.scene_id}/${info.scene_id}_thumb_small.jpg`; 170 | 171 | if (full) { 172 | let json_path = `${e}${landsat_id}_MTL.json`; 173 | return utils.get_l8_info(landsat_bucket, json_path, s3) 174 | .then(data => { 175 | return Object.assign({}, info, data); 176 | }); 177 | } else { 178 | return info; 179 | } 180 | })); 181 | }); 182 | }; 183 | 184 | 185 | const get_cbers = (path, row) => { 186 | const s3 = new AWS.S3({region: 'us-east-1'}); 187 | const cbers_bucket = 'cbers-meta-pds'; 188 | 189 | row = utils.zeroPad(row, 3); 190 | path = utils.zeroPad(path, 3); 191 | 192 | // get list sceneid 193 | const prefix = `CBERS4/MUX/${path}/${row}/`; 194 | return utils.aws_list_directory(cbers_bucket, prefix, s3) 195 | .then(dirs => { 196 | dirs = [].concat.apply([], dirs); 197 | return dirs.map(e => { 198 | let cbers_id = e.split('/').slice(-2,-1)[0]; 199 | let info = utils.parseCBERSid(cbers_id); 200 | let preview_id = cbers_id.split('_').slice(0,-1).join('_'); 201 | info.browseURL = `https://${cbers_bucket}.s3.amazonaws.com/CBERS4/MUX/${path}/${row}/${cbers_id}/${preview_id}_small.jpeg`; 202 | return info; 203 | }); 204 | }); 205 | }; 206 | 207 | const get_sentinel = (utm, lat, grid, full=false) => { 208 | const s3 = new AWS.S3({region: 'eu-central-1'}); 209 | const sentinel_bucket = 'sentinel-s2-l1c'; 210 | const img_year = utils.generate_year_range(2015, moment().year()); 211 | 212 | utm = utm.replace(/^0/, ''); 213 | 214 | // get list of month 215 | return Promise.all(img_year.map(e => { 216 | let prefix = `tiles/${utm}/${lat}/${grid}/${e}/`; 217 | return utils.aws_list_directory(sentinel_bucket, prefix, s3); 218 | })) 219 | .then(dirs => { 220 | // get list of day 221 | dirs = [].concat.apply([], dirs); 222 | return Promise.all(dirs.map(e => { 223 | return utils.aws_list_directory(sentinel_bucket, e, s3); 224 | })); 225 | }) 226 | .then(dirs => { 227 | // get list of version 228 | dirs = [].concat.apply([], dirs); 229 | return Promise.all(dirs.map(e => { 230 | return utils.aws_list_directory(sentinel_bucket, e, s3); 231 | })); 232 | }) 233 | .then(data => { 234 | //create list of image 235 | data = [].concat.apply([], data); 236 | return Promise.all(data.map(e => { 237 | let s2path = e.replace(/\/$/, ''); 238 | let yeah = s2path.split('/')[4]; 239 | let month = utils.zeroPad(s2path.split('/')[5], 2); 240 | let day = utils.zeroPad(s2path.split('/')[6], 2); 241 | 242 | let info = { 243 | path: s2path, 244 | utm_zone: s2path.split('/')[1], 245 | latitude_band: s2path.split('/')[2], 246 | grid_square: s2path.split('/')[3], 247 | num: s2path.split('/')[7], 248 | acquisition_date: `${yeah}${month}${day}`, 249 | browseURL: `https://sentinel-s2-l1c.s3.amazonaws.com/${s2path}/preview.jpg`}; 250 | 251 | const utm = utils.zeroPad(info.utm_zone, 2); 252 | info.scene_id = `S2A_tile_${info.acquisition_date}_${utm}${info.latitude_band}${info.grid_square}_${info.num}`; 253 | 254 | if (full) { 255 | let json_path = `${e}tileInfo.json`; 256 | return utils.get_s2_info(sentinel_bucket, json_path, s3) 257 | .then(data => { 258 | info = Object.assign({}, info, data); 259 | info.scene_id = `${info.sat}_tile_${info.acquisition_date}_${utm}${info.latitude_band}${info.grid_square}_${info.num}`; 260 | return info; 261 | }); 262 | } else { 263 | return info; 264 | } 265 | })); 266 | }); 267 | }; 268 | 269 | 270 | const utils = { 271 | generate_year_range: generate_year_range, 272 | aws_list_directory: aws_list_directory, 273 | parseSceneid_pre: parseSceneid_pre, 274 | parseSceneid_c1: parseSceneid_c1, 275 | parseCBERSid: parseCBERSid, 276 | get_sentinel: get_sentinel, 277 | get_landsat: get_landsat, 278 | get_cbers: get_cbers, 279 | get_s2_info: get_s2_info, 280 | get_l8_info: get_l8_info, 281 | zeroPad: zeroPad 282 | }; 283 | module.exports = utils; 284 | --------------------------------------------------------------------------------