├── .bookignore ├── .circleci └── config.yml ├── .dockerignore ├── .eslintignore ├── .eslintrc.json ├── .gitignore ├── .nvmrc ├── CHANGELOG.md ├── Dockerfile ├── LICENSE ├── README.md ├── docs ├── api.md ├── docs.md ├── images │ └── logo.png └── index.html ├── lerna.json ├── package.json └── packages ├── api-lib ├── README.md ├── api-spec.yaml ├── index.js ├── libs │ ├── ElasticSearchWriteableStream.js │ ├── api.js │ ├── es.js │ ├── ingest.js │ └── logger.js ├── package.json └── tests │ ├── fixtures │ ├── item.json │ ├── itemLinks.json │ └── stac │ │ ├── LC80100102015050LGN00.json │ │ ├── LC80100102015082LGN00.json │ │ ├── badGeometryItem.json │ │ ├── catalog.json │ │ ├── collection.json │ │ ├── collection2.json │ │ ├── collection2_item.json │ │ ├── collectionNoChildren.json │ │ ├── intersectsFeature.json │ │ └── noIntersectsFeature.json │ ├── integration │ ├── docker-compose.yml │ ├── ingestCollections.js │ ├── ingestData.js │ ├── runIntegration.sh │ └── test_api.js │ ├── test_api_extractIntersects.js │ ├── test_api_parsePath.js │ ├── test_api_search.js │ └── test_ingest.js ├── api ├── README.md ├── index.js ├── package.json ├── tests │ └── test_handler.js └── webpack.config.js └── ingest ├── README.md ├── bin └── ingest.js ├── index.js ├── package.json ├── tests └── test_handler.js └── webpack.config.js /.bookignore: -------------------------------------------------------------------------------- 1 | .* 2 | bin 3 | package* 4 | CONTRIBUTING.md 5 | tests 6 | CHANGELOG.md 7 | lerna.json 8 | _book -------------------------------------------------------------------------------- /.circleci/config.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | 3 | references: 4 | restore_repo: &restore_repo 5 | restore_cache: 6 | keys: 7 | - v1-repo-{{ .Branch }}-{{ .Revision }} 8 | - v1-repo-{{ .Branch }} 9 | - v1-repo 10 | save_repo: &save_repo 11 | save_cache: 12 | key: v1-repo-{{ .Branch }}-{{ .Revision }} 13 | paths: 14 | - ~/project 15 | 16 | 17 | jobs: 18 | build_and_test: 19 | docker: 20 | - image: circleci/node:8.11 21 | steps: 22 | - *restore_repo 23 | - checkout 24 | - *save_repo 25 | - run: 26 | name: Install 27 | command: | 28 | yarn 29 | yarn bootstrap 30 | - run: 31 | name: Lint 32 | command: yarn eslint 33 | - run: 34 | name: Build and test 35 | command: | 36 | yarn build 37 | yarn test 38 | - run: 39 | name: Build Documentation 40 | command: | 41 | if [[ "$CIRCLE_BRANCH" == 'master' ]]; then 42 | yarn build-api-docs 43 | fi 44 | - run: 45 | name: Deploy to NPM 46 | command: | 47 | if [[ "$CIRCLE_BRANCH" == 'master' ]]; then 48 | echo "//registry.npmjs.org/:_authToken=${NPM_TOKEN}" > ~/.npmrc 49 | VERSION=$(jq --raw-output .version lerna.json) 50 | node_modules/.bin/lerna publish --skip-git --repo-version $VERSION --yes --force-publish=* 51 | fi 52 | 53 | docker_build_and_test: 54 | machine: 55 | docker_layer_caching: false 56 | steps: 57 | - *restore_repo 58 | - checkout 59 | - *save_repo 60 | - run: 61 | name: Build Docker image 62 | command: | 63 | docker build -t satutils/sat-api:latest . 64 | - run: 65 | name: Test Docker image 66 | command: | 67 | docker run -it satutils/sat-api:latest yarn test 68 | 69 | docker_deploy_production: 70 | machine: 71 | docker_layer_caching: false 72 | steps: 73 | - *restore_repo 74 | - run: 75 | name: Build Docker image 76 | command: | 77 | docker build -t satutils/sat-api:latest . 78 | - run: 79 | name: Deploy Docker image 80 | command: | 81 | docker login -u ${DOCKER_USER} -p ${DOCKER_PASS} 82 | VERSION=$(jq --raw-output .version lerna.json) 83 | docker tag satutils/sat-api:latest satutils/sat-api:${VERSION} 84 | docker push satutils/sat-api:latest 85 | docker push satutils/sat-api:${VERSION} 86 | 87 | docker_deploy_develop: 88 | machine: 89 | docker_layer_caching: false 90 | steps: 91 | - *restore_repo 92 | - run: 93 | name: Build Docker image 94 | command: | 95 | docker build -t satutils/sat-api:develop . 96 | - run: 97 | name: Deploy Docker image 98 | command: | 99 | docker login -u ${DOCKER_USER} -p ${DOCKER_PASS} 100 | VERSION=$(jq --raw-output .version lerna.json) 101 | docker tag satutils/sat-api:develop satutils/sat-api:${VERSION} 102 | docker push satutils/sat-api:develop 103 | docker push satutils/sat-api:${VERSION} 104 | 105 | 106 | workflows: 107 | version: 2 108 | build_and_test: 109 | jobs: 110 | - build_and_test 111 | docker_build_and_test: 112 | jobs: 113 | - docker_build_and_test 114 | - docker_deploy_production: 115 | requires: 116 | - docker_build_and_test 117 | filters: 118 | branches: 119 | only: master 120 | - docker_deploy_develop: 121 | requires: 122 | - docker_build_and_test 123 | filters: 124 | branches: 125 | only: develop 126 | -------------------------------------------------------------------------------- /.dockerignore: -------------------------------------------------------------------------------- 1 | .git 2 | .dockerignore 3 | Dockerfile 4 | docker-compose.yml 5 | node_modules 6 | packages/*/node_modules 7 | -------------------------------------------------------------------------------- /.eslintignore: -------------------------------------------------------------------------------- 1 | **/node_modules/** 2 | **/dist/** 3 | **/webpack.config.js 4 | **/.nyc_output/** 5 | **/coverage/** 6 | tmp/** 7 | -------------------------------------------------------------------------------- /.eslintrc.json: -------------------------------------------------------------------------------- 1 | { 2 | "root": true, 3 | "extends": "airbnb", 4 | "plugins": ["eslint-plugin-jsdoc"], 5 | "env": { 6 | "node": true, 7 | "mocha": true 8 | }, 9 | "rules": { 10 | "semi": [2, "never"], 11 | "quote-props": 0, 12 | "camelcase": 0, 13 | "func-names": 0, 14 | "no-param-reassign": "off", 15 | "no-prototype-builtins": "off", 16 | "radix": "off", 17 | "no-console": "off", 18 | "indent": [ "error", 2 ], 19 | "require-jsdoc": "off", 20 | "valid-jsdoc": [ "error", { 21 | "prefer": { 22 | "arg": "param", 23 | "return": "returns" 24 | }, 25 | "preferType": { 26 | "Boolean": "boolean", 27 | "Number": "number", 28 | "String": "string", 29 | "object": "Object", 30 | "array": "Array", 31 | "date": "Date", 32 | "regexp": "RegExp", 33 | "Regexp": "RegExp", 34 | "promise": "Promise" 35 | }, 36 | "requireReturn": true 37 | }], 38 | "jsdoc/check-param-names": "error", 39 | "jsdoc/check-tag-names": "error", 40 | "jsdoc/check-types": "off", 41 | "jsdoc/newline-after-description": "error", 42 | "jsdoc/require-description-complete-sentence": "off", 43 | "jsdoc/require-example": "off", 44 | "jsdoc/require-hyphen-before-param-description": "error", 45 | "jsdoc/require-param": "error", 46 | "jsdoc/require-param-description": "error", 47 | "jsdoc/require-param-name": "error", 48 | "jsdoc/require-param-type": "error", 49 | "jsdoc/require-returns-description": "error", 50 | "jsdoc/require-returns-type": "error", 51 | "generator-star-spacing": "off", 52 | "import/no-extraneous-dependencies": "off", 53 | "import/newline-after-import": "off", 54 | "class-methods-use-this": "off", 55 | "no-warning-comments": "off", 56 | "no-unused-vars": [ 57 | "error", 58 | { "argsIgnorePattern": "^_" } 59 | ], 60 | "no-useless-escape": "off", 61 | "spaced-comment": "off", 62 | "require-yield": "off", 63 | "prefer-template": "warn", 64 | "prefer-promise-reject-errors": "off", 65 | "no-underscore-dangle": "off", 66 | "comma-dangle": [ 67 | "warn", 68 | "never" 69 | ], 70 | "strict": "off", 71 | "guard-for-in": "off", 72 | "object-shorthand": "off", 73 | "space-before-function-paren": [ 74 | "error", 75 | { 76 | "anonymous": "always", 77 | "named": "never", 78 | "asyncArrow": "always" 79 | } 80 | ], 81 | "max-len": [ 82 | 2, 83 | { 84 | "code": 100, 85 | "ignorePattern": "(https?:|JSON\\.parse|[Uu]rl =)" 86 | } 87 | ], 88 | "arrow-parens": ["error", "always"], 89 | "prefer-destructuring": "off", 90 | "function-paren-newline": ["error", "consistent"] 91 | } 92 | } 93 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | dist 3 | .env 4 | build 5 | dist 6 | cloudformation.yml 7 | ddd.js 8 | package-lock.json 9 | yarn.lock 10 | *.lerna_backup 11 | _book -------------------------------------------------------------------------------- /.nvmrc: -------------------------------------------------------------------------------- 1 | 8.10 2 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | All notable changes to this project will be documented in this file. 3 | 4 | The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) 5 | and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html). 6 | 7 | ## [Unreleased] 8 | 9 | ## [v0.3.0] - 2019-10-16 10 | 11 | ### Added 12 | - /stac/search linked to from /stac 13 | - New `ids` parameter added for searching by IDs 14 | - New `collections` parameter added for searching list of specific collections 15 | - SATAPI_COLLECTION_LIMIT environment variable added for the number of collections to return at the /stac and /collections endpoints. Since pagination is not supported at these endpoints this should be set higher than the number of collections available. Defaults to 100. 16 | 17 | ### Changed 18 | - Fields parameter changed to search on any fields rather than just fields under `properties`. Field under `properties` must now be referenced by `properties.` 19 | - `collection` now a top level field rather than a property 20 | 21 | 22 | ## [v0.2.6] - 2019-08-28 23 | 24 | ### Added 25 | - Support for `in` operator on property fields 26 | - SATAPI_COLLECTION_LIMIT environment variable added for the number of collections to return at the /stac and /collections endpoints. Since pagination is not supported at these endpoints this should be set higher than the number of collections available. Defaults to 100. 27 | 28 | ### Added 29 | - /stac/search linked to from /stac 30 | - New `ids` parameter added for searching by IDs 31 | - New `collections` parameter added for searching list of specific collections 32 | 33 | ### Changed 34 | - Fields parameter changed to search on any fields rather than just fields under `properties`. `property` fields must now be referenced by `property.fieldname` 35 | - `collection` now a top level field rather than a property 36 | 37 | 38 | ## [v0.2.5] - 2019-06-21 39 | 40 | ### Fixed 41 | - Added missing dependency (through2) 42 | 43 | ### Removed 44 | - Removed gzip compression which caused problems with APIGateway. Use APIGateway to enable compression instead rather than in the library. 45 | 46 | ## [v0.2.4] - 2019-06-02 47 | 48 | ### Added 49 | - Add `fields` filter to return only specific fields 50 | - Added SATAPI_URL environment variable for defining a custom root URL used for creating hierarchical links. Otherwise, the root URL will try to be inferred from the headers. 51 | - Gzip compression added for clients that support it (via `Accept-Encoding`) 52 | - Added SATAPI_ES_PRECISION environment variable to change the precision for underlying intersection geometry queries. This will have an adverse impact on performance and storage size. If changed after ingestion, a reindex operation will need to be performed. It defaults to '5mi' (5 miles). 53 | 54 | ### Fixed 55 | - Fix datetime range to be inclusive (i.e., gte and lte) 56 | - Fixed `next` links to properly stringify geometries 57 | 58 | 59 | ## [v0.2.3] - 2019-01-29 60 | 61 | ### Fixed 62 | - Proper handling of bounding box passed as string 63 | 64 | ### Changed 65 | - De-normalize Item properties to include all properties from collection 66 | - Flattened elastic search to simplify query logic 67 | - Items returned will now include all 'Common' properties that are in the Items Collection 68 | 69 | ## [v0.2.2] - 2019-01-21 70 | 71 | ### Fixed 72 | - Fix error handling of single Items written to Elasticsearch. 73 | 74 | ## [v0.2.1] - 2019-01-17 75 | 76 | ### Fixed 77 | - Error handling of Items failing to get written to Elasticsearch (such as when unable to parse geometry). Now will continue traversing catalog. 78 | 79 | 80 | ## [v0.2.0] - 2019-01-16 81 | 82 | ### Changed 83 | - Implemented the changes in [STAC 0.6.0](https://github.com/radiantearth/stac-spec/blob/master/CHANGELOG.md) 84 | - All functionality relating to handling a STAC API path and query parameters has been moved into a the api.js module. The simplified lambda handler now passes query parameters or POST body through to the `api.js` module. 85 | - API now only supports STAC compliant query parameters and filters. 86 | - All functionality and references relating to Elasticsearch have been migrated into the es.js module to faciltate separation of concerns and abstract data storage. 87 | - Refactored Elasticsearch queries and ES mappings to search and share fields with the properties nested type. Now, only fields under a Collections properties are inherited by the Item. 88 | - Elasticsearch queries have been updated to use non-scoring filters to improve performance and return more intuitive results. 89 | - Elasticsearch writing has been modified to use bulk updates whenever possible to improve throughput. 90 | - Ingest has been updated to use concurrent file requests (to a user defined limit) to improve throughput. 91 | - Ingest now supports a Fargate mode to run as a Fargate task rather than as a Lambda. 92 | - API documentation generation now uses OpenAPI definitions processed by widdershins and shins. 93 | - Unit and integration test coverage for all modules. 94 | 95 | ### Removed 96 | - Manager module: Removed in favor of Kibana for Elasticsearch administration and management tasks. 97 | - Landsat and Sentinel lambda functions: Data is now ingested via the ingest Lambda. It can be invoked with individual SNS messages or run in batch mode to ingest larger catalogs. 98 | - Deployment files: Deployment related templates and code have been migrated to [https://github.com/sat-utils/sat-api-deployment](https://github.com/sat-utils/sat-api-deployment) 99 | 100 | ## [v0.1.0] - 2018-09-18 101 | 102 | ### Fixed 103 | - Fixed broken ingests for Landsat and Sentinel 104 | 105 | ## [v0.0.2] 106 | 107 | ### Added 108 | - Added support for [STAC specification](https://github.com/radiantearth/stac-spec/) 109 | - The following packages are released 110 | - @sat-utils/api 111 | - @sat-utils/api-lib 112 | - @sat-utils/ingest 113 | - @sat-utils/landsat 114 | - @sat-utils/sentinel 115 | - @sat-utils/manager 116 | - A new document is added on how to configure and deploy and instance of sat-api 117 | 118 | ### Changed 119 | - All lambdas and packages moved to `/packages` and lerna is used for managing them 120 | - npm packages are published under `@sat-utils` org on npm 121 | 122 | ### Removed 123 | - /geojson endpoint 124 | - /count endpoint 125 | 126 | ## [legacy-v2.0.0] - 2018-01-01 127 | 128 | - Moves all the metadata indexing logics to the same repo 129 | - Uses CloudFormation for full deployment 130 | - Includes a better ApiGateway support 131 | - Use streams to read, transform, and write into elasticsearch 132 | - Use batches of lambdas to speed up processing 133 | - Refactor several modules: metadata, landsat, sentinel 134 | - Refactor and improve splitting 135 | 136 | [Unreleased]: https://github.com/sat-utils/sat-api/compare/master...develop 137 | [v0.3.0]: https://github.com/sat-utils/sat-api/compare/v0.2.6...v0.3.0 138 | [v0.2.6]: https://github.com/sat-utils/sat-api/compare/v0.2.5...v0.2.6 139 | [v0.2.5]: https://github.com/sat-utils/sat-api/compare/v0.2.4...v0.2.5 140 | [v0.2.4]: https://github.com/sat-utils/sat-api/compare/v0.2.3...v0.2.4 141 | [v0.2.3]: https://github.com/sat-utils/sat-api/compare/v0.2.2...v0.2.3 142 | [v0.2.2]: https://github.com/sat-utils/sat-api/compare/v0.2.1...v0.2.2 143 | [v0.2.1]: https://github.com/sat-utils/sat-api/compare/v0.2.0...v0.2.1 144 | [v0.2.0]: https://github.com/sat-utils/sat-api/compare/v0.1.0...v0.2.0 145 | [v0.1.0]: https://github.com/sat-utils/sat-api/compare/v0.0.2...v0.1.0 146 | [v0.0.2]: https://github.com/sat-utils/sat-api/compare/legacy-v2.0.0...v0.0.2 147 | [legacy-v2.0.0]: https://github.com/sat-utils/sat-api/tree/legacy 148 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | # required envvars 2 | # - ES_HOST: Elasticsearch https endpoint 3 | 4 | 5 | FROM node:8 6 | 7 | ENV \ 8 | HOME=/home/sat-utils 9 | 10 | WORKDIR ${HOME}/sat-api 11 | 12 | COPY package.json ./ 13 | 14 | RUN \ 15 | npm install -g lerna; \ 16 | yarn; 17 | 18 | COPY . ./ 19 | 20 | RUN \ 21 | yarn bootstrap; \ 22 | yarn build; \ 23 | yarn linkall 24 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2016 Development Seed 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy 4 | of this software and associated documentation files (the "Software"), to deal 5 | in the Software without restriction, including without limitation the rights 6 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | copies of the Software, and to permit persons to whom the Software is 8 | furnished to do so, subject to the following conditions: 9 | 10 | The above copyright notice and this permission notice shall be included in all 11 | copies or substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 19 | SOFTWARE. 20 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # sat-api 2 | 3 | [![CircleCI](https://circleci.com/gh/sat-utils/sat-api.svg?style=svg)](https://circleci.com/gh/sat-utils/sat-api) 4 | 5 | Sat-api is a STAC compliant web API for searching and serving metadata for geospatial data (including but not limited to satellite imagery). 6 | 7 | Development Seed runs an instance of sat-api for the Landsat-8 and Sentinel-2 imagery that is [hosted on AWS](https://aws.amazon.com/earth/). You can access this at https://sat-api.developmentseed.org using the [API documentation](http://sat-utils.github.io/sat-api/) for reference and examples. 8 | 9 | The STAC version supported by a given version of sat-api is shown in the table below. Additional information can be found in the [CHANGELOG](CHANGELOG.md) 10 | 11 | | sat-api | STAC | 12 | | -------- | ---- | 13 | | 0.1.0 | 0.5.x | 14 | | 0.2.x | 0.6.x | 15 | | 0.3.x | 0.7.x | 16 | 17 | ## Usage 18 | 19 | This repository contains just the Node libraries for running the API. The [sat-api-deployment](https://github.com/sat-utils/sat-api-deployment) repository is for deploying sat-api to AWS. 20 | 21 | ### Environment variables 22 | 23 | There are some environment variables used in the code. Some do not have defaults and must be set. 24 | 25 | | Name | Description | Default Value | 26 | | ---- | ----------- | ------------- | 27 | | STAC_ID | ID of this catalog | - | 28 | | STAC_TITLE | Title of this catalog | - | 29 | | STAC_DESCRIPTION | Description of this catalog | - | 30 | | STAC_DOCS_URL | URL to documentation | - | 31 | | SATAPI_URL | The root endpoint of this API to use for links | Inferred from request | 32 | | ES_BATCH_SIZE | Number of records to ingest in single batch | 500 | 33 | | SATAPI_ES_PRECISION | Precision to use for geometry queries, see [ES documentation](https://www.elastic.co/guide/en/elasticsearch/reference/current/geo-shape.html) | '5mi' | 34 | | LOG_LEVEL | Level for logging | 'info' | 35 | 36 | 37 | ## Development 38 | 39 | Sat-api includes a number of NPM packages (in the packages/ directory) that are used to create and populate an instance of sat-api. See the [sat-utils org on NPM](https://www.npmjs.com/org/sat-utils) for the full list of packages. [Lerna](https://github.com/lerna/lerna) is used for for managing these packages. 40 | 41 | The latest version released is on the [master branch](https://github.com/sat-utils/sat-api/tree/master), and the latest development version is on the [develop](https://github.com/sat-utils/sat-api/tree/develop) branch. 42 | 43 | ### Building local version 44 | 45 | # Install dependencies in package.json 46 | $ yarn 47 | 48 | # Run lerna boostrap to link together packages and install those package dependencies 49 | $ yarn bootstrap 50 | 51 | # Run the build command in each of the packages (runs webpack) 52 | $ yarn build 53 | 54 | # To continually watch and build source files 55 | $ yarn watch 56 | 57 | # To run tests for all packages 58 | $ yarn test 59 | 60 | ### Building API docs 61 | 62 | # To build API docs from the api spec 63 | $ yarn build-api-docs 64 | 65 | ### Creating a release 66 | 67 | To create a new version for npm: 68 | 69 | - create a new branch from master 70 | - `$ yarn update` 71 | - Follow the prompt and select the correct the version, then commit the changes. 72 | - Update [CHANGELOG.md](CHANGELOG.md). 73 | - Tag your branch with the same version number 74 | - Make a PR 75 | - When the PR is merged to master, the npm packages are automatically deployed to npm 76 | - In GitHub create a release with the version (prefixed with 'v') and paste in the CHANGELOG section. This will create a GitHub release and a tag. 77 | 78 | 79 | ## About 80 | 81 | [sat-api](https://github.com/sat-utils/sat-api) was created by [Development Seed]() and is part of a collection of tools called [sat-utils](https://github.com/sat-utils). 82 | -------------------------------------------------------------------------------- /docs/docs.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | # sat-api Documentation 4 | 5 | Sat-api is a STAC compliant web API for searching and serving satellite imagery metadata. Development Seed runs an instance of it for Landsat and Sentinel imagery hosted on AWS. You can access this instance at https://sat-api.developmentseed.org. 6 | 7 | This repo includes a number of npm packages that are used to create and populate an instance of sat-api. For the full list of packages go to: 8 | https://www.npmjs.com/org/sat-utils 9 | 10 | ### Table of Contents 11 | 12 | * [Deployment](https://github.com/sat-utils/sat-api-deployment) 13 | * [API Docs](api.md) 14 | * [Changelog](../CHANGELOG.md) 15 | -------------------------------------------------------------------------------- /docs/images/logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sat-utils/sat-api/63dd52081a101fe8ff1fb44573ed90d36e6b12cc/docs/images/logo.png -------------------------------------------------------------------------------- /lerna.json: -------------------------------------------------------------------------------- 1 | { 2 | "lerna": "2.11.0", 3 | "version": "0.3.0", 4 | "npmClient": "yarn", 5 | "packages": [ 6 | "packages/*" 7 | ] 8 | } 9 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "description": "One API to search public Satellites metadata", 3 | "repository": "https://github.com/sat-utils/sat-api", 4 | "author": "Alireza Jazayeri , Matthew Hanson ", 5 | "license": "MIT", 6 | "scripts": { 7 | "bootstrap": "lerna bootstrap", 8 | "clean": "lerna clean", 9 | "build": "lerna run build", 10 | "linkall": "lerna exec -- yarn link", 11 | "test": "lerna run test", 12 | "update": "lerna publish --skip-git --skip-npm", 13 | "eslint": "eslint packages/* --ext .js", 14 | "build-api-docs": "yarn widdershins --search false --language_tabs 'nodejs:NodeJS' 'python:Python' --summary ./packages/api-lib/api-spec.yaml -o ./docs/api.md & yarn shins --inline --logo ./docs/images/logo.png -o ./docs/index.html ./docs/api.md" 15 | }, 16 | "devDependencies": { 17 | "eslint": "^5.7.0", 18 | "eslint-config-airbnb": "^16.1.0", 19 | "eslint-plugin-import": "^2.9.0", 20 | "eslint-plugin-jsdoc": "^3.5.0", 21 | "eslint-plugin-jsx-a11y": "^6.0.3", 22 | "eslint-plugin-react": "^7.7.0", 23 | "lerna": "^2.11.0", 24 | "shins": "^2.3.2-3", 25 | "widdershins": "^3.6.6" 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /packages/api-lib/README.md: -------------------------------------------------------------------------------- 1 | ## @sat-utils/api-lib 2 | 3 | ### Unit Tests 4 | ``` 5 | $ yarn 6 | $ yarn test 7 | ``` 8 | 9 | ### Integration Tests 10 | Navigate to the integration directory 11 | ``` 12 | $ cd ./tests/integration 13 | ``` 14 | Use the environment variable `DOCKER_NAME` to set your Docker host name. 15 | Normally `localhost`. 16 | ``` 17 | $ export DOCKER_NAME=localhost 18 | ``` 19 | The AWS-SDK library also requires fake key fields to create a connection so set. 20 | ``` 21 | $ export AWS_ACCESS_KEY_ID=none 22 | ``` 23 | ``` 24 | $ export AWS_SECRET_ACCESS_KEY=none 25 | ``` 26 | To run the tests 27 | ``` 28 | $ ./runIntegration.sh 29 | ``` 30 | 31 | ### Environment variables 32 | 33 | `AWS_REGION` 34 | `AWS_ACCESS_KEY_ID` 35 | `AWS_SECRET_ACCESS_KEY` 36 | `ES_HOST` 37 | `ES_BATCH_SIZE` 38 | `STAC_ID` 39 | `STAC_TITLE` 40 | `STAC_DESCRIPTION` 41 | `STAC_VERSION` 42 | `STAC_DOCS_URL` 43 | `SATAPI_URL` 44 | 45 | ### About 46 | [sat-api](https://github.com/sat-utils/sat-api) was created by [Development Seed]() and is part of a collection of tools called [sat-utils](https://github.com/sat-utils). 47 | -------------------------------------------------------------------------------- /packages/api-lib/api-spec.yaml: -------------------------------------------------------------------------------- 1 | openapi: 3.0.1 2 | info: 3 | title: The SAT-API 4 | version: 0.2.0 5 | description: >- 6 | Sat-api is a STAC compliant web API for searching and serving metadata for 7 | geospatial data (including but not limited to satellite imagery). 8 | Development Seed runs an instance of sat-api for the Landsat-8 9 | and Sentinel-2 imagery that is hosted on AWS. 10 | contact: 11 | name: Development Seed 12 | email: info@developmentseed.org 13 | url: 'https://developmentseed.org/contacts/' 14 | license: 15 | name: MIT License 16 | url: 'https://github.com/sat-utils/sat-api/blob/master/LICENSE' 17 | servers: 18 | - url: 'https://sat-api.developmentseed.org/' 19 | description: Production server 20 | - url: 'https://sat-api-dev.developmentseed.org/' 21 | description: Development server 22 | paths: 23 | /stac: 24 | get: 25 | summary: Return the root catalog or collection. 26 | description: >- 27 | Returns the root STAC Catalog or STAC Collection that is the entry point 28 | for users to browse with STAC Browser or for search engines to crawl. 29 | This can either return a single STAC Collection or more commonly a STAC 30 | catalog that usually lists sub-catalogs of STAC Collections, i.e. a 31 | simple catalog that lists all collections available through the API. 32 | tags: 33 | - STAC 34 | responses: 35 | '200': 36 | description: A catalog json definition. Used as an entry point for a crawler. 37 | content: 38 | application/json: 39 | schema: 40 | $ref: '#/components/schemas/catalogDefinition' 41 | 42 | /stac/search: 43 | get: 44 | summary: Search STAC items by simple filtering. 45 | description: >- 46 | Retrieve Items matching filters. Intended as a shorthand API for simple 47 | queries. 48 | operationId: getSearchSTAC 49 | tags: 50 | - STAC 51 | parameters: 52 | - $ref: '#/components/parameters/bbox' 53 | - $ref: '#/components/parameters/time' 54 | - $ref: '#/components/parameters/limit' 55 | - $ref: '#/components/parameters/query' 56 | - $ref: '#/components/parameters/sort' 57 | - $ref: '#/components/parameters/fields' 58 | responses: 59 | '200': 60 | description: A feature collection. 61 | content: 62 | application/geo+json: 63 | schema: 64 | $ref: '#/components/schemas/itemCollection' 65 | default: 66 | description: An error occurred. 67 | content: 68 | application/json: 69 | schema: 70 | $ref: '#/components/schemas/exception' 71 | 72 | post: 73 | summary: Search STAC items by full-featured filtering. 74 | description: >- 75 | retrieve items matching filters. Intended as the standard, full-featured 76 | query API. This method is mandatory. 77 | operationId: postSearchSTAC 78 | tags: 79 | - STAC 80 | requestBody: 81 | content: 82 | application/json: 83 | schema: 84 | $ref: '#/components/schemas/searchBody' 85 | responses: 86 | '200': 87 | description: A feature collection. 88 | content: 89 | application/geo+json: 90 | schema: 91 | $ref: '#/components/schemas/itemCollection' 92 | text/html: 93 | schema: 94 | type: string 95 | default: 96 | description: An error occurred. 97 | content: 98 | application/json: 99 | schema: 100 | $ref: '#/components/schemas/exception' 101 | text/html: 102 | schema: 103 | type: string 104 | /: 105 | get: 106 | summary: landing page of this API 107 | description: >- 108 | The landing page provides links to the API definition, the Conformance 109 | statements and the metadata about the feature data in this dataset. 110 | operationId: getLandingPage 111 | tags: 112 | - Capabilities 113 | responses: 114 | '200': 115 | description: links to the API capabilities 116 | content: 117 | application/json: 118 | schema: 119 | $ref: '#/components/schemas/root' 120 | text/html: 121 | schema: 122 | type: string 123 | 124 | /collections: 125 | get: 126 | summary: describe the feature collections in the dataset 127 | operationId: describeCollections 128 | tags: 129 | - Capabilities 130 | responses: 131 | '200': 132 | description: Metdata about the feature collections shared by this API. 133 | content: 134 | application/json: 135 | schema: 136 | $ref: '#/components/schemas/content' 137 | 138 | '/collections/{collectionId}': 139 | get: 140 | summary: 'describe the {collectionId} feature collection' 141 | operationId: describeCollection 142 | tags: 143 | - Capabilities 144 | parameters: 145 | - $ref: '#/components/parameters/collectionId' 146 | responses: 147 | '200': 148 | description: 'Metadata about the {collectionId} collection shared by this API.' 149 | content: 150 | application/json: 151 | schema: 152 | $ref: '#/components/schemas/collectionInfo' 153 | default: 154 | description: An error occurred. 155 | content: 156 | application/json: 157 | schema: 158 | $ref: '#/components/schemas/exception' 159 | 160 | '/collections/{collectionId}/items': 161 | get: 162 | summary: 'retrieve features of feature collection {collectionId}' 163 | description: >- 164 | Every feature in a dataset belongs to a collection. A dataset may 165 | consist of multiple feature collections. A feature collection is often a 166 | collection of features of a similar type, based on a common schema.\ 167 | 168 | Use content negotiation to request HTML or GeoJSON. 169 | operationId: getFeatures 170 | tags: 171 | - Features 172 | parameters: 173 | - $ref: '#/components/parameters/collectionId' 174 | - $ref: '#/components/parameters/limit' 175 | - $ref: '#/components/parameters/bbox' 176 | - $ref: '#/components/parameters/time' 177 | - $ref: '#/components/parameters/query' 178 | - $ref: '#/components/parameters/sort' 179 | responses: 180 | '200': 181 | description: >- 182 | Information about the feature collection plus the first features 183 | matching the selection parameters. 184 | content: 185 | application/geo+json: 186 | schema: 187 | $ref: '#/components/schemas/itemCollection' 188 | default: 189 | description: An error occurred. 190 | content: 191 | application/json: 192 | schema: 193 | $ref: '#/components/schemas/exception' 194 | 195 | '/collections/{collectionId}/items/{featureId}': 196 | get: 197 | summary: retrieve a feature; use content negotiation to request HTML or GeoJSON 198 | operationId: getFeature 199 | tags: 200 | - Features 201 | parameters: 202 | - $ref: '#/components/parameters/collectionId' 203 | - $ref: '#/components/parameters/featureId' 204 | responses: 205 | '200': 206 | description: A feature. 207 | content: 208 | application/geo+json: 209 | schema: 210 | $ref: '#/components/schemas/item' 211 | default: 212 | description: An error occurred. 213 | content: 214 | application/json: 215 | schema: 216 | $ref: '#/components/schemas/exception' 217 | 218 | components: 219 | parameters: 220 | limit: 221 | name: limit 222 | in: query 223 | description: | 224 | The optional limit parameter limits the number of items that are 225 | presented in the response document. 226 | 227 | Only items are counted that are on the first level of the collection in 228 | the response document. Nested objects contained within the explicitly 229 | requested items shall not be counted. 230 | 231 | * Minimum = 1 232 | * Maximum = 10000 233 | * Default = 10 234 | required: false 235 | schema: 236 | type: integer 237 | minimum: 1 238 | maximum: 10000 239 | default: 10 240 | style: form 241 | explode: false 242 | bbox: 243 | name: bbox 244 | in: query 245 | description: | 246 | Only features that have a geometry that intersects the bounding box are 247 | selected. The bounding box is provided as four or six numbers, 248 | depending on whether the coordinate reference system includes a 249 | vertical axis (elevation or depth): 250 | 251 | * Lower left corner, coordinate axis 1 252 | * Lower left corner, coordinate axis 2 253 | * Lower left corner, coordinate axis 3 (optional) 254 | * Upper right corner, coordinate axis 1 255 | * Upper right corner, coordinate axis 2 256 | * Upper right corner, coordinate axis 3 (optional) 257 | 258 | The coordinate reference system of the values is WGS84 259 | longitude/latitude (http://www.opengis.net/def/crs/OGC/1.3/CRS84) unless 260 | a different coordinate reference system is specified in the parameter 261 | `bbox-crs`. 262 | 263 | For WGS84 longitude/latitude the values are in most cases the sequence 264 | of minimum longitude, minimum latitude, maximum longitude and maximum 265 | latitude. However, in cases where the box spans the antimeridian the 266 | first value (west-most box edge) is larger than the third value 267 | (east-most box edge). 268 | 269 | 270 | If a feature has multiple spatial geometry properties, it is the 271 | decision of the server whether only a single spatial geometry property 272 | is used to determine the extent or all relevant geometries. 273 | required: false 274 | schema: 275 | type: array 276 | minItems: 4 277 | maxItems: 6 278 | items: 279 | type: number 280 | style: form 281 | explode: false 282 | time: 283 | name: time 284 | in: query 285 | description: > 286 | Either a date-time or a period string that adheres to RFC3339. Examples: 287 | 288 | * A date-time: "2018-02-12T23:20:50Z" 289 | * A period: "2018-02-12T00:00:00Z/2018-03-18T12:31:12Z" or "2018-02-12T00:00:00Z/P1M6DT12H31M12S" 290 | 291 | Only features that have a temporal property that intersects the value 292 | of `time` are selected. If a feature has multiple temporal properties, it 293 | is the decision of the server whether only a single temporal property is 294 | used to determine the extent or all relevant temporal properties. 295 | required: false 296 | schema: 297 | type: string 298 | style: form 299 | explode: false 300 | collectionId: 301 | name: collectionId 302 | in: path 303 | required: true 304 | description: Identifier (name) of a specific collection 305 | schema: 306 | type: string 307 | featureId: 308 | name: featureId 309 | in: path 310 | description: Local identifier of a specific feature 311 | required: true 312 | schema: 313 | type: string 314 | query: 315 | name: query 316 | in: query 317 | description: >- 318 | query for properties in items. Use the JSON form of the queryFilter used 319 | in POST. 320 | required: false 321 | schema: 322 | type: string 323 | sort: 324 | name: sort 325 | in: query 326 | description: Allows sorting results by the specified properties 327 | required: false 328 | schema: 329 | $ref: '#/components/schemas/sort' 330 | fields: 331 | name: fields 332 | in: query 333 | description: Determines the shape of the features in the response 334 | required: false 335 | schema: 336 | $ref: '#/components/schemas/fields' 337 | style: form 338 | explode: false 339 | schemas: 340 | exception: 341 | type: object 342 | required: 343 | - code 344 | properties: 345 | code: 346 | type: string 347 | description: 348 | type: string 349 | links: 350 | type: array 351 | items: 352 | $ref: '#/components/schemas/link' 353 | link: 354 | type: object 355 | required: 356 | - href 357 | - rel 358 | additionalProperties: true 359 | properties: 360 | href: 361 | type: string 362 | format: url 363 | example: 'http://www.geoserver.example/stac/naip/child/catalog.json' 364 | rel: 365 | type: string 366 | example: child 367 | type: 368 | type: string 369 | example: application/json 370 | title: 371 | type: string 372 | example: NAIP Child Catalog 373 | searchBody: 374 | description: The search criteria 375 | type: object 376 | allOf: 377 | - $ref: '#/components/schemas/bboxFilter' 378 | - $ref: '#/components/schemas/timeFilter' 379 | - $ref: '#/components/schemas/intersectsFilter' 380 | - $ref: '#/components/schemas/queryFilter' 381 | - $ref: '#/components/schemas/sortFilter' 382 | bbox: 383 | description: | 384 | Only features that have a geometry that intersects the bounding box are 385 | selected. The bounding box is provided as four or six numbers, 386 | depending on whether the coordinate reference system includes a 387 | vertical axis (elevation or depth): 388 | 389 | * Lower left corner, coordinate axis 1 390 | * Lower left corner, coordinate axis 2 391 | * Lower left corner, coordinate axis 3 (optional) 392 | * Upper right corner, coordinate axis 1 393 | * Upper right corner, coordinate axis 2 394 | * Upper right corner, coordinate axis 3 (optional) 395 | 396 | The coordinate reference system of the values is WGS84 397 | longitude/latitude (http://www.opengis.net/def/crs/OGC/1.3/CRS84) unless 398 | a different coordinate reference system is specified in the parameter 399 | `bbox-crs`. 400 | 401 | For WGS84 longitude/latitude the values are in most cases the sequence 402 | of minimum longitude, minimum latitude, maximum longitude and maximum 403 | latitude. However, in cases where the box spans the antimeridian the 404 | first value (west-most box edge) is larger than the third value 405 | (east-most box edge). 406 | 407 | 408 | If a feature has multiple spatial geometry properties, it is the 409 | decision of the server whether only a single spatial geometry property 410 | is used to determine the extent or all relevant geometries. 411 | type: array 412 | minItems: 4 413 | maxItems: 6 414 | items: 415 | type: number 416 | example: 417 | - -110 418 | - 39.5 419 | - -105 420 | - 40.5 421 | bboxFilter: 422 | type: object 423 | description: Only return items that intersect the provided bounding box. 424 | properties: 425 | bbox: 426 | $ref: '#/components/schemas/bbox' 427 | timeFilter: 428 | description: An object representing a time based filter. 429 | type: object 430 | properties: 431 | time: 432 | $ref: '#/components/schemas/time' 433 | intersectsFilter: 434 | type: object 435 | description: Only returns items that intersect with the provided polygon. 436 | properties: 437 | intersects: 438 | $ref: 'http://geojson.org/schema/Geometry.json' 439 | time: 440 | type: string 441 | description: > 442 | Either a date-time or a period string that adheres to RFC 3339. 443 | Examples: 444 | 445 | * A date-time: "2018-02-12T23:20:50Z" 446 | * A period: "2018-02-12T00:00:00Z/2018-03-18T12:31:12Z" or "2018-02-12T00:00:00Z/P1M6DT12H31M12S" 447 | 448 | Only features that have a temporal property that intersects the value of 449 | `time` are selected. 450 | 451 | If a feature has multiple temporal properties, it is the decision of the 452 | server whether only a single temporal property is used to determine the 453 | extent or all relevant temporal properties. 454 | example: '2018-02-12T00:00:00Z/2018-03-18T12:31:12Z' 455 | catalogDefinition: 456 | type: object 457 | required: 458 | - stac_version 459 | - id 460 | - description 461 | - links 462 | additionalProperties: true 463 | properties: 464 | stac_version: 465 | type: string 466 | example: 0.6.0 467 | id: 468 | type: string 469 | example: naip 470 | title: 471 | type: string 472 | example: NAIP Imagery 473 | description: 474 | type: string 475 | example: Catalog of NAIP Imagery. 476 | links: 477 | $ref: '#/components/schemas/links' 478 | itemCollection: 479 | type: object 480 | required: 481 | - features 482 | - type 483 | properties: 484 | type: 485 | type: string 486 | enum: 487 | - FeatureCollection 488 | features: 489 | type: array 490 | items: 491 | $ref: '#/components/schemas/item' 492 | links: 493 | $ref: '#/components/schemas/itemCollectionLinks' 494 | item: 495 | type: object 496 | required: 497 | - id 498 | - type 499 | - geometry 500 | - bbox 501 | - links 502 | - properties 503 | - assets 504 | properties: 505 | id: 506 | $ref: '#/components/schemas/itemId' 507 | bbox: 508 | $ref: '#/components/schemas/bbox' 509 | geometry: 510 | $ref: 'http://geojson.org/schema/Geometry.json' 511 | type: 512 | $ref: '#/components/schemas/itemType' 513 | properties: 514 | $ref: '#/components/schemas/itemProperties' 515 | links: 516 | $ref: '#/components/schemas/links' 517 | assets: 518 | $ref: '#/components/schemas/itemAssets' 519 | example: 520 | type: Feature 521 | id: CS3-20160503_132130_04 522 | bbox: 523 | - -122.59750209 524 | - 37.48803556 525 | - -122.2880486 526 | - 37.613537207 527 | geometry: 528 | type: Polygon 529 | coordinates: 530 | - - - -122.308150179 531 | - 37.488035566 532 | - - -122.597502109 533 | - 37.538869539 534 | - - -122.576687533 535 | - 37.613537207 536 | - - -122.2880486 537 | - 37.562818007 538 | - - -122.308150179 539 | - 37.488035566 540 | properties: 541 | datetime: '2016-05-03T13:21:30.040Z' 542 | links: 543 | - rel: self 544 | href: >- 545 | http://https://sat-api.developmentseed.org/collections/landsat-8-l1/items/LC80100102015050LGN00.json 546 | assets: 547 | analytic: 548 | title: 4-Band Analytic 549 | href: >- 550 | http://cool-sat.com/LC80100102015050LGN00/band4.tiff 551 | type: image/tiff 552 | thumbnail: 553 | title: Thumbnail 554 | href: >- 555 | http://cool-sat.com/LC80100102015050LGN00/thumb.png 556 | type: image/png 557 | itemId: 558 | type: string 559 | example: path/to/example.tif 560 | description: 'Provider identifier, a unique ID, potentially a link to a file.' 561 | itemType: 562 | type: string 563 | description: The GeoJSON type 564 | enum: 565 | - Feature 566 | itemAssets: 567 | type: object 568 | additionalProperties: 569 | type: object 570 | required: 571 | - href 572 | properties: 573 | href: 574 | type: string 575 | format: url 576 | description: Link to the asset object 577 | example: >- 578 | http://cool-sat.com/LC80100102015050LGN00/thumb.png 579 | title: 580 | type: string 581 | description: Displayed title 582 | example: Thumbnail 583 | type: 584 | type: string 585 | description: Media type of the asset 586 | example: image/png 587 | itemProperties: 588 | type: object 589 | required: 590 | - datetime 591 | description: provides the core metatdata fields plus extensions 592 | properties: 593 | datetime: 594 | $ref: '#/components/schemas/time' 595 | additionalProperties: 596 | description: Any additional properties added in via extensions. 597 | itemCollectionLinks: 598 | type: array 599 | description: >- 600 | An array of links. Can be used for pagination, e.g. by providing a link 601 | with the `next` relation type. 602 | items: 603 | $ref: '#/components/schemas/link' 604 | example: 605 | - rel: next 606 | href: >- 607 | http://sat-api.developmentseed.org/collections/landsat-8-l1/items/gasd312fsaeg 608 | root: 609 | type: object 610 | required: 611 | - links 612 | properties: 613 | links: 614 | type: array 615 | items: 616 | $ref: '#/components/schemas/link' 617 | example: 618 | - href: 'http://sat-api.developmentseed.org' 619 | rel: self 620 | type: application/json 621 | title: this document 622 | - href: 'http://sat-api.developmentseed.org/api' 623 | rel: service 624 | type: application/json 625 | title: this document 626 | - href: 'http://sat-api.developmentseed.org/collections' 627 | rel: data 628 | type: application/json 629 | title: Metadata about the feature collections 630 | req-classes: 631 | type: object 632 | required: 633 | - conformsTo 634 | properties: 635 | conformsTo: 636 | type: array 637 | items: 638 | type: string 639 | example: 640 | - 'http://www.opengis.net/spec/wfs-1/3.0/req/core' 641 | - 'http://www.opengis.net/spec/wfs-1/3.0/req/oas30' 642 | - 'http://www.opengis.net/spec/wfs-1/3.0/req/html' 643 | - 'http://www.opengis.net/spec/wfs-1/3.0/req/geojson' 644 | content: 645 | type: object 646 | required: 647 | - links 648 | - collections 649 | properties: 650 | links: 651 | type: array 652 | items: 653 | $ref: '#/components/schemas/link' 654 | example: 655 | - href: 'http://data.example.org/collections.json' 656 | rel: self 657 | type: application/json 658 | title: this document 659 | - href: 'http://data.example.org/collections.html' 660 | rel: alternate 661 | type: text/html 662 | title: this document as HTML 663 | - href: 'http://schemas.example.org/1.0/foobar.xsd' 664 | rel: describedBy 665 | type: application/xml 666 | title: XML schema for Acme Corporation data 667 | collections: 668 | type: array 669 | items: 670 | $ref: '#/components/schemas/collectionInfo' 671 | collectionInfo: 672 | type: object 673 | required: 674 | - name 675 | - links 676 | - stac_version 677 | - id 678 | - description 679 | - license 680 | - extent 681 | properties: 682 | name: 683 | description: 'identifier of the collection used, for example, in URIs' 684 | type: string 685 | example: buildings 686 | title: 687 | description: human readable title of the collection 688 | type: string 689 | example: Buildings 690 | description: 691 | description: a description of the features in the collection 692 | type: string 693 | example: Buildings in the city of Bonn. 694 | links: 695 | type: array 696 | items: 697 | $ref: '#/components/schemas/link' 698 | example: 699 | - href: 'http://data.example.org/collections/buildings/items' 700 | rel: item 701 | type: application/geo+json 702 | title: Buildings 703 | - href: 'http://example.org/concepts/building.html' 704 | rel: describedBy 705 | type: text/html 706 | title: Feature catalogue for buildings 707 | extent: 708 | $ref: '#/components/schemas/extent' 709 | crs: 710 | description: >- 711 | The coordinate reference systems in which geometries may be 712 | retrieved. Coordinate reference systems are identified by a URI. The 713 | first coordinate reference system is the coordinate reference system 714 | that is used by default. This is always 715 | "http://www.opengis.net/def/crs/OGC/1.3/CRS84", i.e. WGS84 716 | longitude/latitude. 717 | type: array 718 | items: 719 | type: string 720 | default: 721 | - 'http://www.opengis.net/def/crs/OGC/1.3/CRS84' 722 | stac_version: 723 | type: string 724 | example: 0.6.0 725 | id: 726 | description: 'identifier of the collection used, for example, in URIs' 727 | type: string 728 | example: buildings 729 | keywords: 730 | title: Keywords 731 | type: array 732 | items: 733 | type: string 734 | example: 735 | - buildings 736 | - properties 737 | - constructions 738 | version: 739 | title: Collection Version 740 | type: string 741 | example: 1 742 | license: 743 | title: Collection License Name 744 | type: string 745 | example: Apache-2.0 746 | providers: 747 | type: array 748 | items: 749 | properties: 750 | name: 751 | title: Organization name 752 | type: string 753 | example: Big Building Corp 754 | description: 755 | title: Provider description 756 | type: string 757 | example: No further processing applied. 758 | roles: 759 | title: Organization roles 760 | type: array 761 | items: 762 | type: string 763 | enum: 764 | - producer 765 | - licensor 766 | - processor 767 | - host 768 | example: 769 | - producer 770 | - licensor 771 | url: 772 | title: Homepage 773 | description: >- 774 | Homepage on which the provider describes the dataset and 775 | publishes contact information. 776 | type: string 777 | format: url 778 | example: 'http://www.big-building.com' 779 | queryFilter: 780 | type: object 781 | description: Allows users to query properties for specific values 782 | properties: 783 | query: 784 | $ref: '#/components/schemas/query' 785 | query: 786 | type: object 787 | description: Define which properties to query and the operatations to apply 788 | additionalProperties: 789 | $ref: '#/components/schemas/queryProp' 790 | example: 791 | 'eo:cloud_cover': 792 | lt: 50 793 | queryProp: 794 | description: Apply query operations to a specific property 795 | anyOf: 796 | - description: >- 797 | if the object doesn't contain any of the operators, it is equivalent 798 | to using the equals operator 799 | - type: object 800 | description: Match using an operator 801 | properties: 802 | eq: 803 | description: >- 804 | Find items with a property that is equal to the specified value. 805 | For strings, a case-insensitive comparison must be performed. 806 | gt: 807 | type: number 808 | description: >- 809 | Find items with a property value greater than the specified 810 | value. 811 | lt: 812 | type: number 813 | description: Find items with a property value less than the specified value. 814 | gte: 815 | type: number 816 | description: >- 817 | Find items with a property value greater than or equal the 818 | specified value. 819 | lte: 820 | type: number 821 | description: >- 822 | Find items with a property value greater than or equal the 823 | specified value. 824 | sortFilter: 825 | type: object 826 | description: Sort the results 827 | properties: 828 | sort: 829 | $ref: '#/components/schemas/sort' 830 | sort: 831 | type: array 832 | description: | 833 | An array of objects containing a property name and sort direction. 834 | minItems: 1 835 | items: 836 | type: object 837 | required: 838 | - field 839 | properties: 840 | field: 841 | type: string 842 | direction: 843 | type: string 844 | default: asc 845 | enum: 846 | - asc 847 | - desc 848 | example: 849 | - field: 'eo:cloud_cover' 850 | direction: desc 851 | extent: 852 | type: object 853 | properties: 854 | crs: 855 | description: >- 856 | Coordinate reference system of the coordinates in the spatial extent 857 | (property `spatial`). In the Core, only WGS84 longitude/latitude is 858 | supported. Extensions may support additional coordinate reference 859 | systems. 860 | type: string 861 | enum: 862 | - 'http://www.opengis.net/def/crs/OGC/1.3/CRS84' 863 | default: 'http://www.opengis.net/def/crs/OGC/1.3/CRS84' 864 | spatial: 865 | description: >- 866 | West, south, east, north edges of the spatial extent. The minimum 867 | and maximum values apply to the coordinate reference system WGS84 868 | longitude/latitude that is supported in the Core. If, for example, a 869 | projected coordinate reference system is used, the minimum and 870 | maximum values need to be adjusted. 871 | type: array 872 | minItems: 4 873 | maxItems: 6 874 | items: 875 | type: number 876 | example: 877 | - -180 878 | - -90 879 | - 180 880 | - 90 881 | trs: 882 | description: >- 883 | Temporal reference system of the coordinates in the temporal extent 884 | (property `temporal`). In the Core, only the Gregorian calendar is 885 | supported. Extensions may support additional temporal reference 886 | systems. 887 | type: string 888 | enum: 889 | - 'http://www.opengis.net/def/uom/ISO-8601/0/Gregorian' 890 | default: 'http://www.opengis.net/def/uom/ISO-8601/0/Gregorian' 891 | temporal: 892 | description: Begin and end times of the temporal extent. 893 | type: array 894 | minItems: 2 895 | maxItems: 2 896 | items: 897 | type: string 898 | format: dateTime 899 | example: 900 | - '2011-11-11T12:22:11Z' 901 | - '2012-11-24T12:32:43Z' 902 | fieldsFilter: 903 | type: object 904 | description: Determines the shape of the features in the response 905 | properties: 906 | fields: 907 | $ref: '#/components/schemas/fields' 908 | fields: 909 | description: > 910 | The geometry member determines whether the geometry is populated or is 911 | null. The 912 | 913 | include and exclude members specify an array of property names that are 914 | either 915 | 916 | included or excluded from the result, respectively. If both include and 917 | exclude 918 | 919 | are specified, include takes precedence. 920 | 921 | id and links are required feature properties and cannot be excluded. 922 | type: object 923 | properties: 924 | geometry: 925 | type: boolean 926 | include: 927 | type: array 928 | items: 929 | type: string 930 | example: 931 | - 'eo:cloud_cover' 932 | exclude: 933 | type: array 934 | items: 935 | type: string 936 | example: 937 | - 'eo:sun_azimuth' 938 | tags: 939 | - name: STAC 940 | description: Extension to WFS3 Core to support STAC metadata model and search API 941 | -------------------------------------------------------------------------------- /packages/api-lib/index.js: -------------------------------------------------------------------------------- 1 | /* eslint-disable global-require */ 2 | 3 | 'use strict' 4 | 5 | module.exports = { 6 | api: require('./libs/api.js'), 7 | es: require('./libs/es.js'), 8 | ingest: require('./libs/ingest.js') 9 | } 10 | -------------------------------------------------------------------------------- /packages/api-lib/libs/ElasticSearchWriteableStream.js: -------------------------------------------------------------------------------- 1 | const stream = require('stream') 2 | const logger = require('./logger') 3 | 4 | class ElasticSearchWritableStream extends stream.Writable { 5 | constructor(config, options) { 6 | super(options) 7 | this.config = config 8 | 9 | this.client = this.config.client 10 | } 11 | 12 | _destroy() { 13 | return this.client.close() 14 | } 15 | 16 | // Allows the flexibility to batch write to multiple indexes. 17 | transformRecords(chunks) { 18 | const operations = chunks.reduce((bulkOperations, chunk) => { 19 | const operation = {} 20 | const { chunk: record } = chunk 21 | operation[record.action] = { 22 | _index: record.index, 23 | _type: record.type, 24 | _id: record.id 25 | } 26 | if (record.parent) { 27 | operation[record.action]._parent = record.parent 28 | } 29 | 30 | bulkOperations.push(operation) 31 | if (record.action !== 'delete') { 32 | bulkOperations.push(record.body) 33 | } 34 | return bulkOperations 35 | }, []) 36 | return operations 37 | } 38 | // Write individual records with update/upsert 39 | async _write(record, enc, next) { 40 | try { 41 | const { index, id, body } = record 42 | await this.client.update({ 43 | index, 44 | type: 'doc', 45 | id, 46 | body 47 | }) 48 | logger.debug(`Wrote document ${id}`) 49 | next() 50 | } catch (err) { 51 | logger.error(err) 52 | next() 53 | } 54 | } 55 | 56 | // Batch write records, use highWaterMark to set batch size. 57 | async _writev(records, next) { 58 | const body = this.transformRecords(records) 59 | try { 60 | const result = await this.client.bulk({ body }) 61 | const { errors, items } = result 62 | if (errors) { 63 | logger.error(items) 64 | } else { 65 | logger.debug(`Wrote batch of documents size ${body.length / 2}`) 66 | } 67 | next() 68 | } catch (err) { 69 | logger.error(err) 70 | next() 71 | } 72 | } 73 | } 74 | 75 | module.exports = ElasticSearchWritableStream 76 | -------------------------------------------------------------------------------- /packages/api-lib/libs/api.js: -------------------------------------------------------------------------------- 1 | const gjv = require('geojson-validation') 2 | const extent = require('@mapbox/extent') 3 | const { feature } = require('@turf/helpers') 4 | const logger = require('./logger') 5 | 6 | const extractIntersects = function (params) { 7 | let intersectsGeometry 8 | const geojsonError = new Error('Invalid GeoJSON Feature or geometry') 9 | const { intersects } = params 10 | if (intersects) { 11 | let geojson 12 | // if we receive a string, try to parse as GeoJSON, otherwise assume it is GeoJSON 13 | if (typeof intersects === 'string') { 14 | try { 15 | geojson = JSON.parse(intersects) 16 | } catch (e) { 17 | throw geojsonError 18 | } 19 | } else { 20 | geojson = Object.assign({}, intersects) 21 | } 22 | 23 | if (gjv.valid(geojson)) { 24 | if (geojson.type === 'FeatureCollection') { 25 | throw geojsonError 26 | } else if (geojson.type !== 'Feature') { 27 | geojson = feature(geojson) 28 | } 29 | intersectsGeometry = geojson 30 | } else { 31 | throw geojsonError 32 | } 33 | } 34 | return intersectsGeometry 35 | } 36 | 37 | const extractBbox = function (params) { 38 | let intersectsGeometry 39 | const { bbox } = params 40 | if (bbox) { 41 | let bboxArray 42 | if (typeof bbox === 'string') { 43 | bboxArray = JSON.parse(bbox) 44 | } else { 45 | bboxArray = bbox 46 | } 47 | const boundingBox = extent(bboxArray) 48 | const geojson = feature(boundingBox.polygon()) 49 | intersectsGeometry = geojson 50 | } 51 | return intersectsGeometry 52 | } 53 | 54 | 55 | const extractStacQuery = function (params) { 56 | let stacQuery 57 | const { query } = params 58 | if (query) { 59 | if (typeof query === 'string') { 60 | const parsed = JSON.parse(query) 61 | stacQuery = parsed 62 | } else { 63 | stacQuery = Object.assign({}, query) 64 | } 65 | } 66 | return stacQuery 67 | } 68 | 69 | const extractSort = function (params) { 70 | let sortRules 71 | const { sort } = params 72 | if (sort) { 73 | if (typeof sort === 'string') { 74 | sortRules = JSON.parse(sort) 75 | } else { 76 | sortRules = sort.slice() 77 | } 78 | } 79 | return sortRules 80 | } 81 | 82 | const extractFields = function (params) { 83 | let fieldRules 84 | const { fields } = params 85 | if (fields) { 86 | if (typeof fields === 'string') { 87 | fieldRules = JSON.parse(fields) 88 | } else { 89 | fieldRules = fields 90 | } 91 | } 92 | return fieldRules 93 | } 94 | 95 | const extractIds = function (params) { 96 | let idsRules 97 | const { ids } = params 98 | if (ids) { 99 | if (typeof ids === 'string') { 100 | idsRules = JSON.parse(ids) 101 | } else { 102 | idsRules = ids.slice() 103 | } 104 | } 105 | return idsRules 106 | } 107 | 108 | const parsePath = function (path) { 109 | const searchFilters = { 110 | stac: false, 111 | collections: false, 112 | search: false, 113 | collectionId: false, 114 | items: false, 115 | itemId: false 116 | } 117 | const stac = 'stac' 118 | const collections = 'collections' 119 | const search = 'search' 120 | const items = 'items' 121 | 122 | const pathComponents = path.split('/').filter((x) => x) 123 | const { length } = pathComponents 124 | searchFilters.stac = pathComponents[0] === stac 125 | searchFilters.collections = pathComponents[0] === collections 126 | searchFilters.collectionId = 127 | pathComponents[0] === collections && length >= 2 ? pathComponents[1] : false 128 | searchFilters.search = pathComponents[1] === search 129 | searchFilters.items = pathComponents[2] === items 130 | searchFilters.itemId = 131 | pathComponents[2] === items && length === 4 ? pathComponents[3] : false 132 | return searchFilters 133 | } 134 | 135 | // Impure - mutates results 136 | const addCollectionLinks = function (results, endpoint) { 137 | results.forEach((result) => { 138 | const { id, links } = result 139 | // self link 140 | links.splice(0, 0, { 141 | rel: 'self', 142 | href: `${endpoint}/collections/${id}` 143 | }) 144 | // parent catalog 145 | links.push({ 146 | rel: 'parent', 147 | href: `${endpoint}/stac` 148 | }) 149 | // root catalog 150 | links.push({ 151 | rel: 'root', 152 | href: `${endpoint}/stac` 153 | }) 154 | // child items 155 | links.push({ 156 | rel: 'items', 157 | href: `${endpoint}/collections/${id}/items` 158 | }) 159 | }) 160 | return results 161 | } 162 | 163 | // Impure - mutates results 164 | const addItemLinks = function (results, endpoint) { 165 | results.forEach((result) => { 166 | let { links } = result 167 | const { id, collection } = result 168 | 169 | links = (links === undefined) ? [] : links 170 | // self link 171 | links.splice(0, 0, { 172 | rel: 'self', 173 | href: `${endpoint}/collections/${collection}/items/${id}` 174 | }) 175 | // parent catalogs 176 | links.push({ 177 | rel: 'parent', 178 | href: `${endpoint}/collections/${collection}` 179 | }) 180 | links.push({ 181 | rel: 'collection', 182 | href: `${endpoint}/collections/${collection}` 183 | }) 184 | // root catalog 185 | links.push({ 186 | rel: 'root', 187 | href: `${endpoint}/stac` 188 | }) 189 | result.type = 'Feature' 190 | return result 191 | }) 192 | return results 193 | } 194 | 195 | const buildRootObject = function (endpoint) { 196 | const stac_docs_url = process.env.STAC_DOCS_URL 197 | const root = { 198 | links: [ 199 | { 200 | href: endpoint, 201 | rel: 'self' 202 | }, 203 | { 204 | href: `${endpoint}/collections`, 205 | rel: 'data' 206 | }, 207 | { 208 | href: stac_docs_url, 209 | rel: 'service' 210 | } 211 | ] 212 | } 213 | return root 214 | } 215 | 216 | const collectionsToCatalogLinks = function (results, endpoint) { 217 | const stac_version = process.env.STAC_VERSION 218 | const stac_id = process.env.STAC_ID 219 | const stac_title = process.env.STAC_TITLE 220 | const stac_description = process.env.STAC_DESCRIPTION 221 | const catalog = { 222 | stac_version, 223 | id: stac_id, 224 | title: stac_title, 225 | description: stac_description 226 | } 227 | catalog.links = results.map((result) => { 228 | const { id } = result 229 | return { 230 | rel: 'child', 231 | href: `${endpoint}/collections/${id}` 232 | } 233 | }) 234 | catalog.links.push({ 235 | rel: 'self', 236 | href: `${endpoint}/stac` 237 | }) 238 | catalog.links.push({ 239 | rel: 'search', 240 | href: `${endpoint}/stac/search` 241 | }) 242 | return catalog 243 | } 244 | 245 | const wrapResponseInFeatureCollection = function ( 246 | meta, features = [], links = [] 247 | ) { 248 | return { 249 | type: 'FeatureCollection', 250 | meta, 251 | features, 252 | links 253 | } 254 | } 255 | 256 | const buildPageLinks = function (meta, parameters, endpoint) { 257 | const pageLinks = [] 258 | 259 | const dictToURI = (dict) => ( 260 | Object.keys(dict).map( 261 | (p) => `${encodeURIComponent(p)}=${encodeURIComponent(JSON.stringify(dict[p]))}` 262 | ).join('&') 263 | ) 264 | const { found, page, limit } = meta 265 | if ((page * limit) < found) { 266 | const newParams = Object.assign({}, parameters, { page: page + 1, limit }) 267 | const nextQueryParameters = dictToURI(newParams) 268 | pageLinks.push({ 269 | rel: 'next', 270 | title: 'Next page of results', 271 | href: `${endpoint}/stac/search?${nextQueryParameters}` 272 | }) 273 | } 274 | return pageLinks 275 | } 276 | 277 | const searchItems = async function (parameters, page, limit, backend, endpoint) { 278 | const { results: itemsResults, meta: itemsMeta } = 279 | await backend.search(parameters, 'items', page, limit) 280 | const pageLinks = buildPageLinks(itemsMeta, parameters, endpoint) 281 | const items = addItemLinks(itemsResults, endpoint) 282 | const response = wrapResponseInFeatureCollection(itemsMeta, items, pageLinks) 283 | return response 284 | } 285 | 286 | const search = async function ( 287 | path = '', queryParameters = {}, backend, endpoint = '' 288 | ) { 289 | let apiResponse 290 | try { 291 | const pathElements = parsePath(path) 292 | const hasPathElement = 293 | Object.keys(pathElements).reduce((accumulator, key) => { 294 | let containsPathElement 295 | if (accumulator) { 296 | containsPathElement = true 297 | } else { 298 | containsPathElement = pathElements[key] 299 | } 300 | return containsPathElement 301 | }, false) 302 | 303 | const { 304 | stac, 305 | search: searchPath, 306 | collections, 307 | collectionId, 308 | items, 309 | itemId 310 | } = pathElements 311 | 312 | const { 313 | limit, 314 | page, 315 | time: datetime 316 | } = queryParameters 317 | const bbox = extractBbox(queryParameters) 318 | const hasIntersects = extractIntersects(queryParameters) 319 | const sort = extractSort(queryParameters) 320 | // Prefer intersects 321 | const intersects = hasIntersects || bbox 322 | const query = extractStacQuery(queryParameters) 323 | const fields = extractFields(queryParameters) 324 | const ids = extractIds(queryParameters) 325 | const parameters = { 326 | datetime, 327 | intersects, 328 | query, 329 | sort, 330 | fields, 331 | ids 332 | } 333 | const colLimit = process.env.SATAPI_COLLECTION_LIMIT || 100 334 | // Keep only existing parameters 335 | const searchParameters = Object.keys(parameters) 336 | .filter((key) => parameters[key]) 337 | .reduce((obj, key) => ({ 338 | ...obj, 339 | [key]: parameters[key] 340 | }), {}) 341 | // Landing page url 342 | if (!hasPathElement) { 343 | apiResponse = buildRootObject(endpoint) 344 | } 345 | // Root catalog with collection links 346 | if (stac && !searchPath) { 347 | const { results } = 348 | await backend.search({}, 'collections', page, colLimit) 349 | apiResponse = collectionsToCatalogLinks(results, endpoint) 350 | } 351 | // STAC Search 352 | if (stac && searchPath) { 353 | apiResponse = await searchItems( 354 | searchParameters, page, limit, backend, endpoint 355 | ) 356 | } 357 | // All collections 358 | if (collections && !collectionId) { 359 | const { results, meta } = 360 | await backend.search({}, 'collections', page, colLimit) 361 | const linkedCollections = addCollectionLinks(results, endpoint) 362 | apiResponse = { meta, collections: linkedCollections } 363 | } 364 | // Specific collection 365 | if (collections && collectionId && !items) { 366 | const collectionQuery = { id: collectionId } 367 | const { results } = await backend.search( 368 | collectionQuery, 'collections', page, limit 369 | ) 370 | const collection = addCollectionLinks(results, endpoint) 371 | if (collection.length > 0) { 372 | apiResponse = collection[0] 373 | } else { 374 | apiResponse = new Error('Collection not found') 375 | } 376 | } 377 | // Items in a collection 378 | if (collections && collectionId && items && !itemId) { 379 | const updatedQuery = Object.assign({}, searchParameters.query, { 380 | collections: [ 381 | collectionId 382 | ] 383 | }) 384 | const itemIdParameters = Object.assign( 385 | {}, searchParameters, { query: updatedQuery } 386 | ) 387 | apiResponse = await searchItems( 388 | itemIdParameters, page, limit, backend, endpoint 389 | ) 390 | } 391 | if (collections && collectionId && items && itemId) { 392 | const itemQuery = { id: itemId } 393 | const { results } = await backend.search(itemQuery, 'items', page, limit) 394 | const [item] = addItemLinks(results, endpoint) 395 | if (item) { 396 | apiResponse = item 397 | } else { 398 | apiResponse = new Error('Item not found') 399 | } 400 | } 401 | } catch (error) { 402 | logger.error(error) 403 | apiResponse = { 404 | code: 500, 405 | description: error.message 406 | } 407 | } 408 | return apiResponse 409 | } 410 | 411 | module.exports = { 412 | search, 413 | parsePath, 414 | searchItems, 415 | extractIntersects 416 | } 417 | -------------------------------------------------------------------------------- /packages/api-lib/libs/es.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | const AWS = require('aws-sdk') 4 | const httpAwsEs = require('http-aws-es') 5 | const elasticsearch = require('elasticsearch') 6 | const through2 = require('through2') 7 | const ElasticsearchWritableStream = require('./ElasticSearchWriteableStream') 8 | const logger = require('./logger') 9 | 10 | let _esClient 11 | /* 12 | This module is used for connecting to an Elasticsearch instance, writing records, 13 | searching records, and managing the indexes. It looks for the ES_HOST environment 14 | variable which is the URL to the elasticsearch host 15 | */ 16 | 17 | // Connect to an Elasticsearch cluster 18 | async function connect() { 19 | let esConfig 20 | let client 21 | 22 | // non-AWS ES 23 | if(!process.env.AWS_ACCESS_KEY_ID || process.env.AWS_SECRET_ACCESS_KEY) { 24 | // https://www.elastic.co/guide/en/elasticsearch/client/javascript-api/16.x/configuration.html 25 | esConfig = { 26 | hosts: process.env.ES_HOST || 'localhost:9200', 27 | apiVersion: process.env.ES_API_VERSION || '6.8', 28 | httpAuth: process.env.ES_HTTP_AUTH || null, 29 | // add further params here as needed. alternatively move to a kwargs/splat type arrangement 30 | requestTimeout: 120000, // milliseconds 31 | } 32 | client = new elasticsearch.Client(esConfig) 33 | } 34 | // AWS managed ES 35 | else { 36 | await new Promise((resolve, reject) => AWS.config.getCredentials((err) => { 37 | if (err) return reject(err) 38 | return resolve() 39 | })) 40 | 41 | AWS.config.update({ 42 | credentials: new AWS.Credentials(process.env.AWS_ACCESS_KEY_ID, 43 | process.env.AWS_SECRET_ACCESS_KEY), 44 | region: process.env.AWS_REGION || 'us-east-1' 45 | }) 46 | 47 | esConfig = { 48 | hosts: [process.env.ES_HOST], 49 | connectionClass: httpAwsEs, 50 | awsConfig: new AWS.Config({ region: process.env.AWS_REGION || 'us-east-1' }), 51 | httpOptions: {}, 52 | // Note that this doesn't abort the query. 53 | requestTimeout: 120000 // milliseconds 54 | } 55 | client = new elasticsearch.Client(esConfig) 56 | } 57 | 58 | await new Promise((resolve, reject) => client.ping({ requestTimeout: 1000 }, 59 | (err) => { 60 | if (err) { 61 | reject('Unable to connect to elasticsearch') 62 | } else { 63 | resolve() 64 | } 65 | })) 66 | return client 67 | } 68 | 69 | // get existing ES client or create a new one 70 | async function esClient() { 71 | if (!_esClient) { 72 | try { 73 | _esClient = await connect() 74 | } catch (error) { 75 | logger.error(error) 76 | } 77 | if (_esClient) { 78 | logger.debug('Connected to Elasticsearch') 79 | } 80 | } else { 81 | logger.debug('Using existing Elasticsearch connection') 82 | } 83 | return _esClient 84 | } 85 | 86 | // Create STAC mappings 87 | async function prepare(index) { 88 | // TODO - different mappings for collection and item 89 | const props = { 90 | 'type': 'object', 91 | properties: { 92 | 'datetime': { type: 'date' }, 93 | 'eo:cloud_cover': { type: 'float' }, 94 | 'eo:gsd': { type: 'float' }, 95 | 'eo:constellation': { type: 'keyword' }, 96 | 'eo:platform': { type: 'keyword' }, 97 | 'eo:instrument': { type: 'keyword' }, 98 | 'eo:epsg': { type: 'integer' }, 99 | 'eo:off_nadir': { type: 'float' }, 100 | 'eo:azimuth': { type: 'float' }, 101 | 'eo:sun_azimuth': { type: 'float' }, 102 | 'eo:sun_elevation': { type: 'float' } 103 | } 104 | } 105 | 106 | const dynamicTemplates = [{ 107 | strings: { 108 | mapping: { 109 | type: 'keyword' 110 | }, 111 | match_mapping_type: 'string' 112 | } 113 | }] 114 | const client = await esClient() 115 | const indexExists = await client.indices.exists({ index }) 116 | if (!indexExists) { 117 | const precision = process.env.SATAPI_ES_PRECISION || '5mi' 118 | const payload = { 119 | index, 120 | body: { 121 | mappings: { 122 | // TODO: this structure different for ElasticSearch 7+, 123 | doc: { 124 | /*'_all': { 125 | enabled: true 126 | },*/ 127 | dynamic_templates: dynamicTemplates, 128 | properties: { 129 | 'id': { type: 'keyword' }, 130 | 'collection': { type: 'keyword' }, 131 | 'properties': props, 132 | geometry: { 133 | type: 'geo_shape', 134 | tree: 'quadtree', 135 | precision: precision 136 | } 137 | } 138 | } 139 | } 140 | } 141 | } 142 | try { 143 | await client.indices.create(payload) 144 | logger.info(`Created index: ${JSON.stringify(payload)}`) 145 | } catch (error) { 146 | logger.debug(`Error creating index '${index}': ${error}`) 147 | } 148 | } 149 | else { 150 | logger.debug(`Index '${index}' exists`) 151 | } 152 | } 153 | 154 | // Given an input stream and a transform, write records to an elasticsearch instance 155 | async function _stream() { 156 | let esStreams 157 | try { 158 | let collections = [] 159 | const client = await esClient() 160 | const indexExists = await client.indices.exists({ index: 'collections' }) 161 | if (indexExists) { 162 | const body = { query: { match_all: {} } } 163 | const searchParams = { 164 | index: 'collections', 165 | body 166 | } 167 | const resultBody = await client.search(searchParams) 168 | collections = resultBody.hits.hits.map((r) => (r._source)) 169 | } 170 | 171 | const toEs = through2.obj({ objectMode: true }, (data, encoding, next) => { 172 | let index = '' 173 | if (data && data.hasOwnProperty('extent')) { 174 | index = 'collections' 175 | } else if (data && data.hasOwnProperty('geometry')) { 176 | index = 'items' 177 | } else { 178 | next() 179 | return 180 | } 181 | // remove any hierarchy links in a non-mutating way 182 | const hlinks = ['self', 'root', 'parent', 'child', 'collection', 'item'] 183 | const links = data.links.filter((link) => hlinks.includes(link)) 184 | let esDataObject = Object.assign({}, data, { links }) 185 | if (index === 'items') { 186 | const collectionId = data.collection 187 | const itemCollection = 188 | collections.find((collection) => (collectionId === collection.id)) 189 | if (itemCollection) { 190 | const flatProperties = 191 | Object.assign({}, itemCollection.properties, data.properties) 192 | esDataObject = Object.assign({}, esDataObject, { properties: flatProperties }) 193 | } else { 194 | logger.error(`${data.id} has no collection`) 195 | } 196 | } 197 | 198 | // create ES record 199 | const record = { 200 | index, 201 | type: 'doc', 202 | id: esDataObject.id, 203 | action: 'update', 204 | _retry_on_conflict: 3, 205 | body: { 206 | doc: esDataObject, 207 | doc_as_upsert: true 208 | } 209 | } 210 | next(null, record) 211 | }) 212 | const esStream = new ElasticsearchWritableStream({ client: client }, { 213 | objectMode: true, 214 | highWaterMark: process.env.ES_BATCH_SIZE || 500 215 | }) 216 | esStreams = { toEs, esStream } 217 | } catch (error) { 218 | logger.error(error) 219 | } 220 | return esStreams 221 | } 222 | 223 | function buildRangeQuery(property, operators, operatorsObject) { 224 | const gt = 'gt' 225 | const lt = 'lt' 226 | const gte = 'gte' 227 | const lte = 'lte' 228 | const comparisons = [gt, lt, gte, lte] 229 | let rangeQuery 230 | if (operators.includes(gt) || operators.includes(lt) || 231 | operators.includes(gte) || operators.includes(lte)) { 232 | const propertyKey = `properties.${property}` 233 | rangeQuery = { 234 | range: { 235 | [propertyKey]: { 236 | } 237 | } 238 | } 239 | // All operators for a property go in a single range query. 240 | comparisons.forEach((comparison) => { 241 | if (operators.includes(comparison)) { 242 | const exisiting = rangeQuery.range[propertyKey] 243 | rangeQuery.range[propertyKey] = Object.assign({}, exisiting, { 244 | [comparison]: operatorsObject[comparison] 245 | }) 246 | } 247 | }) 248 | } 249 | return rangeQuery 250 | } 251 | 252 | function buildDatetimeQuery(parameters) { 253 | let dateQuery 254 | const { datetime } = parameters 255 | if (datetime) { 256 | const dataRange = datetime.split('/') 257 | if (dataRange.length === 2) { 258 | dateQuery = { 259 | range: { 260 | 'properties.datetime': { 261 | gte: dataRange[0], 262 | lte: dataRange[1] 263 | } 264 | } 265 | } 266 | } else { 267 | dateQuery = { 268 | term: { 269 | 'properties.datetime': datetime 270 | } 271 | } 272 | } 273 | } 274 | return dateQuery 275 | } 276 | 277 | function buildQuery(parameters) { 278 | const eq = 'eq' 279 | const inop = 'in' 280 | const { query, intersects } = parameters 281 | let must = [] 282 | const should = [] 283 | if (query) { 284 | const { collections } = query 285 | // Using reduce rather than map as we don't currently support all 286 | // stac query operators. 287 | must = Object.keys(query).reduce((accumulator, property) => { 288 | const operatorsObject = query[property] 289 | const operators = Object.keys(operatorsObject) 290 | if (operators.includes(eq)) { 291 | const termQuery = { 292 | term: { 293 | [`properties.${property}`]: operatorsObject.eq 294 | } 295 | } 296 | accumulator.push(termQuery) 297 | } else if (operators.includes(inop)) { 298 | const termsQuery = { 299 | terms: { 300 | [`properties.${property}`]: operatorsObject.in 301 | } 302 | } 303 | accumulator.push(termsQuery) 304 | } 305 | const rangeQuery = 306 | buildRangeQuery(property, operators, operatorsObject) 307 | if (rangeQuery) { 308 | accumulator.push(rangeQuery) 309 | } 310 | return accumulator 311 | }, must) 312 | 313 | if (collections) { 314 | collections.forEach((collection) => { 315 | should.push({ term: { 'collection': collection } }) 316 | }) 317 | } 318 | } 319 | 320 | 321 | if (intersects) { 322 | const { geometry } = intersects 323 | must.push({ 324 | geo_shape: { 325 | geometry: { shape: geometry } 326 | } 327 | }) 328 | } 329 | 330 | const datetimeQuery = buildDatetimeQuery(parameters) 331 | if (datetimeQuery) { 332 | must.push(datetimeQuery) 333 | } 334 | 335 | const filter = { bool: { must, should } } 336 | const queryBody = { 337 | constant_score: { filter } 338 | } 339 | return { query: queryBody } 340 | } 341 | 342 | function buildIdQuery(id) { 343 | return { 344 | query: { 345 | constant_score: { 346 | filter: { 347 | term: { 348 | id 349 | } 350 | } 351 | } 352 | } 353 | } 354 | } 355 | 356 | function buildIdsQuery(ids) { 357 | return { 358 | query: { 359 | ids: { 360 | values: ids 361 | } 362 | } 363 | } 364 | } 365 | 366 | function buildSort(parameters) { 367 | const { sort } = parameters 368 | let sorting 369 | if (sort && sort.length > 0) { 370 | sorting = sort.map((sortRule) => { 371 | const { field, direction } = sortRule 372 | const propertyKey = `properties.${field}` 373 | return { 374 | [propertyKey]: { 375 | order: direction 376 | } 377 | } 378 | }) 379 | } else { 380 | // Default item sorting 381 | sorting = [ 382 | { 'properties.datetime': { order: 'desc' } } 383 | ] 384 | } 385 | return sorting 386 | } 387 | 388 | function buildFieldsFilter(parameters) { 389 | const id = 'id' 390 | const { fields } = parameters 391 | const _sourceInclude = [] 392 | const _sourceExclude = [] 393 | if (fields) { 394 | const { include, exclude } = fields 395 | if (include && include.length > 0) { 396 | const propertiesIncludes = include.map( 397 | (field) => (`${field}`) 398 | ).concat( 399 | [id] 400 | ) 401 | _sourceInclude.push(...propertiesIncludes) 402 | } 403 | if (exclude && exclude.length > 0) { 404 | const filteredExcludes = exclude.filter((field) => 405 | (![id].includes(field))) 406 | const propertiesExclude = filteredExcludes.map((field) => (`${field}`)) 407 | _sourceExclude.push(...propertiesExclude) 408 | } 409 | } 410 | return { _sourceExclude, _sourceInclude } 411 | } 412 | 413 | async function search(parameters, index = '*', page = 1, limit = 10) { 414 | let body 415 | if (parameters.ids) { 416 | const { ids } = parameters 417 | body = buildIdsQuery(ids) 418 | } else if (parameters.id) { 419 | const { id } = parameters 420 | body = buildIdQuery(id) 421 | } else { 422 | body = buildQuery(parameters) 423 | } 424 | const sort = buildSort(parameters) 425 | body.sort = sort 426 | logger.info(`Elasticsearch query: ${JSON.stringify(body)}`) 427 | 428 | const searchParams = { 429 | index, 430 | body, 431 | size: limit, 432 | from: (page - 1) * limit 433 | } 434 | 435 | const { _sourceExclude, _sourceInclude } = buildFieldsFilter(parameters) 436 | if (_sourceExclude.length > 0) { 437 | searchParams._sourceExclude = _sourceExclude 438 | } 439 | if (_sourceInclude.length > 0) { 440 | searchParams._sourceInclude = _sourceInclude 441 | } 442 | const client = await esClient() 443 | const resultBody = await client.search(searchParams) 444 | const results = resultBody.hits.hits.map((r) => (r._source)) 445 | const response = { 446 | results, 447 | meta: { 448 | page, 449 | limit, 450 | found: resultBody.hits.total, 451 | returned: results.length 452 | } 453 | } 454 | return response 455 | } 456 | 457 | module.exports.prepare = prepare 458 | module.exports.stream = _stream 459 | module.exports.search = search 460 | -------------------------------------------------------------------------------- /packages/api-lib/libs/ingest.js: -------------------------------------------------------------------------------- 1 | const request = require('request-promise-native') 2 | const Bottleneck = require('bottleneck') 3 | const util = require('util') 4 | const path = require('path') 5 | const fs = require('fs') 6 | const isUrl = require('is-url') 7 | const MemoryStream = require('memorystream') 8 | const { Readable } = require('readable-stream') 9 | const pump = require('pump') 10 | const uuid = require('uuid/v4') 11 | const logger = require('./logger') 12 | 13 | const limiter = new Bottleneck({ 14 | maxConcurrent: 500 15 | }) 16 | 17 | const limitedRequest = limiter.wrap(request) 18 | const limitedRead = limiter.wrap(util.promisify(fs.readFile)) 19 | 20 | function getSelfRef(node) { 21 | let ref 22 | const self = node.links.find((link) => (link.rel === 'self')) 23 | if (self && self.href) { 24 | ref = self.href 25 | } 26 | return ref 27 | } 28 | 29 | function getChildLinks(node) { 30 | const links = 31 | node.links.filter((link) => (link.rel === 'child' || link.rel === 'item')) 32 | return links 33 | } 34 | 35 | async function fetchChildren(node, links, basePath) { 36 | const selfHref = getSelfRef(node) 37 | const linkPromises = links.map((link) => { 38 | let urlPath 39 | let returnPromise 40 | if (!selfHref || !link.href) { 41 | return Promise.reject(new Error(`${node.id} has invalid links`)) 42 | } 43 | if (isUrl(link.href) || path.isAbsolute(link.href)) { 44 | urlPath = link.href 45 | } else { 46 | // eslint-disable-next-line 47 | if (basePath) { 48 | urlPath = `${path.dirname(basePath)}/${link.href}` 49 | } else { 50 | urlPath = `${path.dirname(selfHref)}/${link.href}` 51 | } 52 | } 53 | if (isUrl(urlPath)) { 54 | returnPromise = limitedRequest(urlPath) 55 | } else { 56 | returnPromise = limitedRead(urlPath) 57 | } 58 | return returnPromise 59 | }) 60 | let responses 61 | try { 62 | responses = await Promise.all(linkPromises.map((p) => p.catch((e) => e))) 63 | } catch (error) { 64 | logger.error(error) 65 | } 66 | const validResponses = 67 | responses.filter((response) => !(response instanceof Error)) 68 | const failedResponses = 69 | responses.filter((response) => (response instanceof Error)) 70 | failedResponses.forEach((failure) => { 71 | logger.error(failure.message) 72 | }) 73 | const children = validResponses.map((response) => (JSON.parse(response))) 74 | return children 75 | } 76 | 77 | // Mutates stack and visited 78 | async function visitChildren(node, stack, visited, basePath) { 79 | let children 80 | const nodeLinks = getChildLinks(node) 81 | try { 82 | children = await fetchChildren(node, nodeLinks, basePath) 83 | } catch (error) { 84 | logger.error(error) 85 | } 86 | if (children) { 87 | // eslint-disable-next-line 88 | for (const child of children) { 89 | const key = getSelfRef(child) 90 | const childLinks = getChildLinks(child) 91 | if (key) { 92 | if (!visited[key]) { 93 | stack.push(child) 94 | if (childLinks.length) { 95 | visited[key] = true 96 | } 97 | } 98 | } else { 99 | logger.error(`${node.id} has invalid self link`) 100 | } 101 | } 102 | } 103 | } 104 | 105 | async function visit(url, stream, recursive, collectionsOnly) { 106 | const visited = {} 107 | const stack = [] 108 | let root 109 | let basePath 110 | if (isUrl(url)) { 111 | const rootResponse = await limitedRequest(url) 112 | root = JSON.parse(rootResponse) 113 | } else { 114 | const rootResponse = await limitedRead(url) 115 | root = JSON.parse(rootResponse) 116 | // Handles relative root link in file catalog. 117 | basePath = url 118 | } 119 | stack.push(root) 120 | const rootId = getSelfRef(root) 121 | visited[rootId] = true 122 | while (stack.length) { 123 | const node = stack.pop() 124 | const isCollection = node.hasOwnProperty('extent') 125 | const isItem = node.hasOwnProperty('geometry') 126 | if (!(isCollection || isItem)) { 127 | const selfRef = getSelfRef(node) 128 | logger.debug(`catalog ${selfRef}`) 129 | } 130 | stream.write(node) 131 | if (recursive && !(isCollection && collectionsOnly)) { 132 | try { 133 | // eslint-disable-next-line 134 | await visitChildren(node, stack, visited, basePath) 135 | } catch (error) { 136 | logger.error(error) 137 | } 138 | } 139 | } 140 | stream.push(null) 141 | } 142 | 143 | async function ingest(url, backend, recursive = true, collectionsOnly = false) { 144 | const duplexStream = new MemoryStream(null, { 145 | readable: true, 146 | writable: true, 147 | objectMode: true 148 | }) 149 | 150 | await backend.prepare('collections') 151 | await backend.prepare('items') 152 | const { toEs, esStream } = await backend.stream() 153 | const ingestJobId = uuid() 154 | logger.info(`${ingestJobId} Started`) 155 | const promise = new Promise((resolve, reject) => { 156 | pump( 157 | duplexStream, 158 | toEs, 159 | esStream, 160 | (error) => { 161 | if (error) { 162 | logger.error(error) 163 | reject(error) 164 | } else { 165 | logger.info(`${ingestJobId} Completed`) 166 | resolve(true) 167 | } 168 | } 169 | ) 170 | }) 171 | visit(url, duplexStream, recursive, collectionsOnly) 172 | return promise 173 | } 174 | 175 | async function ingestItem(item, backend) { 176 | const readable = new Readable({ objectMode: true }) 177 | await backend.prepare('collections') 178 | await backend.prepare('items') 179 | const { toEs, esStream } = await backend.stream() 180 | const promise = new Promise((resolve, reject) => { 181 | pump( 182 | readable, 183 | toEs, 184 | esStream, 185 | (error) => { 186 | if (error) { 187 | logger.error(error) 188 | reject(error) 189 | } else { 190 | logger.info(`Ingested item ${item.id}`) 191 | resolve(true) 192 | } 193 | } 194 | ) 195 | }) 196 | readable.push(item) 197 | readable.push(null) 198 | return promise 199 | } 200 | 201 | module.exports = { ingest, ingestItem } 202 | -------------------------------------------------------------------------------- /packages/api-lib/libs/logger.js: -------------------------------------------------------------------------------- 1 | const winston = require('winston') 2 | 3 | const logger = new (winston.Logger)({ 4 | level: process.env.LOG_LEVEL || 'info', 5 | transports: [ 6 | new (winston.transports.Console)({ timestamp: true }) 7 | ] 8 | }) 9 | 10 | module.exports = logger 11 | -------------------------------------------------------------------------------- /packages/api-lib/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@sat-utils/api-lib", 3 | "version": "0.3.0", 4 | "description": "A library for creating a search API of public Satellites metadata using Elasticsearch", 5 | "main": "index.js", 6 | "scripts": { 7 | "test": "LOG_LEVEL=3 ava" 8 | }, 9 | "repository": { 10 | "type": "git", 11 | "url": "git+https://github.com/sat-utils/sat-api.git" 12 | }, 13 | "author": "Alireza Jazayeri , Matthew Hanson ", 14 | "license": "MIT", 15 | "ava": { 16 | "files": "tests/*.js", 17 | "verbose": true, 18 | "serial": true 19 | }, 20 | "publishConfig": { 21 | "access": "public" 22 | }, 23 | "bugs": { 24 | "url": "https://github.com/sat-utils/sat-api/issues" 25 | }, 26 | "homepage": "https://github.com/sat-utils/sat-api#readme", 27 | "dependencies": { 28 | "@mapbox/extent": "^0.4.0", 29 | "@turf/helpers": "^6.1.4", 30 | "bottleneck": "^2.13.1", 31 | "elasticsearch": "^16", 32 | "geojson-validation": "^0.1.6", 33 | "http-aws-es": "^6", 34 | "is-url": "^1.2.4", 35 | "memorystream": "^0.3.1", 36 | "pump": "^3.0.0", 37 | "request": "^2.88.0", 38 | "request-promise-native": "^1.0.5", 39 | "through2": "^3.0.1", 40 | "uuid": "^3.3.2", 41 | "winston": "^2.2.0" 42 | }, 43 | "devDependencies": { 44 | "ava": "^0.16.0", 45 | "aws-sdk": "^2.382.0", 46 | "nock": "^8.0.0", 47 | "proxyquire": "^2.1.0", 48 | "sinon": "^7.1.1" 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /packages/api-lib/tests/fixtures/item.json: -------------------------------------------------------------------------------- 1 | { 2 | "assets": { 3 | "ANG": { 4 | "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/L8/010/010/LC80100102015082LGN00/LC80100102015082LGN00_ANG.txt", 5 | "title": "Angle coefficients file", 6 | "type": "text/plain" 7 | }, 8 | "B1": { 9 | "eo:bands": [ 10 | 0 11 | ], 12 | "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/L8/010/010/LC80100102015082LGN00/LC80100102015082LGN00_B1.TIF", 13 | "title": "Band 1 (coastal)", 14 | "type": "image/x.geotiff" 15 | }, 16 | "B10": { 17 | "eo:bands": [ 18 | 9 19 | ], 20 | "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/L8/010/010/LC80100102015082LGN00/LC80100102015082LGN00_B10.TIF", 21 | "title": "Band 10 (lwir)", 22 | "type": "image/x.geotiff" 23 | }, 24 | "B11": { 25 | "eo:bands": [ 26 | 10 27 | ], 28 | "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/L8/010/010/LC80100102015082LGN00/LC80100102015082LGN00_B11.TIF", 29 | "title": "Band 11 (lwir)", 30 | "type": "image/x.geotiff" 31 | }, 32 | "B2": { 33 | "eo:bands": [ 34 | 1 35 | ], 36 | "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/L8/010/010/LC80100102015082LGN00/LC80100102015082LGN00_B2.TIF", 37 | "title": "Band 2 (blue)", 38 | "type": "image/x.geotiff" 39 | }, 40 | "B3": { 41 | "eo:bands": [ 42 | 2 43 | ], 44 | "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/L8/010/010/LC80100102015082LGN00/LC80100102015082LGN00_B3.TIF", 45 | "title": "Band 3 (green)", 46 | "type": "image/x.geotiff" 47 | }, 48 | "B4": { 49 | "eo:bands": [ 50 | 3 51 | ], 52 | "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/L8/010/010/LC80100102015082LGN00/LC80100102015082LGN00_B4.TIF", 53 | "title": "Band 4 (red)", 54 | "type": "image/x.geotiff" 55 | }, 56 | "B5": { 57 | "eo:bands": [ 58 | 4 59 | ], 60 | "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/L8/010/010/LC80100102015082LGN00/LC80100102015082LGN00_B5.TIF", 61 | "title": "Band 5 (nir)", 62 | "type": "image/x.geotiff" 63 | }, 64 | "B6": { 65 | "eo:bands": [ 66 | 5 67 | ], 68 | "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/L8/010/010/LC80100102015082LGN00/LC80100102015082LGN00_B6.TIF", 69 | "title": "Band 6 (swir16)", 70 | "type": "image/x.geotiff" 71 | }, 72 | "B7": { 73 | "eo:bands": [ 74 | 6 75 | ], 76 | "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/L8/010/010/LC80100102015082LGN00/LC80100102015082LGN00_B7.TIF", 77 | "title": "Band 7 (swir22)", 78 | "type": "image/x.geotiff" 79 | }, 80 | "B8": { 81 | "eo:bands": [ 82 | 7 83 | ], 84 | "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/L8/010/010/LC80100102015082LGN00/LC80100102015082LGN00_B8.TIF", 85 | "title": "Band 8 (pan)", 86 | "type": "image/x.geotiff" 87 | }, 88 | "B9": { 89 | "eo:bands": [ 90 | 8 91 | ], 92 | "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/L8/010/010/LC80100102015082LGN00/LC80100102015082LGN00_B9.TIF", 93 | "title": "Band 9 (cirrus)", 94 | "type": "image/x.geotiff" 95 | }, 96 | "BQA": { 97 | "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/L8/010/010/LC80100102015082LGN00/LC80100102015082LGN00_BQA.TIF", 98 | "title": "Band quality data", 99 | "type": "image/x.geotiff" 100 | }, 101 | "MTL": { 102 | "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/L8/010/010/LC80100102015082LGN00/LC80100102015082LGN00_MTL.txt", 103 | "title": "original metadata file", 104 | "type": "text/plain" 105 | }, 106 | "index": { 107 | "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/L8/010/010/LC80100102015082LGN00/index.html", 108 | "title": "HTML index page", 109 | "type": "text/html" 110 | }, 111 | "thumbnail": { 112 | "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/L8/010/010/LC80100102015082LGN00/LC80100102015082LGN00_thumb_large.jpg", 113 | "title": "Thumbnail image", 114 | "type": "image/jpeg" 115 | } 116 | }, 117 | "bbox": [ 118 | -52.70915, 119 | 69.68455, 120 | -45.21422, 121 | 72.07651 122 | ], 123 | "geometry": { 124 | "coordinates": [ 125 | [ 126 | [ 127 | -52.70915, 128 | 72.07651 129 | ], 130 | [ 131 | -45.21422, 132 | 71.9981 133 | ], 134 | [ 135 | -45.8512, 136 | 69.68455 137 | ], 138 | [ 139 | -52.52, 140 | 69.75337 141 | ], 142 | [ 143 | -52.70915, 144 | 72.07651 145 | ] 146 | ] 147 | ], 148 | "type": "Polygon" 149 | }, 150 | "id": "LC80100102015082LGN00", 151 | "links": [], 152 | "collection": "landsat-8-l1", 153 | "properties": { 154 | "datetime": "2015-03-23T15:05:56.200728+00:00", 155 | "eo:cloud_cover": 8.26, 156 | "eo:epsg": 32622, 157 | "eo:sun_azimuth": 175.47823280, 158 | "eo:sun_elevation": 20.20572191, 159 | "landsat:path": "10", 160 | "landsat:processing_level": "L1T", 161 | "landsat:product_id": null, 162 | "landsat:row": "10", 163 | "landsat:scene_id": "LC80100102015082LGN00" 164 | }, 165 | "type": "Feature" 166 | } 167 | -------------------------------------------------------------------------------- /packages/api-lib/tests/fixtures/itemLinks.json: -------------------------------------------------------------------------------- 1 | { 2 | "links": [ 3 | { "rel": "self", "href": "endpoint/collections/landsat-8-l1/items/LC80100102015082LGN00" }, 4 | { "rel": "parent", "href": "endpoint/collections/landsat-8-l1" }, 5 | { "rel": "collection", "href": "endpoint/collections/landsat-8-l1" }, 6 | { "rel": "root", "href": "endpoint/stac" } 7 | ] 8 | } 9 | -------------------------------------------------------------------------------- /packages/api-lib/tests/fixtures/stac/LC80100102015050LGN00.json: -------------------------------------------------------------------------------- 1 | { 2 | "assets": { 3 | "ANG": { 4 | "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/L8/010/010/LC80100102015050LGN00/LC80100102015050LGN00_ANG.txt", 5 | "title": "Angle coefficients file", 6 | "type": "text/plain" 7 | }, 8 | "B1": { 9 | "eo:bands": [ 10 | 0 11 | ], 12 | "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/L8/010/010/LC80100102015050LGN00/LC80100102015050LGN00_B1.TIF", 13 | "title": "Band 1 (coastal)", 14 | "type": "image/x.geotiff" 15 | }, 16 | "B10": { 17 | "eo:bands": [ 18 | 9 19 | ], 20 | "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/L8/010/010/LC80100102015050LGN00/LC80100102015050LGN00_B10.TIF", 21 | "title": "Band 10 (lwir)", 22 | "type": "image/x.geotiff" 23 | }, 24 | "B11": { 25 | "eo:bands": [ 26 | 10 27 | ], 28 | "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/L8/010/010/LC80100102015050LGN00/LC80100102015050LGN00_B11.TIF", 29 | "title": "Band 11 (lwir)", 30 | "type": "image/x.geotiff" 31 | }, 32 | "B2": { 33 | "eo:bands": [ 34 | 1 35 | ], 36 | "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/L8/010/010/LC80100102015050LGN00/LC80100102015050LGN00_B2.TIF", 37 | "title": "Band 2 (blue)", 38 | "type": "image/x.geotiff" 39 | }, 40 | "B3": { 41 | "eo:bands": [ 42 | 2 43 | ], 44 | "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/L8/010/010/LC80100102015050LGN00/LC80100102015050LGN00_B3.TIF", 45 | "title": "Band 3 (green)", 46 | "type": "image/x.geotiff" 47 | }, 48 | "B4": { 49 | "eo:bands": [ 50 | 3 51 | ], 52 | "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/L8/010/010/LC80100102015050LGN00/LC80100102015050LGN00_B4.TIF", 53 | "title": "Band 4 (red)", 54 | "type": "image/x.geotiff" 55 | }, 56 | "B5": { 57 | "eo:bands": [ 58 | 4 59 | ], 60 | "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/L8/010/010/LC80100102015050LGN00/LC80100102015050LGN00_B5.TIF", 61 | "title": "Band 5 (nir)", 62 | "type": "image/x.geotiff" 63 | }, 64 | "B6": { 65 | "eo:bands": [ 66 | 5 67 | ], 68 | "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/L8/010/010/LC80100102015050LGN00/LC80100102015050LGN00_B6.TIF", 69 | "title": "Band 6 (swir16)", 70 | "type": "image/x.geotiff" 71 | }, 72 | "B7": { 73 | "eo:bands": [ 74 | 6 75 | ], 76 | "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/L8/010/010/LC80100102015050LGN00/LC80100102015050LGN00_B7.TIF", 77 | "title": "Band 7 (swir22)", 78 | "type": "image/x.geotiff" 79 | }, 80 | "B8": { 81 | "eo:bands": [ 82 | 7 83 | ], 84 | "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/L8/010/010/LC80100102015050LGN00/LC80100102015050LGN00_B8.TIF", 85 | "title": "Band 8 (pan)", 86 | "type": "image/x.geotiff" 87 | }, 88 | "B9": { 89 | "eo:bands": [ 90 | 8 91 | ], 92 | "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/L8/010/010/LC80100102015050LGN00/LC80100102015050LGN00_B9.TIF", 93 | "title": "Band 9 (cirrus)", 94 | "type": "image/x.geotiff" 95 | }, 96 | "BQA": { 97 | "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/L8/010/010/LC80100102015050LGN00/LC80100102015050LGN00_BQA.TIF", 98 | "title": "Band quality data", 99 | "type": "image/x.geotiff" 100 | }, 101 | "MTL": { 102 | "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/L8/010/010/LC80100102015050LGN00/LC80100102015050LGN00_MTL.txt", 103 | "title": "original metadata file", 104 | "type": "text/plain" 105 | }, 106 | "index": { 107 | "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/L8/010/010/LC80100102015050LGN00/index.html", 108 | "title": "HTML index page", 109 | "type": "text/html" 110 | }, 111 | "thumbnail": { 112 | "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/L8/010/010/LC80100102015050LGN00/LC80100102015050LGN00_thumb_large.jpg", 113 | "title": "Thumbnail image", 114 | "type": "image/jpeg" 115 | } 116 | }, 117 | "bbox": [ 118 | -52.68296, 119 | 69.68409, 120 | -45.19691, 121 | 72.07674 122 | ], 123 | "geometry": { 124 | "coordinates": [ 125 | [ 126 | [ 127 | -52.68296, 128 | 72.07674 129 | ], 130 | [ 131 | -45.19691, 132 | 71.99758 133 | ], 134 | [ 135 | -45.83578, 136 | 69.68409 137 | ], 138 | [ 139 | -52.4967, 140 | 69.75356 141 | ], 142 | [ 143 | -52.68296, 144 | 72.07674 145 | ] 146 | ] 147 | ], 148 | "type": "Polygon" 149 | }, 150 | "id": "LC80100102015050LGN00", 151 | "links": [ 152 | { 153 | "href": "LC80100102015050LGN00.json", 154 | "rel": "self" 155 | }, 156 | { 157 | "href": "catalog.json", 158 | "rel": "root" 159 | }, 160 | { 161 | "href": "collection.json", 162 | "rel": "parent" 163 | }, 164 | { 165 | "href": "collection.json", 166 | "rel": "collection" 167 | } 168 | ], 169 | "collection": "landsat-8-l1", 170 | "properties": { 171 | "datetime": "2015-02-19T15:06:12.565047+00:00", 172 | "eo:cloud_cover": 0.54, 173 | "eo:epsg": 32622, 174 | "eo:sun_azimuth": 174.10674048, 175 | "eo:sun_elevation": 7.85803613, 176 | "landsat:path": "10", 177 | "landsat:processing_level": "L1T", 178 | "landsat:product_id": null, 179 | "landsat:row": "10", 180 | "landsat:scene_id": "LC80100102015050LGN00" 181 | }, 182 | "type": "Feature" 183 | } 184 | -------------------------------------------------------------------------------- /packages/api-lib/tests/fixtures/stac/LC80100102015082LGN00.json: -------------------------------------------------------------------------------- 1 | { 2 | "assets": { 3 | "ANG": { 4 | "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/L8/010/010/LC80100102015082LGN00/LC80100102015082LGN00_ANG.txt", 5 | "title": "Angle coefficients file", 6 | "type": "text/plain" 7 | }, 8 | "B1": { 9 | "eo:bands": [ 10 | 0 11 | ], 12 | "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/L8/010/010/LC80100102015082LGN00/LC80100102015082LGN00_B1.TIF", 13 | "title": "Band 1 (coastal)", 14 | "type": "image/x.geotiff" 15 | }, 16 | "B10": { 17 | "eo:bands": [ 18 | 9 19 | ], 20 | "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/L8/010/010/LC80100102015082LGN00/LC80100102015082LGN00_B10.TIF", 21 | "title": "Band 10 (lwir)", 22 | "type": "image/x.geotiff" 23 | }, 24 | "B11": { 25 | "eo:bands": [ 26 | 10 27 | ], 28 | "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/L8/010/010/LC80100102015082LGN00/LC80100102015082LGN00_B11.TIF", 29 | "title": "Band 11 (lwir)", 30 | "type": "image/x.geotiff" 31 | }, 32 | "B2": { 33 | "eo:bands": [ 34 | 1 35 | ], 36 | "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/L8/010/010/LC80100102015082LGN00/LC80100102015082LGN00_B2.TIF", 37 | "title": "Band 2 (blue)", 38 | "type": "image/x.geotiff" 39 | }, 40 | "B3": { 41 | "eo:bands": [ 42 | 2 43 | ], 44 | "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/L8/010/010/LC80100102015082LGN00/LC80100102015082LGN00_B3.TIF", 45 | "title": "Band 3 (green)", 46 | "type": "image/x.geotiff" 47 | }, 48 | "B4": { 49 | "eo:bands": [ 50 | 3 51 | ], 52 | "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/L8/010/010/LC80100102015082LGN00/LC80100102015082LGN00_B4.TIF", 53 | "title": "Band 4 (red)", 54 | "type": "image/x.geotiff" 55 | }, 56 | "B5": { 57 | "eo:bands": [ 58 | 4 59 | ], 60 | "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/L8/010/010/LC80100102015082LGN00/LC80100102015082LGN00_B5.TIF", 61 | "title": "Band 5 (nir)", 62 | "type": "image/x.geotiff" 63 | }, 64 | "B6": { 65 | "eo:bands": [ 66 | 5 67 | ], 68 | "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/L8/010/010/LC80100102015082LGN00/LC80100102015082LGN00_B6.TIF", 69 | "title": "Band 6 (swir16)", 70 | "type": "image/x.geotiff" 71 | }, 72 | "B7": { 73 | "eo:bands": [ 74 | 6 75 | ], 76 | "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/L8/010/010/LC80100102015082LGN00/LC80100102015082LGN00_B7.TIF", 77 | "title": "Band 7 (swir22)", 78 | "type": "image/x.geotiff" 79 | }, 80 | "B8": { 81 | "eo:bands": [ 82 | 7 83 | ], 84 | "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/L8/010/010/LC80100102015082LGN00/LC80100102015082LGN00_B8.TIF", 85 | "title": "Band 8 (pan)", 86 | "type": "image/x.geotiff" 87 | }, 88 | "B9": { 89 | "eo:bands": [ 90 | 8 91 | ], 92 | "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/L8/010/010/LC80100102015082LGN00/LC80100102015082LGN00_B9.TIF", 93 | "title": "Band 9 (cirrus)", 94 | "type": "image/x.geotiff" 95 | }, 96 | "BQA": { 97 | "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/L8/010/010/LC80100102015082LGN00/LC80100102015082LGN00_BQA.TIF", 98 | "title": "Band quality data", 99 | "type": "image/x.geotiff" 100 | }, 101 | "MTL": { 102 | "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/L8/010/010/LC80100102015082LGN00/LC80100102015082LGN00_MTL.txt", 103 | "title": "original metadata file", 104 | "type": "text/plain" 105 | }, 106 | "index": { 107 | "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/L8/010/010/LC80100102015082LGN00/index.html", 108 | "title": "HTML index page", 109 | "type": "text/html" 110 | }, 111 | "thumbnail": { 112 | "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/L8/010/010/LC80100102015082LGN00/LC80100102015082LGN00_thumb_large.jpg", 113 | "title": "Thumbnail image", 114 | "type": "image/jpeg" 115 | } 116 | }, 117 | "bbox": [ 118 | -52.70915, 119 | 69.68455, 120 | -45.21422, 121 | 72.07651 122 | ], 123 | "geometry": { 124 | "coordinates": [ 125 | [ 126 | [ 127 | -52.70915, 128 | 72.07651 129 | ], 130 | [ 131 | -45.21422, 132 | 71.9981 133 | ], 134 | [ 135 | -45.8512, 136 | 69.68455 137 | ], 138 | [ 139 | -52.52, 140 | 69.75337 141 | ], 142 | [ 143 | -52.70915, 144 | 72.07651 145 | ] 146 | ] 147 | ], 148 | "type": "Polygon" 149 | }, 150 | "id": "LC80100102015082LGN00", 151 | "links": [ 152 | { 153 | "href": "LC80100102015082LGN00.json", 154 | "rel": "self" 155 | }, 156 | { 157 | "href": "catalog.json", 158 | "rel": "root" 159 | }, 160 | { 161 | "href": "collection.json", 162 | "rel": "parent" 163 | }, 164 | { 165 | "href": "collection.json", 166 | "rel": "collection" 167 | } 168 | ], 169 | "collection": "landsat-8-l1", 170 | "properties": { 171 | "datetime": "2015-03-23T15:05:56.200728+00:00", 172 | "eo:cloud_cover": 8.26, 173 | "eo:epsg": 32622, 174 | "eo:sun_azimuth": 175.47823280, 175 | "eo:sun_elevation": 20.20572191, 176 | "landsat:path": "10", 177 | "landsat:processing_level": "L1T", 178 | "landsat:product_id": null, 179 | "landsat:row": "10", 180 | "landsat:scene_id": "LC80100102015082LGN00" 181 | }, 182 | "type": "Feature" 183 | } 184 | -------------------------------------------------------------------------------- /packages/api-lib/tests/fixtures/stac/badGeometryItem.json: -------------------------------------------------------------------------------- 1 | { 2 | "assets": { 3 | "ANG": { 4 | "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/L8/096/008/LC80960082014185LGN00/LC80960082014185LGN00_ANG.txt", 5 | "title": "Angle coefficients file", 6 | "type": "text/plain" 7 | }, 8 | "B1": { 9 | "eo:bands": [ 10 | 0 11 | ], 12 | "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/L8/096/008/LC80960082014185LGN00/LC80960082014185LGN00_B1.TIF", 13 | "title": "Band 1 (coastal)", 14 | "type": "image/x.geotiff" 15 | }, 16 | "B10": { 17 | "eo:bands": [ 18 | 9 19 | ], 20 | "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/L8/096/008/LC80960082014185LGN00/LC80960082014185LGN00_B10.TIF", 21 | "title": "Band 10 (lwir)", 22 | "type": "image/x.geotiff" 23 | }, 24 | "B11": { 25 | "eo:bands": [ 26 | 10 27 | ], 28 | "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/L8/096/008/LC80960082014185LGN00/LC80960082014185LGN00_B11.TIF", 29 | "title": "Band 11 (lwir)", 30 | "type": "image/x.geotiff" 31 | }, 32 | "B2": { 33 | "eo:bands": [ 34 | 1 35 | ], 36 | "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/L8/096/008/LC80960082014185LGN00/LC80960082014185LGN00_B2.TIF", 37 | "title": "Band 2 (blue)", 38 | "type": "image/x.geotiff" 39 | }, 40 | "B3": { 41 | "eo:bands": [ 42 | 2 43 | ], 44 | "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/L8/096/008/LC80960082014185LGN00/LC80960082014185LGN00_B3.TIF", 45 | "title": "Band 3 (green)", 46 | "type": "image/x.geotiff" 47 | }, 48 | "B4": { 49 | "eo:bands": [ 50 | 3 51 | ], 52 | "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/L8/096/008/LC80960082014185LGN00/LC80960082014185LGN00_B4.TIF", 53 | "title": "Band 4 (red)", 54 | "type": "image/x.geotiff" 55 | }, 56 | "B5": { 57 | "eo:bands": [ 58 | 4 59 | ], 60 | "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/L8/096/008/LC80960082014185LGN00/LC80960082014185LGN00_B5.TIF", 61 | "title": "Band 5 (nir)", 62 | "type": "image/x.geotiff" 63 | }, 64 | "B6": { 65 | "eo:bands": [ 66 | 5 67 | ], 68 | "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/L8/096/008/LC80960082014185LGN00/LC80960082014185LGN00_B6.TIF", 69 | "title": "Band 6 (swir16)", 70 | "type": "image/x.geotiff" 71 | }, 72 | "B7": { 73 | "eo:bands": [ 74 | 6 75 | ], 76 | "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/L8/096/008/LC80960082014185LGN00/LC80960082014185LGN00_B7.TIF", 77 | "title": "Band 7 (swir22)", 78 | "type": "image/x.geotiff" 79 | }, 80 | "B8": { 81 | "eo:bands": [ 82 | 7 83 | ], 84 | "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/L8/096/008/LC80960082014185LGN00/LC80960082014185LGN00_B8.TIF", 85 | "title": "Band 8 (pan)", 86 | "type": "image/x.geotiff" 87 | }, 88 | "B9": { 89 | "eo:bands": [ 90 | 8 91 | ], 92 | "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/L8/096/008/LC80960082014185LGN00/LC80960082014185LGN00_B9.TIF", 93 | "title": "Band 9 (cirrus)", 94 | "type": "image/x.geotiff" 95 | }, 96 | "BQA": { 97 | "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/L8/096/008/LC80960082014185LGN00/LC80960082014185LGN00_BQA.TIF", 98 | "title": "Band quality data", 99 | "type": "image/x.geotiff" 100 | }, 101 | "MTL": { 102 | "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/L8/096/008/LC80960082014185LGN00/LC80960082014185LGN00_MTL.txt", 103 | "title": "original metadata file", 104 | "type": "text/plain" 105 | }, 106 | "index": { 107 | "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/L8/096/008/LC80960082014185LGN00/index.html", 108 | "title": "HTML index page", 109 | "type": "text/html" 110 | }, 111 | "thumbnail": { 112 | "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/L8/096/008/LC80960082014185LGN00/LC80960082014185LGN00_thumb_large.jpg", 113 | "title": "Thumbnail image", 114 | "type": "image/jpeg" 115 | } 116 | }, 117 | "bbox": [ 118 | -173.27809, 119 | 72.22415, 120 | 178.89739, 121 | 74.62735 122 | ], 123 | "geometry": { 124 | "coordinates": [ 125 | [ 126 | [ 127 | [ 128 | -180.0, 129 | 73.00696964504284 130 | ], 131 | [ 132 | -180.0, 133 | 73.09357289928789 134 | ], 135 | [ 136 | -180.0, 137 | 73.4022696917215 138 | ], 139 | [ 140 | -180.0, 141 | 73.70257483495145 142 | ], 143 | [ 144 | -180.0, 145 | 73.7893042253521 146 | ], 147 | [ 148 | -179.99650733482093, 149 | 73.79109483116221 150 | ], 151 | [ 152 | -178.66433925079528, 153 | 74.47406551085987 154 | ], 155 | [ 156 | -178.60228746524575, 157 | 74.50587797556413 158 | ], 159 | [ 160 | -178.533, 161 | 74.5414 162 | ], 163 | [ 164 | -178.52333378926406, 165 | 74.53979633032269 166 | ], 167 | [ 168 | -177.0035, 169 | 74.2876487795428 170 | ], 171 | [ 172 | -176.7359660507269, 173 | 74.24326364313104 174 | ], 175 | [ 176 | -174.01065330156592, 177 | 73.79112148149187 178 | ], 179 | [ 180 | -173.371, 181 | 73.685 182 | ], 183 | [ 184 | -173.45037750938653, 185 | 73.64971591347567 186 | ], 187 | [ 188 | -173.60275118996861, 189 | 73.58198430824847 190 | ], 191 | [ 192 | -175.0997578783975, 193 | 72.9165500570324 194 | ], 195 | [ 196 | -175.40543925453161, 197 | 72.78067166770337 198 | ], 199 | [ 200 | -176.18673897068663, 201 | 72.43337623100723 202 | ], 203 | [ 204 | -176.258, 205 | 72.4017 206 | ], 207 | [ 208 | -177.00319088157397, 209 | 72.52223485312118 210 | ], 211 | [ 212 | -177.26950829931573, 213 | 72.56531177693338 214 | ], 215 | [ 216 | -178.60216354768573, 217 | 72.78086917114648 218 | ], 219 | [ 220 | -180.0, 221 | 73.00696964504284 222 | ] 223 | ] 224 | ], 225 | [ 226 | [ 227 | [ 228 | 180.0, 229 | 73.7893042253521 230 | ], 231 | [ 232 | 180.0, 233 | 73.70257483495145 234 | ], 235 | [ 236 | 180.0, 237 | 73.40226969172153 238 | ], 239 | [ 240 | 180.0, 241 | 73.09357289928789 242 | ], 243 | [ 244 | 180.0, 245 | 73.00696964504284 246 | ], 247 | [ 248 | 178.84742274390246, 249 | 73.19339936890243 250 | ], 251 | [ 252 | 178.84, 253 | 73.1946 254 | ], 255 | [ 256 | 178.90331756836056, 257 | 73.227061401244 258 | ], 259 | [ 260 | 178.96865829339555, 261 | 73.2605600264732 262 | ], 263 | [ 264 | 179.59566838898684, 265 | 73.58201308956507 266 | ], 267 | [ 268 | 179.75007781076354, 269 | 73.66117510298298 270 | ], 271 | [ 272 | 180.0, 273 | 73.7893042253521 274 | ] 275 | ] 276 | ] 277 | ], 278 | "type": "Polygon" 279 | }, 280 | "id": "badGeometryItem", 281 | "links": [ 282 | { 283 | "href": "badGeometryItem.json", 284 | "rel": "self" 285 | }, 286 | { 287 | "href": "catalog.json", 288 | "rel": "root" 289 | }, 290 | { 291 | "href": "collection.json", 292 | "rel": "parent" 293 | }, 294 | { 295 | "href": "collection.json", 296 | "rel": "collection" 297 | } 298 | ], 299 | "collection": "landsat-8-l1", 300 | "properties": { 301 | "datetime": "2014-07-04T23:56:54.593296+00:00", 302 | "eo:cloud_cover": 65, 303 | "eo:column": "096", 304 | "eo:epsg": 3261, 305 | "eo:row": "008", 306 | "eo:sun_azimuth": -178.92904262, 307 | "eo:sun_elevation": 39.43839572, 308 | "landsat:processing_level": "L1GT", 309 | "landsat:product_id": null, 310 | "landsat:scene_id": "LC80960082014185LGN00", 311 | "landsat:tier": "pre-collection" 312 | }, 313 | "type": "Feature" 314 | } 315 | -------------------------------------------------------------------------------- /packages/api-lib/tests/fixtures/stac/catalog.json: -------------------------------------------------------------------------------- 1 | { 2 | "description": "Landsat imagery", 3 | "id": "landsat", 4 | "links": [ 5 | { 6 | "href": "catalog.json", 7 | "rel": "self" 8 | }, 9 | { 10 | "href": "catalog.json", 11 | "rel": "root" 12 | }, 13 | { 14 | "href": "collection.json", 15 | "rel": "child" 16 | }, 17 | { 18 | "href": "collection2.json", 19 | "rel": "child" 20 | } 21 | ], 22 | "stac_version": "0.7.0" 23 | } 24 | -------------------------------------------------------------------------------- /packages/api-lib/tests/fixtures/stac/collection.json: -------------------------------------------------------------------------------- 1 | { 2 | "assets": { 3 | "ANG": { 4 | "title": "Angle coefficients file", 5 | "type": "text/plain" 6 | }, 7 | "B1": { 8 | "eo:bands": [ 9 | 0 10 | ], 11 | "title": "Band 1 (coastal)", 12 | "type": "image/x.geotiff" 13 | }, 14 | "B10": { 15 | "eo:bands": [ 16 | 9 17 | ], 18 | "title": "Band 10 (lwir)", 19 | "type": "image/x.geotiff" 20 | }, 21 | "B11": { 22 | "eo:bands": [ 23 | 10 24 | ], 25 | "title": "Band 11 (lwir)", 26 | "type": "image/x.geotiff" 27 | }, 28 | "B2": { 29 | "eo:bands": [ 30 | 1 31 | ], 32 | "title": "Band 2 (blue)", 33 | "type": "image/x.geotiff" 34 | }, 35 | "B3": { 36 | "eo:bands": [ 37 | 2 38 | ], 39 | "title": "Band 3 (green)", 40 | "type": "image/x.geotiff" 41 | }, 42 | "B4": { 43 | "eo:bands": [ 44 | 3 45 | ], 46 | "title": "Band 4 (red)", 47 | "type": "image/x.geotiff" 48 | }, 49 | "B5": { 50 | "eo:bands": [ 51 | 4 52 | ], 53 | "title": "Band 5 (nir)", 54 | "type": "image/x.geotiff" 55 | }, 56 | "B6": { 57 | "eo:bands": [ 58 | 5 59 | ], 60 | "title": "Band 6 (swir16)", 61 | "type": "image/x.geotiff" 62 | }, 63 | "B7": { 64 | "eo:bands": [ 65 | 6 66 | ], 67 | "title": "Band 7 (swir22)", 68 | "type": "image/x.geotiff" 69 | }, 70 | "B8": { 71 | "eo:bands": [ 72 | 7 73 | ], 74 | "title": "Band 8 (pan)", 75 | "type": "image/x.geotiff" 76 | }, 77 | "B9": { 78 | "eo:bands": [ 79 | 8 80 | ], 81 | "title": "Band 9 (cirrus)", 82 | "type": "image/x.geotiff" 83 | }, 84 | "BQA": { 85 | "title": "Band quality data", 86 | "type": "image/x.geotiff" 87 | }, 88 | "MTL": { 89 | "title": "original metadata file", 90 | "type": "text/plain" 91 | }, 92 | "index": { 93 | "title": "HTML index page", 94 | "type": "text/html" 95 | }, 96 | "thumbnail": { 97 | "title": "Thumbnail image", 98 | "type": "image/jpeg" 99 | } 100 | }, 101 | "description": "Landat 8 imagery radiometrically calibrated and orthorectified using gound points and Digital Elevation Model (DEM) data to correct relief displacement.", 102 | "extent": { 103 | "spatial": [ 104 | -180, 105 | -90, 106 | 180, 107 | 90 108 | ], 109 | "temporal": [ 110 | "2013-06-01", 111 | null 112 | ] 113 | }, 114 | "id": "landsat-8-l1", 115 | "keywords": [ 116 | "landsat", 117 | "earth observation", 118 | "usgs" 119 | ], 120 | "license": "PDDL-1.0", 121 | "links": [ 122 | { 123 | "href": "collection.json", 124 | "rel": "self" 125 | }, 126 | { 127 | "href": "catalog.json", 128 | "rel": "root" 129 | }, 130 | { 131 | "href": "catalog.json", 132 | "rel": "parent" 133 | }, 134 | { 135 | "href": "LC80100102015050LGN00.json", 136 | "rel": "item" 137 | }, 138 | { 139 | "href": "LC80100102015082LGN00.json", 140 | "rel": "item" 141 | }, 142 | { 143 | "href": "badGeometryItem.json", 144 | "rel": "item" 145 | } 146 | ], 147 | "properties": { 148 | "collection": "landsat-8-l1", 149 | "eo:bands": [ 150 | { 151 | "center_wavelength": 0.44, 152 | "common_name": "coastal", 153 | "full_width_half_max": 0.02, 154 | "gsd": 30, 155 | "id": "B1" 156 | }, 157 | { 158 | "center_wavelength": 0.48, 159 | "common_name": "blue", 160 | "full_width_half_max": 0.06, 161 | "gsd": 30, 162 | "id": "B2" 163 | }, 164 | { 165 | "center_wavelength": 0.56, 166 | "common_name": "green", 167 | "full_width_half_max": 0.06, 168 | "gsd": 30, 169 | "id": "B3" 170 | }, 171 | { 172 | "center_wavelength": 0.65, 173 | "common_name": "red", 174 | "full_width_half_max": 0.04, 175 | "gsd": 30, 176 | "id": "B4" 177 | }, 178 | { 179 | "center_wavelength": 0.86, 180 | "common_name": "nir", 181 | "full_width_half_max": 0.03, 182 | "gsd": 30, 183 | "id": "B5" 184 | }, 185 | { 186 | "center_wavelength": 1.6, 187 | "common_name": "swir16", 188 | "full_width_half_max": 0.08, 189 | "gsd": 30, 190 | "id": "B6" 191 | }, 192 | { 193 | "center_wavelength": 2.2, 194 | "common_name": "swir22", 195 | "full_width_half_max": 0.2, 196 | "gsd": 30, 197 | "id": "B7" 198 | }, 199 | { 200 | "center_wavelength": 0.59, 201 | "common_name": "pan", 202 | "full_width_half_max": 0.18, 203 | "gsd": 15, 204 | "id": "B8" 205 | }, 206 | { 207 | "center_wavelength": 1.37, 208 | "common_name": "cirrus", 209 | "full_width_half_max": 0.02, 210 | "gsd": 30, 211 | "id": "B9" 212 | }, 213 | { 214 | "center_wavelength": 10.9, 215 | "common_name": "lwir11", 216 | "full_width_half_max": 0.8, 217 | "gsd": 100, 218 | "id": "B10" 219 | }, 220 | { 221 | "center_wavelength": 12, 222 | "common_name": "lwir12", 223 | "full_width_half_max": 1, 224 | "gsd": 100, 225 | "id": "B11" 226 | } 227 | ], 228 | "eo:gsd": 15, 229 | "eo:instrument": "OLI_TIRS", 230 | "eo:off_nadir": 0, 231 | "eo:platform": "landsat-8" 232 | }, 233 | "providers": [ 234 | { 235 | "name": "USGS", 236 | "roles": [ 237 | "producer" 238 | ], 239 | "url": "https://landsat.usgs.gov/" 240 | }, 241 | { 242 | "name": "Planet Labs", 243 | "roles": [ 244 | "processor" 245 | ], 246 | "url": "https://github.com/landsat-pds/landsat_ingestor" 247 | }, 248 | { 249 | "name": "AWS", 250 | "roles": [ 251 | "host" 252 | ], 253 | "url": "https://landsatonaws.com/" 254 | }, 255 | { 256 | "name": "Development Seed", 257 | "roles": [ 258 | "processor" 259 | ], 260 | "url": "https://github.com/sat-utils/sat-api" 261 | } 262 | ], 263 | "stac_version": "0.7.0", 264 | "title": "Landsat 8 L1", 265 | "version": "0.1.0" 266 | } 267 | -------------------------------------------------------------------------------- /packages/api-lib/tests/fixtures/stac/collection2.json: -------------------------------------------------------------------------------- 1 | { 2 | "assets": { 3 | "ANG": { 4 | "title": "Angle coefficients file", 5 | "type": "text/plain" 6 | }, 7 | "B1": { 8 | "eo:bands": [ 9 | 0 10 | ], 11 | "title": "Band 1 (coastal)", 12 | "type": "image/x.geotiff" 13 | }, 14 | "B10": { 15 | "eo:bands": [ 16 | 9 17 | ], 18 | "title": "Band 10 (lwir)", 19 | "type": "image/x.geotiff" 20 | }, 21 | "B11": { 22 | "eo:bands": [ 23 | 10 24 | ], 25 | "title": "Band 11 (lwir)", 26 | "type": "image/x.geotiff" 27 | }, 28 | "B2": { 29 | "eo:bands": [ 30 | 1 31 | ], 32 | "title": "Band 2 (blue)", 33 | "type": "image/x.geotiff" 34 | }, 35 | "B3": { 36 | "eo:bands": [ 37 | 2 38 | ], 39 | "title": "Band 3 (green)", 40 | "type": "image/x.geotiff" 41 | }, 42 | "B4": { 43 | "eo:bands": [ 44 | 3 45 | ], 46 | "title": "Band 4 (red)", 47 | "type": "image/x.geotiff" 48 | }, 49 | "B5": { 50 | "eo:bands": [ 51 | 4 52 | ], 53 | "title": "Band 5 (nir)", 54 | "type": "image/x.geotiff" 55 | }, 56 | "B6": { 57 | "eo:bands": [ 58 | 5 59 | ], 60 | "title": "Band 6 (swir16)", 61 | "type": "image/x.geotiff" 62 | }, 63 | "B7": { 64 | "eo:bands": [ 65 | 6 66 | ], 67 | "title": "Band 7 (swir22)", 68 | "type": "image/x.geotiff" 69 | }, 70 | "B8": { 71 | "eo:bands": [ 72 | 7 73 | ], 74 | "title": "Band 8 (pan)", 75 | "type": "image/x.geotiff" 76 | }, 77 | "B9": { 78 | "eo:bands": [ 79 | 8 80 | ], 81 | "title": "Band 9 (cirrus)", 82 | "type": "image/x.geotiff" 83 | }, 84 | "BQA": { 85 | "title": "Band quality data", 86 | "type": "image/x.geotiff" 87 | }, 88 | "MTL": { 89 | "title": "original metadata file", 90 | "type": "text/plain" 91 | }, 92 | "index": { 93 | "title": "HTML index page", 94 | "type": "text/html" 95 | }, 96 | "thumbnail": { 97 | "title": "Thumbnail image", 98 | "type": "image/jpeg" 99 | } 100 | }, 101 | "description": "Landat 8 imagery radiometrically calibrated and orthorectified using gound points and Digital Elevation Model (DEM) data to correct relief displacement.", 102 | "extent": { 103 | "spatial": [ 104 | -180, 105 | -90, 106 | 180, 107 | 90 108 | ], 109 | "temporal": [ 110 | "2013-06-01", 111 | null 112 | ] 113 | }, 114 | "id": "collection2", 115 | "keywords": [ 116 | "landsat", 117 | "earth observation", 118 | "usgs" 119 | ], 120 | "license": "PDDL-1.0", 121 | "links": [ 122 | { 123 | "href": "collection2.json", 124 | "rel": "self" 125 | }, 126 | { 127 | "href": "catalog.json", 128 | "rel": "root" 129 | }, 130 | { 131 | "href": "catalog.json", 132 | "rel": "parent" 133 | }, 134 | { 135 | "href": "collection2_item.json", 136 | "rel": "item" 137 | } 138 | ], 139 | "properties": { 140 | "collection": "collection2", 141 | "eo:bands": [ 142 | { 143 | "center_wavelength": 0.44, 144 | "common_name": "coastal", 145 | "full_width_half_max": 0.02, 146 | "gsd": 30, 147 | "id": "B1" 148 | }, 149 | { 150 | "center_wavelength": 0.48, 151 | "common_name": "blue", 152 | "full_width_half_max": 0.06, 153 | "gsd": 30, 154 | "id": "B2" 155 | }, 156 | { 157 | "center_wavelength": 0.56, 158 | "common_name": "green", 159 | "full_width_half_max": 0.06, 160 | "gsd": 30, 161 | "id": "B3" 162 | }, 163 | { 164 | "center_wavelength": 0.65, 165 | "common_name": "red", 166 | "full_width_half_max": 0.04, 167 | "gsd": 30, 168 | "id": "B4" 169 | }, 170 | { 171 | "center_wavelength": 0.86, 172 | "common_name": "nir", 173 | "full_width_half_max": 0.03, 174 | "gsd": 30, 175 | "id": "B5" 176 | }, 177 | { 178 | "center_wavelength": 1.6, 179 | "common_name": "swir16", 180 | "full_width_half_max": 0.08, 181 | "gsd": 30, 182 | "id": "B6" 183 | }, 184 | { 185 | "center_wavelength": 2.2, 186 | "common_name": "swir22", 187 | "full_width_half_max": 0.2, 188 | "gsd": 30, 189 | "id": "B7" 190 | }, 191 | { 192 | "center_wavelength": 0.59, 193 | "common_name": "pan", 194 | "full_width_half_max": 0.18, 195 | "gsd": 15, 196 | "id": "B8" 197 | }, 198 | { 199 | "center_wavelength": 1.37, 200 | "common_name": "cirrus", 201 | "full_width_half_max": 0.02, 202 | "gsd": 30, 203 | "id": "B9" 204 | }, 205 | { 206 | "center_wavelength": 10.9, 207 | "common_name": "lwir11", 208 | "full_width_half_max": 0.8, 209 | "gsd": 100, 210 | "id": "B10" 211 | }, 212 | { 213 | "center_wavelength": 12, 214 | "common_name": "lwir12", 215 | "full_width_half_max": 1, 216 | "gsd": 100, 217 | "id": "B11" 218 | } 219 | ], 220 | "eo:gsd": 15, 221 | "eo:instrument": "OLI_TIRS", 222 | "eo:off_nadir": 0, 223 | "eo:platform": "platform2" 224 | }, 225 | "providers": [ 226 | { 227 | "name": "USGS", 228 | "roles": [ 229 | "producer" 230 | ], 231 | "url": "https://landsat.usgs.gov/" 232 | }, 233 | { 234 | "name": "Planet Labs", 235 | "roles": [ 236 | "processor" 237 | ], 238 | "url": "https://github.com/landsat-pds/landsat_ingestor" 239 | }, 240 | { 241 | "name": "AWS", 242 | "roles": [ 243 | "host" 244 | ], 245 | "url": "https://landsatonaws.com/" 246 | }, 247 | { 248 | "name": "Development Seed", 249 | "roles": [ 250 | "processor" 251 | ], 252 | "url": "https://github.com/sat-utils/sat-api" 253 | } 254 | ], 255 | "stac_version": "0.7.0", 256 | "title": "Landsat 8 L1", 257 | "version": "0.1.0" 258 | } 259 | -------------------------------------------------------------------------------- /packages/api-lib/tests/fixtures/stac/collection2_item.json: -------------------------------------------------------------------------------- 1 | { 2 | "assets": { 3 | "ANG": { 4 | "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/L8/010/010/LC80100102015050LGN00/LC80100102015050LGN00_ANG.txt", 5 | "title": "Angle coefficients file", 6 | "type": "text/plain" 7 | }, 8 | "B1": { 9 | "eo:bands": [ 10 | 0 11 | ], 12 | "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/L8/010/010/LC80100102015050LGN00/LC80100102015050LGN00_B1.TIF", 13 | "title": "Band 1 (coastal)", 14 | "type": "image/x.geotiff" 15 | }, 16 | "B10": { 17 | "eo:bands": [ 18 | 9 19 | ], 20 | "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/L8/010/010/LC80100102015050LGN00/LC80100102015050LGN00_B10.TIF", 21 | "title": "Band 10 (lwir)", 22 | "type": "image/x.geotiff" 23 | }, 24 | "B11": { 25 | "eo:bands": [ 26 | 10 27 | ], 28 | "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/L8/010/010/LC80100102015050LGN00/LC80100102015050LGN00_B11.TIF", 29 | "title": "Band 11 (lwir)", 30 | "type": "image/x.geotiff" 31 | }, 32 | "B2": { 33 | "eo:bands": [ 34 | 1 35 | ], 36 | "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/L8/010/010/LC80100102015050LGN00/LC80100102015050LGN00_B2.TIF", 37 | "title": "Band 2 (blue)", 38 | "type": "image/x.geotiff" 39 | }, 40 | "B3": { 41 | "eo:bands": [ 42 | 2 43 | ], 44 | "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/L8/010/010/LC80100102015050LGN00/LC80100102015050LGN00_B3.TIF", 45 | "title": "Band 3 (green)", 46 | "type": "image/x.geotiff" 47 | }, 48 | "B4": { 49 | "eo:bands": [ 50 | 3 51 | ], 52 | "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/L8/010/010/LC80100102015050LGN00/LC80100102015050LGN00_B4.TIF", 53 | "title": "Band 4 (red)", 54 | "type": "image/x.geotiff" 55 | }, 56 | "B5": { 57 | "eo:bands": [ 58 | 4 59 | ], 60 | "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/L8/010/010/LC80100102015050LGN00/LC80100102015050LGN00_B5.TIF", 61 | "title": "Band 5 (nir)", 62 | "type": "image/x.geotiff" 63 | }, 64 | "B6": { 65 | "eo:bands": [ 66 | 5 67 | ], 68 | "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/L8/010/010/LC80100102015050LGN00/LC80100102015050LGN00_B6.TIF", 69 | "title": "Band 6 (swir16)", 70 | "type": "image/x.geotiff" 71 | }, 72 | "B7": { 73 | "eo:bands": [ 74 | 6 75 | ], 76 | "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/L8/010/010/LC80100102015050LGN00/LC80100102015050LGN00_B7.TIF", 77 | "title": "Band 7 (swir22)", 78 | "type": "image/x.geotiff" 79 | }, 80 | "B8": { 81 | "eo:bands": [ 82 | 7 83 | ], 84 | "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/L8/010/010/LC80100102015050LGN00/LC80100102015050LGN00_B8.TIF", 85 | "title": "Band 8 (pan)", 86 | "type": "image/x.geotiff" 87 | }, 88 | "B9": { 89 | "eo:bands": [ 90 | 8 91 | ], 92 | "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/L8/010/010/LC80100102015050LGN00/LC80100102015050LGN00_B9.TIF", 93 | "title": "Band 9 (cirrus)", 94 | "type": "image/x.geotiff" 95 | }, 96 | "BQA": { 97 | "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/L8/010/010/LC80100102015050LGN00/LC80100102015050LGN00_BQA.TIF", 98 | "title": "Band quality data", 99 | "type": "image/x.geotiff" 100 | }, 101 | "MTL": { 102 | "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/L8/010/010/LC80100102015050LGN00/LC80100102015050LGN00_MTL.txt", 103 | "title": "original metadata file", 104 | "type": "text/plain" 105 | }, 106 | "index": { 107 | "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/L8/010/010/LC80100102015050LGN00/index.html", 108 | "title": "HTML index page", 109 | "type": "text/html" 110 | }, 111 | "thumbnail": { 112 | "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/L8/010/010/LC80100102015050LGN00/LC80100102015050LGN00_thumb_large.jpg", 113 | "title": "Thumbnail image", 114 | "type": "image/jpeg" 115 | } 116 | }, 117 | "bbox": [ 118 | -52.68296, 119 | 69.68409, 120 | -45.19691, 121 | 72.07674 122 | ], 123 | "geometry": { 124 | "coordinates": [ 125 | [ 126 | [ 127 | -52.68296, 128 | 72.07674 129 | ], 130 | [ 131 | -45.19691, 132 | 71.99758 133 | ], 134 | [ 135 | -45.83578, 136 | 69.68409 137 | ], 138 | [ 139 | -52.4967, 140 | 69.75356 141 | ], 142 | [ 143 | -52.68296, 144 | 72.07674 145 | ] 146 | ] 147 | ], 148 | "type": "Polygon" 149 | }, 150 | "id": "collection2_item", 151 | "links": [ 152 | { 153 | "href": "collection2_item.json", 154 | "rel": "self" 155 | }, 156 | { 157 | "href": "catalog.json", 158 | "rel": "root" 159 | }, 160 | { 161 | "href": "collection2.json", 162 | "rel": "parent" 163 | }, 164 | { 165 | "href": "collection2.json", 166 | "rel": "collection" 167 | } 168 | ], 169 | "collection": "collection2", 170 | "properties": { 171 | "datetime": "2015-02-19T15:06:12.565047+00:00", 172 | "eo:cloud_cover": 0.54, 173 | "eo:epsg": 32622, 174 | "eo:sun_azimuth": 174.10674048, 175 | "eo:sun_elevation": 7.85803613, 176 | "landsat:path": "10", 177 | "landsat:processing_level": "L1T", 178 | "landsat:product_id": null, 179 | "landsat:row": "10", 180 | "landsat:scene_id": "collection2_item" 181 | }, 182 | "type": "Feature" 183 | } 184 | -------------------------------------------------------------------------------- /packages/api-lib/tests/fixtures/stac/collectionNoChildren.json: -------------------------------------------------------------------------------- 1 | { 2 | "assets": { 3 | "ANG": { 4 | "title": "Angle coefficients file", 5 | "type": "text/plain" 6 | }, 7 | "B1": { 8 | "eo:bands": [ 9 | 0 10 | ], 11 | "title": "Band 1 (coastal)", 12 | "type": "image/x.geotiff" 13 | }, 14 | "B10": { 15 | "eo:bands": [ 16 | 9 17 | ], 18 | "title": "Band 10 (lwir)", 19 | "type": "image/x.geotiff" 20 | }, 21 | "B11": { 22 | "eo:bands": [ 23 | 10 24 | ], 25 | "title": "Band 11 (lwir)", 26 | "type": "image/x.geotiff" 27 | }, 28 | "B2": { 29 | "eo:bands": [ 30 | 1 31 | ], 32 | "title": "Band 2 (blue)", 33 | "type": "image/x.geotiff" 34 | }, 35 | "B3": { 36 | "eo:bands": [ 37 | 2 38 | ], 39 | "title": "Band 3 (green)", 40 | "type": "image/x.geotiff" 41 | }, 42 | "B4": { 43 | "eo:bands": [ 44 | 3 45 | ], 46 | "title": "Band 4 (red)", 47 | "type": "image/x.geotiff" 48 | }, 49 | "B5": { 50 | "eo:bands": [ 51 | 4 52 | ], 53 | "title": "Band 5 (nir)", 54 | "type": "image/x.geotiff" 55 | }, 56 | "B6": { 57 | "eo:bands": [ 58 | 5 59 | ], 60 | "title": "Band 6 (swir16)", 61 | "type": "image/x.geotiff" 62 | }, 63 | "B7": { 64 | "eo:bands": [ 65 | 6 66 | ], 67 | "title": "Band 7 (swir22)", 68 | "type": "image/x.geotiff" 69 | }, 70 | "B8": { 71 | "eo:bands": [ 72 | 7 73 | ], 74 | "title": "Band 8 (pan)", 75 | "type": "image/x.geotiff" 76 | }, 77 | "B9": { 78 | "eo:bands": [ 79 | 8 80 | ], 81 | "title": "Band 9 (cirrus)", 82 | "type": "image/x.geotiff" 83 | }, 84 | "BQA": { 85 | "title": "Band quality data", 86 | "type": "image/x.geotiff" 87 | }, 88 | "MTL": { 89 | "title": "original metadata file", 90 | "type": "text/plain" 91 | }, 92 | "index": { 93 | "title": "HTML index page", 94 | "type": "text/html" 95 | }, 96 | "thumbnail": { 97 | "title": "Thumbnail image", 98 | "type": "image/jpeg" 99 | } 100 | }, 101 | "description": "Landat 8 imagery radiometrically calibrated and orthorectified using gound points and Digital Elevation Model (DEM) data to correct relief displacement.", 102 | "extent": { 103 | "spatial": [ 104 | -180, 105 | -90, 106 | 180, 107 | 90 108 | ], 109 | "temporal": [ 110 | "2013-06-01", 111 | null 112 | ] 113 | }, 114 | "id": "landsat-8-l1", 115 | "keywords": [ 116 | "landsat", 117 | "earth observation", 118 | "usgs" 119 | ], 120 | "license": "PDDL-1.0", 121 | "links": [ 122 | { 123 | "href": "collection.json", 124 | "rel": "self" 125 | }, 126 | { 127 | "href": "catalog.json", 128 | "rel": "root" 129 | }, 130 | { 131 | "href": "catalog.json", 132 | "rel": "parent" 133 | } 134 | ], 135 | "properties": { 136 | "collection": "landsat-8-l1", 137 | "eo:bands": [ 138 | { 139 | "center_wavelength": 0.44, 140 | "common_name": "coastal", 141 | "full_width_half_max": 0.02, 142 | "gsd": 30, 143 | "id": "B1" 144 | }, 145 | { 146 | "center_wavelength": 0.48, 147 | "common_name": "blue", 148 | "full_width_half_max": 0.06, 149 | "gsd": 30, 150 | "id": "B2" 151 | }, 152 | { 153 | "center_wavelength": 0.56, 154 | "common_name": "green", 155 | "full_width_half_max": 0.06, 156 | "gsd": 30, 157 | "id": "B3" 158 | }, 159 | { 160 | "center_wavelength": 0.65, 161 | "common_name": "red", 162 | "full_width_half_max": 0.04, 163 | "gsd": 30, 164 | "id": "B4" 165 | }, 166 | { 167 | "center_wavelength": 0.86, 168 | "common_name": "nir", 169 | "full_width_half_max": 0.03, 170 | "gsd": 30, 171 | "id": "B5" 172 | }, 173 | { 174 | "center_wavelength": 1.6, 175 | "common_name": "swir16", 176 | "full_width_half_max": 0.08, 177 | "gsd": 30, 178 | "id": "B6" 179 | }, 180 | { 181 | "center_wavelength": 2.2, 182 | "common_name": "swir22", 183 | "full_width_half_max": 0.2, 184 | "gsd": 30, 185 | "id": "B7" 186 | }, 187 | { 188 | "center_wavelength": 0.59, 189 | "common_name": "pan", 190 | "full_width_half_max": 0.18, 191 | "gsd": 15, 192 | "id": "B8" 193 | }, 194 | { 195 | "center_wavelength": 1.37, 196 | "common_name": "cirrus", 197 | "full_width_half_max": 0.02, 198 | "gsd": 30, 199 | "id": "B9" 200 | }, 201 | { 202 | "center_wavelength": 10.9, 203 | "common_name": "lwir11", 204 | "full_width_half_max": 0.8, 205 | "gsd": 100, 206 | "id": "B10" 207 | }, 208 | { 209 | "center_wavelength": 12, 210 | "common_name": "lwir12", 211 | "full_width_half_max": 1, 212 | "gsd": 100, 213 | "id": "B11" 214 | } 215 | ], 216 | "eo:gsd": 15, 217 | "eo:instrument": "OLI_TIRS", 218 | "eo:off_nadir": 0, 219 | "eo:platform": "landsat-8" 220 | }, 221 | "providers": [ 222 | { 223 | "name": "USGS", 224 | "roles": [ 225 | "producer" 226 | ], 227 | "url": "https://landsat.usgs.gov/" 228 | }, 229 | { 230 | "name": "Planet Labs", 231 | "roles": [ 232 | "processor" 233 | ], 234 | "url": "https://github.com/landsat-pds/landsat_ingestor" 235 | }, 236 | { 237 | "name": "AWS", 238 | "roles": [ 239 | "host" 240 | ], 241 | "url": "https://landsatonaws.com/" 242 | }, 243 | { 244 | "name": "Development Seed", 245 | "roles": [ 246 | "processor" 247 | ], 248 | "url": "https://github.com/sat-utils/sat-api" 249 | } 250 | ], 251 | "stac_version": "0.7.0", 252 | "title": "Landsat 8 L1", 253 | "version": "0.1.0" 254 | } 255 | -------------------------------------------------------------------------------- /packages/api-lib/tests/fixtures/stac/intersectsFeature.json: -------------------------------------------------------------------------------- 1 | { 2 | "type": "Feature", 3 | "properties": {}, 4 | "geometry": { 5 | "type": "Polygon", 6 | "coordinates": [ 7 | [ 8 | [ 9 | -54.58007812499999, 10 | 69.39578308847753 11 | ], 12 | [ 13 | -49.0869140625, 14 | 69.39578308847753 15 | ], 16 | [ 17 | -49.0869140625, 18 | 70.72897946208789 19 | ], 20 | [ 21 | -54.58007812499999, 22 | 70.72897946208789 23 | ], 24 | [ 25 | -54.58007812499999, 26 | 69.39578308847753 27 | ] 28 | ] 29 | ] 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /packages/api-lib/tests/fixtures/stac/noIntersectsFeature.json: -------------------------------------------------------------------------------- 1 | { 2 | "type": "Feature", 3 | "properties": {}, 4 | "geometry": { 5 | "type": "Polygon", 6 | "coordinates": [ 7 | [ 8 | [ 9 | -71.4111328125, 10 | 67.1016555307692 11 | ], 12 | [ 13 | -67.67578124999999, 14 | 67.1016555307692 15 | ], 16 | [ 17 | -67.67578124999999, 18 | 68.12248241161676 19 | ], 20 | [ 21 | -71.4111328125, 22 | 68.12248241161676 23 | ], 24 | [ 25 | -71.4111328125, 26 | 67.1016555307692 27 | ] 28 | ] 29 | ] 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /packages/api-lib/tests/integration/docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '2' 2 | services: 3 | local: 4 | image: localstack/localstack:latest 5 | environment: 6 | SERVICES: 'elasticsearch' 7 | ports: 8 | - 4571:4571 # elasticsearch 9 | -------------------------------------------------------------------------------- /packages/api-lib/tests/integration/ingestCollections.js: -------------------------------------------------------------------------------- 1 | process.env.ES_HOST = `http://${process.env.DOCKER_NAME}:4571` 2 | const ingest = require('../../libs/ingest').ingest 3 | const backend = require('../../libs/es') 4 | 5 | async function doIngest() { 6 | await ingest('../fixtures/stac/catalog.json', backend, true, true) 7 | console.log('Collections done') 8 | } 9 | doIngest() 10 | -------------------------------------------------------------------------------- /packages/api-lib/tests/integration/ingestData.js: -------------------------------------------------------------------------------- 1 | process.env.ES_HOST = `http://${process.env.DOCKER_NAME}:4571` 2 | 3 | const ingest = require('../../libs/ingest').ingest 4 | const backend = require('../../libs/es') 5 | 6 | async function doIngest() { 7 | try { 8 | await ingest('../fixtures/stac/catalog.json', backend) 9 | console.log('Items done') 10 | } catch (error) { 11 | console.log(error.message) 12 | } 13 | } 14 | doIngest() 15 | 16 | -------------------------------------------------------------------------------- /packages/api-lib/tests/integration/runIntegration.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | docker-compose up & while ! nc -z $DOCKER_NAME 4571; do sleep 1; done; 3 | sleep 20; 4 | node ./ingestCollections.js && node ./ingestData.js && yarn ava ./tests/integration/test_api.js 5 | -------------------------------------------------------------------------------- /packages/api-lib/tests/integration/test_api.js: -------------------------------------------------------------------------------- 1 | const test = require('ava') 2 | process.env.ES_HOST = `http://${process.env.DOCKER_NAME}:4571` 3 | process.env.AWS_ACCESS_KEY_ID = 'none' 4 | process.env.AWS_SECRET_ACCESS_KEY = 'none' 5 | const backend = require('../../libs/es') 6 | const api = require('../../libs/api') 7 | const intersectsFeature = require('../fixtures/stac/intersectsFeature.json') 8 | const noIntersectsFeature = require('../fixtures/stac/noIntersectsFeature.json') 9 | 10 | const { search } = api 11 | const endpoint = 'endpoint' 12 | 13 | test('collections', async (t) => { 14 | const response = await search('/collections', {}, backend, endpoint) 15 | t.is(response.collections.length, 2) 16 | t.is(response.meta.returned, 2) 17 | }) 18 | 19 | test('collections/{collectionId}', async (t) => { 20 | let response = await search('/collections/landsat-8-l1', {}, backend, endpoint) 21 | t.is(response.id, 'landsat-8-l1') 22 | response = await search('/collections/collection2', {}, backend, endpoint) 23 | t.is(response.id, 'collection2') 24 | }) 25 | 26 | test('collections/{collectionId}/items', async (t) => { 27 | const response = await search('/collections/landsat-8-l1/items', 28 | {}, backend, endpoint) 29 | t.is(response.type, 'FeatureCollection') 30 | t.is(response.features.length, 2) 31 | t.is(response.features[0].id, 'LC80100102015082LGN00') 32 | t.is(response.features[1].id, 'LC80100102015050LGN00') 33 | }) 34 | 35 | test('collections/{collectionId}/items/{itemId}', async (t) => { 36 | const response = 37 | await search('/collections/landsat-8-l1/items/LC80100102015082LGN00', 38 | {}, backend, endpoint) 39 | t.is(response.type, 'Feature') 40 | t.is(response.id, 'LC80100102015082LGN00') 41 | }) 42 | 43 | test('collections/{collectionId}/items with bbox', async (t) => { 44 | let response = await search('/collections/landsat-8-l1/items', { 45 | bbox: [-180, -90, 180, 90] 46 | }, backend, endpoint) 47 | t.is(response.type, 'FeatureCollection') 48 | t.is(response.features[0].id, 'LC80100102015082LGN00') 49 | t.is(response.features[1].id, 'LC80100102015050LGN00') 50 | 51 | response = await search('/collections/landsat-8-l1/items', { 52 | bbox: [-5, -5, 5, 5] 53 | }, backend, endpoint) 54 | t.is(response.features.length, 0) 55 | }) 56 | 57 | test('collections/{collectionId}/items with time', async (t) => { 58 | let response = await search('/collections/landsat-8-l1/items', { 59 | time: '2015-02-19T15:06:12.565047+00:00' 60 | }, backend, endpoint) 61 | t.is(response.type, 'FeatureCollection') 62 | t.is(response.features[0].id, 'LC80100102015050LGN00') 63 | 64 | response = await search('/collections/landsat-8-l1/items', { 65 | time: '2015-02-17/2015-02-20' 66 | }, backend, endpoint) 67 | t.is(response.type, 'FeatureCollection') 68 | t.is(response.features[0].id, 'LC80100102015050LGN00') 69 | 70 | response = await search('/collections/landsat-8-l1/items', { 71 | time: '2015-02-19/2015-02-20' 72 | }, backend, endpoint) 73 | t.is(response.features[0].id, 'LC80100102015050LGN00', 74 | 'Handles date range without times inclusion issue') 75 | }) 76 | 77 | test('collections/{collectionId}/items with limit', async (t) => { 78 | const response = await search('/collections/landsat-8-l1/items', { 79 | limit: 1 80 | }, backend, endpoint) 81 | t.is(response.type, 'FeatureCollection') 82 | t.is(response.features.length, 1) 83 | }) 84 | 85 | test('collections/{collectionId}/items with intersects', async (t) => { 86 | let response = await search('/collections/landsat-8-l1/items', { 87 | intersects: intersectsFeature 88 | }, backend, endpoint) 89 | t.is(response.type, 'FeatureCollection') 90 | t.is(response.features[0].id, 'LC80100102015082LGN00') 91 | t.is(response.features[1].id, 'LC80100102015050LGN00') 92 | 93 | response = await search('/collections/landsat-8-l1/items', { 94 | intersects: noIntersectsFeature 95 | }, backend, endpoint) 96 | t.is(response.features.length, 0) 97 | }) 98 | 99 | test('collections/{collectionId}/items with eq query', async (t) => { 100 | const response = await search('/collections/landsat-8-l1/items', { 101 | query: { 102 | 'eo:cloud_cover': { 103 | eq: 0.54 104 | } 105 | } 106 | }, backend, endpoint) 107 | t.is(response.features.length, 1) 108 | t.is(response.features[0].id, 'LC80100102015050LGN00') 109 | }) 110 | 111 | test('collections/{collectionId}/items with gt lt query', async (t) => { 112 | const response = await search('/collections/landsat-8-l1/items', { 113 | query: { 114 | 'eo:cloud_cover': { 115 | gt: 0.5, 116 | lt: 0.6 117 | } 118 | } 119 | }, backend, endpoint) 120 | t.is(response.features.length, 1) 121 | t.is(response.features[0].id, 'LC80100102015050LGN00') 122 | }) 123 | 124 | 125 | test('stac', async (t) => { 126 | const response = await search('/stac', {}, backend, endpoint) 127 | t.is(response.links.length, 4) 128 | }) 129 | 130 | test('stac/search bbox', async (t) => { 131 | let response = await search('/stac/search', { 132 | bbox: [-180, -90, 180, 90] 133 | }, backend, endpoint) 134 | t.is(response.type, 'FeatureCollection') 135 | t.is(response.features[0].id, 'LC80100102015082LGN00') 136 | t.is(response.features[1].id, 'collection2_item') 137 | response = await search('/stac/search', { 138 | bbox: [-5, -5, 5, 5] 139 | }, backend, endpoint) 140 | t.is(response.features.length, 0) 141 | }) 142 | 143 | test('stac/search default sort', async (t) => { 144 | const response = await search('/stac/search', {}, backend, endpoint) 145 | t.is(response.features[0].id, 'LC80100102015082LGN00') 146 | }) 147 | 148 | test('stac/search sort', async (t) => { 149 | let response = await search('/stac/search', { 150 | sort: [{ 151 | field: 'eo:cloud_cover', 152 | direction: 'desc' 153 | }] 154 | }, backend, endpoint) 155 | t.is(response.features[0].id, 'LC80100102015082LGN00') 156 | 157 | response = await search('/stac/search', { 158 | sort: '[{ "field": "eo:cloud_cover", "direction": "desc" }]' 159 | }, backend, endpoint) 160 | t.is(response.features[0].id, 'LC80100102015082LGN00') 161 | }) 162 | 163 | test('stac/search flattened collection properties', async (t) => { 164 | let response = await search('/stac/search', { 165 | query: { 166 | 'eo:platform': { 167 | eq: 'platform2' 168 | } 169 | } 170 | }, backend, endpoint) 171 | t.is(response.features[0].id, 'collection2_item') 172 | 173 | response = await search('/stac/search', { 174 | query: { 175 | 'eo:platform': { 176 | eq: 'landsat-8' 177 | } 178 | } 179 | }, backend, endpoint) 180 | const havePlatform = 181 | response.features.filter( 182 | (item) => (item.properties['eo:platform'] === 'landsat-8') 183 | ) 184 | t.is(havePlatform.length, response.features.length) 185 | }) 186 | 187 | test('stac/search fields filter', async (t) => { 188 | let response = await search('/stac/search', { 189 | fields: { 190 | exclude: ['collection'] 191 | } 192 | }, backend, endpoint) 193 | t.falsy(response.features[0].collection) 194 | 195 | response = await search('/stac/search', { 196 | fields: { 197 | exclude: ['geometry'] 198 | } 199 | }, backend, endpoint) 200 | t.falsy(response.features[0].geometry) 201 | 202 | response = await search('/stac/search', { 203 | }, backend, endpoint) 204 | t.truthy(response.features[0].geometry) 205 | 206 | response = await search('/stac/search', { 207 | fields: { 208 | include: ['collection', 'properties.eo:epsg'] 209 | } 210 | }, backend, endpoint) 211 | t.truthy(response.features[0].collection) 212 | t.truthy(response.features[0].properties['eo:epsg']) 213 | t.falsy(response.features[0].properties['eo:cloud_cover']) 214 | 215 | response = await search('/stac/search', { 216 | fields: { 217 | exclude: ['id', 'links'] 218 | } 219 | }, backend, endpoint) 220 | t.truthy(response.features.length, 'Does not exclude required fields') 221 | }) 222 | 223 | test('stac/search in query', async (t) => { 224 | const response = await search('/stac/search', { 225 | query: { 226 | 'landsat:path': { 227 | in: ['10'] 228 | } 229 | } 230 | }, backend, endpoint) 231 | t.is(response.features.length, 3) 232 | }) 233 | 234 | test('stac/search ids', async (t) => { 235 | const response = await search('/stac/search', { 236 | ids: ['collection2_item', 'LC80100102015050LGN00'] 237 | }, backend, endpoint) 238 | t.is(response.features.length, 2) 239 | t.is(response.features[0].id, 'collection2_item') 240 | t.is(response.features[1].id, 'LC80100102015050LGN00') 241 | }) 242 | 243 | test('stac/search collections', async (t) => { 244 | let response = await search('/stac/search', { 245 | query: { 246 | collections: ['collection2'] 247 | } 248 | }, backend, endpoint) 249 | t.is(response.features.length, 1) 250 | t.is(response.features[0].id, 'collection2_item') 251 | 252 | response = await search('/stac/search', { 253 | query: { 254 | collections: ['landsat-8-l1'] 255 | } 256 | }, backend, endpoint) 257 | t.is(response.features.length, 2) 258 | t.is(response.features[0].id, 'LC80100102015082LGN00') 259 | t.is(response.features[1].id, 'LC80100102015050LGN00') 260 | 261 | response = await search('/stac/search', { 262 | query: { 263 | collections: ['collection2', 'landsat-8-l1'] 264 | } 265 | }, backend, endpoint) 266 | t.is(response.features.length, 3) 267 | }) 268 | -------------------------------------------------------------------------------- /packages/api-lib/tests/test_api_extractIntersects.js: -------------------------------------------------------------------------------- 1 | const test = require('ava') 2 | const sinon = require('sinon') 3 | const proxyquire = require('proxyquire') 4 | const api = require('../libs/api') 5 | 6 | test('extractIntersects', (t) => { 7 | const params = {} 8 | const intersectsGeometry = api.extractIntersects(params) 9 | t.falsy(intersectsGeometry, 10 | 'Returns undefined when no intersects parameter') 11 | }) 12 | 13 | test('extractIntersects', (t) => { 14 | const valid = sinon.stub().returns(false) 15 | const proxyApi = proxyquire('../libs/api', { 16 | 'geojson-validation': { valid } 17 | }) 18 | t.throws(() => { 19 | proxyApi.extractIntersects({ intersects: {} }) 20 | }, null, 'Throws exception when GeoJSON is invalid') 21 | }) 22 | 23 | test('extractIntersects', (t) => { 24 | const valid = sinon.stub().returns(true) 25 | const proxyApi = proxyquire('../libs/api', { 26 | 'geojson-validation': { valid } 27 | }) 28 | t.throws(() => { 29 | proxyApi.extractIntersects({ 30 | intersects: { type: 'FeatureCollection' } 31 | }) 32 | }, null, 'Throws exception when GeoJSON type is FeatureCollection') 33 | }) 34 | -------------------------------------------------------------------------------- /packages/api-lib/tests/test_api_parsePath.js: -------------------------------------------------------------------------------- 1 | const test = require('ava') 2 | const api = require('../libs/api') 3 | 4 | test('parsePath', (t) => { 5 | let expected = { 6 | stac: true, 7 | collections: false, 8 | search: false, 9 | collectionId: false, 10 | items: false, 11 | itemId: false 12 | } 13 | let actual = api.parsePath('/stac') 14 | t.deepEqual(actual, expected) 15 | 16 | expected = { 17 | stac: true, 18 | collections: false, 19 | search: true, 20 | collectionId: false, 21 | items: false, 22 | itemId: false 23 | } 24 | actual = api.parsePath('/stac/search') 25 | t.deepEqual(actual, expected) 26 | 27 | expected = { 28 | stac: false, 29 | collections: true, 30 | search: false, 31 | collectionId: false, 32 | items: false, 33 | itemId: false 34 | } 35 | actual = api.parsePath('/collections') 36 | t.deepEqual(actual, expected) 37 | 38 | expected = { 39 | stac: false, 40 | collections: true, 41 | search: false, 42 | collectionId: 'id', 43 | items: false, 44 | itemId: false 45 | } 46 | actual = api.parsePath('/collections/id') 47 | t.deepEqual(actual, expected) 48 | 49 | expected = { 50 | stac: false, 51 | collections: true, 52 | search: false, 53 | collectionId: 'id', 54 | items: true, 55 | itemId: false 56 | } 57 | actual = api.parsePath('/collections/id/items') 58 | t.deepEqual(actual, expected) 59 | 60 | expected = { 61 | stac: false, 62 | collections: true, 63 | search: false, 64 | collectionId: 'id', 65 | items: true, 66 | itemId: 'id' 67 | } 68 | actual = api.parsePath('/collections/id/items/id') 69 | t.deepEqual(actual, expected) 70 | }) 71 | 72 | -------------------------------------------------------------------------------- /packages/api-lib/tests/test_api_search.js: -------------------------------------------------------------------------------- 1 | const test = require('ava') 2 | const sinon = require('sinon') 3 | const proxquire = require('proxyquire') 4 | const api = require('../libs/api') 5 | const item = require('./fixtures/item.json') 6 | const itemLinks = require('./fixtures/itemLinks.json') 7 | 8 | function cloneMutatedItem() { 9 | return Object.assign({}, item, { links: item.links.slice(0) }) 10 | } 11 | 12 | test('search es error', async (t) => { 13 | const error = sinon.spy() 14 | const proxyApi = proxquire('../libs/api', { 15 | './logger': { 16 | error 17 | } 18 | }) 19 | const errorMessage = 'errorMessage' 20 | const search = sinon.stub().throws(new Error(errorMessage)) 21 | const backend = { search } 22 | const response = await proxyApi.search('/stac', undefined, backend, 'endpoint') 23 | t.is(error.firstCall.args[0].message, errorMessage, 24 | 'Logs Elasticsearch error via Winston transport') 25 | t.is(response.description, errorMessage) 26 | t.is(response.code, 500) 27 | }) 28 | 29 | test('search /', async (t) => { 30 | process.env.STAC_DOCS_URL = 'test' 31 | const endpoint = 'endpoint' 32 | const expected = { 33 | links: [ 34 | { 35 | href: endpoint, 36 | rel: 'self' 37 | }, 38 | { 39 | href: `${endpoint}/collections`, 40 | rel: 'data' 41 | }, 42 | { 43 | href: 'test', 44 | rel: 'service' 45 | } 46 | ] 47 | } 48 | const actual = await api.search('/', undefined, {}, endpoint) 49 | t.deepEqual(actual, expected, 'Returns root API node') 50 | }) 51 | 52 | test('search /stac', async (t) => { 53 | const collection = 'collection' 54 | const results = { results: [{ id: collection }] } 55 | const search = sinon.stub().resolves(results) 56 | const backend = { search } 57 | const actual = await api.search('/stac', undefined, backend, 'endpoint') 58 | const expectedLinks = [ 59 | { 60 | rel: 'child', 61 | href: 'endpoint/collections/collection' 62 | }, 63 | { 64 | rel: 'self', 65 | href: 'endpoint/stac' 66 | }, 67 | { 68 | rel: 'search', 69 | href: 'endpoint/stac/search' 70 | } 71 | ] 72 | t.is(search.firstCall.args[1], 'collections') 73 | t.deepEqual(actual.links, expectedLinks, 74 | 'Returns STAC catalog with links to collections') 75 | }) 76 | 77 | test('search /stac/search wraps results', async (t) => { 78 | const limit = 10 79 | const page = 1 80 | const meta = { 81 | limit, 82 | page, 83 | found: 1, 84 | returned: 1 85 | } 86 | const clonedItem = cloneMutatedItem() 87 | const results = [clonedItem] 88 | 89 | const itemsResults = { meta, results } 90 | const search = sinon.stub() 91 | search.resolves(itemsResults) 92 | const backend = { search } 93 | const actual = await api.search('/stac/search', {}, backend, 'endpoint') 94 | t.deepEqual(actual.features[0].links, itemLinks.links, 95 | 'Adds correct relative STAC links') 96 | 97 | const expectedMeta = { 98 | limit, 99 | page, 100 | found: 1, 101 | returned: 1 102 | } 103 | t.deepEqual(actual.meta, expectedMeta, 'Adds correct response metadata fields') 104 | t.is(actual.type, 'FeatureCollection', 'Wraps response as FeatureCollection') 105 | }) 106 | 107 | test('search /stac/search query parameters', async (t) => { 108 | const search = sinon.stub().resolves({ results: [], meta: {} }) 109 | const backend = { search } 110 | const query = { 'test': true } 111 | const queryParams = { 112 | page: 1, 113 | limit: 2, 114 | query 115 | } 116 | api.search('/stac/search', queryParams, backend, 'endpoint') 117 | t.deepEqual(search.firstCall.args[0], { query }, 118 | 'Extracts query to use in search parameters') 119 | }) 120 | 121 | test('search /stac/search intersects parameter', async (t) => { 122 | const search = sinon.stub().resolves({ results: [], meta: {} }) 123 | const backend = { search } 124 | const queryParams = { 125 | intersects: item, 126 | page: 1, 127 | limit: 1 128 | } 129 | api.search('/stac/search', queryParams, backend, 'endpoint') 130 | t.deepEqual(search.firstCall.args[0].intersects, item, 131 | 'Uses valid GeoJSON as intersects search parameter') 132 | 133 | search.resetHistory() 134 | queryParams.intersects = JSON.stringify(item) 135 | api.search('/stac/search', queryParams, backend, 'endpoint') 136 | t.deepEqual(search.firstCall.args[0].intersects, item, 137 | 'Handles stringified GeoJSON intersects parameter') 138 | }) 139 | 140 | test('search /stac/search bbox parameter', async (t) => { 141 | const search = sinon.stub().resolves({ results: [], meta: {} }) 142 | const backend = { search } 143 | const w = -10 144 | const s = -10 145 | const e = 10 146 | const n = 10 147 | const bbox = [w, s, e, n] 148 | const queryParams = { 149 | bbox, 150 | page: 1, 151 | limit: 1 152 | } 153 | const expected = { 154 | type: 'Feature', 155 | properties: {}, 156 | geometry: { 157 | type: 'Polygon', 158 | coordinates: [[ 159 | [s, w], 160 | [n, w], 161 | [n, e], 162 | [s, e], 163 | [s, w] 164 | ]] 165 | } 166 | } 167 | await api.search('/stac/search', queryParams, backend, 'endpoint') 168 | t.deepEqual(search.firstCall.args[0].intersects, expected, 169 | 'Converts a [w,s,e,n] bbox to an intersects search parameter') 170 | search.resetHistory() 171 | queryParams.bbox = `[${bbox.toString()}]` 172 | await api.search('/stac/search', queryParams, backend, 'endpoint') 173 | t.deepEqual(search.firstCall.args[0].intersects, expected, 174 | 'Converts stringified [w,s,e,n] bbox to an intersects search parameter') 175 | }) 176 | 177 | test('search /stac/search time parameter', async (t) => { 178 | const search = sinon.stub().resolves({ results: [], meta: {} }) 179 | const backend = { search } 180 | const range = '2007-03-01T13:00:00Z/2008-05-11T15:30:00Z' 181 | const queryParams = { 182 | page: 1, 183 | limit: 2, 184 | time: range 185 | } 186 | await api.search('/stac/search', queryParams, backend, 'endpoint') 187 | t.deepEqual(search.firstCall.args[0], { datetime: range }, 188 | 'Extracts time query parameter and transforms it into ' + 189 | 'datetime search parameter') 190 | }) 191 | 192 | test('search /collections', async (t) => { 193 | const meta = { 194 | limit: 1, 195 | page: 1, 196 | found: 1, 197 | returned: 1 198 | } 199 | const search = sinon.stub().resolves({ 200 | meta, 201 | results: [{ 202 | id: 1, 203 | links: [] 204 | }] 205 | }) 206 | const backend = { search } 207 | const actual = await api.search('/collections', {}, backend, 'endpoint') 208 | t.is(search.firstCall.args[1], 'collections') 209 | t.is(actual.collections.length, 1) 210 | t.is(actual.collections[0].links.length, 4, 'Adds STAC links to each collection') 211 | }) 212 | 213 | test('search /collections/collectionId', async (t) => { 214 | const meta = { 215 | limit: 1, 216 | page: 1, 217 | found: 1, 218 | returned: 1 219 | } 220 | const search = sinon.stub().resolves({ 221 | meta, 222 | results: [{ 223 | id: 1, 224 | links: [] 225 | }] 226 | }) 227 | const backend = { search } 228 | const collectionId = 'collectionId' 229 | let actual = await api.search( 230 | `/collections/${collectionId}`, { test: 'test' }, backend, 'endpoint' 231 | ) 232 | t.deepEqual(search.firstCall.args[0], { id: collectionId }, 233 | 'Calls search with the collectionId path element as id parameter' + 234 | ' and ignores other passed filter parameters') 235 | t.is(actual.links.length, 4, 'Returns the first found collection as object') 236 | 237 | search.reset() 238 | search.resolves({ 239 | meta, 240 | results: [] 241 | }) 242 | actual = await api.search( 243 | `/collections/${collectionId}`, {}, backend, 'endpoint' 244 | ) 245 | t.is(actual.message, 'Collection not found', 246 | 'Sends error when not collections are found in search') 247 | }) 248 | 249 | test('search /collections/collectionId/items', async (t) => { 250 | const meta = { 251 | limit: 1, 252 | page: 1, 253 | found: 1, 254 | returned: 1 255 | } 256 | 257 | const search = sinon.stub().resolves({ 258 | meta, 259 | results: [] 260 | }) 261 | const backend = { search } 262 | const collectionId = 'collectionId' 263 | await api.search( 264 | `/collections/${collectionId}/items`, {}, backend, 'endpoint' 265 | ) 266 | const expectedParameters = { 267 | query: { 268 | collections: [collectionId] 269 | } 270 | } 271 | t.deepEqual(search.firstCall.args[0], expectedParameters, 272 | 'Calls search with the collectionId as part of the query parameter') 273 | }) 274 | 275 | test('search /collections/collectionId/items/itemId', async (t) => { 276 | const meta = { 277 | limit: 1, 278 | page: 1, 279 | found: 1, 280 | returned: 1 281 | } 282 | const clonedItem = cloneMutatedItem() 283 | const results = [clonedItem] 284 | const search = sinon.stub().resolves({ 285 | meta, 286 | results 287 | }) 288 | const backend = { search } 289 | const itemId = 'itemId' 290 | const actual = await api.search( 291 | `/collections/collectionId/items/${itemId}`, {}, backend, 'endpoint' 292 | ) 293 | t.deepEqual(search.firstCall.args[0], { id: itemId }, 294 | 'Calls search with the itemId path element as id parameter' + 295 | ' and ignores other passed filter parameters') 296 | 297 | t.is(actual.type, 'Feature') 298 | t.is(actual.links.length, 4, 'Adds STAC links to response object') 299 | }) 300 | -------------------------------------------------------------------------------- /packages/api-lib/tests/test_ingest.js: -------------------------------------------------------------------------------- 1 | const test = require('ava') 2 | const sinon = require('sinon') 3 | const MemoryStream = require('memorystream') 4 | const proxquire = require('proxyquire') 5 | const fs = require('fs') 6 | const { ingest, ingestItem } = require('../libs/ingest') 7 | const firstItem = require('./fixtures/stac/LC80100102015050LGN00.json') 8 | 9 | const setup = () => { 10 | const dupOptions = { 11 | readable: true, 12 | writable: true, 13 | objectMode: true 14 | } 15 | const writeOptions = { 16 | writable: true, 17 | readable: false, 18 | objectMode: true 19 | } 20 | // Catalog is filtered by real toEs transform stream but is left in here. 21 | const toEs = new MemoryStream(null, dupOptions) 22 | const esStream = new MemoryStream(null, writeOptions) 23 | const backend = { 24 | stream: () => ({ toEs, esStream }), 25 | prepare: sinon.stub().resolves(true) 26 | } 27 | return { 28 | toEs, 29 | esStream, 30 | backend 31 | } 32 | } 33 | 34 | test('ingest traverses the entire STAC tree', async (t) => { 35 | const { esStream, backend } = setup() 36 | await ingest('./fixtures/stac/catalog.json', backend) 37 | 38 | const itemIds = [ 39 | 'landsat-8-l1', 40 | 'collection2', 41 | 'collection2_item', 42 | 'LC80100102015050LGN00', 43 | 'LC80100102015082LGN00' 44 | ] 45 | const queudIds = esStream.queue.map((queued) => queued.id) 46 | const hasItems = itemIds.map((itemId) => (queudIds.includes(itemId))) 47 | t.falsy(hasItems.includes(false)) 48 | }) 49 | 50 | test('ingest does not recurse', async (t) => { 51 | const { esStream, backend } = setup() 52 | await ingest('./fixtures/stac/catalog.json', backend, false) 53 | t.is(esStream.queue.length, 1) 54 | }) 55 | 56 | test('ingest consumes item with no children and closes stream', async (t) => { 57 | const { esStream, backend } = setup() 58 | await ingest('./fixtures/stac/collectionNoChildren.json', backend) 59 | t.is(esStream.queue.length, 1) 60 | }) 61 | 62 | test('ingest stops at collections when collectionsOnly is true', async (t) => { 63 | const { esStream, backend } = setup() 64 | await ingest('./fixtures/stac/catalog.json', backend, true, true) 65 | const itemIds = [ 66 | 'LC80100102015050LGN00', 67 | 'LC80100102015082LGN00' 68 | ] 69 | const hasItems = esStream.queue.map((queued) => (itemIds.includes(queued.id))) 70 | t.falsy(hasItems.includes(true)) 71 | }) 72 | 73 | test('ingest logs request error and continues', async (t) => { 74 | const error = sinon.spy() 75 | const stubFsRead = sinon.stub(fs, 'readFile') 76 | stubFsRead.callThrough() 77 | const errorMessage = 'errorMessage' 78 | stubFsRead.withArgs('./fixtures/stac/LC80100102015050LGN00.json') 79 | .throws(new Error(errorMessage)) 80 | const proxyIngest = proxquire('../libs/ingest', { 81 | './logger': { 82 | error, 83 | info: () => {} 84 | }, 85 | fs: stubFsRead 86 | }) 87 | const { esStream, backend } = setup() 88 | await proxyIngest.ingest('./fixtures/stac/catalog.json', backend) 89 | t.is(error.firstCall.args[0], errorMessage, 90 | 'Logs error via Winston transport') 91 | t.is(esStream.queue.length, 6, 'Skips errored request and continues') 92 | }) 93 | 94 | test('ingestItem passes item through transform stream', async (t) => { 95 | const { esStream, backend } = setup() 96 | await ingestItem(firstItem, backend) 97 | t.deepEqual(esStream.queue[0], firstItem) 98 | }) 99 | -------------------------------------------------------------------------------- /packages/api/README.md: -------------------------------------------------------------------------------- 1 | ## @sat-utils/api 2 | 3 | 4 | ### Unit Tests 5 | ``` 6 | $ yarn 7 | $ yarn test 8 | ``` 9 | 10 | ### About 11 | [sat-api](https://github.com/sat-utils/sat-api) was created by [Development Seed]() and is part of a collection of tools called [sat-utils](https://github.com/sat-utils). 12 | -------------------------------------------------------------------------------- /packages/api/index.js: -------------------------------------------------------------------------------- 1 | /* eslint-disable new-cap, no-lonely-if */ 2 | 3 | 'use strict' 4 | 5 | const satlib = require('@sat-utils/api-lib') 6 | 7 | module.exports.handler = async (event) => { 8 | // determine endpoint 9 | let endpoint = process.env.SATAPI_URL 10 | if (typeof endpoint === 'undefined') { 11 | if ('X-Forwarded-Host' in event.headers) { 12 | endpoint = `${event.headers['X-Forwarded-Proto']}://${event.headers['X-Forwarded-Host']}` 13 | } else { 14 | endpoint = `${event.headers['X-Forwarded-Proto']}://${event.headers.Host}` 15 | if ('stage' in event.requestContext) { 16 | endpoint = `${endpoint}/${event.requestContext.stage}` 17 | } 18 | } 19 | } 20 | 21 | const buildResponse = async (statusCode, result) => { 22 | const response = { 23 | statusCode, 24 | headers: { 25 | 'Content-Type': 'application/json', 26 | 'Access-Control-Allow-Origin': '*', // Required for CORS support to work 27 | 'Access-Control-Allow-Credentials': true 28 | }, 29 | body: result 30 | } 31 | return response 32 | } 33 | 34 | // get payload 35 | const method = event.httpMethod 36 | let query = {} 37 | if (method === 'POST' && event.body) { 38 | query = JSON.parse(event.body) 39 | } else if (method === 'GET' && event.queryStringParameters) { 40 | query = event.queryStringParameters 41 | } 42 | 43 | const result = await satlib.api.search(event.path, query, satlib.es, endpoint) 44 | let returnResponse 45 | if (result instanceof Error) { 46 | returnResponse = buildResponse(404, result.message) 47 | } else { 48 | returnResponse = buildResponse(200, JSON.stringify(result)) 49 | } 50 | return returnResponse 51 | } 52 | -------------------------------------------------------------------------------- /packages/api/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@sat-utils/api", 3 | "version": "0.3.0", 4 | "description": "The api lambda function for sat-api", 5 | "main": "index.js", 6 | "repository": { 7 | "type": "git", 8 | "url": "git+https://github.com/sat-utils/sat-api.git" 9 | }, 10 | "author": "Alireza Jazayeri , Matthew Hanson ", 11 | "license": "MIT", 12 | "ava": { 13 | "files": "tests/*.js", 14 | "verbose": true, 15 | "serial": true 16 | }, 17 | "bugs": { 18 | "url": "https://github.com/sat-utils/sat-api/issues" 19 | }, 20 | "scripts": { 21 | "build": "webpack", 22 | "test": "ava" 23 | }, 24 | "publishConfig": { 25 | "access": "public" 26 | }, 27 | "homepage": "https://github.com/sat-utils/sat-api#readme", 28 | "dependencies": { 29 | "@sat-utils/api-lib": "^0.3.0" 30 | }, 31 | "devDependencies": { 32 | "ava": "^0.25.0", 33 | "aws-event-mocks": "0.0.0", 34 | "proxyquire": "^2.1.0", 35 | "sinon": "^7.1.1", 36 | "webpack": "~4.5.0", 37 | "webpack-cli": "~2.0.14" 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /packages/api/tests/test_handler.js: -------------------------------------------------------------------------------- 1 | const test = require('ava') 2 | const sinon = require('sinon') 3 | const proxyquire = require('proxyquire') 4 | const createEvent = require('aws-event-mocks') 5 | 6 | test('handler calls search with parameters', async (t) => { 7 | const result = { value: 'value' } 8 | const search = sinon.stub().resolves(result) 9 | const satlib = { 10 | api: { 11 | search 12 | } 13 | } 14 | const lambda = proxyquire('../index.js', { 15 | '@sat-utils/api-lib': satlib 16 | }) 17 | const host = 'host' 18 | const httpMethod = 'GET' 19 | const path = 'path' 20 | 21 | const queryStringParameters = { 22 | test: 'test' 23 | } 24 | const event = createEvent({ 25 | template: 'aws:apiGateway', 26 | merge: { 27 | headers: { 28 | Host: host, 29 | 'Accept-Encoding': '' 30 | }, 31 | requestContext: {}, 32 | httpMethod, 33 | queryStringParameters, 34 | path 35 | } 36 | }) 37 | 38 | const actual = await lambda.handler(event) 39 | const { args } = search.firstCall 40 | t.is(args[0], path) 41 | t.deepEqual(args[1], queryStringParameters) 42 | t.is(args[3], `https://${host}`) 43 | t.is(actual.statusCode, 200) 44 | t.is(actual.body, JSON.stringify(result)) 45 | }) 46 | 47 | test('handler returns 404 for error', async (t) => { 48 | const errorMessage = 'errorMessage' 49 | const result = new Error(errorMessage) 50 | const search = sinon.stub().resolves(result) 51 | const satlib = { 52 | api: { 53 | search 54 | } 55 | } 56 | const lambda = proxyquire('../index.js', { 57 | '@sat-utils/api-lib': satlib 58 | }) 59 | const host = 'host' 60 | const httpMethod = 'GET' 61 | const path = 'path' 62 | 63 | const queryStringParameters = { 64 | test: 'test' 65 | } 66 | const event = createEvent({ 67 | template: 'aws:apiGateway', 68 | merge: { 69 | headers: { 70 | Host: host, 71 | 'Accept-Encoding': '' 72 | }, 73 | requestContext: {}, 74 | httpMethod, 75 | queryStringParameters, 76 | path 77 | } 78 | }) 79 | 80 | const actual = await lambda.handler(event) 81 | t.is(actual.statusCode, 404) 82 | t.is(actual.body, errorMessage) 83 | }) 84 | -------------------------------------------------------------------------------- /packages/api/webpack.config.js: -------------------------------------------------------------------------------- 1 | 2 | const path = require('path'); 3 | 4 | let mode = 'development'; 5 | let devtool = 'inline-source-map'; 6 | 7 | if(process.env.PRODUCTION) { 8 | mode = 'production', 9 | devtool = false 10 | } 11 | 12 | module.exports = { 13 | mode, 14 | entry: './index.js', 15 | output: { 16 | libraryTarget: 'commonjs2', 17 | filename: 'index.js', 18 | path: path.resolve(__dirname, 'dist') 19 | }, 20 | externals: [ 21 | 'aws-sdk', 22 | 'electron', 23 | {'formidable': 'url'} 24 | ], 25 | devtool, 26 | target: 'node' 27 | }; -------------------------------------------------------------------------------- /packages/ingest/README.md: -------------------------------------------------------------------------------- 1 | ## @sat-utils/ingest 2 | 3 | For additional information on ingesting data into sat-api, see the README in the [sat-api-deployment repository](https://github.com/sat-utils/sat-api-deployment). 4 | 5 | 6 | ### Unit Tests 7 | ``` 8 | $ yarn 9 | $ yarn test 10 | ``` 11 | 12 | ### Environment variables 13 | 14 | `CLUSTER_ARN` 15 | `TASK_ARN` 16 | `SECURITY_GROUPS` 17 | `ECS_ROLE_ARN` 18 | `ES_HOST` 19 | `ES_BATCH_SIZE` 20 | 21 | ### About 22 | [sat-api](https://github.com/sat-utils/sat-api) was created by [Development Seed]() and is part of a collection of tools called [sat-utils](https://github.com/sat-utils). 23 | -------------------------------------------------------------------------------- /packages/ingest/bin/ingest.js: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | 3 | const handler = require('../').handler 4 | 5 | // call handler 6 | const event = JSON.parse(process.argv.slice(2)) 7 | console.log(event) 8 | handler(event) 9 | -------------------------------------------------------------------------------- /packages/ingest/index.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | const AWS = require('aws-sdk') 4 | const satlib = require('@sat-utils/api-lib') 5 | 6 | // Runs on Fargate 7 | const runIngestTask = async function (input, envvars) { 8 | const ecs = new AWS.ECS() 9 | const params = { 10 | cluster: process.env.CLUSTER_ARN, 11 | taskDefinition: process.env.TASK_ARN, 12 | launchType: 'FARGATE', 13 | networkConfiguration: { 14 | awsvpcConfiguration: { 15 | subnets: process.env.SUBNETS.split(' '), 16 | assignPublicIp: 'ENABLED', 17 | securityGroups: process.env.SECURITY_GROUPS.split(' ') 18 | } 19 | }, 20 | overrides: { 21 | containerOverrides: [ 22 | { 23 | command: [ 24 | 'node', 25 | 'packages/ingest/bin/ingest.js', 26 | JSON.stringify(input) 27 | ], 28 | environment: envvars, 29 | name: 'SatApi' 30 | } 31 | ], 32 | executionRoleArn: process.env.ECS_ROLE_ARN, 33 | taskRoleArn: process.env.ECS_ROLE_ARN 34 | } 35 | } 36 | return ecs.runTask(params).promise() 37 | } 38 | 39 | module.exports.handler = async function handler(event) { 40 | console.log(`Ingest Event: ${JSON.stringify(event)}`) 41 | try { 42 | if (event.Records && (event.Records[0].EventSource === 'aws:sns')) { 43 | // event is SNS message of updated file on s3 44 | const message = JSON.parse(event.Records[0].Sns.Message) 45 | if (message.type && message.type === 'Feature') { 46 | // event is a STAC Item 47 | await satlib.ingest.ingestItem(message, satlib.es) 48 | } else { 49 | // updated s3 50 | const { Records: s3Records } = message 51 | const promises = s3Records.map((s3Record) => { 52 | const { 53 | s3: { 54 | bucket: { name: bucketName }, 55 | object: { key } 56 | } 57 | } = s3Record 58 | const url = `https://${bucketName}.s3.amazonaws.com/${key}` 59 | console.log(`Ingesting catalog file ${url}`) 60 | const recursive = false 61 | return satlib.ingest.ingest(url, satlib.es, recursive) 62 | }) 63 | await Promise.all(promises) 64 | } 65 | } else if ((event.type && event.type === 'Feature') || (event.id && event.extent)) { 66 | // event is STAC Item or Collection JSON 67 | await satlib.ingest.ingestItem(event, satlib.es) 68 | } else if (event.url) { 69 | // event is URL to a catalog node 70 | const { url, recursive, collectionsOnly } = event 71 | const recurse = recursive === undefined ? true : recursive 72 | const collections = collectionsOnly === undefined ? false : collectionsOnly 73 | await satlib.ingest.ingest(url, satlib.es, recurse, collections) 74 | } else if (event.fargate) { 75 | // event is URL to a catalog node - start a Fargate instance to process 76 | console.log(`Starting Fargate ingesttask ${JSON.stringify(event.fargate)}`) 77 | const envvars = [ 78 | { 'name': 'ES_HOST', 'value': process.env.ES_HOST }, 79 | { 'name': 'ES_BATCH_SIZE', 'value': process.env.ES_BATCH_SIZE }, 80 | { 'name': 'LOG_LEVEL', 'value': process.env.LOG_LEVEL || 'info' } 81 | ] 82 | await runIngestTask(event.fargate, envvars) 83 | } 84 | } catch (error) { 85 | console.log(error) 86 | } 87 | } 88 | 89 | -------------------------------------------------------------------------------- /packages/ingest/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@sat-utils/ingest", 3 | "version": "0.3.0", 4 | "description": "ingest lambda function of sat-api", 5 | "main": "index.js", 6 | "bin": { 7 | "sat-api-ingest": "bin/ingest.js" 8 | }, 9 | "repository": { 10 | "type": "git", 11 | "url": "git+https://github.com/sat-utils/sat-api.git" 12 | }, 13 | "author": "Matthew Hanson ", 14 | "license": "MIT", 15 | "bugs": { 16 | "url": "https://github.com/sat-utils/sat-api/issues" 17 | }, 18 | "scripts": { 19 | "build": "webpack", 20 | "test": "ava" 21 | }, 22 | "ava": { 23 | "files": "tests/*.js", 24 | "verbose": true, 25 | "serial": true 26 | }, 27 | "publishConfig": { 28 | "access": "public" 29 | }, 30 | "homepage": "https://github.com/sat-utils/sat-api#readme", 31 | "dependencies": { 32 | "@sat-utils/api-lib": "^0.3.0" 33 | }, 34 | "devDependencies": { 35 | "ava": "^0.25.0", 36 | "aws-event-mocks": "^0.0.0", 37 | "aws-sdk": "^2.382.0", 38 | "proxyquire": "^2.1.0", 39 | "sinon": "^7.1.1", 40 | "webpack": "~4.5.0", 41 | "webpack-cli": "~2.0.14" 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /packages/ingest/tests/test_handler.js: -------------------------------------------------------------------------------- 1 | const test = require('ava') 2 | const sinon = require('sinon') 3 | const proxyquire = require('proxyquire') 4 | const createEvent = require('aws-event-mocks') 5 | 6 | const setup = () => { 7 | const ingest = sinon.stub().resolves(true) 8 | const ingestItem = sinon.stub().resolves(true) 9 | const elasticsearch = 'elasticsearch' 10 | const ECS = sinon.stub() 11 | const runTask = sinon.stub().resolves(true).returns({ 12 | promise: () => (Promise.resolve(true)) 13 | }) 14 | ECS.prototype.runTask = runTask 15 | const AWS = { 16 | ECS 17 | } 18 | const satlib = { 19 | ingest: { 20 | ingest, 21 | ingestItem 22 | }, 23 | es: elasticsearch 24 | } 25 | const lambda = proxyquire('../index.js', { 26 | '@sat-utils/api-lib': satlib, 27 | 'aws-sdk': AWS 28 | }) 29 | return { 30 | ingest, 31 | ingestItem, 32 | elasticsearch, 33 | lambda, 34 | runTask 35 | } 36 | } 37 | 38 | test('handler uses non-recursive ingest for S3 SNS Event', async (t) => { 39 | const { ingest, lambda, elasticsearch } = setup() 40 | const bucket = 'bucket' 41 | const key = 'key' 42 | const s3Event = createEvent({ 43 | template: 'aws:s3', 44 | merge: { 45 | Records: [{ 46 | eventName: 'ObjectCreated:Put', 47 | s3: { 48 | bucket: { 49 | name: bucket 50 | }, 51 | object: { 52 | key: key 53 | } 54 | } 55 | }] 56 | } 57 | }) 58 | const message = JSON.stringify(s3Event) 59 | 60 | const snsEvent = createEvent({ 61 | template: 'aws:sns', 62 | merge: { 63 | Records: [{ 64 | Sns: { 65 | Message: message 66 | } 67 | }] 68 | } 69 | }) 70 | await lambda.handler(snsEvent) 71 | const expectedUrl = `https://${bucket}.s3.amazonaws.com/${key}` 72 | const expectedRecursive = false 73 | t.is(ingest.firstCall.args[0], expectedUrl, 'S3 Url is parsed correctly') 74 | t.is(ingest.firstCall.args[1], elasticsearch, 'ES library passed as parameter') 75 | t.is(ingest.firstCall.args[2], expectedRecursive, 'Recursive is false') 76 | }) 77 | 78 | test('handler calls ingestItem when event payload is a feature', async (t) => { 79 | const { ingestItem, lambda, elasticsearch } = setup() 80 | const event = { 81 | type: 'Feature' 82 | } 83 | await lambda.handler(event) 84 | t.deepEqual(ingestItem.firstCall.args[0], event, 'Calls ingestItem with event') 85 | t.is(ingestItem.firstCall.args[1], elasticsearch, 'ES library passed as a parameter') 86 | }) 87 | 88 | test('handler call ingest when event payload contains url', async (t) => { 89 | const { ingest, lambda, elasticsearch } = setup() 90 | const url = 'url' 91 | const recursive = false 92 | const collectionsOnly = true 93 | const event = { 94 | url, 95 | recursive, 96 | collectionsOnly 97 | } 98 | await lambda.handler(event) 99 | t.truthy(ingest.calledOnceWith(url, elasticsearch, recursive, collectionsOnly), 100 | 'Calls ingest with url and correct parameters.') 101 | }) 102 | 103 | test('ingest with fargate event creates ecs task with command', async (t) => { 104 | process.env.SUBNETS = '{}' 105 | process.env.SECURITY_GROUPS = '{}' 106 | const { lambda, runTask } = setup() 107 | const event = { 108 | fargate: { 109 | url: 'url' 110 | } 111 | } 112 | await lambda.handler(event) 113 | const params = runTask.firstCall.args[0] 114 | const command = params.overrides.containerOverrides[0].command 115 | t.is(command[2], JSON.stringify(event.fargate)) 116 | }) 117 | -------------------------------------------------------------------------------- /packages/ingest/webpack.config.js: -------------------------------------------------------------------------------- 1 | 2 | const path = require('path'); 3 | 4 | let mode = 'development'; 5 | let devtool = 'inline-source-map'; 6 | 7 | if(process.env.PRODUCTION) { 8 | mode = 'production', 9 | devtool = false 10 | } 11 | 12 | module.exports = { 13 | mode, 14 | entry: './index.js', 15 | output: { 16 | libraryTarget: 'commonjs2', 17 | filename: 'index.js', 18 | path: path.resolve(__dirname, 'dist') 19 | }, 20 | externals: [ 21 | 'aws-sdk', 22 | 'electron', 23 | {'formidable': 'url'} 24 | ], 25 | devtool, 26 | target: 'node' 27 | }; --------------------------------------------------------------------------------