├── .eslintignore ├── .eslintrc.json ├── .github ├── pull_request_template.md └── workflows │ ├── lint.yaml │ └── run_tests.yaml ├── .gitignore ├── .mocharc.json ├── .nvmrc ├── .prettierignore ├── .prettierrc ├── LICENSE.txt ├── README.md ├── adr-log ├── adr-00000000-template.md ├── adr-20190926-initial-design.md ├── adr-20221201-graphql-backend.md └── adr-20230323-entry-id-title-identifiers.md ├── cdk ├── cmr-stac-dev │ ├── .gitignore │ ├── .npmignore │ ├── README.md │ ├── bin │ │ └── cmr-stac-dev.ts │ ├── cdk.json │ ├── jest.config.js │ ├── lib │ │ └── cmr-stac-dev-stack.ts │ ├── package-lock.json │ ├── package.json │ ├── test │ │ └── cmr-stac-dev.test.ts │ └── tsconfig.json └── cmr-stac │ ├── .gitignore │ ├── .npmignore │ ├── README.md │ ├── bin │ └── cmr-stac.ts │ ├── cdk.json │ ├── jest.config.js │ ├── lib │ └── cmr-stac-stack.ts │ ├── package-lock.json │ ├── package.json │ ├── test │ └── cmr-stac.test.ts │ └── tsconfig.json ├── deploy.sh ├── docs ├── README.md ├── index │ └── index.html └── usage │ ├── images │ ├── cmr-stac-cal.png │ ├── cmr-stac-cal2.png │ └── cmr-stac-table.png │ ├── notebooks │ └── ndvi_demo.ipynb │ └── usage.md ├── package-lock.json ├── package.json ├── resources ├── Feature.json ├── Geometry.json ├── catalog-spec │ └── json-schema │ │ └── catalog.json ├── collection-spec │ └── json-schema │ │ └── collection.json ├── item-spec │ └── json-schema │ │ ├── basics.json │ │ ├── datetime.json │ │ ├── instrument.json │ │ ├── item.json │ │ ├── licensing.json │ │ └── provider.json └── swagger.json ├── sam_local_envs.json ├── scripts ├── mirror-api │ ├── README.md │ ├── mirror-api.py │ └── requirements.txt └── validation │ ├── README.md │ ├── requirements.txt │ └── validate.py ├── src ├── @types │ ├── StacCatalog.d.ts │ ├── StacCollection.d.ts │ └── StacItem.d.ts ├── __tests__ │ ├── geojsonGeometry.ts │ ├── items.spec.ts │ ├── providerBrowse.spec.ts │ ├── providerCatalog.spec.ts │ ├── providerCollection.spec.ts │ ├── providerSearch.spec.ts │ └── rootCatalog.spec.ts ├── app.ts ├── domains │ ├── __tests__ │ │ ├── bounding-box.spec.ts │ │ ├── collections.spec.ts │ │ ├── geojson.spec.ts │ │ ├── items.spec.ts │ │ ├── providers.spec.ts │ │ └── stac.spec.ts │ ├── bounding-box.ts │ ├── cache.ts │ ├── cmr.ts │ ├── collections.ts │ ├── geojson.ts │ ├── items.ts │ ├── providers.ts │ └── stac.ts ├── handler.ts ├── index.d.ts ├── middleware │ ├── __tests__ │ │ └── index.spec.ts │ └── index.ts ├── models │ ├── CmrModels.ts │ ├── GraphQLModels.ts │ ├── StacModels.ts │ └── errors.ts ├── routes │ ├── __tests__ │ │ └── browse.spec.ts │ ├── browse.ts │ ├── catalog.ts │ ├── conformance.ts │ ├── healthcheck.ts │ ├── index.ts │ ├── items.ts │ ├── providerConformance.ts │ ├── rootCatalog.ts │ └── search.ts └── utils │ ├── __tests__ │ ├── datetime.spec.ts │ ├── sort.spec.ts │ └── utils.spec.ts │ ├── datetime.ts │ ├── index.ts │ ├── sort.ts │ └── testUtils.ts ├── start.js └── tsconfig.json /.eslintignore: -------------------------------------------------------------------------------- 1 | cdk.out 2 | -------------------------------------------------------------------------------- /.eslintrc.json: -------------------------------------------------------------------------------- 1 | { 2 | "parser": "@typescript-eslint/parser", 3 | "parserOptions": { 4 | "ecmaVersion": 12, 5 | "sourceType": "module" 6 | }, 7 | "plugins": ["@typescript-eslint", "import"], 8 | "extends": ["eslint:recommended", "plugin:@typescript-eslint/recommended", "prettier"], 9 | "rules": { 10 | "@typescript-eslint/no-unused-vars": [ 11 | "warn", 12 | { 13 | "argsIgnorePattern": "^_", 14 | "varsIgnorePattern": "^_" 15 | } 16 | ], 17 | 18 | // Allow shadow declarations 19 | "@typescript-eslint/no-shadow": "off", 20 | 21 | // Require spacing in object literals 22 | "@typescript-eslint/object-curly-spacing": ["error", "always"], 23 | 24 | // Allow class methods that dont use 'this' 25 | "class-methods-use-this": "off", 26 | 27 | // Allow console log messages 28 | "no-console": "off", 29 | 30 | // Allow extraneous dependencies 31 | "import/no-extraneous-dependencies": ["error", {"devDependencies": true}], 32 | 33 | // Allow named default exports 34 | "import/no-named-as-default": "off", 35 | 36 | // Allow files with no default export 37 | "import/prefer-default-export": "off", 38 | 39 | // Allowing cyclic dependencies 40 | "import/no-cycle": "off" 41 | }, 42 | "ignorePatterns": ["**/*.spec.ts"], 43 | "env": { 44 | "browser": false, 45 | "es2021": true, 46 | "mocha": true 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /.github/pull_request_template.md: -------------------------------------------------------------------------------- 1 | # Overview 2 | 3 | ### What is the feature? 4 | 5 | Please summarize the feature or fix. 6 | 7 | ### What is the Solution? 8 | 9 | Summarize what you changed. 10 | 11 | ### What areas of the application does this impact? 12 | 13 | List impacted areas. 14 | 15 | # Testing 16 | 17 | ### Reproduction steps 18 | 19 | - **Environment for testing:** 20 | - **Collection to test with:** 21 | 22 | 1. Step 1 23 | 2. Step 2... 24 | 25 | ### Attachments 26 | 27 | Please include relevant screenshots or files that would be helpful in reviewing and verifying this change. 28 | 29 | # Checklist 30 | 31 | - [ ] I have added automated tests that prove my fix is effective or that my feature works 32 | - [ ] New and existing unit tests pass locally with my changes 33 | - [ ] I have performed a self-review of my own code 34 | - [ ] I have commented my code, particularly in hard-to-understand areas 35 | - [ ] I have made corresponding changes to the documentation 36 | - [ ] My changes generate no new warnings 37 | 38 | -------------------------------------------------------------------------------- /.github/workflows/lint.yaml: -------------------------------------------------------------------------------- 1 | name: Linting 2 | 3 | on: 4 | pull_request: 5 | branches: ["master"] 6 | 7 | jobs: 8 | Lint: 9 | runs-on: ubuntu-latest 10 | strategy: 11 | matrix: 12 | node_version: [lts/jod] 13 | 14 | steps: 15 | - uses: actions/checkout@v4 16 | - name: Setup Node 17 | uses: actions/setup-node@v4 18 | with: 19 | node-version: ${{ matrix.node_version }} 20 | 21 | - name: Install Dependencies 22 | run: npm install 23 | 24 | - name: Run Linter (check only) 25 | run: npm run lint 26 | 27 | Prettier: 28 | runs-on: ubuntu-latest 29 | strategy: 30 | matrix: 31 | node_version: [lts/jod] 32 | 33 | steps: 34 | - uses: actions/checkout@v4 35 | - name: Setup Node 36 | uses: actions/setup-node@v4 37 | with: 38 | node-version: ${{ matrix.node_version }} 39 | 40 | - name: Install Dependencies 41 | run: npm install 42 | 43 | - name: Run Prettier (check only) 44 | run: npm run prettier 45 | -------------------------------------------------------------------------------- /.github/workflows/run_tests.yaml: -------------------------------------------------------------------------------- 1 | name: STAC Tests 2 | 3 | on: 4 | push: 5 | branches: 6 | - "master" 7 | - "[0-9]+\\.[0-9]+\\.[0-9]+-r[0-9]{2}\\.[0-9]\\.[0-9]" 8 | - "[0-9]+\\.[0-9]+\\.x" 9 | pull_request: 10 | branches: ["master"] 11 | 12 | jobs: 13 | UnitTests: 14 | runs-on: ubuntu-latest 15 | strategy: 16 | fail-fast: true 17 | matrix: 18 | node_version: [lts/jod] 19 | 20 | steps: 21 | - uses: actions/checkout@v4 22 | - name: Setup Node 23 | uses: actions/setup-node@v4 24 | with: 25 | node-version: ${{ matrix.node_version }} 26 | 27 | - name: Install Dependencies 28 | run: npm ci 29 | 30 | - name: Run Tests with Coverage 31 | run: npm run test:coverage 32 | env: 33 | CI: "true" 34 | - name: Upload coverage to codecov 35 | uses: codecov/codecov-action@v4 36 | env: 37 | CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} 38 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | serverless-output.yml 2 | config.sh 3 | 4 | ### Linux ### 5 | *~ 6 | 7 | # temporary files which can be created if a process still has a handle open of a deleted file 8 | .fuse_hidden* 9 | 10 | # KDE directory preferences 11 | .directory 12 | 13 | # Linux trash folder which might appear on any partition or disk 14 | .Trash-* 15 | 16 | # .nfs files are created when an open file is removed but is still being accessed 17 | .nfs* 18 | 19 | ### Node ### 20 | # Logs 21 | logs 22 | *.log 23 | npm-debug.log* 24 | yarn-debug.log* 25 | yarn-error.log* 26 | 27 | # Runtime data 28 | pids 29 | *.pid 30 | *.seed 31 | *.pid.lock 32 | 33 | # Directory for instrumented libs generated by jscoverage/JSCover 34 | lib-cov 35 | 36 | # Coverage directory used by tools like istanbul 37 | coverage 38 | 39 | # jest junit xml output 40 | junit.xml 41 | 42 | # nyc test coverage 43 | .nyc_output 44 | 45 | # Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files) 46 | .grunt 47 | 48 | # Bower dependency directory (https://bower.io/) 49 | bower_components 50 | 51 | # node-waf configuration 52 | .lock-wscript 53 | 54 | # Compiled binary addons (http://nodejs.org/api/addons.html) 55 | build/Release 56 | 57 | # Dependency directories 58 | node_modules/ 59 | jspm_packages/ 60 | 61 | # Typescript v1 declaration files 62 | typings/ 63 | 64 | # Optional npm cache directory 65 | .npm 66 | 67 | # Optional eslint cache 68 | .eslintcache 69 | 70 | # Optional REPL history 71 | .node_repl_history 72 | 73 | # Output of 'npm pack' 74 | *.tgz 75 | 76 | # Yarn Integrity file 77 | .yarn-integrity 78 | 79 | # dotenv environment variables file 80 | .env 81 | 82 | ### OSX ### 83 | *.DS_Store 84 | .AppleDouble 85 | .LSOverride 86 | 87 | # Icon must end with two \r 88 | Icon 89 | 90 | # Thumbnails 91 | ._* 92 | 93 | # Files that might appear in the root of a volume 94 | .DocumentRevisions-V100 95 | .fseventsd 96 | .Spotlight-V100 97 | .TemporaryItems 98 | .Trashes 99 | .VolumeIcon.icns 100 | .com.apple.timemachine.donotpresent 101 | 102 | # Directories potentially created on remote AFP share 103 | .AppleDB 104 | .AppleDesktop 105 | Network Trash Folder 106 | Temporary Items 107 | .apdisk 108 | 109 | ### Windows ### 110 | # Windows thumbnail cache files 111 | Thumbs.db 112 | ehthumbs.db 113 | ehthumbs_vista.db 114 | 115 | # Folder config file 116 | Desktop.ini 117 | 118 | # Recycle Bin used on file shares 119 | $RECYCLE.BIN/ 120 | 121 | # Windows Installer files 122 | *.cab 123 | *.msi 124 | *.msm 125 | *.msp 126 | 127 | # Windows shortcuts 128 | *.lnk 129 | 130 | # VS Code 131 | .vscode/ 132 | 133 | # WebStorm 134 | .idea/ 135 | 136 | # AWS SAM CLI 137 | .aws-sam/ 138 | 139 | # Serverless 140 | .serverless/ 141 | 142 | # IntelliJ 143 | .idea/ 144 | 145 | /search/.dynamodb/ 146 | *.out 147 | 148 | .build/ 149 | 150 | # emacs 151 | \#*\# 152 | .\#* 153 | 154 | tmp 155 | 156 | # Ignore docker file items 157 | Dockerfile 158 | .dockerignore 159 | -------------------------------------------------------------------------------- /.mocharc.json: -------------------------------------------------------------------------------- 1 | { 2 | "extension": ["ts"], 3 | "spec": "src/**/*.spec.ts", 4 | "require": ["ts-node/register", "mocha-suppress-logs"] 5 | } 6 | -------------------------------------------------------------------------------- /.nvmrc: -------------------------------------------------------------------------------- 1 | lts/jod 2 | -------------------------------------------------------------------------------- /.prettierignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | node_modules 3 | /build 4 | /.svelte-kit 5 | /package 6 | .env 7 | .env.* 8 | !.env.example 9 | 10 | # Ignore files for PNPM, NPM and YARN 11 | pnpm-lock.yaml 12 | package-lock.json 13 | yarn.lock 14 | 15 | cdk.out 16 | -------------------------------------------------------------------------------- /.prettierrc: -------------------------------------------------------------------------------- 1 | { 2 | "useTabs": false, 3 | "singleQuote": false, 4 | "trailingComma": "es5", 5 | "semi": true, 6 | "printWidth": 100 7 | } 8 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # NASA CMR STAC 2 | 3 | NASA's [Common Metadata Repository (CMR)](https://cmr.earthdata.nasa.gov/search) is a metadata 4 | catalog of NASA Earth Science data. [STAC, or SpatioTemporal Asset Catalog](https://stacspec.org/), is a 5 | [Specification](https://github.com/radiantearth/stac-spec) for describing geospatial data with 6 | [JSON](https://www.json.org/) and [GeoJSON](http://geojson.io/). The related 7 | [STAC-API Specification](https://github.com/radiantearth/stac-api-spec) defines an API 8 | for searching and browsing STAC catalogs. 9 | 10 | ## CMR-STAC 11 | 12 | CMR-STAC acts as a proxy between the CMR repository and STAC API queries. 13 | The goal is to expose CMR's vast collections of geospatial data as a STAC-compliant API. 14 | Even though the core metadata remains the same, a benefit of the CMR-STAC proxy is the ability 15 | to use the growing ecosystem of STAC software. Underneath, STAC API queries are translated into 16 | CMR queries which are sent to CMR and the responses are translated into STAC Collections and Items. 17 | This entire process happens dynamically at runtime, so responses will always be representative of 18 | whatever data is currently stored in CMR. If there are any deletions of data in CMR by data providers, 19 | those deletions are represented in CMR-STAC immediately. 20 | 21 | CMR-STAC follows the STAC API 1.0.0-beta.1 specification, see the 22 | [OpenAPI Documentation](https://api.stacspec.org/v1.0.0-beta.1/index.html). 23 | 24 | ## Usage 25 | 26 | ### Endpoints 27 | 28 | - [CMR-STAC](https://cmr.earthdata.nasa.gov/stac): The entire catalog of NASA CMR data, organized by provider. 29 | 30 | - [CMR-CLOUDSTAC](https://cmr.earthdata.nasa.gov/cloudstac): Also organized by provider, this API only contains STAC Collections where the Item Assets are available "in the cloud" (i.e., on S3). 31 | 32 | ### Navigating 33 | 34 | CMR-STAC can be navigated manually using the endpoints provided above, or you can utilize available STAC software to browse and use the API. 35 | 36 | A common STAC utility is Radiant Earth's `stac-browser` to use this tool against your development server navigate to 37 | ```radiantearth.github.io/stac-browser/#/external/http:/localhost:3000/stac?.language=en``` 38 | 39 | See the [Usage Documentation](docs/usage/usage.md) for examples of how to interact with the API and search for data. 40 | 41 | ### Limitations 42 | 43 | While CMR-STAC provides some advantages over the CMR, there are some limitations that you should be aware of: 44 | 45 | - Limited search functionality: CMR-STAC does not support all of the search capabilities that CMR provides. For example, with CMR, you can search for data based on temporal and spatial criteria, as well as specific parameters such as platform, instrument, and granule size. However, with CMR-STAC, you can only search based on the STAC standard. 46 | - Limited metadata availability: CMR-STAC only provides metadata that follows the STAC specification. While this metadata is very rich and comprehensive, it may not provide all of the information that you need for your specific use case. 47 | 48 | ## For Developers 49 | 50 | [Developer README](docs/README.md) 51 | 52 | ## License 53 | 54 | NASA Open Source Agreement v1.3 (NASA-1.3) 55 | See [LICENSE.txt](./LICENSE.txt) 56 | -------------------------------------------------------------------------------- /adr-log/adr-00000000-template.md: -------------------------------------------------------------------------------- 1 | # Title 2 | 3 | Table of Contents: 4 | * [Status](#status) 5 | * [Context](#context) 6 | * [Decision](#decision) 7 | * [Consequences](#consequences) 8 | 9 | __In each ADR file, write these sections:__ 10 | 11 | ## Status 12 | 13 | What is the status, such as proposed, accepted, rejected, deprecated, superseded, etc.? 14 | 15 | ## Context 16 | 17 | What is the issue that we're seeing that is motivating this decision or change? 18 | 19 | ## Decision 20 | 21 | What is the change that we're proposing and/or doing? 22 | 23 | ## Consequences 24 | 25 | What becomes easier or more difficult to do because of this change? 26 | -------------------------------------------------------------------------------- /adr-log/adr-20190926-initial-design.md: -------------------------------------------------------------------------------- 1 | # Create a STAC compliant service for the Common Metadata Repository 2 | 3 | Table of Contents: 4 | * [Status](#status) 5 | * [Context](#context) 6 | * [Decision](#decision) 7 | * [Consequences](#consequences) 8 | 9 | ## Status 10 | 11 | __Accepted__ 12 | 13 | ## Context 14 | 15 | There is a community desire to be able to use the [Spatio-Temporal Asset Catalog](https://stacspec.org/) specification to browse data holdings within NASA's Common Metadata Repository. 16 | 17 | ## Decision 18 | 19 | Create this application to implement a STAC compliant wrapper to access CMR data. 20 | 21 | ## Consequences 22 | 23 | This application has been created. 24 | -------------------------------------------------------------------------------- /adr-log/adr-20221201-graphql-backend.md: -------------------------------------------------------------------------------- 1 | # Replace direct calls to the Common Metadata Repository with calls to GraphQL 2 | 3 | Table of Contents: 4 | * [Status](#status) 5 | * [Context](#context) 6 | * [Decision](#decision) 7 | * [Consequences](#consequences) 8 | 9 | ## Status 10 | 11 | __Accepted__ 12 | 13 | ## Context 14 | 15 | Using direct calls to CMR make it necessary to combine multiple call results to produce STAC items. There is also no mechanism to call for only what fields are needed. This results in extra overhead, both in terms of number of calls needed and extra data being returned that was ultimately discarded. 16 | 17 | ## Decision 18 | 19 | Replace the logic in CMR-STAC to query GraphQL for data instead of querying directly to CMR. 20 | 21 | ## Consequences 22 | 23 | ### Pros 24 | * Constructing STAC responses requires far fewer calls to generate the same responses. 25 | * Calls to GraphQL return only what is is needed, and can be aliased resulting in simplified query logic. 26 | 27 | ### Cons 28 | * There is now an extra layer between STAC users and CMR, resulting in possible additional latency for calls. 29 | * GraphQL is now a direct dependency of STAC. 30 | * Any new queries must be supported by GraphQL first. 31 | * GraphQL via CMR search does not support queries for Producrs, leaving at least one direct call to CMR in place. 32 | -------------------------------------------------------------------------------- /adr-log/adr-20230323-entry-id-title-identifiers.md: -------------------------------------------------------------------------------- 1 | # Continue using entry_id and title as the respective Collection and Granule/Item identifiers 2 | 3 | Table of Contents: 4 | 5 | * [Status](#status) 6 | * [Context](#context) 7 | * [Decision](#decision) 8 | * [Consequences](#consequences) 9 | 10 | ## Status 11 | 12 | __Accepted__ 13 | 14 | ## Context 15 | 16 | The initial implementation of [CMR-STAC using GraphQL](./adr-20221201-graphql-backend.md "graphql-backend") had converted the collection and granule identifiers to using CMR conceptIds. This resulted in breaking the functionality of many CMR-STAC community users workflows, scripts, and tools. 17 | 18 | ## Decision 19 | 20 | To maintain functionality of existing scripts and queries with a minimal amount of changes, CMR-STAC will continue to support ".v" as a separator for searches. 21 | 22 | ## Consequences 23 | 24 | There is ambiguousness regarding collections when the shortName contains a ".v" separator. The `.v` must be converted to an underscore to be transformed into a valid CMR `entry_id`. This transformation is only required for navigating directly to a collection or item 25 | 26 | ### Bookmarked Collections 27 | 28 | Bookmarked collection pages will need to be updated to change the ".v" to "_": 29 | 30 | * INCORRECT `/stac/provider/collections/myCollection.v1` 31 | * CORRECT `/stac/provider/collections/myCollection_1` 32 | 33 | This fixes ambiguousness of separators, and this currently affects the existing deployed PROD STAC, and has since the initial implementation of CMR-STAC. If there was a collection called `myColl.volume` with a version of `2` it would be listed as `myColl.volume.v2` which CMR-STAC incorrectly would convert to the equivalent `{shortName: "myColl", version: "olume.v2"}` because it only split on the first instance of the `.v` separator. This issue has affected all versions of CMR-STAC. 34 | 35 | By switching to the `entry_id` syntax no parsing or guessing is needed to retrieve the correct collection. 36 | 37 | ### Searches 38 | Searches using collections with a `.v` separator will continue to work, but with the consequence of added uncertentainty. 39 | Collections created without a version or set with an ambiguous version will need to have the appropriate placeholder appended. Examples of such non-versions are `Not applicable`, `Not provided` and `None`. 40 | Such an example of a collection would be `10.3334/cdiac/otg.vos_alligatorhope_1999-2001_Not applicable`. 41 | 42 | The unencoded URL would be `/stac/NOAA_NCEI/collections/10.3334/cdiac/otg.vos_alligatorhope_1999-2001_Not applicable` and would result in 404 exception. 43 | 44 | The encoded `entry_id` would be transformed into `/stac/NOAA_NCEI/collections/10.3334%2Fcdiac%2Fotg.vos_alligatorhope_1999-2001_Not+applicable` which would yield the correct collection. 45 | 46 | ### URI Encoding 47 | Because CMR shortNames can contain characters that would break URLs, it is recommended that the collection ID and item IDs be URI encoded. 48 | * If browsing or searching using CMR-STAC the values returned will be automatically encoded. 49 | * If manually entering values containing special characters, they must be manually URI encoded. 50 | -------------------------------------------------------------------------------- /cdk/cmr-stac-dev/.gitignore: -------------------------------------------------------------------------------- 1 | *.js 2 | !jest.config.js 3 | *.d.ts 4 | node_modules 5 | 6 | # CDK asset staging directory 7 | .cdk.staging 8 | cdk.out 9 | -------------------------------------------------------------------------------- /cdk/cmr-stac-dev/.npmignore: -------------------------------------------------------------------------------- 1 | *.ts 2 | !*.d.ts 3 | 4 | # CDK asset staging directory 5 | .cdk.staging 6 | cdk.out 7 | -------------------------------------------------------------------------------- /cdk/cmr-stac-dev/README.md: -------------------------------------------------------------------------------- 1 | # Welcome to your CDK TypeScript project 2 | 3 | This is a blank project for CDK development with TypeScript. 4 | 5 | The `cdk.json` file tells the CDK Toolkit how to execute your app. 6 | 7 | ## Useful commands 8 | 9 | - `npm run build` compile typescript to js 10 | - `npm run watch` watch for changes and compile 11 | - `npm run test` perform the jest unit tests 12 | - `npx cdk deploy` deploy this stack to your default AWS account/region 13 | - `npx cdk diff` compare deployed stack with current state 14 | - `npx cdk synth` emits the synthesized CloudFormation template 15 | -------------------------------------------------------------------------------- /cdk/cmr-stac-dev/bin/cmr-stac-dev.ts: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | import * as cdk from "aws-cdk-lib"; 3 | import { CmrStacDevStack } from "../lib/cmr-stac-dev-stack"; 4 | 5 | const { AWS_ACCOUNT = "1234567890", AWS_REGION = "us-east-1", STAGE_NAME = "dev" } = process.env; 6 | const app = new cdk.App(); 7 | new CmrStacDevStack(app, `cmr-stac-${STAGE_NAME}`, { 8 | /* If you don't specify 'env', this stack will be environment-agnostic. 9 | * Account/Region-dependent features and context lookups will not work, 10 | * but a single synthesized template can be deployed anywhere. */ 11 | 12 | /* Uncomment the next line to specialize this stack for the AWS Account 13 | * and Region that are implied by the current CLI configuration. */ 14 | // env: { account: process.env.CDK_DEFAULT_ACCOUNT, region: process.env.CDK_DEFAULT_REGION }, 15 | 16 | /* Uncomment the next line if you know exactly what Account and Region you 17 | * want to deploy the stack to. */ 18 | env: { account: AWS_ACCOUNT, region: AWS_REGION }, 19 | 20 | /* For more information, see https://docs.aws.amazon.com/cdk/latest/guide/environments.html */ 21 | }); 22 | -------------------------------------------------------------------------------- /cdk/cmr-stac-dev/cdk.json: -------------------------------------------------------------------------------- 1 | { 2 | "app": "npx ts-node --prefer-ts-exts bin/cmr-stac-dev.ts", 3 | "watch": { 4 | "include": ["**"], 5 | "exclude": [ 6 | "README.md", 7 | "cdk*.json", 8 | "**/*.js", 9 | "tsconfig.json", 10 | "package*.json", 11 | "yarn.lock", 12 | "node_modules", 13 | "test" 14 | ] 15 | }, 16 | "context": { 17 | "@aws-cdk/aws-lambda:recognizeLayerVersion": true, 18 | "@aws-cdk/core:checkSecretUsage": true, 19 | "@aws-cdk/core:target-partitions": ["aws", "aws-cn"], 20 | "@aws-cdk-containers/ecs-service-extensions:enableDefaultLogDriver": true, 21 | "@aws-cdk/aws-ec2:uniqueImdsv2TemplateName": true, 22 | "@aws-cdk/aws-ecs:arnFormatIncludesClusterName": true, 23 | "@aws-cdk/aws-iam:minimizePolicies": true, 24 | "@aws-cdk/core:validateSnapshotRemovalPolicy": true, 25 | "@aws-cdk/aws-codepipeline:crossAccountKeyAliasStackSafeResourceName": true, 26 | "@aws-cdk/aws-s3:createDefaultLoggingPolicy": true, 27 | "@aws-cdk/aws-sns-subscriptions:restrictSqsDescryption": true, 28 | "@aws-cdk/aws-apigateway:disableCloudWatchRole": true, 29 | "@aws-cdk/core:enablePartitionLiterals": true, 30 | "@aws-cdk/aws-events:eventsTargetQueueSameAccount": true, 31 | "@aws-cdk/aws-ecs:disableExplicitDeploymentControllerForCircuitBreaker": true, 32 | "@aws-cdk/aws-iam:importedRoleStackSafeDefaultPolicyName": true, 33 | "@aws-cdk/aws-s3:serverAccessLogsUseBucketPolicy": true, 34 | "@aws-cdk/aws-route53-patters:useCertificate": true, 35 | "@aws-cdk/customresources:installLatestAwsSdkDefault": false, 36 | "@aws-cdk/aws-rds:databaseProxyUniqueResourceName": true, 37 | "@aws-cdk/aws-codedeploy:removeAlarmsFromDeploymentGroup": true, 38 | "@aws-cdk/aws-apigateway:authorizerChangeDeploymentLogicalId": true, 39 | "@aws-cdk/aws-ec2:launchTemplateDefaultUserData": true, 40 | "@aws-cdk/aws-secretsmanager:useAttachedSecretResourcePolicyForSecretTargetAttachments": true, 41 | "@aws-cdk/aws-redshift:columnId": true, 42 | "@aws-cdk/aws-stepfunctions-tasks:enableEmrServicePolicyV2": true, 43 | "@aws-cdk/aws-ec2:restrictDefaultSecurityGroup": true, 44 | "@aws-cdk/aws-apigateway:requestValidatorUniqueId": true, 45 | "@aws-cdk/aws-kms:aliasNameRef": true, 46 | "@aws-cdk/aws-autoscaling:generateLaunchTemplateInsteadOfLaunchConfig": true, 47 | "@aws-cdk/core:includePrefixInUniqueNameGeneration": true, 48 | "@aws-cdk/aws-efs:denyAnonymousAccess": true, 49 | "@aws-cdk/aws-opensearchservice:enableOpensearchMultiAzWithStandby": true, 50 | "@aws-cdk/aws-lambda-nodejs:useLatestRuntimeVersion": true, 51 | "@aws-cdk/aws-efs:mountTargetOrderInsensitiveLogicalId": true, 52 | "@aws-cdk/aws-rds:auroraClusterChangeScopeOfInstanceParameterGroupWithEachParameters": true, 53 | "@aws-cdk/aws-appsync:useArnForSourceApiAssociationIdentifier": true, 54 | "@aws-cdk/aws-rds:preventRenderingDeprecatedCredentials": true, 55 | "@aws-cdk/aws-codepipeline-actions:useNewDefaultBranchForCodeCommitSource": true, 56 | "@aws-cdk/aws-cloudwatch-actions:changeLambdaPermissionLogicalIdForLambdaAction": true, 57 | "@aws-cdk/aws-codepipeline:crossAccountKeysDefaultValueToFalse": true, 58 | "@aws-cdk/aws-codepipeline:defaultPipelineTypeToV2": true, 59 | "@aws-cdk/aws-kms:reduceCrossAccountRegionPolicyScope": true, 60 | "@aws-cdk/aws-eks:nodegroupNameAttribute": true, 61 | "@aws-cdk/aws-ec2:ebsDefaultGp3Volume": true, 62 | "@aws-cdk/aws-ecs:removeDefaultDeploymentAlarm": true, 63 | "@aws-cdk/custom-resources:logApiResponseDataPropertyTrueDefault": false, 64 | "@aws-cdk/aws-s3:keepNotificationInImportedBucket": false, 65 | "@aws-cdk/aws-ecs:reduceEc2FargateCloudWatchPermissions": true, 66 | "@aws-cdk/aws-dynamodb:resourcePolicyPerReplica": true, 67 | "@aws-cdk/aws-ec2:ec2SumTImeoutEnabled": true, 68 | "@aws-cdk/aws-appsync:appSyncGraphQLAPIScopeLambdaPermission": true, 69 | "@aws-cdk/aws-rds:setCorrectValueForDatabaseInstanceReadReplicaInstanceResourceId": true, 70 | "@aws-cdk/core:cfnIncludeRejectComplexResourceUpdateCreatePolicyIntrinsics": true, 71 | "@aws-cdk/aws-lambda-nodejs:sdkV3ExcludeSmithyPackages": true, 72 | "@aws-cdk/aws-stepfunctions-tasks:fixRunEcsTaskPolicy": true, 73 | "@aws-cdk/aws-ec2:bastionHostUseAmazonLinux2023ByDefault": true, 74 | "cdk-migrate": true 75 | } 76 | } 77 | -------------------------------------------------------------------------------- /cdk/cmr-stac-dev/jest.config.js: -------------------------------------------------------------------------------- 1 | export const testEnvironment = "node"; 2 | export const roots = ["/test"]; 3 | export const testMatch = ["**/*.test.ts"]; 4 | export const transform = { 5 | "^.+\\.tsx?$": "ts-jest", 6 | }; 7 | -------------------------------------------------------------------------------- /cdk/cmr-stac-dev/lib/cmr-stac-dev-stack.ts: -------------------------------------------------------------------------------- 1 | import * as cdk from "aws-cdk-lib"; 2 | import * as iam from "aws-cdk-lib/aws-iam"; 3 | import * as lambda from "aws-cdk-lib/aws-lambda"; 4 | import * as ec2 from "aws-cdk-lib/aws-ec2"; 5 | 6 | import { application } from "@edsc/cdk-utils"; 7 | 8 | export type CmrStacStackProps = cdk.StackProps; 9 | 10 | const logGroupSuffix = ""; 11 | const LOG_DESTINATION_ARN = "local-arn"; 12 | const STAGE_NAME = "dev"; 13 | const SUBNET_ID_A = "local-subnet-a"; 14 | const SUBNET_ID_B = "local-subnet-b"; 15 | const SUBNET_ID_C = "local-subnet-c"; 16 | const VPC_ID = "local-vpc"; 17 | const runtime = lambda.Runtime.NODEJS_22_X; 18 | const memorySize = 1024; 19 | 20 | // Don't bundle lambda assets since this is the dev stack 21 | const bundling = { 22 | // Only minify in production 23 | minify: false, 24 | externalModules: ["@aws-sdk/*"], 25 | }; 26 | /** 27 | * The AWS CloudFormation template for this Serverless application 28 | */ 29 | export class CmrStacDevStack extends cdk.Stack { 30 | /** 31 | * URL of the service endpoint 32 | */ 33 | public readonly serviceEndpoint; 34 | /** 35 | * Current Lambda function version 36 | */ 37 | 38 | public constructor(scope: cdk.App, id: string, props: CmrStacStackProps = {}) { 39 | super(scope, id, props); 40 | 41 | const environment = { 42 | CMR_URL: (process.env.CMR_URL = ""), 43 | CMR_LB_URL: (process.env.CMR_LB_URL = ""), 44 | GRAPHQL_URL: (process.env.URS_ROOT_URL = ""), 45 | NODE_ENV: "development", 46 | STAC_VERSION: "1.0.0", 47 | LOG_LEVEL: "INFO", 48 | PAGE_SIZE: "100", 49 | }; 50 | 51 | const cmrStacRole = new iam.CfnRole(this, "cmrStacRole", { 52 | roleName: `stacRole-${STAGE_NAME}`, 53 | permissionsBoundary: ["arn:aws:iam::", this.account, ":policy/NGAPShRoleBoundary"].join(""), 54 | managedPolicyArns: ["arn:aws:iam::aws:policy/service-role/AWSLambdaVPCAccessExecutionRole"], 55 | assumeRolePolicyDocument: { 56 | Version: "2012-10-17", 57 | Statement: [ 58 | { 59 | Effect: "Allow", 60 | Principal: { 61 | Service: ["lambda.amazonaws.com"], 62 | }, 63 | Action: ["sts:AssumeRole"], 64 | }, 65 | ], 66 | }, 67 | }); 68 | 69 | // Get lambda role from application role 70 | const lambdaRole = iam.Role.fromRoleArn(this, "CmrStacLambdaRole", cmrStacRole.attrArn); 71 | 72 | const vpc = ec2.Vpc.fromVpcAttributes(this, "Vpc", { 73 | availabilityZones: ["us-east-1a", "us-east-1b", "us-east-1c"], 74 | privateSubnetIds: [SUBNET_ID_A, SUBNET_ID_B, SUBNET_ID_C], 75 | vpcId: VPC_ID, 76 | }); 77 | 78 | const lambdaSecurityGroup = ec2.SecurityGroup.fromSecurityGroupId( 79 | this, 80 | "lambdaSecurityGroup", 81 | "cmrStacLambdaSecurityGroup" 82 | ); 83 | 84 | const defaultLambdaConfig = { 85 | bundling, 86 | environment, 87 | logDestinationArn: LOG_DESTINATION_ARN, 88 | logGroupSuffix, 89 | memorySize, 90 | role: lambdaRole, 91 | runtime, 92 | securityGroups: [lambdaSecurityGroup], 93 | stageName: STAGE_NAME, 94 | vpc, 95 | }; 96 | 97 | const apiGateway = new application.ApiGateway(this, "ApiGateway", { 98 | apiName: `${STAGE_NAME}-cmr-stac`, 99 | stageName: STAGE_NAME, 100 | }); 101 | 102 | const { apiGatewayDeployment, apiGatewayRestApi } = apiGateway; 103 | 104 | new application.NodeJsFunction(this, "StacLambdaFunction", { 105 | ...defaultLambdaConfig, 106 | api: { 107 | apiGatewayDeployment, 108 | apiGatewayRestApi, 109 | methods: ["GET", "POST"], 110 | path: "{proxy+}", 111 | }, 112 | entry: "../../src/handler.ts", 113 | functionName: `cmr-stac-api-${STAGE_NAME}`, 114 | }); 115 | 116 | this.serviceEndpoint = [ 117 | "https://", 118 | apiGatewayRestApi.ref, 119 | ".execute-api.", 120 | this.region, 121 | ".", 122 | this.urlSuffix, 123 | `/${STAGE_NAME}`, 124 | ].join(""); 125 | 126 | new cdk.CfnOutput(this, "CfnOutputServiceEndpoint", { 127 | key: "ServiceEndpoint", 128 | description: "URL of the service endpoint", 129 | exportName: `sls-${this.stackName}-ServiceEndpoint`, 130 | value: this.serviceEndpoint.toString(), 131 | }); 132 | } 133 | } 134 | -------------------------------------------------------------------------------- /cdk/cmr-stac-dev/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "cmr-stac", 3 | "version": "0.1.0", 4 | "bin": { 5 | "cmr-stac": "bin/cmr-stac-dev.js" 6 | }, 7 | "scripts": { 8 | "build": "tsc", 9 | "watch": "tsc -w", 10 | "test": "jest", 11 | "cdk": "cdk" 12 | }, 13 | "devDependencies": { 14 | "@types/jest": "^29.5.14", 15 | "@types/node": "22.7.9", 16 | "aws-cdk": "2.173.4", 17 | "esbuild": "^0.24.2", 18 | "jest": "^29.7.0", 19 | "ts-jest": "^29.2.5", 20 | "ts-node": "^10.9.2", 21 | "typescript": "~5.6.3" 22 | }, 23 | "dependencies": { 24 | "@edsc/cdk-utils": "^0.0.6", 25 | "aws-cdk-lib": "2.173.4", 26 | "constructs": "^10.0.0" 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /cdk/cmr-stac-dev/test/cmr-stac-dev.test.ts: -------------------------------------------------------------------------------- 1 | // import * as cdk from 'aws-cdk-lib'; 2 | // import { Template } from 'aws-cdk-lib/assertions'; 3 | // import * as CmrStac from '../lib/cmr-stac-stack'; 4 | 5 | // example test. To run these tests, uncomment this file along with the 6 | // example resource in lib/cmr-stac-stack.ts 7 | test("SQS Queue Created", () => { 8 | // const app = new cdk.App(); 9 | // // WHEN 10 | // const stack = new CmrStac.CmrStacStack(app, 'MyTestStack'); 11 | // // THEN 12 | // const template = Template.fromStack(stack); 13 | // template.hasResourceProperties('AWS::SQS::Queue', { 14 | // VisibilityTimeout: 300 15 | // }); 16 | }); 17 | -------------------------------------------------------------------------------- /cdk/cmr-stac-dev/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "target": "ES2020", 4 | "module": "commonjs", 5 | "lib": ["es2020", "dom"], 6 | "declaration": true, 7 | "strict": true, 8 | "noImplicitAny": true, 9 | "strictNullChecks": true, 10 | "noImplicitThis": true, 11 | "alwaysStrict": true, 12 | "noUnusedLocals": false, 13 | "noUnusedParameters": false, 14 | "noImplicitReturns": true, 15 | "noFallthroughCasesInSwitch": false, 16 | "inlineSourceMap": true, 17 | "inlineSources": true, 18 | "experimentalDecorators": true, 19 | "strictPropertyInitialization": false, 20 | "typeRoots": ["./node_modules/@types"] 21 | }, 22 | "exclude": ["node_modules", "cdk.out"] 23 | } 24 | -------------------------------------------------------------------------------- /cdk/cmr-stac/.gitignore: -------------------------------------------------------------------------------- 1 | *.js 2 | !jest.config.js 3 | *.d.ts 4 | node_modules 5 | 6 | # CDK asset staging directory 7 | .cdk.staging 8 | cdk.out 9 | -------------------------------------------------------------------------------- /cdk/cmr-stac/.npmignore: -------------------------------------------------------------------------------- 1 | *.ts 2 | !*.d.ts 3 | 4 | # CDK asset staging directory 5 | .cdk.staging 6 | cdk.out 7 | -------------------------------------------------------------------------------- /cdk/cmr-stac/README.md: -------------------------------------------------------------------------------- 1 | # Welcome to your CDK TypeScript project 2 | 3 | This is a blank project for CDK development with TypeScript. 4 | 5 | The `cdk.json` file tells the CDK Toolkit how to execute your app. 6 | 7 | ## Useful commands 8 | 9 | - `npm run build` compile typescript to js 10 | - `npm run watch` watch for changes and compile 11 | - `npm run test` perform the jest unit tests 12 | - `npx cdk deploy` deploy this stack to your default AWS account/region 13 | - `npx cdk diff` compare deployed stack with current state 14 | - `npx cdk synth` emits the synthesized CloudFormation template 15 | -------------------------------------------------------------------------------- /cdk/cmr-stac/bin/cmr-stac.ts: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | import * as cdk from "aws-cdk-lib"; 3 | import { CmrStacStack } from "../lib/cmr-stac-stack"; 4 | 5 | const { AWS_ACCOUNT = "1234567890", AWS_REGION = "us-east-1", STAGE_NAME = "dev" } = process.env; 6 | const app = new cdk.App(); 7 | new CmrStacStack(app, `cmr-stac-api-${STAGE_NAME}`, { 8 | /* If you don't specify 'env', this stack will be environment-agnostic. 9 | * Account/Region-dependent features and context lookups will not work, 10 | * but a single synthesized template can be deployed anywhere. */ 11 | 12 | /* Uncomment the next line to specialize this stack for the AWS Account 13 | * and Region that are implied by the current CLI configuration. */ 14 | // env: { account: process.env.CDK_DEFAULT_ACCOUNT, region: process.env.CDK_DEFAULT_REGION }, 15 | 16 | /* Uncomment the next line if you know exactly what Account and Region you 17 | * want to deploy the stack to. */ 18 | env: { account: AWS_ACCOUNT, region: AWS_REGION }, 19 | 20 | /* For more information, see https://docs.aws.amazon.com/cdk/latest/guide/environments.html */ 21 | }); 22 | -------------------------------------------------------------------------------- /cdk/cmr-stac/cdk.json: -------------------------------------------------------------------------------- 1 | { 2 | "app": "npx ts-node --prefer-ts-exts bin/cmr-stac.ts", 3 | "watch": { 4 | "include": ["**"], 5 | "exclude": [ 6 | "README.md", 7 | "cdk*.json", 8 | "**/*.js", 9 | "tsconfig.json", 10 | "package*.json", 11 | "yarn.lock", 12 | "node_modules", 13 | "test" 14 | ] 15 | }, 16 | "context": { 17 | "@aws-cdk/aws-lambda:recognizeLayerVersion": true, 18 | "@aws-cdk/core:checkSecretUsage": true, 19 | "@aws-cdk/core:target-partitions": ["aws", "aws-cn"], 20 | "@aws-cdk-containers/ecs-service-extensions:enableDefaultLogDriver": true, 21 | "@aws-cdk/aws-ec2:uniqueImdsv2TemplateName": true, 22 | "@aws-cdk/aws-ecs:arnFormatIncludesClusterName": true, 23 | "@aws-cdk/aws-iam:minimizePolicies": true, 24 | "@aws-cdk/core:validateSnapshotRemovalPolicy": true, 25 | "@aws-cdk/aws-codepipeline:crossAccountKeyAliasStackSafeResourceName": true, 26 | "@aws-cdk/aws-s3:createDefaultLoggingPolicy": true, 27 | "@aws-cdk/aws-sns-subscriptions:restrictSqsDescryption": true, 28 | "@aws-cdk/aws-apigateway:disableCloudWatchRole": true, 29 | "@aws-cdk/core:enablePartitionLiterals": true, 30 | "@aws-cdk/aws-events:eventsTargetQueueSameAccount": true, 31 | "@aws-cdk/aws-ecs:disableExplicitDeploymentControllerForCircuitBreaker": true, 32 | "@aws-cdk/aws-iam:importedRoleStackSafeDefaultPolicyName": true, 33 | "@aws-cdk/aws-s3:serverAccessLogsUseBucketPolicy": true, 34 | "@aws-cdk/aws-route53-patters:useCertificate": true, 35 | "@aws-cdk/customresources:installLatestAwsSdkDefault": false, 36 | "@aws-cdk/aws-rds:databaseProxyUniqueResourceName": true, 37 | "@aws-cdk/aws-codedeploy:removeAlarmsFromDeploymentGroup": true, 38 | "@aws-cdk/aws-apigateway:authorizerChangeDeploymentLogicalId": true, 39 | "@aws-cdk/aws-ec2:launchTemplateDefaultUserData": true, 40 | "@aws-cdk/aws-secretsmanager:useAttachedSecretResourcePolicyForSecretTargetAttachments": true, 41 | "@aws-cdk/aws-redshift:columnId": true, 42 | "@aws-cdk/aws-stepfunctions-tasks:enableEmrServicePolicyV2": true, 43 | "@aws-cdk/aws-ec2:restrictDefaultSecurityGroup": true, 44 | "@aws-cdk/aws-apigateway:requestValidatorUniqueId": true, 45 | "@aws-cdk/aws-kms:aliasNameRef": true, 46 | "@aws-cdk/aws-autoscaling:generateLaunchTemplateInsteadOfLaunchConfig": true, 47 | "@aws-cdk/core:includePrefixInUniqueNameGeneration": true, 48 | "@aws-cdk/aws-efs:denyAnonymousAccess": true, 49 | "@aws-cdk/aws-opensearchservice:enableOpensearchMultiAzWithStandby": true, 50 | "@aws-cdk/aws-lambda-nodejs:useLatestRuntimeVersion": true, 51 | "@aws-cdk/aws-efs:mountTargetOrderInsensitiveLogicalId": true, 52 | "@aws-cdk/aws-rds:auroraClusterChangeScopeOfInstanceParameterGroupWithEachParameters": true, 53 | "@aws-cdk/aws-appsync:useArnForSourceApiAssociationIdentifier": true, 54 | "@aws-cdk/aws-rds:preventRenderingDeprecatedCredentials": true, 55 | "@aws-cdk/aws-codepipeline-actions:useNewDefaultBranchForCodeCommitSource": true, 56 | "@aws-cdk/aws-cloudwatch-actions:changeLambdaPermissionLogicalIdForLambdaAction": true, 57 | "@aws-cdk/aws-codepipeline:crossAccountKeysDefaultValueToFalse": true, 58 | "@aws-cdk/aws-codepipeline:defaultPipelineTypeToV2": true, 59 | "@aws-cdk/aws-kms:reduceCrossAccountRegionPolicyScope": true, 60 | "@aws-cdk/aws-eks:nodegroupNameAttribute": true, 61 | "@aws-cdk/aws-ec2:ebsDefaultGp3Volume": true, 62 | "@aws-cdk/aws-ecs:removeDefaultDeploymentAlarm": true, 63 | "@aws-cdk/custom-resources:logApiResponseDataPropertyTrueDefault": false, 64 | "@aws-cdk/aws-s3:keepNotificationInImportedBucket": false, 65 | "@aws-cdk/aws-ecs:reduceEc2FargateCloudWatchPermissions": true, 66 | "@aws-cdk/aws-dynamodb:resourcePolicyPerReplica": true, 67 | "@aws-cdk/aws-ec2:ec2SumTImeoutEnabled": true, 68 | "@aws-cdk/aws-appsync:appSyncGraphQLAPIScopeLambdaPermission": true, 69 | "@aws-cdk/aws-rds:setCorrectValueForDatabaseInstanceReadReplicaInstanceResourceId": true, 70 | "@aws-cdk/core:cfnIncludeRejectComplexResourceUpdateCreatePolicyIntrinsics": true, 71 | "@aws-cdk/aws-lambda-nodejs:sdkV3ExcludeSmithyPackages": true, 72 | "@aws-cdk/aws-stepfunctions-tasks:fixRunEcsTaskPolicy": true, 73 | "@aws-cdk/aws-ec2:bastionHostUseAmazonLinux2023ByDefault": true, 74 | "cdk-migrate": true 75 | } 76 | } 77 | -------------------------------------------------------------------------------- /cdk/cmr-stac/jest.config.js: -------------------------------------------------------------------------------- 1 | export const testEnvironment = "node"; 2 | export const roots = ["/test"]; 3 | export const testMatch = ["**/*.test.ts"]; 4 | export const transform = { 5 | "^.+\\.tsx?$": "ts-jest", 6 | }; 7 | -------------------------------------------------------------------------------- /cdk/cmr-stac/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "cmr-stac", 3 | "version": "0.1.0", 4 | "bin": { 5 | "cmr-stac": "bin/cmr-stac.js" 6 | }, 7 | "scripts": { 8 | "build": "tsc", 9 | "watch": "tsc -w", 10 | "test": "jest", 11 | "cdk": "cdk" 12 | }, 13 | "devDependencies": { 14 | "@types/jest": "^29.5.14", 15 | "@types/node": "22.7.9", 16 | "aws-cdk": "2.173.4", 17 | "esbuild": "^0.24.2", 18 | "jest": "^29.7.0", 19 | "ts-jest": "^29.2.5", 20 | "ts-node": "^10.9.2", 21 | "typescript": "~5.6.3" 22 | }, 23 | "dependencies": { 24 | "@edsc/cdk-utils": "^0.0.5", 25 | "aws-cdk-lib": "2.173.4", 26 | "constructs": "^10.0.0" 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /cdk/cmr-stac/test/cmr-stac.test.ts: -------------------------------------------------------------------------------- 1 | // import * as cdk from 'aws-cdk-lib'; 2 | // import { Template } from 'aws-cdk-lib/assertions'; 3 | // import * as CmrStac from '../lib/cmr-stac-stack'; 4 | 5 | // example test. To run these tests, uncomment this file along with the 6 | // example resource in lib/cmr-stac-stack.ts 7 | test("SQS Queue Created", () => { 8 | // const app = new cdk.App(); 9 | // // WHEN 10 | // const stack = new CmrStac.CmrStacStack(app, 'MyTestStack'); 11 | // // THEN 12 | // const template = Template.fromStack(stack); 13 | // template.hasResourceProperties('AWS::SQS::Queue', { 14 | // VisibilityTimeout: 300 15 | // }); 16 | }); 17 | -------------------------------------------------------------------------------- /cdk/cmr-stac/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "target": "ES2020", 4 | "module": "commonjs", 5 | "lib": ["es2020", "dom"], 6 | "declaration": true, 7 | "strict": true, 8 | "noImplicitAny": true, 9 | "strictNullChecks": true, 10 | "noImplicitThis": true, 11 | "alwaysStrict": true, 12 | "noUnusedLocals": false, 13 | "noUnusedParameters": false, 14 | "noImplicitReturns": true, 15 | "noFallthroughCasesInSwitch": false, 16 | "inlineSourceMap": true, 17 | "inlineSources": true, 18 | "experimentalDecorators": true, 19 | "strictPropertyInitialization": false, 20 | "typeRoots": ["./node_modules/@types"] 21 | }, 22 | "exclude": ["node_modules", "cdk.out"] 23 | } 24 | -------------------------------------------------------------------------------- /deploy.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Bail on unset variables, errors and trace execution 4 | set -eux 5 | 6 | 7 | # Set up Docker image 8 | ##################### 9 | 10 | cat < .dockerignore 11 | .DS_Store 12 | .git 13 | .github 14 | .serverless 15 | .webpack 16 | cypress 17 | dist 18 | node_modules 19 | tmp 20 | EOF 21 | 22 | cat < Dockerfile 23 | FROM node:22 24 | COPY . /build 25 | WORKDIR /build 26 | RUN npm ci 27 | EOF 28 | 29 | dockerTag=cmr-stac-$bamboo_STAGE_NAME 30 | stageOpts="--stage $bamboo_STAGE_NAME " 31 | 32 | docker build -t $dockerTag . 33 | 34 | # Convenience function to invoke `docker run` with appropriate env vars instead of baking them into image 35 | dockerRun() { 36 | docker run \ 37 | --rm \ 38 | -e "CMR_URL=$bamboo_CMR_URL" \ 39 | -e "CMR_LB_URL=$bamboo_CMR_LB_URL" \ 40 | -e "GRAPHQL_URL=$bamboo_GRAPHQL_URL" \ 41 | -e "STAC_VERSION=$bamboo_STAC_VERSION" \ 42 | -e "PAGE_SIZE=$bamboo_PAGE_SIZE" \ 43 | -e "LOG_LEVEL=$bamboo_LOG_LEVEL" \ 44 | -e "AWS_ACCESS_KEY_ID=$bamboo_AWS_ACCESS_KEY_ID_PASSWORD" \ 45 | -e "AWS_SECRET_ACCESS_KEY=$bamboo_AWS_SECRET_ACCESS_KEY_PASSWORD" \ 46 | -e "AWS_DEFAULT_REGION=$bamboo_AWS_DEFAULT_REGION" \ 47 | -e "AWS_ACCOUNT=$bamboo_AWS_ACCOUNT" \ 48 | -e "LISTENER_ARN=$bamboo_LISTENER_ARN" \ 49 | -e "LOG_DESTINATION_ARN=$bamboo_LOG_DESTINATION_ARN" \ 50 | -e "AWS_ORG_USER=$bamboo_AWS_ORG_USER" \ 51 | -e "AWS_ORG_ID=$bamboo_AWS_ORG_ID" \ 52 | -e "NODE_ENV=production" \ 53 | -e "STAGE_NAME=$bamboo_STAGE_NAME" \ 54 | -e "SUBNET_ID_A=$bamboo_SUBNET_ID_A" \ 55 | -e "SUBNET_ID_B=$bamboo_SUBNET_ID_B" \ 56 | -e "SUBNET_ID_C=$bamboo_SUBNET_ID_C" \ 57 | -e "CMR_SERVICE_SECURITY_GROUP_ID=$bamboo_CMR_SERVICE_SECURITY_GROUP_ID" \ 58 | -e "VPC_ID=$bamboo_VPC_ID" \ 59 | $dockerTag "$@" 60 | } 61 | 62 | # Execute deployment commands in Docker 63 | ####################################### 64 | 65 | # Deploy to AWS 66 | echo 'Deploying to AWS Resources...' 67 | dockerRun npm run deploy-application 68 | 69 | -------------------------------------------------------------------------------- /docs/README.md: -------------------------------------------------------------------------------- 1 | # CMR-STAC Development 2 | 3 | CMR-STAC is a Node.js application built on the [Express.js framework](https://expressjs.com/) and deployed as an AWS serverless application using API Gateway + Lambda. This README is intended for developers who want to contribute to CMR-STAC, or set up a development environment for it. 4 | 5 | The remainder of this README is documentation for developing, testing, and deploying CMR-STAC. See the [Usage documentation](../docs/usage/usage.md) if you are interested in using the CMR-STAC API. 6 | 7 | ## Repository Structure 8 | 9 | | Directory | Description | 10 | | -------------------- | ------------ | 11 | | docs | The `docs` directory is where the combined specification document made from the STAC and WFS3 specification documents is held. Paths and component schemas are defined here. The generated STAC documentation file is also located in this directory. | 12 | | [src](../src) | The `src` directory contains the main logic of the application. It is broken down into modules pertaining to areas of responsibility. 13 | | [scripts](../scripts) | Utility (Python) scripts for validating and crawling CMR-STAC | 14 | | [usage](../docs/usage/usage.md) | Documentation on usage of the CMR-STAC endpoint(s) | 15 | 16 | ## Getting Started 17 | 18 | ### Setup 19 | 20 | - Set the correct NodeJS version (specified in [.nvmrc](../.nvmrc) required 21 | by CMR-STAC with [`nvm`](https://github.com/nvm-sh/nvm) (recommended for managing NodeJS versions)): 22 | 23 | - install aws-sam-cli (`brew install aws-sam-cli`) 24 | 25 | ```bash 26 | nvm use 27 | ``` 28 | 29 | Then install dependencies with npm: 30 | 31 | ```bash 32 | npm install 33 | ``` 34 | 35 | To run the unit test suite associated with CMR-STAC: 36 | 37 | ```bash 38 | npm test 39 | ``` 40 | 41 | To lint your developed code: 42 | 43 | ```bash 44 | npm run prettier:fix 45 | ``` 46 | 47 | To run the CMR-STAC server locally: 48 | 49 | ```bash 50 | npm run start 51 | ``` 52 | 53 | This will run the process in the current terminal session start up the necessary docker container where the the local server will be available from: 54 | 55 | ```bash 56 | http://localhost:3000/stac 57 | http://localhost:3000/cloudstac 58 | ``` 59 | 60 | To configure environment variables for this application such as point to `uat` or `prod` update the values in `sam_local_envs.json` 61 | 62 | ### Creating index.html from Swagger.json 63 | 64 | To Create the index.html located in docs/index we can use the `redocly` service 65 | the most straightforward way to do this is to use the cli tool against our `swagger.json` file 66 | 67 | ```bash 68 | npx @redocly/cli build-docs swagger.json 69 | ``` 70 | 71 | ### Testing STAC validation 72 | 73 | We can test our API both locally and on deployed instance against the on a stac-validation service using the tool 74 | 75 | The tool can be installed using pip and requires a Python runtime 76 | 77 | ```bash 78 | pip install stac-api-validator 79 | ``` 80 | 81 | ```bash 82 | stac-api-validator\ 83 | --root-url http://localhost:3000/stac/CMR_ONLY \ 84 | --conformance core 85 | ``` 86 | 87 | this can be extended to validate against additional conformance APIs 88 | 89 | ### Deploying 90 | 91 | The deployment is handled via the [AWS CDK Framework](https://aws.amazon.com/cdk/) 92 | 93 | To deploy the CMR-STAC application to AWS, you will need to set up a set of AWS credentials for the account where the application is being deployed, with the following permissions: 94 | 95 | - manage cloud formation 96 | - manage S3 buckets 97 | - manage lambda function 98 | - manage api gateway 99 | 100 | Running the `deploy.sh` script with teh accompanying environment variables will run the `cdk synth` command on `cdk/crm-stac` to build the application and start the deployment 101 | 102 | ## License 103 | 104 | NASA Open Source Agreement v1.3 (NASA-1.3) 105 | See [LICENSE.txt](../LICENSE.txt) 106 | -------------------------------------------------------------------------------- /docs/usage/images/cmr-stac-cal.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nasa/cmr-stac/ab030f1ec66c9006007940d7caea77351c9fb866/docs/usage/images/cmr-stac-cal.png -------------------------------------------------------------------------------- /docs/usage/images/cmr-stac-cal2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nasa/cmr-stac/ab030f1ec66c9006007940d7caea77351c9fb866/docs/usage/images/cmr-stac-cal2.png -------------------------------------------------------------------------------- /docs/usage/images/cmr-stac-table.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nasa/cmr-stac/ab030f1ec66c9006007940d7caea77351c9fb866/docs/usage/images/cmr-stac-table.png -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "CMR-STAC", 3 | "version": "2.0.0", 4 | "maintainers": [ 5 | { 6 | "name": "Earthdata Search", 7 | "email": "support@earthdata.nasa.gov" 8 | } 9 | ], 10 | "description": "NASA' Common Metadata Repository STAC implementation.", 11 | "repository": "https://github.com/NASA/cmr-stac", 12 | "license": "NASA-1.3", 13 | "scripts": { 14 | "preinstall": "cd cdk/cmr-stac && npm ci --target_arch=x64 && cd ../cmr-stac-dev && npm ci --target_arch=x64", 15 | "run-synth": "cd cdk/cmr-stac && npm run cdk synth -- --quiet", 16 | "run-synth-dev": "cd cdk/cmr-stac-dev && npm run cdk synth -- --quiet", 17 | "watch": "onchange 'src/**' -- npm run run-synth-dev", 18 | "start:api": "sam local start-api -t ./cdk/cmr-stac/cdk.out/cmr-stac-dev.template.json --warm-containers LAZY --port 3000 --docker-network host", 19 | "prestart": "npm run run-synth-dev ", 20 | "start": "node start.js", 21 | "test": "cross-env TS_NODE_FILES=true NODE_ENV=development mocha", 22 | "test:pattern": "cross-env TS_NODE_FILES=true NODE_ENV=development mocha --grep", 23 | "test:coverage": "cross-env TS_NODE_FILES=true NODE_ENV=development nyc --reporter=html --reporter=json --reporter=text mocha", 24 | "test:watch": "cross-env TS_NODE_FILES=true NODE_ENV=development mocha --watch", 25 | "deploy-application": "cd cdk/cmr-stac && npm ci --include=dev && npm run cdk deploy -- --progress events --require-approval never", 26 | "lint": "eslint src cdk", 27 | "lint:fix": "eslint src cdk --fix", 28 | "prettier": "prettier src cdk --check", 29 | "prettier:fix": "prettier src cdk --write", 30 | "format": "npm run prettier:fix && npm run lint:fix" 31 | }, 32 | "devDependencies": { 33 | "@faker-js/faker": "^7.6.0", 34 | "@serverless/event-mocks": "^1.1.1", 35 | "@serverless/typescript": "^3.21.0", 36 | "@types/aws-lambda": "^8.10.108", 37 | "@types/chai": "^4.3.4", 38 | "@types/chai-string": "^1.4.2", 39 | "@types/compression": "^1.7.2", 40 | "@types/cookie-parser": "^1.4.3", 41 | "@types/cors": "^2.8.12", 42 | "@types/express": "^4.17.14", 43 | "@types/mocha": "^10.0.0", 44 | "@types/morgan": "^1.9.3", 45 | "@types/node": "^18.11.9", 46 | "@types/serverless": "^3.12.8", 47 | "@types/sinon": "^10.0.13", 48 | "@types/sinon-chai": "^3.2.9", 49 | "@types/supertest": "^2.0.12", 50 | "@typescript-eslint/eslint-plugin": "^5.55.0", 51 | "@typescript-eslint/parser": "^5.55.0", 52 | "ajv": "^8.11.0", 53 | "ajv-formats": "^3.0.1", 54 | "ajv-formats-draft2019": "^1.6.1", 55 | "chai": "^4.3.7", 56 | "chai-string": "^1.5.0", 57 | "concurrently": "^9.1.2", 58 | "cross-env": "^7.0.3", 59 | "eslint": "^8.36.0", 60 | "eslint-config-prettier": "^8.7.0", 61 | "json-schema-to-typescript": "^11.0.2", 62 | "mocha": "^10.2.0", 63 | "mocha-junit-reporter": "^2.2.0", 64 | "mocha-suppress-logs": "^0.3.1", 65 | "nyc": "^15.1.0", 66 | "onchange": "^7.1.0", 67 | "prettier": "^2.8.4", 68 | "sinon": "^14.0.2", 69 | "sinon-chai": "^3.7.0", 70 | "supertest": "^6.3.1", 71 | "ts-node": "^10.9.1", 72 | "typescript": "^4.8.4" 73 | }, 74 | "dependencies": { 75 | "@vendia/serverless-express": "^4.10.4", 76 | "aws-lambda": "^1.0.7", 77 | "aws-sdk": "^2.1692.0", 78 | "axios": "^1.7.4", 79 | "body-parser": "^1.20.2", 80 | "compression": "^1.7.4", 81 | "cookie-parser": "^1.4.6", 82 | "cors": "^2.8.5", 83 | "dotenv": "^16.1.4", 84 | "eslint-plugin-import": "^2.28.0", 85 | "express": "^4.21.2", 86 | "graphql": "^16.8.1", 87 | "graphql-request": "^5.2.0", 88 | "helmet": "^6.2.0", 89 | "lodash": "^4.17.21", 90 | "morgan": "^1.10.0", 91 | "qs": "^6.13.0", 92 | "semver": "^6.3.1", 93 | "source-map-support": "^0.5.21" 94 | } 95 | } 96 | -------------------------------------------------------------------------------- /resources/Geometry.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "http://json-schema.org/draft-07/schema#", 3 | "$id": "https://geojson.org/schema/Geometry.json", 4 | "title": "GeoJSON Geometry", 5 | "oneOf": [ 6 | { 7 | "title": "GeoJSON Point", 8 | "type": "object", 9 | "required": ["type", "coordinates"], 10 | "properties": { 11 | "type": { 12 | "type": "string", 13 | "enum": ["Point"] 14 | }, 15 | "coordinates": { 16 | "type": "array", 17 | "minItems": 2, 18 | "items": { 19 | "type": "number" 20 | } 21 | }, 22 | "bbox": { 23 | "type": "array", 24 | "minItems": 4, 25 | "items": { 26 | "type": "number" 27 | } 28 | } 29 | } 30 | }, 31 | { 32 | "title": "GeoJSON LineString", 33 | "type": "object", 34 | "required": ["type", "coordinates"], 35 | "properties": { 36 | "type": { 37 | "type": "string", 38 | "enum": ["LineString"] 39 | }, 40 | "coordinates": { 41 | "type": "array", 42 | "minItems": 2, 43 | "items": { 44 | "type": "array", 45 | "minItems": 2, 46 | "items": { 47 | "type": "number" 48 | } 49 | } 50 | }, 51 | "bbox": { 52 | "type": "array", 53 | "minItems": 4, 54 | "items": { 55 | "type": "number" 56 | } 57 | } 58 | } 59 | }, 60 | { 61 | "title": "GeoJSON Polygon", 62 | "type": "object", 63 | "required": ["type", "coordinates"], 64 | "properties": { 65 | "type": { 66 | "type": "string", 67 | "enum": ["Polygon"] 68 | }, 69 | "coordinates": { 70 | "type": "array", 71 | "items": { 72 | "type": "array", 73 | "minItems": 4, 74 | "items": { 75 | "type": "array", 76 | "minItems": 2, 77 | "items": { 78 | "type": "number" 79 | } 80 | } 81 | } 82 | }, 83 | "bbox": { 84 | "type": "array", 85 | "minItems": 4, 86 | "items": { 87 | "type": "number" 88 | } 89 | } 90 | } 91 | }, 92 | { 93 | "title": "GeoJSON MultiPoint", 94 | "type": "object", 95 | "required": ["type", "coordinates"], 96 | "properties": { 97 | "type": { 98 | "type": "string", 99 | "enum": ["MultiPoint"] 100 | }, 101 | "coordinates": { 102 | "type": "array", 103 | "items": { 104 | "type": "array", 105 | "minItems": 2, 106 | "items": { 107 | "type": "number" 108 | } 109 | } 110 | }, 111 | "bbox": { 112 | "type": "array", 113 | "minItems": 4, 114 | "items": { 115 | "type": "number" 116 | } 117 | } 118 | } 119 | }, 120 | { 121 | "title": "GeoJSON MultiLineString", 122 | "type": "object", 123 | "required": ["type", "coordinates"], 124 | "properties": { 125 | "type": { 126 | "type": "string", 127 | "enum": ["MultiLineString"] 128 | }, 129 | "coordinates": { 130 | "type": "array", 131 | "items": { 132 | "type": "array", 133 | "minItems": 2, 134 | "items": { 135 | "type": "array", 136 | "minItems": 2, 137 | "items": { 138 | "type": "number" 139 | } 140 | } 141 | } 142 | }, 143 | "bbox": { 144 | "type": "array", 145 | "minItems": 4, 146 | "items": { 147 | "type": "number" 148 | } 149 | } 150 | } 151 | }, 152 | { 153 | "title": "GeoJSON MultiPolygon", 154 | "type": "object", 155 | "required": ["type", "coordinates"], 156 | "properties": { 157 | "type": { 158 | "type": "string", 159 | "enum": ["MultiPolygon"] 160 | }, 161 | "coordinates": { 162 | "type": "array", 163 | "items": { 164 | "type": "array", 165 | "items": { 166 | "type": "array", 167 | "minItems": 4, 168 | "items": { 169 | "type": "array", 170 | "minItems": 2, 171 | "items": { 172 | "type": "number" 173 | } 174 | } 175 | } 176 | } 177 | }, 178 | "bbox": { 179 | "type": "array", 180 | "minItems": 4, 181 | "items": { 182 | "type": "number" 183 | } 184 | } 185 | } 186 | } 187 | ] 188 | } 189 | -------------------------------------------------------------------------------- /resources/catalog-spec/json-schema/catalog.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "http://json-schema.org/draft-07/schema#", 3 | "$id": "https://schemas.stacspec.org/v1.0.0/catalog-spec/json-schema/catalog.json#", 4 | "title": "STAC Catalog Specification", 5 | "description": "This object represents Catalogs in a SpatioTemporal Asset Catalog.", 6 | "allOf": [ 7 | { 8 | "$ref": "#/definitions/catalog" 9 | } 10 | ], 11 | "definitions": { 12 | "catalog": { 13 | "title": "STAC Catalog", 14 | "type": "object", 15 | "required": [ 16 | "stac_version", 17 | "type", 18 | "id", 19 | "description", 20 | "links" 21 | ], 22 | "properties": { 23 | "stac_version": { 24 | "title": "STAC version", 25 | "type": "string", 26 | "const": "1.0.0" 27 | }, 28 | "stac_extensions": { 29 | "title": "STAC extensions", 30 | "type": "array", 31 | "uniqueItems": true, 32 | "items": { 33 | "title": "Reference to a JSON Schema", 34 | "type": "string", 35 | "format": "iri" 36 | } 37 | }, 38 | "type": { 39 | "title": "Type of STAC entity", 40 | "const": "Catalog" 41 | }, 42 | "id": { 43 | "title": "Identifier", 44 | "type": "string", 45 | "minLength": 1 46 | }, 47 | "title": { 48 | "title": "Title", 49 | "type": "string" 50 | }, 51 | "description": { 52 | "title": "Description", 53 | "type": "string", 54 | "minLength": 1 55 | }, 56 | "links": { 57 | "title": "Links", 58 | "type": "array", 59 | "items": { 60 | "$ref": "#/definitions/link" 61 | } 62 | } 63 | } 64 | }, 65 | "link": { 66 | "type": "object", 67 | "required": [ 68 | "rel", 69 | "href" 70 | ], 71 | "properties": { 72 | "href": { 73 | "title": "Link reference", 74 | "type": "string", 75 | "format": "iri-reference", 76 | "minLength": 1 77 | }, 78 | "rel": { 79 | "title": "Link relation type", 80 | "type": "string", 81 | "minLength": 1 82 | }, 83 | "type": { 84 | "title": "Link type", 85 | "type": "string" 86 | }, 87 | "title": { 88 | "title": "Link title", 89 | "type": "string" 90 | } 91 | } 92 | } 93 | } 94 | } 95 | -------------------------------------------------------------------------------- /resources/collection-spec/json-schema/collection.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "http://json-schema.org/draft-07/schema#", 3 | "$id": "https://schemas.stacspec.org/v1.0.0/collection-spec/json-schema/collection.json#", 4 | "title": "STAC Collection Specification", 5 | "description": "This object represents Collections in a SpatioTemporal Asset Catalog.", 6 | "allOf": [ 7 | { 8 | "$ref": "#/definitions/collection" 9 | } 10 | ], 11 | "definitions": { 12 | "collection": { 13 | "title": "STAC Collection", 14 | "description": "These are the fields specific to a STAC Collection. All other fields are inherited from STAC Catalog.", 15 | "type": "object", 16 | "required": [ 17 | "stac_version", 18 | "type", 19 | "id", 20 | "description", 21 | "license", 22 | "extent", 23 | "links" 24 | ], 25 | "properties": { 26 | "stac_version": { 27 | "title": "STAC version", 28 | "type": "string", 29 | "const": "1.0.0" 30 | }, 31 | "stac_extensions": { 32 | "title": "STAC extensions", 33 | "type": "array", 34 | "uniqueItems": true, 35 | "items": { 36 | "title": "Reference to a JSON Schema", 37 | "type": "string", 38 | "format": "iri" 39 | } 40 | }, 41 | "type": { 42 | "title": "Type of STAC entity", 43 | "const": "Collection" 44 | }, 45 | "id": { 46 | "title": "Identifier", 47 | "type": "string", 48 | "minLength": 1 49 | }, 50 | "title": { 51 | "title": "Title", 52 | "type": "string" 53 | }, 54 | "description": { 55 | "title": "Description", 56 | "type": "string", 57 | "minLength": 1 58 | }, 59 | "keywords": { 60 | "title": "Keywords", 61 | "type": "array", 62 | "items": { 63 | "type": "string" 64 | } 65 | }, 66 | "license": { 67 | "title": "Collection License Name", 68 | "type": "string", 69 | "pattern": "^[\\w\\-\\.\\+]+$" 70 | }, 71 | "providers": { 72 | "type": "array", 73 | "items": { 74 | "type": "object", 75 | "required": [ 76 | "name" 77 | ], 78 | "properties": { 79 | "name": { 80 | "title": "Organization name", 81 | "type": "string" 82 | }, 83 | "description": { 84 | "title": "Organization description", 85 | "type": "string" 86 | }, 87 | "roles": { 88 | "title": "Organization roles", 89 | "type": "array", 90 | "items": { 91 | "type": "string", 92 | "enum": [ 93 | "producer", 94 | "licensor", 95 | "processor", 96 | "host" 97 | ] 98 | } 99 | }, 100 | "url": { 101 | "title": "Organization homepage", 102 | "type": "string", 103 | "format": "iri" 104 | } 105 | } 106 | } 107 | }, 108 | "extent": { 109 | "title": "Extents", 110 | "type": "object", 111 | "required": [ 112 | "spatial", 113 | "temporal" 114 | ], 115 | "properties": { 116 | "spatial": { 117 | "title": "Spatial extent object", 118 | "type": "object", 119 | "required": [ 120 | "bbox" 121 | ], 122 | "properties": { 123 | "bbox": { 124 | "title": "Spatial extents", 125 | "type": "array", 126 | "minItems": 1, 127 | "items": { 128 | "title": "Spatial extent", 129 | "type": "array", 130 | "oneOf": [ 131 | { 132 | "minItems":4, 133 | "maxItems":4 134 | }, 135 | { 136 | "minItems":6, 137 | "maxItems":6 138 | } 139 | ], 140 | "items": { 141 | "type": "number" 142 | } 143 | } 144 | } 145 | } 146 | }, 147 | "temporal": { 148 | "title": "Temporal extent object", 149 | "type": "object", 150 | "required": [ 151 | "interval" 152 | ], 153 | "properties": { 154 | "interval": { 155 | "title": "Temporal extents", 156 | "type": "array", 157 | "minItems": 1, 158 | "items": { 159 | "title": "Temporal extent", 160 | "type": "array", 161 | "minItems": 2, 162 | "maxItems": 2, 163 | "items": { 164 | "type": [ 165 | "string", 166 | "null" 167 | ], 168 | "format": "date-time", 169 | "pattern": "(\\+00:00|Z)$" 170 | } 171 | } 172 | } 173 | } 174 | } 175 | } 176 | }, 177 | "assets": { 178 | "$ref": "../../item-spec/json-schema/item.json#/definitions/assets" 179 | }, 180 | "links": { 181 | "title": "Links", 182 | "type": "array", 183 | "items": { 184 | "$ref": "#/definitions/link" 185 | } 186 | }, 187 | "summaries": { 188 | "$ref": "#/definitions/summaries" 189 | } 190 | } 191 | }, 192 | "link": { 193 | "type": "object", 194 | "required": [ 195 | "rel", 196 | "href" 197 | ], 198 | "properties": { 199 | "href": { 200 | "title": "Link reference", 201 | "type": "string", 202 | "format": "iri-reference", 203 | "minLength": 1 204 | }, 205 | "rel": { 206 | "title": "Link relation type", 207 | "type": "string", 208 | "minLength": 1 209 | }, 210 | "type": { 211 | "title": "Link type", 212 | "type": "string" 213 | }, 214 | "title": { 215 | "title": "Link title", 216 | "type": "string" 217 | } 218 | } 219 | }, 220 | "summaries": { 221 | "type": "object", 222 | "additionalProperties": { 223 | "anyOf": [ 224 | { 225 | "title": "JSON Schema", 226 | "type": "object", 227 | "minProperties": 1, 228 | "allOf": [ 229 | { 230 | "$ref": "http://json-schema.org/draft-07/schema" 231 | } 232 | ] 233 | }, 234 | { 235 | "title": "Range", 236 | "type": "object", 237 | "required": [ 238 | "minimum", 239 | "maximum" 240 | ], 241 | "properties": { 242 | "minimum": { 243 | "title": "Minimum value", 244 | "type": [ 245 | "number", 246 | "string" 247 | ] 248 | }, 249 | "maximum": { 250 | "title": "Maximum value", 251 | "type": [ 252 | "number", 253 | "string" 254 | ] 255 | } 256 | } 257 | }, 258 | { 259 | "title": "Set of values", 260 | "type": "array", 261 | "minItems": 1, 262 | "items": { 263 | "description": "For each field only the original data type of the property can occur (except for arrays), but we can't validate that in JSON Schema yet. See the sumamry description in the STAC specification for details." 264 | } 265 | } 266 | ] 267 | } 268 | } 269 | } 270 | } -------------------------------------------------------------------------------- /resources/item-spec/json-schema/basics.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "http://json-schema.org/draft-07/schema#", 3 | "$id": "https://schemas.stacspec.org/v1.0.0/item-spec/json-schema/basics.json#", 4 | "title": "Basic Descriptive Fields", 5 | "type": "object", 6 | "properties": { 7 | "title": { 8 | "title": "Item Title", 9 | "description": "A human-readable title describing the Item.", 10 | "type": "string" 11 | }, 12 | "description": { 13 | "title": "Item Description", 14 | "description": "Detailed multi-line description to fully explain the Item.", 15 | "type": "string" 16 | } 17 | } 18 | } -------------------------------------------------------------------------------- /resources/item-spec/json-schema/datetime.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "http://json-schema.org/draft-07/schema#", 3 | "$id": "https://schemas.stacspec.org/v1.0.0/item-spec/json-schema/datetime.json#", 4 | "title": "Date and Time Fields", 5 | "type": "object", 6 | "dependencies": { 7 | "start_datetime": { 8 | "required": [ 9 | "end_datetime" 10 | ] 11 | }, 12 | "end_datetime": { 13 | "required": [ 14 | "start_datetime" 15 | ] 16 | } 17 | }, 18 | "properties": { 19 | "datetime": { 20 | "title": "Date and Time", 21 | "description": "The searchable date/time of the assets, in UTC (Formatted in RFC 3339) ", 22 | "type": ["string", "null"], 23 | "format": "date-time", 24 | "pattern": "(\\+00:00|Z)$" 25 | }, 26 | "start_datetime": { 27 | "title": "Start Date and Time", 28 | "description": "The searchable start date/time of the assets, in UTC (Formatted in RFC 3339) ", 29 | "type": "string", 30 | "format": "date-time", 31 | "pattern": "(\\+00:00|Z)$" 32 | }, 33 | "end_datetime": { 34 | "title": "End Date and Time", 35 | "description": "The searchable end date/time of the assets, in UTC (Formatted in RFC 3339) ", 36 | "type": "string", 37 | "format": "date-time", 38 | "pattern": "(\\+00:00|Z)$" 39 | }, 40 | "created": { 41 | "title": "Creation Time", 42 | "type": "string", 43 | "format": "date-time", 44 | "pattern": "(\\+00:00|Z)$" 45 | }, 46 | "updated": { 47 | "title": "Last Update Time", 48 | "type": "string", 49 | "format": "date-time", 50 | "pattern": "(\\+00:00|Z)$" 51 | } 52 | } 53 | } -------------------------------------------------------------------------------- /resources/item-spec/json-schema/instrument.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "http://json-schema.org/draft-07/schema#", 3 | "$id": "https://schemas.stacspec.org/v1.0.0/item-spec/json-schema/instrument.json#", 4 | "title": "Instrument Fields", 5 | "type": "object", 6 | "properties": { 7 | "platform": { 8 | "title": "Platform", 9 | "type": "string" 10 | }, 11 | "instruments": { 12 | "title": "Instruments", 13 | "type": "array", 14 | "items": { 15 | "type": "string" 16 | } 17 | }, 18 | "constellation": { 19 | "title": "Constellation", 20 | "type": "string" 21 | }, 22 | "mission": { 23 | "title": "Mission", 24 | "type": "string" 25 | }, 26 | "gsd": { 27 | "title": "Ground Sample Distance", 28 | "type": "number", 29 | "exclusiveMinimum": 0 30 | } 31 | } 32 | } -------------------------------------------------------------------------------- /resources/item-spec/json-schema/item.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "http://json-schema.org/draft-07/schema#", 3 | "$id": "https://schemas.stacspec.org/v1.0.0/item-spec/json-schema/item.json#", 4 | "title": "STAC Item", 5 | "type": "object", 6 | "description": "This object represents the metadata for an item in a SpatioTemporal Asset Catalog.", 7 | "allOf": [ 8 | { 9 | "$ref": "#/definitions/core" 10 | } 11 | ], 12 | "definitions": { 13 | "common_metadata": { 14 | "allOf": [ 15 | { 16 | "$ref": "basics.json" 17 | }, 18 | { 19 | "$ref": "datetime.json" 20 | }, 21 | { 22 | "$ref": "instrument.json" 23 | }, 24 | { 25 | "$ref": "licensing.json" 26 | }, 27 | { 28 | "$ref": "provider.json" 29 | } 30 | ] 31 | }, 32 | "core": { 33 | "allOf": [ 34 | { 35 | "$ref": "https://geojson.org/schema/Feature.json" 36 | }, 37 | { 38 | "oneOf": [ 39 | { 40 | "type": "object", 41 | "required": [ 42 | "geometry", 43 | "bbox" 44 | ], 45 | "properties": { 46 | "geometry": { 47 | "$ref": "https://geojson.org/schema/Geometry.json" 48 | }, 49 | "bbox": { 50 | "type": "array", 51 | "oneOf": [ 52 | { 53 | "minItems": 4, 54 | "maxItems": 4 55 | }, 56 | { 57 | "minItems": 6, 58 | "maxItems": 6 59 | } 60 | ], 61 | "items": { 62 | "type": "number" 63 | } 64 | } 65 | } 66 | }, 67 | { 68 | "type": "object", 69 | "required": [ 70 | "geometry" 71 | ], 72 | "properties": { 73 | "geometry": { 74 | "type": "null" 75 | }, 76 | "bbox": { 77 | "not": {} 78 | } 79 | } 80 | } 81 | ] 82 | }, 83 | { 84 | "type": "object", 85 | "required": [ 86 | "stac_version", 87 | "id", 88 | "links", 89 | "assets", 90 | "properties" 91 | ], 92 | "properties": { 93 | "stac_version": { 94 | "title": "STAC version", 95 | "type": "string", 96 | "const": "1.0.0" 97 | }, 98 | "stac_extensions": { 99 | "title": "STAC extensions", 100 | "type": "array", 101 | "uniqueItems": true, 102 | "items": { 103 | "title": "Reference to a JSON Schema", 104 | "type": "string", 105 | "format": "iri" 106 | } 107 | }, 108 | "id": { 109 | "title": "Provider ID", 110 | "description": "Provider item ID", 111 | "type": "string", 112 | "minLength": 1 113 | }, 114 | "links": { 115 | "title": "Item links", 116 | "description": "Links to item relations", 117 | "type": "array", 118 | "items": { 119 | "$ref": "#/definitions/link" 120 | } 121 | }, 122 | "assets": { 123 | "$ref": "#/definitions/assets" 124 | }, 125 | "properties": { 126 | "allOf": [ 127 | { 128 | "$ref": "#/definitions/common_metadata" 129 | }, 130 | { 131 | "anyOf": [ 132 | { 133 | "required": [ 134 | "datetime" 135 | ], 136 | "properties": { 137 | "datetime": { 138 | "not": { 139 | "type": "null" 140 | } 141 | } 142 | } 143 | }, 144 | { 145 | "required": [ 146 | "datetime", 147 | "start_datetime", 148 | "end_datetime" 149 | ] 150 | } 151 | ] 152 | } 153 | ] 154 | } 155 | }, 156 | "if": { 157 | "properties": { 158 | "links": { 159 | "contains": { 160 | "required": [ 161 | "rel" 162 | ], 163 | "properties": { 164 | "rel": { 165 | "const": "collection" 166 | } 167 | } 168 | } 169 | } 170 | } 171 | }, 172 | "then": { 173 | "required": [ 174 | "collection" 175 | ], 176 | "properties": { 177 | "collection": { 178 | "title": "Collection ID", 179 | "description": "The ID of the STAC Collection this Item references to.", 180 | "type": "string", 181 | "minLength": 1 182 | } 183 | } 184 | }, 185 | "else": { 186 | "properties": { 187 | "collection": { 188 | "not": {} 189 | } 190 | } 191 | } 192 | } 193 | ] 194 | }, 195 | "link": { 196 | "type": "object", 197 | "required": [ 198 | "rel", 199 | "href" 200 | ], 201 | "properties": { 202 | "href": { 203 | "title": "Link reference", 204 | "type": "string", 205 | "format": "iri-reference", 206 | "minLength": 1 207 | }, 208 | "rel": { 209 | "title": "Link relation type", 210 | "type": "string", 211 | "minLength": 1 212 | }, 213 | "type": { 214 | "title": "Link type", 215 | "type": "string" 216 | }, 217 | "title": { 218 | "title": "Link title", 219 | "type": "string" 220 | } 221 | } 222 | }, 223 | "assets": { 224 | "title": "Asset links", 225 | "description": "Links to assets", 226 | "type": "object", 227 | "additionalProperties": { 228 | "$ref": "#/definitions/asset" 229 | } 230 | }, 231 | "asset": { 232 | "allOf": [ 233 | { 234 | "type": "object", 235 | "required": [ 236 | "href" 237 | ], 238 | "properties": { 239 | "href": { 240 | "title": "Asset reference", 241 | "type": "string", 242 | "format": "iri-reference", 243 | "minLength": 1 244 | }, 245 | "title": { 246 | "title": "Asset title", 247 | "type": "string" 248 | }, 249 | "description": { 250 | "title": "Asset description", 251 | "type": "string" 252 | }, 253 | "type": { 254 | "title": "Asset type", 255 | "type": "string" 256 | }, 257 | "roles": { 258 | "title": "Asset roles", 259 | "type": "array", 260 | "items": { 261 | "type": "string" 262 | } 263 | } 264 | } 265 | }, 266 | { 267 | "$ref": "#/definitions/common_metadata" 268 | } 269 | ] 270 | } 271 | } 272 | } 273 | -------------------------------------------------------------------------------- /resources/item-spec/json-schema/licensing.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "http://json-schema.org/draft-07/schema#", 3 | "$id": "https://schemas.stacspec.org/v1.0.0/item-spec/json-schema/licensing.json#", 4 | "title": "Licensing Fields", 5 | "type": "object", 6 | "properties": { 7 | "license": { 8 | "type": "string", 9 | "pattern": "^[\\w\\-\\.\\+]+$" 10 | } 11 | } 12 | } -------------------------------------------------------------------------------- /resources/item-spec/json-schema/provider.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "http://json-schema.org/draft-07/schema#", 3 | "$id": "https://schemas.stacspec.org/v1.0.0/item-spec/json-schema/provider.json#", 4 | "title": "Provider Fields", 5 | "type": "object", 6 | "properties": { 7 | "providers": { 8 | "title": "Providers", 9 | "type": "array", 10 | "items": { 11 | "type": "object", 12 | "required": [ 13 | "name" 14 | ], 15 | "properties": { 16 | "name": { 17 | "title": "Organization name", 18 | "type": "string", 19 | "minLength": 1 20 | }, 21 | "description": { 22 | "title": "Organization description", 23 | "type": "string" 24 | }, 25 | "roles": { 26 | "title": "Organization roles", 27 | "type": "array", 28 | "items": { 29 | "type": "string", 30 | "enum": [ 31 | "producer", 32 | "licensor", 33 | "processor", 34 | "host" 35 | ] 36 | } 37 | }, 38 | "url": { 39 | "title": "Organization homepage", 40 | "type": "string", 41 | "format": "iri" 42 | } 43 | } 44 | } 45 | } 46 | } 47 | } -------------------------------------------------------------------------------- /sam_local_envs.json: -------------------------------------------------------------------------------- 1 | { 2 | "Parameters": { 3 | "CMR_URL": "https://cmr.earthdata.nasa.gov", 4 | "CMR_LB_URL": "https://cmr.earthdata.nasa.gov", 5 | "GRAPHQL_URL": "https://graphql.earthdata.nasa.gov/api", 6 | "NODE_ENV": "development", 7 | "STAC_VERSION": "1.0.0", 8 | "LOG_LEVEL": "1.0.0", 9 | "PAGE_SIZE": "100", 10 | "STAGE_NAME": "dev" 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /scripts/mirror-api/README.md: -------------------------------------------------------------------------------- 1 | # CMR-STAC API Mirror 2 | 3 | The script in this directory is for creating a static catalog from the CMR-STAC API. It can be run as a command line program and meant to be run on each collection separately (so that each collection could run in parallel). 4 | 5 | First, create the initial catalog which contains the root catalog which links to the child catalogs for providers. Each of the provider catalogs links to the collections for that provider. Providers with no collections will be automatically removed from the static catalog. 6 | 7 | ``` 8 | $ mirror-api.py create --url https://cmr.earthdata.nasa.gov/cloudstac --path 9 | ``` 10 | 11 | After the static catalog is created, use the CLI to mirror each collection. 12 | 13 | ``` 14 | $ mirror-api.py update --url https://cmr.earthdata.nasa.gov/cloudstac /catalog.json 15 | ``` 16 | 17 | This command will query for Items in the Collection, 1 page at a time. Once it has all the pages it will create the subcatalogs for year, month, and day, then save all files to disk. 18 | 19 | mirror-api can also take in a datetime range to limit the granules that are copied: 20 | 21 | ``` 22 | $ mirror-api.py update --url https://cmr.earthdata.nasa.gov/cloudstac /catalog.json --datetime 23 | ``` -------------------------------------------------------------------------------- /scripts/mirror-api/requirements.txt: -------------------------------------------------------------------------------- 1 | asyncio 2 | aioboto3 3 | aiofiles 4 | boto3 5 | httpx 6 | git+git://github.com/stac-utils/pystac@mah/async 7 | requests 8 | -------------------------------------------------------------------------------- /scripts/validation/README.md: -------------------------------------------------------------------------------- 1 | # CMR-STAC Validator 2 | 3 | The script in this directory will crawl the provided CMR-STAC root catalog, validating all provider endpoints, all collections within a provider, and one Item from each Collection. 4 | 5 | First, install the requirements: 6 | 7 | ``` 8 | $ pip install -r requirements.txt 9 | ``` 10 | 11 | Then run the script providing the URL to the CMR-STAC root catalog, e.g., 12 | 13 | ``` 14 | $ ./validate.py https://cmr.earthdata.nasa.gov/stac 15 | ``` 16 | 17 | The script will print the progress by printing out each CMR Provider, Collection, and Item it validates. In the case of an error it will print a message that says "INVALID", along with the URL to the invalid STAC object and the error message. 18 | 19 | The best way to use the script is to pipe the output to a log file so that it can be looked at and searched for afterwards: 20 | 21 | ``` 22 | $ ./validate.py https://cmr.earthdata.nasa.gov/stac > cmr-stac-validation.log 23 | ``` 24 | -------------------------------------------------------------------------------- /scripts/validation/requirements.txt: -------------------------------------------------------------------------------- 1 | stac-validator==2.2.0 2 | -------------------------------------------------------------------------------- /scripts/validation/validate.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import argparse 4 | import logging 5 | import os 6 | import sys 7 | from datetime import datetime 8 | 9 | import requests 10 | 11 | from stac_validator.stac_validator import StacValidate 12 | 13 | 14 | VERBOSE = True 15 | 16 | 17 | def validate(obj): 18 | validator = StacValidate() 19 | validator.validate_dict(obj) 20 | if not validator.valid: 21 | self_link = [l['href'] for l in obj['links'] if l['rel'] == 'self'][0] 22 | print(f"INVALID {self_link}: {validator.message[-1]['error_message']}") 23 | return validator.valid 24 | 25 | 26 | def crawl_collection(collection, nitems=1): 27 | print(f"{datetime.now()} Collection: {collection['id']}", flush=True) 28 | valid = validate(collection) 29 | if valid: 30 | items_link = [l['href'] for l in collection['links'] if l['rel'] == 'items'][0] + f"?limit={nitems}" 31 | items = requests.get(items_link).json() 32 | for i, item in enumerate(items['features']): 33 | print(f"{datetime.now()} Item: {item['id']}") 34 | if i == nitems: 35 | break 36 | valid = validate(item) 37 | return valid 38 | 39 | 40 | def get_collections(provider): 41 | for url in [l['href'] for l in provider['links'] if l['rel'] == 'child']: 42 | yield requests.get(url).json() 43 | next_link = [l['href'] for l in provider['links'] if l['rel'] == 'next'] 44 | if len(next_link) == 1: 45 | next_provider = requests.get(next_link[0]).json() 46 | yield from get_collections(next_provider) 47 | 48 | 49 | def crawl_provider(url, nitems=1): 50 | provider = requests.get(url).json() 51 | print(f"{datetime.now()} Provider {provider['id']}") 52 | validate(provider) 53 | count = 0 54 | for collection in get_collections(provider): 55 | crawl_collection(collection, nitems=nitems) 56 | count += 1 57 | print(f"{datetime.now()} Provider {provider.id}: {count} collections", flush=True) 58 | 59 | 60 | def read_json(url): 61 | resp = requests.get(url).json() 62 | 63 | 64 | # crawl from root catalog 65 | def crawl(url, nitems=1): 66 | cat = requests.get(url).json() 67 | for provider in [l['href'] for l in cat['links'] if l['rel'] == 'child']: 68 | crawl_provider(provider, nitems=nitems) 69 | 70 | 71 | def parse_args(args): 72 | desc = 'STAC API to Static Catalog Utility' 73 | dhf = argparse.ArgumentDefaultsHelpFormatter 74 | parser0 = argparse.ArgumentParser(description=desc) 75 | 76 | parser0.add_argument('url', help='Root API URL to copy', default=os.getenv('STAC_URL', None)) 77 | parser0.add_argument('--nitems', help='Items per collection to validate', type=int, default=1) 78 | 79 | return vars(parser0.parse_args(args)) 80 | 81 | 82 | def cli(): 83 | args = parse_args(sys.argv[1:]) 84 | 85 | url = args.pop('url') 86 | crawl(url, **args) 87 | 88 | 89 | if __name__ == "__main__": 90 | cli() -------------------------------------------------------------------------------- /src/@types/StacCatalog.d.ts: -------------------------------------------------------------------------------- 1 | /* tslint:disable */ 2 | /** 3 | * This file was automatically generated by json-schema-to-typescript. 4 | * DO NOT MODIFY IT BY HAND. Instead, modify the source JSONSchema file, 5 | * and run json-schema-to-typescript to regenerate this file. 6 | */ 7 | 8 | /** 9 | * This object represents Catalogs in a SpatioTemporal Asset Catalog. 10 | */ 11 | export type STACCatalogSpecification = STACCatalog; 12 | export type STACVersion = "1.0.0"; 13 | export type ReferenceToAJSONSchema = string; 14 | export type STACExtensions = ReferenceToAJSONSchema[]; 15 | export type TypeOfSTACEntity = "Catalog"; 16 | export type Identifier = string; 17 | export type Title = string; 18 | export type Description = string; 19 | export type LinkReference = string; 20 | export type LinkRelationType = string; 21 | export type LinkType = string; 22 | export type LinkTitle = string; 23 | export type Links = Link[]; 24 | 25 | export type STACCatalog = { 26 | stac_version: STACVersion; 27 | stac_extensions?: STACExtensions; 28 | type: TypeOfSTACEntity; 29 | id: Identifier; 30 | title?: Title; 31 | description: Description; 32 | links: Links; 33 | [k: string]: unknown; 34 | }; 35 | export type Link = { 36 | href: LinkReference; 37 | rel: LinkRelationType; 38 | type?: LinkType; 39 | title?: LinkTitle; 40 | [k: string]: unknown; 41 | }; 42 | -------------------------------------------------------------------------------- /src/__tests__/geojsonGeometry.ts: -------------------------------------------------------------------------------- 1 | export const polygon = { 2 | type: "Polygon", 3 | coordinates: [ 4 | [ 5 | [-11.4859874, 29.877883], 6 | [-12.9884453, 40.0118589], 7 | [-26.1081458, 40], 8 | [-23.0765216, 29.8680955], 9 | [-11.4859874, 29.877883], 10 | ], 11 | ], 12 | }; 13 | 14 | export const multiPolygon = { 15 | coordinates: [ 16 | [ 17 | [-21.152908074967627, 34.92093468588722], 18 | [-20.260872487986973, 33.851892421194805], 19 | [-19.765330688701084, 35.36028306350546], 20 | [-21.152908074967627, 34.92093468588722], 21 | ], 22 | [ 23 | [-20.54559139845611, 34.80198482887522], 24 | [-20.16046734162748, 34.80198482887522], 25 | [-20.16046734162748, 34.950650786370204], 26 | [-20.54559139845611, 34.950650786370204], 27 | [-20.54559139845611, 34.80198482887522], 28 | ], 29 | ], 30 | type: "MultiPolygon", 31 | }; 32 | 33 | export const linestring = { 34 | coordinates: [ 35 | [-23.35647537634162, 33.46159084335139], 36 | [-19.08936398604652, 37.17575287065196], 37 | [-17.04460596151685, 34.13571428435793], 38 | [-13.591391595092915, 36.43055248148188], 39 | ], 40 | type: "LineString", 41 | }; 42 | 43 | export const multiLinestring = { 44 | type: "MultiLineString", 45 | coordinates: [ 46 | [ 47 | [-105.021443, 39.578057], 48 | [-105.021507, 39.577809], 49 | [-105.021572, 39.577495], 50 | [-105.021572, 39.577164], 51 | [-105.021572, 39.577032], 52 | [-105.021529, 39.576784], 53 | ], 54 | [ 55 | [-105.019898, 39.574997], 56 | [-105.019598, 39.574898], 57 | [-105.019061, 39.574782], 58 | ], 59 | [ 60 | [-105.017173, 39.574402], 61 | [-105.01698, 39.574385], 62 | [-105.016636, 39.574385], 63 | [-105.016508, 39.574402], 64 | [-105.01595, 39.57427], 65 | ], 66 | [ 67 | [-105.014276, 39.573972], 68 | [-105.014126, 39.574038], 69 | [-105.013825, 39.57417], 70 | [-105.01331, 39.574452], 71 | ], 72 | ], 73 | }; 74 | 75 | export const point = { 76 | coordinates: [-19.968067373479073, 36.813771775705405], 77 | type: "Point", 78 | }; 79 | 80 | export const multiPoint = { 81 | type: "MultiPoint", 82 | coordinates: [ 83 | [-105.01621, 39.57422], 84 | [-80.666513, 35.053994], 85 | ], 86 | }; 87 | -------------------------------------------------------------------------------- /src/__tests__/items.spec.ts: -------------------------------------------------------------------------------- 1 | import request from "supertest"; 2 | import * as sinon from "sinon"; 3 | import chai from "chai"; 4 | import chaiString from "chai-string"; 5 | import sinonChai from "sinon-chai"; 6 | 7 | chai.use(chaiString); 8 | chai.use(sinonChai); 9 | 10 | const { expect } = chai; 11 | 12 | import Ajv from "ajv"; 13 | const apply = require("ajv-formats-draft2019"); 14 | const ajv = new Ajv(); 15 | apply(ajv); 16 | 17 | import { createApp } from "../app"; 18 | const app = createApp(); 19 | 20 | import * as Providers from "../domains/providers"; 21 | import * as Collections from "../domains/collections"; 22 | import * as Items from "../domains/items"; 23 | import { STACCollection } from "../@types/StacCollection"; 24 | import { STACItem } from "../@types/StacItem"; 25 | import { Link } from "../@types/StacCatalog"; 26 | 27 | const cmrCollectionsResponse = { 28 | items: [ 29 | { 30 | id: "TEST_COLL", 31 | } as STACCollection, 32 | ], 33 | cursor: "TEST_COLL_CURSOR", 34 | count: 1, 35 | }; 36 | const cmrItemsResponse = { 37 | items: [ 38 | { 39 | id: "TEST ITEM", 40 | } as STACItem, 41 | ], 42 | cursor: "TEST_GRAN_CURSOR", 43 | count: 1, 44 | }; 45 | 46 | const sandbox = sinon.createSandbox(); 47 | 48 | afterEach(() => { 49 | sandbox.restore(); 50 | }); 51 | 52 | describe("GET /PROVIDER/collections/COLLECTION/items", () => { 53 | beforeEach(() => { 54 | sandbox 55 | .stub(Providers, "getProviders") 56 | .resolves([null, [{ "provider-id": "TEST", "short-name": "TEST" }]]); 57 | sandbox.stub(Collections, "getCollections").resolves(cmrCollectionsResponse); 58 | sandbox.stub(Items, "getItems").resolves(cmrItemsResponse); 59 | }); 60 | 61 | describe("given the provider does not exist", () => { 62 | it("should return 404", async () => { 63 | const { statusCode } = await request(app).get("/BAD_PROVIDER/collections/COLLECTION/items"); 64 | expect(statusCode).to.equal(404); 65 | }); 66 | }); 67 | 68 | describe("given the collection does not exist", () => { 69 | it("should return 404", async () => { 70 | const { statusCode } = await request(app).get("/TEST/collections/COLLECTION/items"); 71 | expect(statusCode).to.equal(404); 72 | }); 73 | }); 74 | 75 | describe("given the provider and collection exist", () => { 76 | it("should return 200", async () => { 77 | const { statusCode, body } = await request(app).get("/stac/TEST/collections/TEST_COLL/items"); 78 | expect(statusCode, JSON.stringify(body, null, 2)).to.equal(200); 79 | }); 80 | }); 81 | 82 | describe("given there are query parameters on the path", () => { 83 | it("excludes query parameters", async () => { 84 | const { statusCode, body } = await request(app).get( 85 | "/stac/TEST/collections/TEST_COLL/items?cursor=pageonemillion" 86 | ); 87 | expect(statusCode, JSON.stringify(body, null, 2)).to.equal(200); 88 | 89 | const selfLink = body.features[0].links.find((lnk: Link) => lnk.rel === "self"); 90 | expect(selfLink.href).endsWith("/stac/TEST/collections/TEST_COLL/items/TEST%20ITEM"); 91 | }); 92 | 93 | it("URI encodes names", async () => { 94 | const { statusCode, body } = await request(app).get( 95 | "/stac/TEST/collections/TEST_COLL/items?cursor=pageonemillion" 96 | ); 97 | expect(statusCode, JSON.stringify(body, null, 2)).to.equal(200); 98 | 99 | const selfLink = body.features[0].links.find((lnk: Link) => lnk.rel === "self"); 100 | expect(selfLink.href).endsWith("/stac/TEST/collections/TEST_COLL/items/TEST%20ITEM"); 101 | }); 102 | }); 103 | }); 104 | 105 | describe("GET /PROVIDER/collections/COLLECTION/items/ITEM", () => { 106 | describe("given the provider does not exist", () => { 107 | it("should return 404", async () => { 108 | sandbox.stub(Providers, "getProviders").resolves([null, []]); 109 | 110 | const { statusCode, body } = await request(app).get( 111 | "/BAD_PROVIDER/collections/COLLECTION/items/ITEM" 112 | ); 113 | expect(statusCode, JSON.stringify(body, null, 2)).to.equal(404); 114 | }); 115 | }); 116 | 117 | describe("given the collection does not exist", () => { 118 | it("should return 404", async () => { 119 | sandbox 120 | .stub(Providers, "getProviders") 121 | .resolves([null, [{ "provider-id": "TEST", "short-name": "TEST" }]]); 122 | sandbox.stub(Collections, "getCollections").resolves({ cursor: null, items: [], count: 0 }); 123 | 124 | const { statusCode } = await request(app).get("/stac/TEST/collections/COLLECTION/items/ITEM"); 125 | expect(statusCode).to.equal(404); 126 | }); 127 | }); 128 | 129 | describe("given the item does not exist", () => { 130 | it("should return 404", async () => { 131 | sandbox 132 | .stub(Providers, "getProviders") 133 | .resolves([null, [{ "provider-id": "TEST", "short-name": "TEST" }]]); 134 | 135 | sandbox.stub(Collections, "getCollections").resolves({ 136 | cursor: null, 137 | items: [{ id: "TEST_COLL" } as STACCollection], 138 | count: 1, 139 | }); 140 | 141 | sandbox.stub(Items, "getItems").resolves({ 142 | cursor: null, 143 | items: [], 144 | count: 0, 145 | }); 146 | 147 | const { statusCode } = await request(app).get("/stac/TEST/collections/COLLECTION/items/ITEM"); 148 | expect(statusCode).to.equal(404); 149 | }); 150 | }); 151 | 152 | describe("given the provider, collection, and item exist", () => { 153 | it("should return 200", async () => { 154 | sandbox 155 | .stub(Providers, "getProviders") 156 | .resolves([null, [{ "provider-id": "TEST", "short-name": "TEST" }]]); 157 | sandbox.stub(Collections, "getCollections").resolves({ 158 | cursor: null, 159 | items: [{ id: "TEST_COLL" } as STACCollection], 160 | count: 1, 161 | }); 162 | sandbox.stub(Items, "getItems").resolves({ 163 | cursor: "cursor", 164 | items: [{ id: "TEST_ITEM" } as STACItem], 165 | count: 1, 166 | }); 167 | 168 | const { statusCode } = await request(app).get( 169 | "/stac/TEST/collections/TEST_COLL/items/TEST_ITEM" 170 | ); 171 | expect(statusCode).to.equal(200); 172 | }); 173 | }); 174 | }); 175 | 176 | describe("GET /ALL/collections/:collection/items/", () => { 177 | it("should return a 404", async () => { 178 | sandbox 179 | .stub(Providers, "getProviders") 180 | .resolves([null, [{ "provider-id": "TEST", "short-name": "TEST" }]]); 181 | 182 | const { statusCode, body } = await request(app).get("/stac/ALL/collections/foo/items"); 183 | 184 | expect(statusCode).to.equal(404); 185 | expect(body).to.deep.equal({ 186 | errors: ["This operation is not allowed for the ALL Catalog."], 187 | }); 188 | }); 189 | }); 190 | 191 | describe("GET /ALL/collections/:collection/items/:item", () => { 192 | it("should return a 404", async () => { 193 | sandbox 194 | .stub(Providers, "getProviders") 195 | .resolves([null, [{ "provider-id": "TEST", "short-name": "TEST" }]]); 196 | 197 | const { statusCode, body } = await request(app).get("/stac/ALL/collections/foo/items/bar"); 198 | 199 | expect(statusCode).to.equal(404); 200 | expect(body).to.deep.equal({ 201 | errors: ["This operation is not allowed for the ALL Catalog."], 202 | }); 203 | }); 204 | }); 205 | -------------------------------------------------------------------------------- /src/__tests__/providerBrowse.spec.ts: -------------------------------------------------------------------------------- 1 | import * as sinon from "sinon"; 2 | import { expect } from "chai"; 3 | import request from "supertest"; 4 | 5 | import Ajv from "ajv"; 6 | const apply = require("ajv-formats-draft2019"); 7 | const ajv = new Ajv(); 8 | apply(ajv); 9 | 10 | import { createApp } from "../app"; 11 | const app = createApp(); 12 | import * as Providers from "../domains/providers"; 13 | import * as Collections from "../domains/collections"; 14 | 15 | const emptyCollections = { count: 0, cursor: null, items: [] }; 16 | const emptyCollectionIds = { count: 0, cursor: null, items: [] }; 17 | 18 | const sandbox = sinon.createSandbox(); 19 | 20 | afterEach(() => { 21 | sandbox.restore(); 22 | }); 23 | 24 | describe("GET /:provider/collections", () => { 25 | describe("given an invalid provider", () => { 26 | it("should return a 404", async () => { 27 | sandbox 28 | .stub(Providers, "getProviders") 29 | .resolves([null, [{ "provider-id": "TEST", "short-name": "TEST" }]]); 30 | sandbox.stub(Collections, "getCollections").resolves(emptyCollections); 31 | 32 | const { statusCode, body } = await request(app).get("/stac/BAD_PROVIDER/collections"); 33 | 34 | expect(statusCode).to.equal(404); 35 | expect(body).to.deep.equal({ 36 | errors: ["Provider [BAD_PROVIDER] not found."], 37 | }); 38 | }); 39 | }); 40 | 41 | describe("given a valid provider", () => { 42 | it("should return a 200", async () => { 43 | sandbox 44 | .stub(Providers, "getProviders") 45 | .resolves([null, [{ "provider-id": "PROV", "short-name": "PROV" }]]); 46 | sandbox.stub(Collections, "getCollections").resolves(emptyCollections); 47 | 48 | const { statusCode } = await request(app).get("/stac/PROV/collections"); 49 | 50 | expect(statusCode).to.equal(200); 51 | }); 52 | 53 | describe("given an invalid datetime parameter", () => { 54 | it("should return a 400", async () => { 55 | sandbox 56 | .stub(Providers, "getProviders") 57 | .resolves([null, [{ "provider-id": "PROV", "short-name": "PROV" }]]); 58 | sandbox.stub(Collections, "getCollectionIds").resolves(emptyCollectionIds); 59 | 60 | const { statusCode, body } = await request(app) 61 | .get("/stac/PROV/collections") 62 | .query({ datetime: "1234-56-789" }); 63 | 64 | expect(statusCode).to.equal(400); 65 | expect(body).to.deep.equal({ 66 | errors: [ 67 | "Query param datetime does not match a valid date format. Please use RFC3339 or ISO8601 formatted datetime strings.", 68 | ], 69 | }); 70 | }); 71 | }); 72 | 73 | describe("given a datetime parameter", () => { 74 | describe("where a single date is provided", () => { 75 | ["2000-12-31T23:59:59.000Z"].forEach((dateString) => { 76 | it(`should handle ${dateString} and return a 200`, async () => { 77 | sandbox 78 | .stub(Providers, "getProviders") 79 | .resolves([null, [{ "provider-id": "PROV", "short-name": "PROV" }]]); 80 | sandbox.stub(Collections, "getCollections").resolves(emptyCollections); 81 | 82 | const { statusCode } = await request(app) 83 | .get(`/stac/PROV/collections`) 84 | .query({ datetime: dateString }); 85 | expect(statusCode).to.equal(200); 86 | }); 87 | }); 88 | }); 89 | 90 | describe("where an open ended date window is provided", () => { 91 | ["2000-12-31T23:59:59.000Z/..", "../2000-12-31T23:59:59.000Z"].forEach((dateString) => { 92 | it(`should handle ${dateString} and return a 200`, async () => { 93 | sandbox 94 | .stub(Providers, "getProviders") 95 | .resolves([null, [{ "provider-id": "PROV", "short-name": "PROV" }]]); 96 | sandbox.stub(Collections, "getCollections").resolves(emptyCollections); 97 | 98 | const { statusCode } = await request(app) 99 | .get(`/stac/PROV/collections`) 100 | .query({ datetime: dateString }); 101 | expect(statusCode).to.equal(200); 102 | }); 103 | }); 104 | }); 105 | 106 | describe("where a closed date window is provided", () => { 107 | ["2019-04-28T06:14:50.000Z,2020-04-28T06:14:50.000Z"].forEach((dateString) => { 108 | it(`should handle ${dateString} and return a 200`, async () => { 109 | sandbox 110 | .stub(Providers, "getProviders") 111 | .resolves([null, [{ "provider-id": "PROV", "short-name": "PROV" }]]); 112 | 113 | sandbox.stub(Collections, "getCollections").resolves(emptyCollections); 114 | 115 | const { statusCode } = await request(app) 116 | .get(`/stac/PROV/collections`) 117 | .query({ datetime: dateString }); 118 | expect(statusCode).to.equal(200); 119 | }); 120 | }); 121 | }); 122 | }); 123 | }); 124 | }); 125 | -------------------------------------------------------------------------------- /src/__tests__/rootCatalog.spec.ts: -------------------------------------------------------------------------------- 1 | import request from "supertest"; 2 | import * as sinon from "sinon"; 3 | import { expect } from "chai"; 4 | 5 | import axios, { AxiosRequestConfig } from "axios"; 6 | 7 | import CatalogSpec from "../../resources/catalog-spec/json-schema/catalog.json"; 8 | import { Link } from "../@types/StacCatalog"; 9 | 10 | import Ajv from "ajv"; 11 | const apply = require("ajv-formats-draft2019"); 12 | const ajv = new Ajv(); 13 | apply(ajv); 14 | 15 | import { createApp } from "../app"; 16 | const app = createApp(); 17 | 18 | import * as Providers from "../domains/providers"; 19 | 20 | const cmrProvidersResponse = [null, [{ "provider-id": "TEST", "short-name": "TEST" }]]; 21 | 22 | const sandbox = sinon.createSandbox(); 23 | 24 | afterEach(() => { 25 | sandbox.restore(); 26 | }); 27 | 28 | describe("GET /stac", () => { 29 | before(() => { 30 | sandbox.stub(Providers, "getProviders").resolves([null, []]); 31 | }); 32 | 33 | it("should return a catalog response", async () => { 34 | const { body } = await request(app).get("/stac"); 35 | 36 | const validate = ajv.compile(CatalogSpec); 37 | const stacSchemaValid = validate(body); 38 | 39 | expect(body).to.have.property("id", "CMR-STAC"); 40 | expect(stacSchemaValid).to.be.true; 41 | expect(body).to.have.property("links"); 42 | 43 | expect(body.links.find((l: Link) => l.rel === "self")).to.have.property( 44 | "title", 45 | `NASA CMR-STAC Root Catalog` 46 | ); 47 | expect(body.links.find((l: Link) => l.rel === "root")).to.have.property( 48 | "title", 49 | `NASA CMR-STAC Root Catalog` 50 | ); 51 | expect(body.links.find((l: Link) => l.rel === "service-doc")).to.have.property( 52 | "title", 53 | `NASA CMR-STAC Documentation` 54 | ); 55 | }); 56 | 57 | describe("given CMR responds with providers", () => { 58 | beforeEach(() => { 59 | sandbox 60 | .stub(Providers, "getProviders") 61 | .resolves([null, [{ "provider-id": "TEST", "short-name": "TEST" }]]); 62 | }); 63 | 64 | it("should have an entry for each provider in the links", async () => { 65 | const { statusCode, body } = await request(app).get("/stac"); 66 | 67 | expect(statusCode).to.equal(200); 68 | const [, expectedProviders] = cmrProvidersResponse; 69 | 70 | expectedProviders!.forEach((provider) => { 71 | const providerLink = body.links.find((l: Link) => l.href.includes(provider["provider-id"])); 72 | 73 | expect(providerLink.href).to.match(/^(http)s?:\/\/.*\w+/); 74 | expect(providerLink.rel).to.equal("child"); 75 | expect(providerLink.type).to.equal("application/json"); 76 | expect(providerLink.title).to.equal(provider["provider-id"]); 77 | }); 78 | }); 79 | 80 | it("should have an entry for each provider in the links without query parameters", async () => { 81 | const { statusCode, body } = await request(app).get("/stac?param=value"); 82 | 83 | expect(statusCode).to.equal(200); 84 | const [, expectedProviders] = cmrProvidersResponse; 85 | 86 | expectedProviders!.forEach((provider) => { 87 | const providerLink = body.links.find((l: Link) => l.href.includes(provider["provider-id"])); 88 | 89 | expect(providerLink.href).to.match(/^(http)s?:\/\/.*\w+/); 90 | expect(providerLink.href).to.not.contain("?param=value"); 91 | expect(providerLink.rel).to.equal("child"); 92 | expect(providerLink.type).to.equal("application/json"); 93 | expect(providerLink.title).to.equal(provider["provider-id"]); 94 | }); 95 | }); 96 | 97 | it("should have an entry for the 'ALL' catalog", async () => { 98 | const { statusCode, body } = await request(app).get("/stac"); 99 | 100 | expect(statusCode).to.equal(200); 101 | const [, expectedProviders] = cmrProvidersResponse; 102 | 103 | const allLink = body.links.find((l: Link) => l.title === "all"); 104 | 105 | expect(allLink.href).to.match(/^(http)s?:\/\/.*\w+/); 106 | expect(allLink.href).to.endWith("/stac/ALL"); 107 | expect(allLink.href).to.not.contain("?param=value"); 108 | expect(allLink.rel).to.equal("child"); 109 | expect(allLink.type).to.equal("application/json"); 110 | expect(allLink.title).to.equal("all"); 111 | }); 112 | }); 113 | 114 | describe("given CMR providers endpoint responds with an error", () => { 115 | it("should return a 503 response", async () => { 116 | sandbox.stub(Providers, "getProviders").resolves(["No upstream connection", null]); 117 | 118 | const { statusCode, body } = await request(app).get("/stac"); 119 | 120 | expect(statusCode, JSON.stringify(body, null, 2)).to.equal(503); 121 | expect(body).to.have.property("errors"); 122 | }); 123 | }); 124 | }); 125 | 126 | describe("/cloudstac", () => { 127 | let mockCmrHits: (s: string, c: AxiosRequestConfig | undefined) => Promise; 128 | 129 | before(() => { 130 | sandbox.stub(Providers, "getProviders").resolves([ 131 | null, 132 | [ 133 | { "short-name": "CLOUD_PROV", "provider-id": "CLOUD_PROV" }, 134 | { "short-name": "NOT_CLOUD", "provider-id": "NOT_CLOUD" }, 135 | ], 136 | ]); 137 | 138 | mockCmrHits = sandbox 139 | .stub(axios, "get") 140 | .callsFake(async (_url: string, config: AxiosRequestConfig | undefined) => 141 | config!.params!.provider!.startsWith("CLOUD") 142 | ? Promise.resolve({ headers: { "cmr-hits": "99" } }) 143 | : Promise.resolve({ headers: { "cmr-hits": "0" } }) 144 | ); 145 | }); 146 | 147 | it("only lists providers with cloud holdings", async () => { 148 | const { statusCode, body } = await request(app).get("/cloudstac"); 149 | 150 | expect(statusCode).to.equal(200); 151 | expect(body.links.find((l: { title: string }) => l.title === "CLOUD_PROV")).have.property( 152 | "title", 153 | "CLOUD_PROV" 154 | ); 155 | expect(body.links.find((l: { title: string }) => l.title === "CLOUD_PROV")).have.property( 156 | "rel", 157 | "child" 158 | ); 159 | expect(body.links.find((l: { title: string }) => l.title === "CLOUD_PROV")).have.property( 160 | "type", 161 | "application/json" 162 | ); 163 | 164 | expect(body.links.find((l: { title: string }) => l.title === "NOT_CLOUD")).to.be.undefined; 165 | 166 | expect(mockCmrHits).to.have.been.calledThrice; 167 | }); 168 | }); 169 | -------------------------------------------------------------------------------- /src/app.ts: -------------------------------------------------------------------------------- 1 | import cors from "cors"; 2 | import morgan from "morgan"; 3 | import helmet from "helmet"; 4 | import express from "express"; 5 | import compression from "compression"; 6 | import cookieParser from "cookie-parser"; 7 | import qs from "qs"; 8 | 9 | import * as dotenv from "dotenv"; 10 | dotenv.config(); 11 | 12 | import routes from "./routes"; 13 | import { notFoundHandler, errorHandler } from "./middleware"; 14 | 15 | const createApp = () => { 16 | const app = express(); 17 | 18 | // This allows the query parser to parse up to 100 coordinates without adding indices. 19 | // Anything over 100 would error out because indices are added. See CMR-10296 and 20 | // https://github.com/ljharb/qs for more details. 21 | app.set("query parser", function (str: string) { 22 | return qs.parse(str, { arrayLimit: 100 }); 23 | }); 24 | 25 | app.use(helmet()); 26 | app.use(cors()); 27 | app.use(express.json()); 28 | app.use(express.urlencoded({ extended: true })); 29 | app.use(cookieParser()); 30 | 31 | const logger = process.env.NODE_ENV === "development" ? morgan("dev") : morgan("combined"); 32 | app.use(logger); 33 | 34 | if (process.env.NODE_ENV != "development") { 35 | app.use(compression()); 36 | } 37 | 38 | app.use(/\/(cloud)?stac?/, routes); 39 | 40 | app.use(notFoundHandler); 41 | app.use(errorHandler); 42 | 43 | return app; 44 | }; 45 | 46 | export { createApp }; 47 | -------------------------------------------------------------------------------- /src/domains/__tests__/bounding-box.spec.ts: -------------------------------------------------------------------------------- 1 | import { expect } from "chai"; 2 | import { SpatialExtent } from "../../@types/StacCollection"; 3 | import { 4 | addPointsToBbox, 5 | mergeBoxes, 6 | crossesAntimeridian, 7 | reorderBoxValues, 8 | WHOLE_WORLD_BBOX_STAC, 9 | } from "../bounding-box"; 10 | 11 | describe("reorderBoxValues", () => { 12 | describe("given a box in CMR format", () => { 13 | it("should return a box in STAC format", () => { 14 | expect(reorderBoxValues([1, 2, 3, 4])).to.deep.equal([2, 1, 4, 3]); 15 | }); 16 | }); 17 | 18 | describe("given null", () => { 19 | it("should return null", () => { 20 | expect(reorderBoxValues(null)).to.be.null; 21 | }); 22 | }); 23 | 24 | describe("given a 3d bounding box", () => { 25 | it("should return a 2d bounding box", () => { 26 | expect(reorderBoxValues([1, 2, 0, 4, 5, 99])).to.be.deep.equal([2, 1, 5, 4]); 27 | }); 28 | }); 29 | }); 30 | 31 | describe("bbox", () => { 32 | const testBbox: SpatialExtent = [-10, -10, 10, 10]; 33 | const testBbox2: SpatialExtent = [-20, 10, 44, 7]; 34 | const points: { lat: number; lon: number }[] = [ 35 | { lon: 100, lat: 20 }, 36 | { lon: 5, lat: -5 }, 37 | ]; 38 | const lotsOfPoints: { lat: number; lon: number }[] = [ 39 | { lon: 100, lat: 20 }, 40 | { lon: 5, lat: -5 }, 41 | { lon: -40, lat: 73 }, 42 | ]; 43 | 44 | describe("addPointsToBbox", () => { 45 | it("should create the largest bbox", () => { 46 | expect(addPointsToBbox([...testBbox], points)).to.deep.equal([-10, -10, 100, 20]); 47 | }); 48 | 49 | it("should return the largest box possible from points", () => { 50 | expect(addPointsToBbox(null, points)).to.deep.equal([5, -5, 100, 20]); 51 | }); 52 | 53 | it("should return the biggest box possible from lotsOfPoints", () => { 54 | expect(addPointsToBbox(null, lotsOfPoints)).to.deep.equal([-40, -5, 100, 73]); 55 | }); 56 | 57 | it("should return the largest box possible from points", () => { 58 | expect(addPointsToBbox(null, points)).to.deep.equal([5, -5, 100, 20]); 59 | }); 60 | 61 | it("should return the biggest box possible from lotsOfPoints", () => { 62 | expect(addPointsToBbox(null, lotsOfPoints)).to.deep.equal([-40, -5, 100, 73]); 63 | }); 64 | 65 | it("should return the largest box", () => { 66 | expect(addPointsToBbox(testBbox, lotsOfPoints)).to.deep.equal([-40, -10, 100, 73]); 67 | }); 68 | 69 | it("should return the WHOLE_WORLD_BOX", () => { 70 | expect(addPointsToBbox(WHOLE_WORLD_BBOX_STAC, lotsOfPoints)).to.deep.equal( 71 | WHOLE_WORLD_BBOX_STAC 72 | ); 73 | }); 74 | 75 | it("should merge the boxes", () => { 76 | expect( 77 | addPointsToBbox( 78 | [1, 2, 3, 4], 79 | [ 80 | { lat: 5, lon: 6 }, 81 | { lat: 7, lon: 8 }, 82 | ] 83 | ) 84 | ).to.deep.equal([1, 2, 8, 7]); 85 | }); 86 | }); 87 | 88 | describe("mergeBoxes", () => { 89 | describe("given the first box is null", () => { 90 | it("should return the second box", () => { 91 | expect(mergeBoxes(null, testBbox)).to.deep.equal(testBbox); 92 | }); 93 | }); 94 | 95 | describe("given the second box is null", () => { 96 | it("should return the first box", () => { 97 | expect(mergeBoxes(testBbox, null)).to.deep.equal(testBbox); 98 | }); 99 | }); 100 | 101 | describe("both boxes are null", () => { 102 | it("should return null", () => { 103 | expect(mergeBoxes(null, null)).to.be.null; 104 | }); 105 | }); 106 | 107 | it("should return the WHOLE_WORLD_BBOX", () => { 108 | expect(mergeBoxes(testBbox, WHOLE_WORLD_BBOX_STAC)).to.deep.equal(WHOLE_WORLD_BBOX_STAC); 109 | }); 110 | 111 | it("should return a mix of the two testBoxes, making the largest possible box", () => { 112 | expect(mergeBoxes(testBbox, testBbox2)).to.have.ordered.members([-20, -10, 44, 10]); 113 | }); 114 | }); 115 | 116 | describe("crossesAntimeridian", () => { 117 | const amBox: SpatialExtent = [170, -10, -175, 5]; 118 | const nonAmBox: SpatialExtent = [-150, -60, -130, -40]; 119 | 120 | it("should return true if box crosses the antimeridian", () => { 121 | expect(crossesAntimeridian(amBox)).to.equal(true); 122 | }); 123 | 124 | it("should return false if box does not cross antimeridian", () => { 125 | expect(crossesAntimeridian(nonAmBox)).to.equal(false); 126 | }); 127 | }); 128 | }); 129 | -------------------------------------------------------------------------------- /src/domains/__tests__/geojson.spec.ts: -------------------------------------------------------------------------------- 1 | import { expect } from "chai"; 2 | import { cmrPolygonToGeoJsonCoordinates } from "../geojson"; 3 | 4 | describe("cmrPolygonToGeoJsonCoordinates", () => { 5 | describe("given a single polygon string", () => { 6 | it("should return a valid set of coordinates", () => { 7 | // lat1 lon1 lat2 lon2... 8 | const polygonStr = ["-10 -10 -10 10 10 10 10 -10 -10 -10"]; 9 | expect(cmrPolygonToGeoJsonCoordinates(polygonStr)).to.deep.equal([ 10 | [ 11 | [-10, -10], 12 | [10, -10], 13 | [10, 10], 14 | [-10, 10], 15 | [-10, -10], 16 | ], 17 | ]); 18 | }); 19 | }); 20 | }); 21 | -------------------------------------------------------------------------------- /src/domains/__tests__/providers.spec.ts: -------------------------------------------------------------------------------- 1 | import * as sinon from "sinon"; 2 | import chai from "chai"; 3 | import sinonChai from "sinon-chai"; 4 | 5 | chai.use(sinonChai); 6 | 7 | const { expect } = chai; 8 | 9 | import axios from "axios"; 10 | import { getProviders, getCloudProviders } from "../providers"; 11 | 12 | const sandbox = sinon.createSandbox(); 13 | 14 | afterEach(() => { 15 | sandbox.restore(); 16 | }); 17 | 18 | describe("getProvider", () => { 19 | it("should return a list of providers", async () => { 20 | sandbox.stub(axios, "get").resolves({ 21 | data: [ 22 | { 23 | "provider-id": "PROV1", 24 | "short-name": "PROV1", 25 | "cmr-only": false, 26 | small: false, 27 | consortiums: ["EOSDIS FEDEO"], 28 | }, 29 | ], 30 | }); 31 | 32 | const providers = await getProviders(); 33 | expect(providers).to.exist; 34 | }); 35 | }); 36 | 37 | describe("getCloudProviders", () => { 38 | it("should providers with cloud holdings", async () => { 39 | sandbox 40 | .stub(axios, "get") 41 | .onFirstCall() 42 | .resolves({ 43 | data: [ 44 | { 45 | "provider-id": "PROV1", 46 | "short-name": "PROV1", 47 | }, 48 | { 49 | "provider-id": "PROV2", 50 | "short-name": "PROV2", 51 | }, 52 | ], 53 | }) 54 | .onSecondCall() 55 | .resolves({ status: 200, headers: { "cmr-hits": "0" } }) 56 | .onThirdCall() 57 | .resolves({ status: 200, headers: { "cmr-hits": "1" } }); 58 | 59 | const [, providers] = await getCloudProviders(); 60 | // the calls happen in parallel and async, so we don't know which may resolve first 61 | // but we know we have only a single provider with cloud holdings. 62 | expect(providers).to.have.length(1); 63 | }); 64 | }); 65 | -------------------------------------------------------------------------------- /src/domains/bounding-box.ts: -------------------------------------------------------------------------------- 1 | import { chunk } from "lodash"; 2 | import { SpatialExtent } from "../@types/StacCollection"; 3 | import { Collection, Granule } from "../models/GraphQLModels"; 4 | 5 | const { max, min } = Math; 6 | 7 | export const WHOLE_WORLD_BBOX_CMR: SpatialExtent = [-90, -180, 90, 180]; 8 | export const WHOLE_WORLD_BBOX_STAC: SpatialExtent = [-180, -90, 180, 90]; 9 | 10 | /** 11 | * Convert box coordintes between CMR format and GeoJSON. 12 | * See https://www.rfc-editor.org/rfc/rfc7946#section-5 13 | * 14 | * @param cmrBox Bounding box in CMR order [S, E, N, W] 15 | * @returns Bounding box in GeoJSON order [E, S, W, N] 16 | */ 17 | export const reorderBoxValues = (cmrBox: SpatialExtent) => { 18 | if (!cmrBox) return null; 19 | 20 | let east, west, north, south; 21 | 22 | if (cmrBox.length === 6) { 23 | // CMR doesn't currently support 3d bounding boxes but handle it just in case 24 | // extra comma and spaces are intentional placeholders for elevation values 25 | [south, east /* elevation */, , north, west /* elevation */] = cmrBox; 26 | } 27 | 28 | if (cmrBox.length === 4) { 29 | [south, east, north, west] = cmrBox; 30 | } 31 | 32 | if (east) return [east, south, west, north]; 33 | else return null; 34 | }; 35 | 36 | /** 37 | * Determines whether a bounding box crosses over the antimeridian 38 | * 39 | * @param bbox in STAC/GeoJSON `[E, S, W, N]` format 40 | * @returns true if the box crosses the antimeridian, false otherwise 41 | */ 42 | export const crossesAntimeridian = (bbox: SpatialExtent) => { 43 | if (!bbox) return false; 44 | if (bbox.length === 6) { 45 | // 3d bbox 46 | return bbox[0] > bbox[3]; 47 | } 48 | // 2d bbox 49 | return bbox[0] > bbox[2]; 50 | }; 51 | 52 | /** 53 | * 54 | * @param bbox - An array of float coordinates in the format `[E, S, W, N]` 55 | * @param points - A single or array of coordinate objects 56 | * @returns SpatialExtent - A single array of float coordinates in the format `[E, S, W, N]` 57 | */ 58 | export const addPointsToBbox = ( 59 | bbox: SpatialExtent, 60 | points: { lat: number; lon: number }[] | { lat: number; lon: number } 61 | ) => { 62 | const pointsList = Array.isArray(points) ? [...points] : [points]; 63 | 64 | return pointsList 65 | .map(({ lon, lat }) => [lon, lat, lon, lat] as SpatialExtent) 66 | .reduce((extent: SpatialExtent, box: SpatialExtent) => { 67 | return mergeBoxes(extent, box); 68 | }, bbox); 69 | }; 70 | 71 | /** 72 | * Join two bounding boxes to create a single box that is the minimal bounding box 73 | * encompassing the two. 74 | * 75 | * @param box1 - A bounding-box array of floats in the `[W, S, E, N]` format 76 | * @param box2 - A bounding-box array of floats in the `[W, S, E, N]` format 77 | * @returns SpatialExtent - A single combined bounding-box in the `[W, S, E, N]` format 78 | */ 79 | export const mergeBoxes = (box1: SpatialExtent, box2: SpatialExtent): SpatialExtent => { 80 | if ((!box1 || box1.length !== 4) && (!box2 || box2.length !== 4)) { 81 | return null; 82 | } 83 | 84 | // only merge 2d bboxes 85 | if (!box1 || box1.length !== 4) { 86 | return box2; 87 | } 88 | 89 | if (!box2 || box2.length !== 4) { 90 | return box1; 91 | } 92 | 93 | let w; 94 | let e; 95 | if (crossesAntimeridian(box1) && crossesAntimeridian(box2)) { 96 | // both cross the antimeridian 97 | w = min(box1[0], box2[0]); 98 | e = max(box1[2], box2[2]); 99 | if (w <= e) { 100 | // if the result covers the whole world then we'll set it to that. 101 | w = -180.0; 102 | e = 180.0; 103 | } 104 | } else if (crossesAntimeridian(box1) || crossesAntimeridian(box2)) { 105 | // one crosses the antimeridian 106 | let b1; 107 | let b2; 108 | if (crossesAntimeridian(box2)) { 109 | b1 = box2; 110 | b2 = box1; 111 | } else { 112 | b1 = box1; 113 | b2 = box2; 114 | } 115 | const w1 = b1[0]; 116 | const w2 = b2[0]; 117 | const e1 = b1[2]; 118 | const e2 = b2[2]; 119 | // We could expand b1 to the east or to the west. Pick the shorter of the two 120 | const westDist = w1 - w2; 121 | const eastDist = e1 - e2; 122 | if (westDist <= 0 || eastDist >= 0) { 123 | w = w1; 124 | e = e1; 125 | } else if (eastDist < westDist) { 126 | w = w1; 127 | e = e2; 128 | } else { 129 | w = w2; 130 | e = e1; 131 | } 132 | 133 | if (w <= e) { 134 | // if the result covers the whole world then we'll set it to that. 135 | w = -180.0; 136 | e = 180.0; 137 | } 138 | } else { 139 | // neither cross the Antimeridian 140 | let b1; 141 | let b2; 142 | if (box1[0] > box2[0]) { 143 | b1 = box2; 144 | b2 = box1; 145 | } else { 146 | b1 = box1; 147 | b2 = box2; 148 | } 149 | const w1 = b1[0]; 150 | const w2 = b2[0]; 151 | const e1 = b1[2]; 152 | const e2 = b2[2]; 153 | 154 | w = min(w1, w2); 155 | e = max(e1, e2); 156 | 157 | // Check if it's shorter to cross the AM 158 | const dist = e - w; 159 | const altWest = w2; 160 | const altEast = e1; 161 | const altDist = 180.0 - altWest + (altEast + 180.0); 162 | 163 | if (altDist < dist) { 164 | w = altWest; 165 | e = altEast; 166 | } 167 | } 168 | 169 | // latitude range union 170 | const n = max(box1[3], box2[3]); 171 | const s = min(box1[1], box2[1]); 172 | 173 | return [w, s, e, n]; 174 | }; 175 | 176 | /** 177 | * Return a list of floats from a array of string values 178 | * @param ordString - String consisting of numeric values 179 | * @returns An array of float values 180 | */ 181 | export const parseOrdinateString = (ordString: string) => { 182 | return ordString.split(/\s+|,/).map(parseFloat); 183 | }; 184 | 185 | /** 186 | * 187 | * @param latLonPoints - A string of coordinate values in CMR `lat lon...` format 188 | * @returns An array of coordinates 189 | */ 190 | export const pointStringToPoints = (latLonPoints: string) => { 191 | return chunk(parseOrdinateString(latLonPoints), 2).map(([lat, lon]) => ({ 192 | lat, 193 | lon, 194 | })); 195 | }; 196 | 197 | /** 198 | * Extract a spatial extent from a CMR concept. 199 | * The returned geometry type is mutually exclusive. 200 | */ 201 | export const cmrSpatialToExtent = (concept: Collection | Granule): SpatialExtent => { 202 | const { polygons, lines, points, boxes } = concept; 203 | 204 | if (polygons) { 205 | return polygons 206 | .map((rings: string[]) => rings[0]) // outer rings only 207 | .map(pointStringToPoints) 208 | .reduce(addPointsToBbox, null); 209 | } 210 | 211 | if (points) { 212 | return points.map(pointStringToPoints).reduce(addPointsToBbox, null); 213 | } 214 | 215 | if (lines) { 216 | return lines.flatMap(pointStringToPoints).reduce(addPointsToBbox, null); 217 | } 218 | 219 | if (boxes) { 220 | return boxes 221 | .map(parseOrdinateString) 222 | .map((box) => reorderBoxValues(box as SpatialExtent)) // CMR returns box coordinates in lon/lat 223 | .reduce( 224 | (merged, current) => mergeBoxes(merged as SpatialExtent, current as SpatialExtent), 225 | null 226 | ) as SpatialExtent; 227 | } 228 | 229 | return WHOLE_WORLD_BBOX_STAC as SpatialExtent; 230 | }; 231 | -------------------------------------------------------------------------------- /src/domains/cache.ts: -------------------------------------------------------------------------------- 1 | import { Provider } from "../models/CmrModels"; 2 | import { getProvider } from "../domains/providers"; 3 | 4 | type Cache = { 5 | data: T; 6 | expiration: number; 7 | }; 8 | 9 | /** 10 | * An in-memory cache that will live for the life of a warm lambda instance. 11 | * An optional TTL in seconds value may be used to expire items sooner. 12 | */ 13 | export abstract class WarmCache { 14 | ttl: number; 15 | store: { [key: string]: Cache } = {}; 16 | 17 | constructor(ttlInSeconds = -1) { 18 | this.ttl = ttlInSeconds; 19 | } 20 | 21 | private expireItems(): void { 22 | if (this.ttl <= 0) return; 23 | 24 | const now = new Date().getTime(); 25 | for (const key in this.store) { 26 | if (this.store[key].expiration < now) { 27 | delete this.store[key]; 28 | } 29 | } 30 | } 31 | 32 | abstract fetchFromRemote(key: string): Promise<[string, null] | [null, T | null]>; 33 | 34 | public size(): number { 35 | this.expireItems(); 36 | 37 | return Object.keys(this.store).length; 38 | } 39 | 40 | public isEmpty(): boolean { 41 | return this.size() === 0; 42 | } 43 | 44 | public async get(key: string): Promise { 45 | this.expireItems(); 46 | 47 | const localData = this.store[key]; 48 | if (localData) return localData.data; 49 | 50 | const [err, remoteData] = await this.fetchFromRemote(key); 51 | 52 | if (err) return; 53 | if (remoteData) return this.set(key, remoteData); 54 | return undefined; 55 | } 56 | 57 | public getAll(): T[] { 58 | this.expireItems(); 59 | 60 | return Object.keys(this.store).reduce((acc, key) => [...acc, this.store[key].data], [] as T[]); 61 | } 62 | 63 | public set(key: string, data: T): T { 64 | this.store[key] = { 65 | data, 66 | expiration: this.ttl > 0 ? new Date().getTime() + this.ttl * 1000 : -1, 67 | } as Cache; 68 | return data; 69 | } 70 | 71 | public clear(): void { 72 | for (const key in this.store) { 73 | delete this.store[key]; 74 | } 75 | } 76 | 77 | public unset(key: string): void { 78 | delete this.store[key]; 79 | } 80 | } 81 | 82 | export class WarmProviderCache extends WarmCache { 83 | async fetchFromRemote(key: string) { 84 | return await getProvider(key); 85 | } 86 | } 87 | -------------------------------------------------------------------------------- /src/domains/cmr.ts: -------------------------------------------------------------------------------- 1 | import { IncomingHttpHeaders } from "http"; 2 | 3 | /** 4 | * Return request headers. 5 | * Can be used for GraphQL or CMR requests. 6 | */ 7 | export const cmrRequestHeaders = (headers: IncomingHttpHeaders) => { 8 | const defaultHeaders = { 9 | "client-id": headers["client-id"] ?? "cmr-stac", 10 | via: "cmr-stac", 11 | }; 12 | 13 | return { ...headers, ...defaultHeaders }; 14 | }; 15 | -------------------------------------------------------------------------------- /src/domains/geojson.ts: -------------------------------------------------------------------------------- 1 | import { GeoJSONGeometry } from "../@types/StacItem"; 2 | import { SpatialExtent } from "../@types/StacCollection"; 3 | import { chunk } from "lodash"; 4 | import { Collection, Granule } from "../models/GraphQLModels"; 5 | import { InvalidParameterError } from "../models/errors"; 6 | import { pointStringToPoints, parseOrdinateString } from "./bounding-box"; 7 | 8 | /** 9 | * Convert a list of polygon strings into a GeoJSON Polygon coordinates. 10 | * STAC GeoJSON requires LAT then LON, or easting and northing 11 | */ 12 | export const cmrPolygonToGeoJsonCoordinates = (polygons: string[]) => { 13 | return polygons 14 | .map(pointStringToPoints) 15 | .map((coords) => coords.map(({ lat, lon }) => [lon, lat])); 16 | }; 17 | 18 | /** 19 | * Convert a 2D bbox string into GeoJSON Polygon coordinates format. 20 | */ 21 | export const cmrBoxToGeoJsonPolygonCoordinates = (box: string): number[][][] => { 22 | const coordinates = parseOrdinateString(box) as SpatialExtent; 23 | 24 | if (!coordinates || coordinates.length !== 4) 25 | // a 6 coordinate box is technically valid if elevation is included but CMR only supports 2d boxes 26 | throw new Error(`Invalid bbox [${box}], exactly 4 coordinates are required.`); 27 | 28 | const [s, w, n, e] = coordinates; 29 | return [ 30 | [ 31 | [w, s], 32 | [e, s], 33 | [e, n], 34 | [w, n], 35 | [w, s], 36 | ], 37 | ]; 38 | }; 39 | 40 | /** 41 | * Convert an array of polygon strings into a GeoJSON geometry. 42 | */ 43 | export const polygonToGeoJSON = (polygons: string[][]): GeoJSONGeometry | null => { 44 | const geometries = polygons.map(cmrPolygonToGeoJsonCoordinates); 45 | 46 | if (geometries.length > 1) { 47 | return { 48 | type: "MultiPolygon", 49 | coordinates: geometries, 50 | } as GeoJSONGeometry; 51 | } 52 | 53 | if (geometries.length === 1) { 54 | return { 55 | type: "Polygon", 56 | coordinates: geometries[0], 57 | } as GeoJSONGeometry; 58 | } 59 | 60 | return null; 61 | }; 62 | 63 | /** 64 | * Convert an array of boxes to a GeoJSON Geometry. 65 | */ 66 | export const boxToGeoJSON = (boxes: string[]): GeoJSONGeometry | null => { 67 | const geometries = boxes.map(cmrBoxToGeoJsonPolygonCoordinates); 68 | 69 | if (geometries.length > 1) { 70 | return { 71 | type: "MultiPolygon", 72 | coordinates: geometries, 73 | } as GeoJSONGeometry; 74 | } 75 | 76 | if (geometries.length === 1) { 77 | return { 78 | type: "Polygon", 79 | coordinates: geometries[0], 80 | } as GeoJSONGeometry; 81 | } 82 | return null; 83 | }; 84 | 85 | /** 86 | * Convert an array of points into a GeoJSON Geometry. 87 | */ 88 | export const pointsToGeoJSON = (points: string[]): GeoJSONGeometry | null => { 89 | const geometries = points 90 | .map(parseOrdinateString) 91 | .flatMap((points) => chunk(points, 2).map(([lat, lon]) => [lon, lat])); 92 | 93 | if (geometries.length > 1) { 94 | return { 95 | type: "MultiPoint", 96 | coordinates: geometries, 97 | } as GeoJSONGeometry; 98 | } 99 | 100 | if (geometries.length === 1) { 101 | return { 102 | type: "Point", 103 | coordinates: geometries[0], 104 | } as GeoJSONGeometry; 105 | } 106 | 107 | return null; 108 | }; 109 | 110 | /** 111 | * Convert an array of lines to GeoJSON Geometry. 112 | */ 113 | export const linesToGeoJSON = (lines: string[]): GeoJSONGeometry | null => { 114 | const geometry = lines 115 | .map(parseOrdinateString) 116 | .map((line) => chunk(line, 2).map(([lat, lon]) => [lon, lat])); 117 | 118 | if (geometry.length > 1) { 119 | return { 120 | type: "MultiLineString", 121 | coordinates: geometry, 122 | } as GeoJSONGeometry; 123 | } 124 | 125 | if (geometry.length === 1) { 126 | return { 127 | type: "LineString", 128 | coordinates: geometry[0], 129 | } as GeoJSONGeometry; 130 | } 131 | return null; 132 | }; 133 | 134 | export const stringToGeoJSON = (geometry: string): GeoJSONGeometry => { 135 | try { 136 | const geoJson = JSON.parse(geometry); 137 | if (!geoJson.type) { 138 | console.info(`Missing 'type' from GeoJSON geometry [${geometry}]`); 139 | throw new InvalidParameterError( 140 | "Invalid intersects parameter detected. Missing ['type']. Please verify all intersects are a valid GeoJSON geometry." 141 | ); 142 | } 143 | 144 | if (!geoJson.coordinates) { 145 | console.info(`Missing 'coordinates' from GeoJSON geometry [${geometry}]`); 146 | throw new InvalidParameterError( 147 | "Invalid intersects parameter detected. Missing ['coordinates'] Please verify all intersects are a valid GeoJSON geometry." 148 | ); 149 | } 150 | return geoJson as GeoJSONGeometry; 151 | } catch (err) { 152 | console.info(`Failed to parse GeoJSON [${geometry}] : ${(err as Error).message}`); 153 | throw new InvalidParameterError( 154 | "Invalid intersects parameter detected. Unable to parse. Please verify it is a valid GeoJSON geometry." 155 | ); 156 | } 157 | }; 158 | 159 | /** 160 | * Return a GeoJSON geometry from CMR Spatial data. 161 | * Null returned if no applicable geometry data exists. 162 | * 163 | * CMR/GraphQL returns geometry strings in `lat1 lon1 lat2 lon2...` format 164 | * GeoJSON requires LAT then LON, or easting then northing 165 | * 166 | * @see https://www.rfc-editor.org/rfc/rfc7946#section-3.1.1 167 | */ 168 | export const cmrSpatialToGeoJSONGeometry = ( 169 | cmrData: Granule | Collection 170 | ): GeoJSONGeometry | null => { 171 | const { boxes, lines, polygons, points } = cmrData; 172 | 173 | if (polygons) return polygonToGeoJSON(polygons); 174 | if (boxes) return boxToGeoJSON(boxes); 175 | if (points) return pointsToGeoJSON(points); 176 | if (lines) return linesToGeoJSON(lines); 177 | return null; 178 | }; 179 | -------------------------------------------------------------------------------- /src/domains/items.ts: -------------------------------------------------------------------------------- 1 | import { Request } from "express"; 2 | import { IncomingHttpHeaders } from "http"; 3 | import { gql } from "graphql-request"; 4 | 5 | import { AssetLinks, STACItem } from "../@types/StacItem"; 6 | import { Granule, GranulesInput, GraphQLHandler, GraphQLResults } from "../models/GraphQLModels"; 7 | import { StacExtension, StacExtensions } from "../models/StacModels"; 8 | 9 | import { cmrSpatialToExtent } from "./bounding-box"; 10 | import { cmrSpatialToGeoJSONGeometry } from "./geojson"; 11 | import { mergeMaybe, stacContext } from "../utils"; 12 | import { extractAssets, paginateQuery } from "./stac"; 13 | import { ItemNotFound } from "../models/errors"; 14 | 15 | const STAC_VERSION = process.env.STAC_VERSION ?? "1.0.0"; 16 | const CMR_URL = process.env.CMR_URL; 17 | 18 | const granulesQuery = gql` 19 | query getGranules($params: GranulesInput) { 20 | granules(params: $params) { 21 | count 22 | cursor 23 | items { 24 | title 25 | conceptId 26 | collection { 27 | conceptId 28 | entryId 29 | title 30 | } 31 | cloudCover 32 | lines 33 | boxes 34 | polygons 35 | points 36 | links 37 | timeStart 38 | timeEnd 39 | relatedUrls 40 | } 41 | } 42 | } 43 | `; 44 | 45 | const granuleIdsQuery = gql` 46 | query getGranules($params: GranulesInput) { 47 | granules(params: $params) { 48 | count 49 | cursor 50 | items { 51 | conceptId 52 | title 53 | } 54 | } 55 | } 56 | `; 57 | 58 | const filterUnique = (val: string, idx: number, arr: string[]) => arr.indexOf(val) === idx; 59 | 60 | /** 61 | * Return the cloudCover extension schema and properties for a granule. 62 | */ 63 | const cloudCoverExtension = (granule: Granule) => { 64 | // purposely using == 65 | if (granule.cloudCover == null) return; 66 | return { 67 | extension: "https://stac-extensions.github.io/eo/v1.0.0/schema.json", 68 | properties: { "eo:cloud_cover": granule.cloudCover }, 69 | }; 70 | }; 71 | 72 | /** 73 | * Returns the self-links for a STACItem. 74 | * 75 | * @param root URL root of the STAC catalog. 76 | * @param providerId Provider ID 77 | * @param item The STAC Item 78 | */ 79 | const selfLinks = (req: Request, item: STACItem) => { 80 | const { provider } = req; 81 | const { stacRoot } = stacContext(req); 82 | 83 | if (!provider) { 84 | throw new ItemNotFound("No provider detected in path."); 85 | } 86 | 87 | const { id, collection } = item; 88 | 89 | const providerId = provider["provider-id"]; 90 | const itemId = encodeURIComponent(id); 91 | const collectionId = encodeURIComponent(collection as string); 92 | 93 | return [ 94 | { 95 | rel: "self", 96 | href: `${stacRoot}/${providerId}/collections/${collectionId}/items/${itemId}`, 97 | type: "application/geo+json", 98 | }, 99 | { 100 | rel: "parent", 101 | href: `${stacRoot}/${providerId}/collections/${collectionId}/`, 102 | type: "application/geo+json", 103 | }, 104 | { 105 | rel: "collection", 106 | href: `${stacRoot}/${providerId}/collections/${collectionId}/`, 107 | type: "application/geo+json", 108 | }, 109 | { 110 | rel: "root", 111 | href: `${stacRoot}`, 112 | type: "application/json", 113 | }, 114 | { 115 | rel: "provider", 116 | href: `${stacRoot}/${providerId}`, 117 | type: "application/json", 118 | }, 119 | ]; 120 | }; 121 | 122 | /** 123 | * Build a list of STAC extensions and properties for the given granule. 124 | * 125 | * Extension builder functions must take a granule as input and 126 | * should return an array with the Schema of the extension 127 | * as the first element, and the associated property map as the second. 128 | * 129 | * @example 130 | * deriveExtensions(granule, [cloudCoverBldr, projectionBldr]) => 131 | * [ 132 | * ["https://stac-extensions.github.io/eo/v1.0.0/schema.json", 133 | * "https://stac-extensions.github.io/projection/v1.0.0/schema.json"], 134 | * { "eo:cloud_cover": 50, 135 | * "proj:epsg" : 32659 136 | * "proj:shape" : [ 5558, 9559 ]} 137 | * ] 138 | */ 139 | const deriveExtensions = ( 140 | granule: Granule, 141 | extensionBuilders: ((g: Granule) => StacExtension | undefined)[] 142 | ): StacExtensions => { 143 | return extensionBuilders.reduce( 144 | ({ extensions, properties }, extBldr) => { 145 | const ext = extBldr(granule); 146 | if (!ext) return { extensions, properties }; 147 | 148 | return { 149 | extensions: [...extensions, ext.extension].filter(filterUnique), 150 | properties: mergeMaybe(properties, ext.properties), 151 | }; 152 | }, 153 | { extensions: [], properties: {} } as StacExtensions 154 | ); 155 | }; 156 | 157 | /** 158 | * Convert a granule to a STAC Item. 159 | */ 160 | export const granuleToStac = (granule: Granule): STACItem => { 161 | if (!granule.collection) { 162 | throw new Error(`Cannot have a granule without a collection, [${granule.conceptId}]`); 163 | } 164 | 165 | const { extensions, properties: extensionProperties } = deriveExtensions(granule, [ 166 | cloudCoverExtension, 167 | ]); 168 | 169 | const properties: { [key: string]: string } = mergeMaybe( 170 | {}, 171 | { 172 | datetime: granule.timeStart, 173 | ...extensionProperties, 174 | } 175 | ); 176 | 177 | if (granule.timeStart && granule.timeEnd) { 178 | // BOTH are required if available 179 | properties.start_datetime = granule.timeStart; 180 | properties.end_datetime = granule.timeEnd; 181 | } 182 | 183 | const geometry = cmrSpatialToGeoJSONGeometry(granule); 184 | const bbox = cmrSpatialToExtent(granule); 185 | const assets: AssetLinks = extractAssets(granule); 186 | 187 | const links = [ 188 | { 189 | rel: "via", 190 | href: `${CMR_URL}/search/concepts/${granule.conceptId}.json`, 191 | title: "CMR JSON metadata for item", 192 | type: "application/json", 193 | }, 194 | { 195 | rel: "via", 196 | href: `${CMR_URL}/search/concepts/${granule.conceptId}.umm_json`, 197 | title: "CMR UMM_JSON metadata for item", 198 | type: "application/vnd.nasa.cmr.umm+json", 199 | }, 200 | ]; 201 | 202 | // core STACItem 203 | const item = { 204 | type: "Feature", 205 | id: granule.title, 206 | stac_version: STAC_VERSION, 207 | stac_extensions: extensions, 208 | properties, 209 | geometry, 210 | bbox, 211 | assets, 212 | links, 213 | } as STACItem; 214 | 215 | return { 216 | ...item, 217 | collection: granule.collection.entryId, 218 | }; 219 | }; 220 | 221 | const granulesQueryHandler: GraphQLHandler = (response: unknown) => { 222 | try { 223 | const { 224 | granules: { count, items, cursor }, 225 | } = response as { granules: GraphQLResults }; 226 | 227 | return [ 228 | null, 229 | { 230 | count, 231 | cursor, 232 | items: (items as Granule[]).map(granuleToStac), 233 | }, 234 | ]; 235 | } catch (err) { 236 | return [(err as Error).message, null]; 237 | } 238 | }; 239 | 240 | /** 241 | * Return an object containing list of STAC Items matching the given query. 242 | */ 243 | export const getItems = async ( 244 | params: GranulesInput, 245 | opts: { 246 | headers?: IncomingHttpHeaders; 247 | } = {} 248 | ): Promise<{ 249 | count: number; 250 | cursor: string | null; 251 | items: STACItem[]; 252 | }> => { 253 | const { count, cursor, items } = await paginateQuery( 254 | granulesQuery, 255 | params, 256 | opts, 257 | granulesQueryHandler 258 | ); 259 | return { count, cursor, items: items as STACItem[] }; 260 | }; 261 | 262 | const granuleIdsQueryHandler: GraphQLHandler = (response: unknown) => { 263 | try { 264 | const { 265 | granules: { count, items, cursor }, 266 | } = response as { granules: GraphQLResults }; 267 | 268 | return [ 269 | null, 270 | { 271 | count, 272 | cursor, 273 | items: items as Granule[], 274 | }, 275 | ]; 276 | } catch (err) { 277 | return [(err as Error).message, null]; 278 | } 279 | }; 280 | 281 | /** 282 | * Return an object containing list of STAC Items Ids matching the given query. 283 | */ 284 | export const getItemIds = async ( 285 | params: GranulesInput, 286 | opts: { 287 | headers?: { "client-id"?: string; authorization?: string }; 288 | } = {} 289 | ): Promise<{ 290 | count: number; 291 | cursor: string | null; 292 | ids: string[]; 293 | }> => { 294 | const { count, cursor, items } = await paginateQuery( 295 | granuleIdsQuery, 296 | params, 297 | opts, 298 | granuleIdsQueryHandler 299 | ); 300 | 301 | return { count, cursor, ids: items as string[] }; 302 | }; 303 | 304 | /** 305 | * Add or append self links to an item. 306 | */ 307 | export const addProviderLinks = (req: Request, item: STACItem): STACItem => { 308 | const providerLinks = selfLinks(req, item); 309 | 310 | item.links = [...providerLinks, ...(item.links ?? [])]; 311 | 312 | return item; 313 | }; 314 | -------------------------------------------------------------------------------- /src/domains/providers.ts: -------------------------------------------------------------------------------- 1 | import axios from "axios"; 2 | 3 | import { Provider } from "../models/CmrModels"; 4 | import { mergeMaybe } from "../utils"; 5 | 6 | const CMR_LB_URL = process.env.CMR_LB_URL; 7 | const CMR_LB_INGEST = `${CMR_LB_URL}/ingest`; 8 | const CMR_LB_SEARCH = `${CMR_LB_URL}/search`; 9 | const CMR_LB_SEARCH_COLLECTIONS = `${CMR_LB_SEARCH}/collections`; 10 | 11 | export const ALL_PROVIDER = "ALL"; 12 | export const ALL_PROVIDERS = { 13 | "provider-id": ALL_PROVIDER.toUpperCase(), 14 | "short-name": ALL_PROVIDER.toLowerCase(), 15 | }; 16 | 17 | export const conformance = [ 18 | "https://api.stacspec.org/v1.0.0-rc.2/core", 19 | "https://api.stacspec.org/v1.0.0-rc.2/item-search", 20 | "https://api.stacspec.org/v1.0.0-rc.2/ogcapi-features", 21 | "https://api.stacspec.org/v1.0.0-rc.2/item-search#fields", 22 | "https://api.stacspec.org/v1.0.0-rc.2/item-search#features", 23 | "https://api.stacspec.org/v1.0.0-rc.2/item-search#query", 24 | "https://api.stacspec.org/v1.0.0-rc.2/item-search#sort", 25 | "https://api.stacspec.org/v1.0.0-rc.2/item-search#context", 26 | "http://www.opengis.net/spec/ogcapi-features-1/1.0/conf/core", 27 | "http://www.opengis.net/spec/ogcapi-features-1/1.0/conf/oas30", 28 | "http://www.opengis.net/spec/ogcapi-features-1/1.0/conf/geojson", 29 | "https://api.stacspec.org/v1.0.0-rc.2/collection-search", 30 | "https://api.stacspec.org/v1.0.0-rc.2/collection-search#free-text", 31 | "https://api.stacspec.org/v1.0.0-rc.2/collection-search#sort", 32 | "http://www.opengis.net/spec/ogcapi-common-1/1.0/conf/simple-query", 33 | ]; 34 | 35 | /** 36 | * Return an array of providers found in CMR. 37 | */ 38 | export const getProviders = async (): Promise<[string, null] | [null, Provider[]]> => { 39 | try { 40 | console.debug(`GET ${CMR_LB_INGEST}/providers`); 41 | const { data: providers } = await axios.get(`${CMR_LB_INGEST}/providers`); 42 | return [null, providers]; 43 | } catch (err) { 44 | console.error("A problem occurred fetching providers", err); 45 | return [(err as Error).message, null]; 46 | } 47 | }; 48 | 49 | /** 50 | * Return provider with `providerId`. 51 | */ 52 | export const getProvider = async ( 53 | providerId: string 54 | ): Promise<[string, null] | [null, Provider | null]> => { 55 | const [errs, providers] = await getProviders(); 56 | 57 | if (errs) { 58 | return [errs as string, null]; 59 | } 60 | providers?.push(ALL_PROVIDERS); 61 | return [ 62 | null, 63 | (providers ?? []).find((provider) => provider["provider-id"] === providerId) ?? null, 64 | ]; 65 | }; 66 | 67 | /** 68 | * Return providers with cloud_hosted collections. 69 | * If given an array of providers, it will return those with cloud_hosted collections. 70 | */ 71 | export const getCloudProviders = async ( 72 | providerCandidates?: Provider[], 73 | opts: { [key: string]: unknown } = {} 74 | ): Promise<[null | string[], Provider[]]> => { 75 | const [err, candidates] = providerCandidates ? [null, providerCandidates] : await getProviders(); 76 | 77 | if (err) { 78 | return [[err], []]; 79 | } 80 | 81 | const { authorization } = opts; 82 | 83 | const searchErrs: string[] = []; 84 | const cloudProviders: Provider[] = []; 85 | 86 | await Promise.all( 87 | (candidates ?? []).map(async (provider) => { 88 | try { 89 | console.debug(`GET ${CMR_LB_SEARCH_COLLECTIONS}`); 90 | const { headers } = await axios.get(CMR_LB_SEARCH_COLLECTIONS, { 91 | headers: mergeMaybe({}, { authorization }), 92 | params: { provider: provider["short-name"], cloud_hosted: true }, 93 | }); 94 | 95 | if (headers["cmr-hits"] !== "0") { 96 | cloudProviders.push(provider); 97 | } 98 | } catch (e) { 99 | console.error( 100 | `A problem occurred checking provider [${provider["provider-id"]}] for cloud holdings.`, 101 | e 102 | ); 103 | searchErrs.push((e as Error).message); 104 | } 105 | }) 106 | ); 107 | return [searchErrs.length ? searchErrs : null, cloudProviders]; 108 | }; 109 | -------------------------------------------------------------------------------- /src/handler.ts: -------------------------------------------------------------------------------- 1 | import "source-map-support/register"; 2 | import serverlessExpress from "@vendia/serverless-express"; 3 | import { createApp } from "./app"; 4 | 5 | const app = createApp(); 6 | const handler = serverlessExpress({ app }); 7 | export default handler; 8 | -------------------------------------------------------------------------------- /src/index.d.ts: -------------------------------------------------------------------------------- 1 | import { WarmProviderCache } from "./domains/cache"; 2 | import { Provider } from "./models/CmrModels"; 3 | import { STACCollection } from "./@types/StacCollection"; 4 | 5 | /** 6 | * Extend the Express Request object to allow for re-use of query results. 7 | */ 8 | declare global { 9 | namespace Express { 10 | // eslint-disable-next-line @typescript-eslint/consistent-type-definitions 11 | export interface Request { 12 | cache?: { 13 | providers: WarmProviderCache; 14 | cloudProviders: WarmProviderCache; 15 | }; 16 | provider?: Provider; 17 | collection?: STACCollection; 18 | } 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /src/middleware/__tests__/index.spec.ts: -------------------------------------------------------------------------------- 1 | import chai from "chai"; 2 | const { expect } = chai; 3 | 4 | import { validBbox } from "../index"; 5 | 6 | describe("validBBOX", () => { 7 | describe("when bbox is a string", () => { 8 | it("returns a valid bbox", async () => { 9 | const bbox = "-122.09,39.89,-122.03,39.92"; 10 | expect(validBbox(bbox)).to.equal(true); 11 | }); 12 | }); 13 | 14 | describe("when bbox is a string array", () => { 15 | it("returns a valid bbox", async () => { 16 | const bbox = ["-122.09", "39.89", "-122.03", "39.92"]; 17 | expect(validBbox(bbox)).to.equal(true); 18 | }); 19 | }); 20 | 21 | describe("when bbox is an invalid string array with negative numbers", () => { 22 | it("returns an invalid bbox", async () => { 23 | const bbox = ["-122.03", "39.89", "-122.09", "39.92"]; 24 | expect(validBbox(bbox)).to.equal(false); 25 | }); 26 | }); 27 | 28 | describe("when bbox is a number array", () => { 29 | it("returns a valid bbox", async () => { 30 | const bbox = [-122.09, 39.89, -122.03, 39.92]; 31 | expect(validBbox(bbox)).to.equal(true); 32 | }); 33 | }); 34 | }); 35 | -------------------------------------------------------------------------------- /src/models/CmrModels.ts: -------------------------------------------------------------------------------- 1 | export type Provider = { 2 | "provider-id": string; 3 | "short-name": string; 4 | "cmr-only"?: boolean; 5 | small?: boolean; 6 | consortiums?: string; 7 | }; 8 | -------------------------------------------------------------------------------- /src/models/GraphQLModels.ts: -------------------------------------------------------------------------------- 1 | export type GraphQLInput = { 2 | // pagination 3 | limit?: number; 4 | cursor?: string; 5 | 6 | // sorting 7 | sortKey?: string | string[]; 8 | }; 9 | 10 | export type GranulesInput = GraphQLInput & { 11 | // filtering 12 | provider?: string; 13 | 14 | // ids 15 | conceptIds?: string[]; 16 | readableGranuleName?: string[]; 17 | 18 | // collections 19 | collectionConceptIds?: string[]; 20 | 21 | // bbox 22 | boundingBox?: string; 23 | 24 | // intersects 25 | polygon?: string[]; 26 | line?: string[]; 27 | point?: string[]; 28 | 29 | // datetime 30 | temporal?: string; 31 | 32 | // extensions 33 | cloudCover?: { 34 | min?: number; 35 | max?: number; 36 | }; 37 | }; 38 | 39 | export type CollectionsInput = GraphQLInput & { 40 | // filtering 41 | cloudHosted?: boolean; 42 | conceptIds?: string[]; 43 | entryId?: string[]; 44 | hasGranules?: boolean; 45 | includeFacets?: string; 46 | keyword?: string; 47 | providers?: string[]; 48 | }; 49 | 50 | export type FacetFilter = { 51 | title: string; 52 | type: string; 53 | applied: boolean; 54 | count: number; 55 | links: { 56 | apply?: string; 57 | remove?: string; 58 | }; 59 | hasChildren: boolean; 60 | children?: FacetFilter[]; 61 | }; 62 | 63 | export type FacetGroup = { 64 | title: string; 65 | type: string; 66 | hasChildren: boolean; 67 | children: FacetFilter[] | FacetGroup[]; 68 | }; 69 | 70 | export enum RelatedUrlType { 71 | DATA_SET_LANDING_PAGE = "DATA SET LANDING PAGE", 72 | VIEW_RELATED_INFORMATION = "VIEW RELATED INFORMATION", 73 | GET_DATA = "GET DATA", 74 | GET_SERVICE = "GET SERVICE", 75 | GET_RELATED_VISUALIZATION = "GET RELATED VISUALIZATION", 76 | PROJECT_HOME_PAGE = "PROJECT HOME PAGE", 77 | GET_CAPABILITIES = "GET CAPABILITIES", 78 | EXTENDED_METADATA = "EXTENDED METADATA", 79 | THUMBNAIL = "Thumbnail", 80 | } 81 | 82 | export enum UrlContentType { 83 | VISUALIZATION_URL = "VisualizationURL", 84 | DISTRIBUTION_URL = "DistributionURL", 85 | PUBLICATION_URL = "PublicationURL", 86 | COLLECTION_URL = "CollectionURL", 87 | } 88 | 89 | export enum RelatedUrlSubType { 90 | HOW_TO = "HOW-TO", 91 | GENERAL_DOCUMENTATION = "GENERAL DOCUMENTATION", 92 | DATA_TREE = "DATA TREE", 93 | EARTHDATA_SEARCH = "Earthdata Search", 94 | GIOVANNI = "GIOVANNI", 95 | STAC = "STAC", 96 | } 97 | 98 | export type UseConstraints = 99 | | { description: string; licenseUrl?: string; freeAndOpenData?: boolean } 100 | | { licenseText: string } 101 | | { 102 | licenseUrl: { 103 | linkage: string; 104 | name: string; 105 | description: string; 106 | mimeType: string; 107 | }; 108 | }; 109 | 110 | export type RelatedUrls = { 111 | description: string; 112 | urlContentType: UrlContentType; 113 | url: string; 114 | subtype?: RelatedUrlSubType | string; 115 | type?: RelatedUrlType | string; 116 | [key: string]: unknown; 117 | getData?: { 118 | format: string; 119 | mimeType: string; 120 | size: number; 121 | unit: string; 122 | checksum?: string; 123 | fees?: string; 124 | }; 125 | getService?: { 126 | format: string; 127 | mimeType: string; 128 | protocol: string; 129 | fullName: string; 130 | dataId: string; 131 | dataType: string; 132 | uri: string[]; 133 | }; 134 | }[]; 135 | 136 | export type Instrument = { 137 | shortName: string; 138 | longName: string; 139 | }; 140 | 141 | export type Platform = { 142 | type: string; 143 | shortName: string; 144 | longName: string; 145 | instruments?: Instrument[]; 146 | }; 147 | 148 | export type ScienceKeywords = { 149 | category: string; 150 | topic: string; 151 | term: string; 152 | variableLevel1?: string; 153 | variableLevel2?: string; 154 | variableLevel3?: string; 155 | detailedVariable?: string; 156 | }; 157 | 158 | export type DirectDistributionInformation = { 159 | region: string; 160 | s3BucketAndObjectPrefixNames: string[]; 161 | s3CredentialsApiEndpoint: string; 162 | s3CredentialsApiDocumentationUrl: string; 163 | }; 164 | 165 | export type CollectionBase = { 166 | conceptId: string; 167 | entryId: string; 168 | title: string; 169 | }; 170 | 171 | export type Collection = CollectionBase & { 172 | provider: string; 173 | description: string; 174 | 175 | polygons: string[][] | null; 176 | lines: string[] | null; 177 | boxes: string[] | null; 178 | points: string[] | null; 179 | 180 | timeStart: string | null; 181 | timeEnd: string | null; 182 | useConstraints: UseConstraints | null; 183 | relatedUrls: RelatedUrls | null; 184 | directDistributionInformation: DirectDistributionInformation | null; 185 | 186 | scienceKeywords: ScienceKeywords[]; 187 | platforms: Platform[]; 188 | }; 189 | 190 | export type GranuleBase = { 191 | title: string | null; 192 | conceptId: string | null; 193 | }; 194 | 195 | export type Granule = GranuleBase & { 196 | collection: CollectionBase | null; 197 | cloudCover: number | null; 198 | 199 | polygons: string[][] | null; 200 | lines: string[] | null; 201 | points: string[] | null; 202 | boxes: string[] | null; 203 | 204 | timeStart: string | null; 205 | timeEnd: string | null; 206 | relatedUrls: RelatedUrls | null; 207 | }; 208 | 209 | export type GraphQLHandlerResponse = 210 | | [error: string, data: null] 211 | | [ 212 | error: null, 213 | data: { 214 | count: number; 215 | cursor: string | null; 216 | items: object[]; 217 | } 218 | ]; 219 | 220 | export type GraphQLHandler = (response: unknown) => GraphQLHandlerResponse; 221 | 222 | export type GraphQLResults = { 223 | count: number; 224 | items: unknown[]; 225 | cursor: string | null; 226 | }; 227 | -------------------------------------------------------------------------------- /src/models/StacModels.ts: -------------------------------------------------------------------------------- 1 | import { GeoJSONGeometry, GeoJSONGeometryCollection } from "../@types/StacItem"; 2 | 3 | export type PropertyQuery = { 4 | lt?: number; 5 | lte?: number; 6 | gt?: number; 7 | gte?: number; 8 | // TODO: Add full support for STAC property query extension, see CMR-9010 9 | }; 10 | 11 | export type SortObject = { 12 | field: string; 13 | direction: "asc" | "desc"; 14 | }; 15 | 16 | export type StacQuery = { 17 | cursor?: string; 18 | sortby?: string | SortObject[]; 19 | limit?: string; 20 | bbox?: string; 21 | datetime?: string; 22 | intersects?: GeoJSONGeometry | GeoJSONGeometryCollection | string | string[]; 23 | ids?: string | string[]; 24 | collections?: string | string[]; 25 | query?: { 26 | [key: string]: PropertyQuery; 27 | }; 28 | q?: string; //query for free text search 29 | }; 30 | 31 | export type StacExtension = { 32 | extension: string; 33 | properties: { [key: string]: unknown }; 34 | }; 35 | 36 | export type StacExtensions = { 37 | extensions: string[]; 38 | properties: { [key: string]: unknown }; 39 | }; 40 | -------------------------------------------------------------------------------- /src/models/errors.ts: -------------------------------------------------------------------------------- 1 | import { Request, Response, NextFunction } from "express"; 2 | 3 | /** 4 | * Mixin helper 5 | * see https://www.digitalocean.com/community/tutorials/typescript-mixins 6 | */ 7 | // eslint-disable-next-line @typescript-eslint/no-explicit-any 8 | const applyMixins = (derivedCtor: any, constructors: any[]) => { 9 | constructors.forEach((baseCtor) => { 10 | Object.getOwnPropertyNames(baseCtor.prototype).forEach((name) => { 11 | Object.defineProperty( 12 | derivedCtor.prototype, 13 | name, 14 | Object.getOwnPropertyDescriptor(baseCtor.prototype, name) || Object.create(null) 15 | ); 16 | }); 17 | }); 18 | }; 19 | 20 | export abstract class ErrorHandler { 21 | abstract handle(err: Error, req: Request, res: Response, next: NextFunction): void; 22 | } 23 | 24 | export class ItemNotFound extends Error { 25 | __proto__ = Error; 26 | 27 | constructor(m: string) { 28 | super(m); 29 | Object.setPrototypeOf(this, ItemNotFound.prototype); 30 | } 31 | } 32 | 33 | class ItemNotFoundHandler extends ErrorHandler { 34 | handle(err: Error, _req: Request, res: Response, _next: NextFunction): void { 35 | res.status(404).json({ errors: [err.message] }); 36 | } 37 | } 38 | 39 | export class InvalidParameterError extends Error { 40 | __proto__ = Error; 41 | 42 | constructor(m: string) { 43 | super(m); 44 | Object.setPrototypeOf(this, InvalidParameterError.prototype); 45 | } 46 | } 47 | 48 | class InvalidParameterErrorHandler extends ErrorHandler { 49 | handle(err: Error, _req: Request, res: Response, _next: NextFunction): void { 50 | res.status(400).json({ errors: [err.message] }); 51 | } 52 | } 53 | 54 | export class ServiceUnavailableError extends Error { 55 | __proto__ = Error; 56 | 57 | constructor(m: string) { 58 | super(m); 59 | Object.setPrototypeOf(this, ServiceUnavailableError.prototype); 60 | } 61 | } 62 | 63 | class ServiceUnavailableErrorHandler extends ErrorHandler { 64 | handle(err: Error, _req: Request, res: Response, _next: NextFunction): void { 65 | res.status(503).json({ errors: [err.message] }); 66 | } 67 | } 68 | 69 | applyMixins(ItemNotFound, [ItemNotFoundHandler]); 70 | applyMixins(InvalidParameterError, [InvalidParameterErrorHandler]); 71 | applyMixins(ServiceUnavailableError, [ServiceUnavailableErrorHandler]); 72 | -------------------------------------------------------------------------------- /src/routes/__tests__/browse.spec.ts: -------------------------------------------------------------------------------- 1 | import * as sinon from "sinon"; 2 | import chai from "chai"; 3 | import sinonChai from "sinon-chai"; 4 | 5 | chai.use(sinonChai); 6 | 7 | const { expect } = chai; 8 | 9 | import * as gql from "graphql-request"; 10 | import { 11 | collectionHandler, 12 | collectionsHandler, 13 | addItemLinkIfNotPresent, 14 | generateBaseUrlForCollection, 15 | generateCollectionResponse, 16 | } from "../browse"; 17 | import { generateSTACCollections } from "../../utils/testUtils"; 18 | 19 | describe("addItemLinkIfNotPresent", () => { 20 | it("will add an item link if no item link is present", async () => { 21 | // Create a STACCollection with no item link 22 | let stacCollection = generateSTACCollections(1)[0]; 23 | // Add a non-item link 24 | stacCollection.links.push({ 25 | rel: "via", 26 | href: "https://example.com/foo", 27 | type: "application/html", 28 | title: "foo", 29 | }); 30 | 31 | const numberoOfLinks = stacCollection.links.length; 32 | // Invoke method 33 | addItemLinkIfNotPresent(stacCollection, "https://foo.com"); 34 | // Observe an addiitonal link in the STAC Collection with rel=items etc. 35 | expect(stacCollection.links.length).to.equal(numberoOfLinks + 1); 36 | expect(stacCollection).to.have.deep.property("links", [ 37 | { 38 | rel: "via", 39 | href: "https://example.com/foo", 40 | type: "application/html", 41 | title: "foo", 42 | }, 43 | { 44 | rel: "items", 45 | href: "https://foo.com/items", 46 | type: "application/geo+json", 47 | title: "Collection Items", 48 | }, 49 | ]); 50 | }); 51 | it("will not add an item link if an item link is present", async () => { 52 | // Create a STACCollection with no item link 53 | let stacCollection = generateSTACCollections(1)[0]; 54 | 55 | // Manually add an item link 56 | stacCollection.links.push({ 57 | rel: "items", 58 | href: "https://example.com/items", 59 | type: "application/geo+json", 60 | title: "Collection Items", 61 | }); 62 | // Add a non-item link 63 | stacCollection.links.push({ 64 | rel: "via", 65 | href: "https://example.com/foo", 66 | type: "application/html", 67 | title: "foo", 68 | }); 69 | const numberoOfLinks = stacCollection.links.length; 70 | // Invoke method 71 | addItemLinkIfNotPresent(stacCollection, "https://foo.com/items"); 72 | // Observe no addiitonal link in the STAC Collection and that the item link remains a CMR link 73 | expect(stacCollection.links.length).to.equal(numberoOfLinks); 74 | expect(stacCollection).to.have.deep.property("links", [ 75 | { 76 | rel: "items", 77 | href: "https://example.com/items", 78 | type: "application/geo+json", 79 | title: "Collection Items", 80 | }, 81 | { 82 | rel: "via", 83 | href: "https://example.com/foo", 84 | type: "application/html", 85 | title: "foo", 86 | }, 87 | ]); 88 | }); 89 | }); 90 | 91 | describe("generateBaseUrlForCollection", () => { 92 | it("will use the provider name for an ALL collection result", async () => { 93 | let stacCollection = generateSTACCollections(1)[0]; 94 | 95 | const baseUrl = generateBaseUrlForCollection( 96 | "http://localhost:3000/stac/ALL/collections/Test%201_1.2", 97 | stacCollection 98 | ); 99 | expect(baseUrl).to.equal("http://localhost:3000/stac/PROV1/collections/Test%201_1.2"); 100 | }); 101 | it("will use the same provider name for any other collection result", async () => { 102 | let stacCollection = generateSTACCollections(1)[0]; 103 | 104 | const baseUrl = generateBaseUrlForCollection( 105 | "http://localhost:3000/stac/PROV1/collections/Test%201_1.2", 106 | stacCollection 107 | ); 108 | expect(baseUrl).to.equal("http://localhost:3000/stac/PROV1/collections/Test%201_1.2"); 109 | }); 110 | }); 111 | 112 | describe("generateCollectionResponse", () => { 113 | it("will add the correct description if the provider is 'ALL'", async () => { 114 | let stacCollections = generateSTACCollections(1); 115 | const baseUrl = "http://localhost:3000/stac/ALL/collections"; 116 | const collectionsResponse = generateCollectionResponse(baseUrl, [], stacCollections); 117 | expect(collectionsResponse.description).to.equal("All collections provided by CMR"); 118 | }); 119 | it("will add the correct description if the provider is a real provider", async () => { 120 | let stacCollections = generateSTACCollections(1); 121 | const baseUrl = "http://localhost:3000/stac/PROV1/collections"; 122 | const collectionsResponse = generateCollectionResponse(baseUrl, [], stacCollections); 123 | expect(collectionsResponse.description).to.equal("All collections provided by PROV1"); 124 | }); 125 | }); 126 | -------------------------------------------------------------------------------- /src/routes/browse.ts: -------------------------------------------------------------------------------- 1 | import { Request, Response } from "express"; 2 | import { stringify as stringifyQuery } from "qs"; 3 | 4 | import { Links } from "../@types/StacCatalog"; 5 | 6 | import { getCollections } from "../domains/collections"; 7 | import { buildQuery } from "../domains/stac"; 8 | import { ItemNotFound } from "../models/errors"; 9 | import { getBaseUrl, mergeMaybe, stacContext } from "../utils"; 10 | import { STACCollection } from "../@types/StacCollection"; 11 | import { ALL_PROVIDER } from "../domains/providers"; 12 | 13 | const collectionLinks = (req: Request, nextCursor?: string | null): Links => { 14 | const { stacRoot, self } = stacContext(req); 15 | 16 | const parent = self.split("/").slice(0, -1).join("/"); 17 | 18 | const links = [ 19 | { 20 | rel: "self", 21 | href: self, 22 | type: "application/json", 23 | }, 24 | { 25 | rel: "root", 26 | href: stacRoot, 27 | type: "application/json", 28 | title: `Root Catalog`, 29 | }, 30 | { 31 | rel: "parent", 32 | href: parent, 33 | type: "application/json", 34 | title: "Provider Collections", 35 | }, 36 | ]; 37 | 38 | const originalQuery = mergeMaybe(req.query, req.body); 39 | 40 | if (nextCursor) { 41 | const nextResultsQuery = { ...originalQuery, cursor: nextCursor }; 42 | 43 | links.push({ 44 | rel: "next", 45 | href: `${stacRoot}${req.path}?${stringifyQuery(nextResultsQuery)}`, 46 | type: "application/geo+json", 47 | }); 48 | } 49 | return links; 50 | }; 51 | 52 | export const collectionsHandler = async (req: Request, res: Response): Promise => { 53 | const { headers } = req; 54 | req.params.searchType = "collection"; 55 | const query = await buildQuery(req); 56 | 57 | // If the query contains a "provider": "ALL" clause, we need to remove it as 58 | // this is a 'special' provider that means 'all providers'. The absence 59 | // of a provider clause gives the right query. 60 | if ("provider" in query && query.provider == ALL_PROVIDER) delete query.provider; 61 | 62 | const { cursor, items: collections } = await getCollections(query, { 63 | headers, 64 | }); 65 | 66 | const { stacRoot, self } = stacContext(req); 67 | 68 | collections.forEach((collection) => { 69 | const baseUrl = generateBaseUrlForCollection(getBaseUrl(self), collection); 70 | collection.links.push({ 71 | rel: "self", 72 | href: `${baseUrl}/${encodeURIComponent(collection.id)}`, 73 | type: "application/json", 74 | }); 75 | collection.links.push({ 76 | rel: "root", 77 | href: encodeURI(stacRoot), 78 | type: "application/json", 79 | }); 80 | addItemLinkIfNotPresent(collection, `${baseUrl}/${encodeURIComponent(collection.id)}`); 81 | }); 82 | 83 | const links = collectionLinks(req, cursor); 84 | 85 | const collectionsResponse = generateCollectionResponse(self, links, collections); 86 | 87 | res.json(collectionsResponse); 88 | }; 89 | 90 | /** 91 | * Returns a STACCollection as the body. 92 | */ 93 | export const collectionHandler = async (req: Request, res: Response): Promise => { 94 | const { 95 | collection, 96 | params: { collectionId, providerId }, 97 | } = req; 98 | 99 | if (!collection) { 100 | throw new ItemNotFound( 101 | `Could not find collection [${collectionId}] in provider [${providerId}]` 102 | ); 103 | } 104 | 105 | collection.links = collection.links 106 | ? [...collectionLinks(req), ...(collection.links ?? [])] 107 | : [...collectionLinks(req)]; 108 | const { path } = stacContext(req); 109 | addItemLinkIfNotPresent(collection, path); 110 | res.json(collection); 111 | }; 112 | 113 | /** 114 | * Marshall the description, links and collections into a valid response 115 | * This catalog may be 'ALL'. If so, we need to override the description 116 | * property to convey that this results represents all of CMR 117 | * 118 | * @param self the base url 119 | * @param links the urls associated with this response 120 | * @param collections the STAC collection object that contains the provider of the collection 121 | * 122 | */ 123 | 124 | export function generateCollectionResponse( 125 | self: string, 126 | links: Links, 127 | collections: STACCollection[] 128 | ): { description: string; links: Links; collections: STACCollection[] } { 129 | // Special case. If provider is 'ALL' use description of 'provided by CMR' 130 | let provider = self.split("/").at(-2); 131 | if (provider == ALL_PROVIDER) provider = "CMR"; 132 | 133 | const collectionsResponse = { 134 | description: `All collections provided by ${provider}`, 135 | links, 136 | collections, 137 | }; 138 | return collectionsResponse; 139 | } 140 | 141 | /** 142 | * This catalog may be 'ALL' but the link to the collection's items must reference 143 | * the catalog associated with the collection's provider 144 | * 145 | * @param self the context of the STAC urls 146 | * @param collection the STAC collection object that contains the provider of the collection 147 | */ 148 | 149 | export function generateBaseUrlForCollection(baseUrl: string, collection: STACCollection): string { 150 | // Extract the actual provider of the collection 151 | const provider = collection.providers.find((p) => p.roles?.includes("producer")); 152 | // Construct the items url from that provider 153 | if (provider) baseUrl = baseUrl.replace("/ALL/", `/${provider.name}/`); 154 | return baseUrl; 155 | } 156 | 157 | /** 158 | * A CMR collection can now indicate to consumers that it has a STAC API. 159 | * If that is the case then we use that link instead of a generic CMR one. 160 | * This is useful of collections that do not index their granule 161 | * metadata in CMR, like CWIC collection. 162 | * If the list of links of does not contain a link of type 'items' then 163 | * add the default items element 164 | * 165 | * @param collection the STAC collection object containing links 166 | * @param url the generic link to a CMR STAC API 167 | */ 168 | 169 | export function addItemLinkIfNotPresent(collection: STACCollection, url: string) { 170 | const itemsLink = collection.links.find((link) => link.rel === "items"); 171 | 172 | if (!itemsLink) { 173 | collection.links.push({ 174 | rel: "items", 175 | href: `${url}/items`, 176 | type: "application/geo+json", 177 | title: "Collection Items", 178 | }); 179 | } 180 | } 181 | -------------------------------------------------------------------------------- /src/routes/catalog.ts: -------------------------------------------------------------------------------- 1 | import { Request, Response } from "express"; 2 | import { stringify as stringifyQuery } from "qs"; 3 | 4 | import { Links, STACCatalog } from "../@types/StacCatalog"; 5 | 6 | import { getAllCollectionIds } from "../domains/collections"; 7 | import { conformance } from "../domains/providers"; 8 | import { ServiceUnavailableError } from "../models/errors"; 9 | import { getBaseUrl, mergeMaybe, stacContext } from "../utils"; 10 | import { CMR_QUERY_MAX } from "../domains/stac"; 11 | import { ALL_PROVIDER } from "../domains/providers"; 12 | 13 | const STAC_VERSION = process.env.STAC_VERSION ?? "1.0.0"; 14 | 15 | const generateSelfLinks = (req: Request, nextCursor?: string | null, count?: number): Links => { 16 | const { stacRoot, path, self } = stacContext(req); 17 | 18 | const links = [ 19 | { 20 | rel: "self", 21 | href: self, 22 | type: "application/geo+json", 23 | title: "Provider Catalog", 24 | }, 25 | { 26 | rel: "root", 27 | href: stacRoot, 28 | type: "application/geo+json", 29 | title: `Root Catalog`, 30 | }, 31 | { 32 | rel: "data", 33 | href: `${path}/collections`, 34 | type: "application/json", 35 | title: "Provider Collections", 36 | method: "GET", 37 | }, 38 | { 39 | rel: "data", 40 | href: `${path}/collections`, 41 | type: "application/json", 42 | title: "Provider Collections", 43 | method: "POST", 44 | }, 45 | { 46 | rel: "conformance", 47 | href: `${path}/conformance`, 48 | type: "application/json", 49 | title: "Conformance Classes", 50 | }, 51 | { 52 | rel: "service-desc", 53 | href: "https://api.stacspec.org/v1.0.0-beta.1/openapi.yaml", 54 | type: "application/yaml", 55 | title: "OpenAPI Doc", 56 | }, 57 | { 58 | rel: "service-doc", 59 | href: "https://api.stacspec.org/v1.0.0-beta.1/index.html", 60 | type: "text/html", 61 | title: "HTML documentation", 62 | }, 63 | ]; 64 | 65 | const { provider } = req; 66 | if (provider && provider["provider-id"] != ALL_PROVIDER) { 67 | links.push({ 68 | rel: "search", 69 | href: `${path}/search`, 70 | type: "application/geo+json", 71 | title: "Provider Item Search", 72 | method: "GET", 73 | }); 74 | links.push({ 75 | rel: "search", 76 | href: `${path}/search`, 77 | type: "application/geo+json", 78 | title: "Provider Item Search", 79 | method: "POST", 80 | }); 81 | } 82 | 83 | const originalQuery = mergeMaybe(req.query, req.body); 84 | 85 | // Add a 'next' link if there are more results available 86 | // This is determined by: 87 | // 1. The presence of a nextCursor (indicating more results) 88 | // 2. The number of collection equaling CMR_QUERY_MAX (100) 89 | // The 'next' link includes the original query parameters plus the new cursor 90 | if (nextCursor && count === CMR_QUERY_MAX) { 91 | const nextResultsQuery = { ...originalQuery, cursor: nextCursor }; 92 | 93 | links.push({ 94 | rel: "next", 95 | href: `${stacRoot}${req.path}?${stringifyQuery(nextResultsQuery)}`, 96 | type: "application/json", 97 | title: "Next page of results", 98 | }); 99 | } 100 | 101 | return links; 102 | }; 103 | 104 | const providerCollections = async ( 105 | req: Request 106 | ): Promise< 107 | [null, { id: string; title: string; provider: string }[], string | null] | [string, null] 108 | > => { 109 | const { headers, provider, query } = req; 110 | 111 | const cloudOnly = headers["cloud-stac"] === "true" ? { cloudHosted: true } : {}; 112 | 113 | const mergedQuery = mergeMaybe( 114 | { 115 | provider: provider?.["provider-id"], 116 | cursor: query?.cursor, 117 | }, 118 | { ...cloudOnly } 119 | ); 120 | 121 | try { 122 | if ("provider" in mergedQuery && mergedQuery.provider == ALL_PROVIDER) 123 | delete mergedQuery.provider; 124 | const { items, cursor } = await getAllCollectionIds(mergedQuery, { headers }); 125 | return [null, items, cursor]; 126 | } catch (err) { 127 | console.error("A problem occurred querying for collections.", err); 128 | return [(err as Error).message, null]; 129 | } 130 | }; 131 | 132 | export const providerCatalogHandler = async (req: Request, res: Response) => { 133 | const { provider } = req; 134 | 135 | if (!provider) throw new ServiceUnavailableError("Could not retrieve provider information"); 136 | 137 | const [err, collections, cursor] = await providerCollections(req); 138 | 139 | if (err) throw new ServiceUnavailableError(err as string); 140 | 141 | const { self } = stacContext(req); 142 | 143 | const selfLinks = generateSelfLinks(req, cursor, collections?.length); 144 | 145 | const childLinks = (collections ?? []).map(({ id, title, provider }) => ({ 146 | rel: "child", 147 | href: `${getBaseUrl(self) 148 | .concat("/") 149 | .replace("/ALL/", "/" + provider + "/")}collections/${encodeURIComponent(id)}`, 150 | title, 151 | type: "application/json", 152 | })); 153 | 154 | const providerCatalog = { 155 | type: "Catalog", 156 | id: provider["provider-id"], 157 | title: `${provider["provider-id"]} STAC Catalog`, 158 | stac_version: STAC_VERSION, 159 | description: `Root STAC catalog for ${provider["provider-id"]}`, 160 | conformsTo: conformance, 161 | links: [...selfLinks, ...childLinks], 162 | } as STACCatalog; 163 | 164 | res.json(providerCatalog); 165 | }; 166 | -------------------------------------------------------------------------------- /src/routes/conformance.ts: -------------------------------------------------------------------------------- 1 | import { Request, Response } from "express"; 2 | 3 | export const rootConformance = ["https://api.stacspec.org/v1.0.0-rc.2/core"]; 4 | 5 | export const rootConformanceHandler = async (_req: Request, res: Response): Promise => { 6 | res.json({ conformsTo: rootConformance }); 7 | }; 8 | -------------------------------------------------------------------------------- /src/routes/healthcheck.ts: -------------------------------------------------------------------------------- 1 | import { Request, Response } from "express"; 2 | import axios from "axios"; 3 | 4 | const CMR_LB_URL = process.env.CMR_LB_URL; 5 | const CMR_INGEST_HEALTH = `${CMR_LB_URL}/ingest/health`; 6 | const CMR_SEARCH_HEALTH = `${CMR_LB_URL}/search/health`; 7 | 8 | export const healthcheckHandler = async (_req: Request, res: Response) => { 9 | console.debug(`GET ${CMR_INGEST_HEALTH}`); 10 | const { status: ingestStatus } = await axios.get(CMR_INGEST_HEALTH); 11 | console.debug(`GET ${CMR_SEARCH_HEALTH}`); 12 | const { status: searchStatus } = await axios.get(CMR_SEARCH_HEALTH); 13 | 14 | if ([ingestStatus, searchStatus].every((status) => status === 200)) { 15 | res.json({ message: "healthy" }); 16 | } else { 17 | res.status(503).json({ message: "unhealthy" }); 18 | } 19 | }; 20 | -------------------------------------------------------------------------------- /src/routes/index.ts: -------------------------------------------------------------------------------- 1 | import express from "express"; 2 | 3 | import { collectionHandler, collectionsHandler } from "./browse"; 4 | import { healthcheckHandler } from "./healthcheck"; 5 | import { multiItemHandler, singleItemHandler } from "./items"; 6 | import { providerCatalogHandler } from "./catalog"; 7 | import { providerConformanceHandler } from "./providerConformance"; 8 | import { rootCatalogHandler } from "./rootCatalog"; 9 | import { rootConformanceHandler } from "./conformance"; 10 | import { searchHandler } from "./search"; 11 | import { wrapErrorHandler } from "../utils"; 12 | 13 | import { 14 | cacheMiddleware, 15 | cloudStacMiddleware, 16 | logFullRequestMiddleware, 17 | refreshProviderCache, 18 | validateCollection, 19 | validateProvider, 20 | validateStacQuery, 21 | validateNotAllProvider, 22 | } from "../middleware"; 23 | 24 | const router = express.Router(); 25 | 26 | router.use(cloudStacMiddleware, cacheMiddleware, logFullRequestMiddleware); 27 | 28 | router.get("/", refreshProviderCache, wrapErrorHandler(rootCatalogHandler)); 29 | router.get("/health", wrapErrorHandler(healthcheckHandler)); 30 | router.get("/conformance", wrapErrorHandler(rootConformanceHandler)); 31 | 32 | router.get( 33 | "/:providerId", 34 | refreshProviderCache, 35 | validateProvider, 36 | wrapErrorHandler(providerCatalogHandler) 37 | ); 38 | router.get( 39 | "/:providerId/conformance", 40 | refreshProviderCache, 41 | validateProvider, 42 | wrapErrorHandler(providerConformanceHandler) 43 | ); 44 | 45 | router 46 | .route("/:providerId/search") 47 | .get( 48 | refreshProviderCache, 49 | validateNotAllProvider, 50 | validateProvider, 51 | validateStacQuery, 52 | wrapErrorHandler(searchHandler) 53 | ) 54 | .post( 55 | refreshProviderCache, 56 | validateNotAllProvider, 57 | validateProvider, 58 | validateStacQuery, 59 | wrapErrorHandler(searchHandler) 60 | ); 61 | 62 | router 63 | .route("/:providerId/collections") 64 | .get( 65 | refreshProviderCache, 66 | validateProvider, 67 | validateStacQuery, 68 | wrapErrorHandler(collectionsHandler) 69 | ) 70 | .post( 71 | refreshProviderCache, 72 | validateProvider, 73 | validateStacQuery, 74 | wrapErrorHandler(collectionsHandler) 75 | ); 76 | 77 | router.get( 78 | "/:providerId/collections/:collectionId", 79 | refreshProviderCache, 80 | validateNotAllProvider, 81 | validateProvider, 82 | validateStacQuery, 83 | validateCollection, 84 | wrapErrorHandler(collectionHandler) 85 | ); 86 | 87 | router.get( 88 | "/:providerId/collections/:collectionId/items", 89 | refreshProviderCache, 90 | validateNotAllProvider, 91 | validateProvider, 92 | validateStacQuery, 93 | validateCollection, 94 | wrapErrorHandler(multiItemHandler) 95 | ); 96 | 97 | router.get( 98 | "/:providerId/collections/:collectionId/items/:itemId", 99 | refreshProviderCache, 100 | validateNotAllProvider, 101 | validateProvider, 102 | validateStacQuery, 103 | validateCollection, 104 | wrapErrorHandler(singleItemHandler) 105 | ); 106 | 107 | export default router; 108 | -------------------------------------------------------------------------------- /src/routes/items.ts: -------------------------------------------------------------------------------- 1 | import { Request, Response } from "express"; 2 | import { stringify as stringifyQuery } from "qs"; 3 | 4 | import { addProviderLinks, getItems } from "../domains/items"; 5 | import { buildQuery } from "../domains/stac"; 6 | import { ItemNotFound } from "../models/errors"; 7 | import { mergeMaybe, stacContext, WEEK_IN_MS } from "../utils/index"; 8 | 9 | const STAC_VERSION = process.env.STAC_VERSION ?? "1.0.0"; 10 | 11 | const generateLinks = (req: Request) => { 12 | const { stacRoot, self } = stacContext(req); 13 | 14 | return [ 15 | { 16 | rel: "self", 17 | href: encodeURI(self), 18 | type: "application/geo+json", 19 | }, 20 | { 21 | rel: "root", 22 | href: stacRoot, 23 | type: "application/json", 24 | }, 25 | { 26 | rel: "parent", 27 | href: encodeURI(self.split("/").slice(0, -1).join("/")), 28 | type: "application/json", 29 | }, 30 | ]; 31 | }; 32 | 33 | /** 34 | * Handle requests for a collection's items. 35 | * Return a FeatureCollection as response. 36 | */ 37 | export const singleItemHandler = async (req: Request, res: Response) => { 38 | const { 39 | headers, 40 | params: { collectionId, itemId }, 41 | } = req; 42 | 43 | req.params.searchType = "item"; 44 | const itemQuery = await buildQuery(req); 45 | const { 46 | items: [item], 47 | } = await getItems(itemQuery, { headers }); 48 | 49 | if (!item) 50 | throw new ItemNotFound( 51 | `Could not find item with ID [${itemId}] in collection [${collectionId}]` 52 | ); 53 | 54 | const itemResponse = addProviderLinks(req, item); 55 | 56 | res.contentType("application/geo+json").json(itemResponse); 57 | }; 58 | 59 | /** 60 | * Handle requests for a collection' items. 61 | * 62 | * Returns a FeatureCollection as response 63 | */ 64 | export const multiItemHandler = async (req: Request, res: Response) => { 65 | const { 66 | query: { cursor }, 67 | collection, 68 | params: { collectionId, providerId }, 69 | } = req; 70 | 71 | if (!collection) { 72 | throw new ItemNotFound( 73 | `Could not find parent collection [${collectionId}] in provider [${providerId}]` 74 | ); 75 | } 76 | 77 | req.params.searchType = "item"; 78 | const itemQuery = await buildQuery(req); 79 | const links = generateLinks(req); 80 | 81 | const { 82 | cursor: nextCursor, 83 | count, 84 | items, 85 | } = await getItems(itemQuery, { 86 | headers: req.headers, 87 | }); 88 | 89 | const { stacRoot } = stacContext(req); 90 | const originalQuery = mergeMaybe(req.query, req.body); 91 | 92 | if (cursor && req.cookies[`prev-${cursor}`]) { 93 | const prevResultsQuery = { ...originalQuery, cursor: req.cookies[`prev-${cursor}`] }; 94 | 95 | links.push({ 96 | rel: "prev", 97 | href: encodeURI(`${stacRoot}${req.path}`) + `?${stringifyQuery(prevResultsQuery)}`, 98 | type: "application/geo+json", 99 | }); 100 | } 101 | 102 | if (nextCursor && nextCursor !== cursor) { 103 | const nextResultsQuery = { ...originalQuery, cursor: nextCursor }; 104 | 105 | links.push({ 106 | rel: "next", 107 | href: encodeURI(`${stacRoot}${req.path}`) + `?${stringifyQuery(nextResultsQuery)}`, 108 | type: "application/geo+json", 109 | }); 110 | } 111 | 112 | if (cursor && nextCursor && "cursor" in originalQuery && nextCursor !== cursor) { 113 | res.cookie(`prev-${nextCursor}`, originalQuery.cursor, { 114 | maxAge: WEEK_IN_MS, 115 | }); 116 | } 117 | 118 | const { path } = stacContext(req); 119 | 120 | const itemsResponse = { 121 | type: "FeatureCollection", 122 | description: `Items in the collection ${collection.id}`, 123 | id: `${collection.id}-items`, 124 | license: collection.license, 125 | extent: collection.extent, 126 | stac_version: STAC_VERSION, 127 | numberMatched: count, 128 | numberReturned: items.length, 129 | features: items.map((item) => { 130 | item.links = [ 131 | { 132 | rel: "self", 133 | href: encodeURI(`${path}/${item.id}`), 134 | type: "application/geo+json", 135 | title: item.id, 136 | }, 137 | ]; 138 | 139 | return item; 140 | }), 141 | links, 142 | context: { 143 | returned: items.length, 144 | limit: itemQuery.limit, 145 | matched: count, 146 | }, 147 | }; 148 | 149 | res.contentType("application/geo+json").json(itemsResponse); 150 | }; 151 | -------------------------------------------------------------------------------- /src/routes/providerConformance.ts: -------------------------------------------------------------------------------- 1 | import { Request, Response } from "express"; 2 | import { conformance } from "../domains/providers"; 3 | 4 | export const providerConformanceHandler = async (_req: Request, res: Response): Promise => { 5 | res.json({ conformsTo: conformance }); 6 | }; 7 | -------------------------------------------------------------------------------- /src/routes/rootCatalog.ts: -------------------------------------------------------------------------------- 1 | import { Request, Response } from "express"; 2 | import { Link, STACCatalog } from "../@types/StacCatalog"; 3 | 4 | import { conformance } from "../domains/providers"; 5 | import { Provider } from "../models/CmrModels"; 6 | import { getBaseUrl, stacContext } from "../utils"; 7 | 8 | const STAC_VERSION = process.env.STAC_VERSION ?? "1.0.0"; 9 | 10 | const selfLinks = (req: Request): Link[] => { 11 | const { stacRoot, id } = stacContext(req); 12 | 13 | return [ 14 | { 15 | rel: "self", 16 | href: stacRoot, 17 | type: "application/json", 18 | title: `NASA CMR-${id} Root Catalog`, 19 | }, 20 | { 21 | rel: "root", 22 | href: stacRoot, 23 | title: `NASA CMR-${id} Root Catalog`, 24 | type: "application/geo+json", 25 | }, 26 | { 27 | rel: "service-desc", 28 | href: `https://api.stacspec.org/v1.0.0-beta.1/openapi.yaml`, 29 | title: "OpenAPI Documentation", 30 | type: "application/yaml", 31 | }, 32 | { 33 | rel: "service-doc", 34 | href: "https://wiki.earthdata.nasa.gov/display/ED/CMR+SpatioTemporal+Asset+Catalog+%28CMR-STAC%29+Documentation", 35 | title: `NASA CMR-${id} Documentation`, 36 | type: "text/html", 37 | }, 38 | ]; 39 | }; 40 | 41 | const providerLinks = (req: Request, providers: Provider[]): Link[] => { 42 | const { self } = stacContext(req); 43 | 44 | return providers.map(({ "short-name": title, "provider-id": providerId }) => ({ 45 | rel: "child", 46 | title, 47 | type: "application/json", 48 | href: `${getBaseUrl(self)}/${providerId}`, 49 | })); 50 | }; 51 | 52 | export const rootCatalogHandler = async (req: Request, res: Response) => { 53 | const isCloudStac = req.headers["cloud-stac"] === "true"; 54 | const id = isCloudStac ? "CMR-CLOUDSTAC" : "CMR-STAC"; 55 | 56 | const providers = isCloudStac 57 | ? req.cache?.cloudProviders.getAll() 58 | : req.cache?.providers.getAll(); 59 | 60 | const _selfLinks = selfLinks(req); 61 | const _providerLinks = providerLinks(req, providers ?? []); 62 | 63 | const rootCatalog = { 64 | type: "Catalog", 65 | id, 66 | stac_version: STAC_VERSION, 67 | conformsTo: conformance, 68 | links: [..._selfLinks, ..._providerLinks], 69 | title: `NASA Common Metadata Repository ${id} API`, 70 | description: `This is the landing page for ${id}. Each provider link contains a STAC endpoint.`, 71 | } as STACCatalog; 72 | 73 | res.json(rootCatalog); 74 | }; 75 | -------------------------------------------------------------------------------- /src/routes/search.ts: -------------------------------------------------------------------------------- 1 | import { Request, Response } from "express"; 2 | import { stringify as stringifyQuery } from "qs"; 3 | 4 | import { Link, STACItem } from "../@types/StacItem"; 5 | 6 | import { addProviderLinks, getItems } from "../domains/items"; 7 | import { buildQuery } from "../domains/stac"; 8 | import { mergeMaybe, stacContext } from "../utils"; 9 | 10 | const searchLinks = (req: Request, nextCursor: string | null): Link[] => { 11 | const { 12 | params: { providerId }, 13 | } = req; 14 | 15 | const { stacRoot, self } = stacContext(req); 16 | const currentQuery = mergeMaybe(req.query, req.body); 17 | 18 | const firstPageQuery = { ...currentQuery }; 19 | if ("cursor" in firstPageQuery) delete firstPageQuery["cursor"]; 20 | 21 | let links = [ 22 | { 23 | rel: "self", 24 | href: self, 25 | type: "application/geo+json", 26 | title: "This search", 27 | }, 28 | { 29 | rel: "root", 30 | href: `${stacRoot}`, 31 | type: "application/json", 32 | title: `Root Catalog`, 33 | }, 34 | { 35 | rel: "parent", 36 | href: `${stacRoot}/${providerId}`, 37 | type: "application/json", 38 | title: `Provider Catalog`, 39 | }, 40 | { 41 | rel: "first", 42 | href: `${stacRoot}/${providerId}/search?${stringifyQuery(firstPageQuery)}`, 43 | type: "application/geo+json", 44 | title: "First page of results", 45 | }, 46 | ]; 47 | 48 | if (nextCursor) { 49 | const nextPageQuery = { ...currentQuery, cursor: nextCursor }; 50 | links = [ 51 | ...links, 52 | { 53 | rel: "next", 54 | href: `${stacRoot}/${providerId}/search?${stringifyQuery(nextPageQuery)}`, 55 | type: "application/geo+json", 56 | title: "Next page of results", 57 | }, 58 | ]; 59 | } 60 | 61 | return links; 62 | }; 63 | 64 | export const searchHandler = async (req: Request, res: Response): Promise => { 65 | const { headers } = req; 66 | req.params.searchType = "item"; 67 | const gqlQuery = await buildQuery(req); 68 | 69 | const itemsResponse = await getItems(gqlQuery, { headers }); 70 | 71 | const { count, cursor, items } = itemsResponse; 72 | const features = items.map((item: STACItem) => addProviderLinks(req, item)); 73 | 74 | const links = searchLinks(req, cursor); 75 | 76 | const context = { 77 | returned: items.length, 78 | limit: gqlQuery.limit, 79 | matched: count, 80 | }; 81 | 82 | res.contentType("application/geo+json").json({ 83 | type: "FeatureCollection", 84 | features, 85 | links, 86 | context, 87 | }); 88 | }; 89 | -------------------------------------------------------------------------------- /src/utils/__tests__/datetime.spec.ts: -------------------------------------------------------------------------------- 1 | import { expect } from "chai"; 2 | 3 | import { validDateTime } from "../datetime"; 4 | 5 | describe("validDateTime", () => { 6 | describe("given a valid datetime string", () => { 7 | [ 8 | "1985-04-12T23:20:50.52Z", 9 | "1996-12-19T16:39:57-00:00", 10 | "1996-12-19T16:39:57+00:00", 11 | "1996-12-19T16:39:57-08:00", 12 | "1996-12-19T16:39:57+08:00", 13 | "../1985-04-12T23:20:50.52Z", 14 | "1985-04-12T23:20:50.52Z/..", 15 | "/1985-04-12T23:20:50.52Z", 16 | "1985-04-12T23:20:50.52Z/", 17 | "1985-04-12T23:20:50.52Z/1986-04-12T23:20:50.52Z", 18 | "1985-04-12T23:20:50.52+01:00/1986-04-12T23:20:50.52+01:00", 19 | "1985-04-12T23:20:50.52-01:00/1986-04-12T23:20:50.52-01:00", 20 | "1937-01-01T12:00:27.87+01:00", 21 | "1985-04-12T23:20:50.52Z", 22 | "1937-01-01T12:00:27.8710+01:00", 23 | "1937-01-01T12:00:27.8+01:00", 24 | "1937-01-01T12:00:27.8Z", 25 | "2020-07-23T00:00:00.000+03:00", 26 | "2020-07-23T00:00:00+03:00", 27 | "1985-04-12t23:20:50.000z", 28 | "2020-07-23T00:00:00Z", 29 | "2020-07-23T00:00:00.0Z", 30 | "2020-07-23T00:00:00.01Z", 31 | "2020-07-23T00:00:00.012Z", 32 | "2020-07-23T00:00:00.0123Z", 33 | "2020-07-23T00:00:00.01234Z", 34 | "2020-07-23T00:00:00.012345Z", 35 | "2020-07-23T00:00:00.0123456Z", 36 | "2020-07-23T00:00:00.01234567Z", 37 | "2020-07-23T00:00:00.012345678Z", 38 | ].forEach((input) => { 39 | it(`${input} should return true`, () => { 40 | expect(validDateTime(input)).to.be.true; 41 | }); 42 | }); 43 | }); 44 | 45 | describe("given an invalid datetime string", () => { 46 | [ 47 | ["unbounded slash", "/"], 48 | ["unbounded slash and dots", "../.."], 49 | ["unbounded slash and future dots", "/.."], 50 | ["unbounded past and slash", "../"], 51 | ["extra delimiter front", "/1984-04-12T23:20:50.52Z/1985-04-12T23:20:50.52Z"], 52 | ["extra delimiter end", "1984-04-12T23:20:50.52Z/1985-04-12T23:20:50.52Z/"], 53 | ["extra delimiter front and end", "/1984-04-12T23:20:50.52Z/1985-04-12T23:20:50.52Z/"], 54 | ["date only", "1985-04-12"], 55 | ["invalid TZ format", "1937-01-01T12:00:27.87+0100"], 56 | ["invalid year", "37-01-01T12:00:27.87Z"], 57 | ["no TZ", "1985-12-12T23:20:50.52"], 58 | ["not 4 digit year", "21985-12-12T23:20:50.52Z"], 59 | ["month out of range", "1985-13-12T23:20:50.52Z"], 60 | ["day out of range", "1985-12-32T23:20:50.52Z"], 61 | ["hour out of range", "1985-12-01T25:20:50.52Z"], 62 | ["minute out of range", "1985-12-01T00:60:50.52Z"], 63 | ["secound out of range", "1985-12-01T00:06:61.52Z"], 64 | ["franctional sec but no value, dot", "1985-04-12T23:20:50.Z"], 65 | ["franctional sec but no value, comma", "1985-04-12T23:20:50,Z"], 66 | ["second out of range without fractional", "1990-12-31T23:59:61Z"], 67 | ["end before start", "1986-04-12T23:20:50.52Z/1985-04-12T23:20:50.52Z"], 68 | ["comma as frac sec sep allowed in ISO8601 but not RFC3339", "1985-04-12T23:20:50,52Z"], 69 | ].forEach(([label, input]) => { 70 | it(`${label} [${input}] should return false`, () => { 71 | expect(validDateTime(input)).to.be.false; 72 | }); 73 | }); 74 | }); 75 | }); 76 | -------------------------------------------------------------------------------- /src/utils/__tests__/sort.spec.ts: -------------------------------------------------------------------------------- 1 | import { expect } from "chai"; 2 | import { parseSortFields, mapIdSortKey } from "../sort"; 3 | import { SortObject } from "../../models/StacModels"; 4 | 5 | describe("parseSortFields", () => { 6 | it("should return an empty array for undefined input", () => { 7 | const parsedField = parseSortFields(); 8 | 9 | expect(parsedField).to.deep.equal([]); 10 | }); 11 | 12 | it("should handle a single field in string based sorting (GET)", () => { 13 | const parsedField = parseSortFields("field1"); 14 | expect(parsedField).to.deep.equal(["field1"]); 15 | }); 16 | 17 | it("should handle multi field string based sorting (GET)", () => { 18 | const parsedField = parseSortFields("field1, -field2, field3"); 19 | 20 | expect(parsedField).to.deep.equal(["field1", "-field2", "field3"]); 21 | }); 22 | 23 | it("should handle a single object in object based sorting (POST)", () => { 24 | const input: SortObject[] = [{ field: "field1", direction: "desc" }]; 25 | expect(parseSortFields(input)).to.deep.equal(["-field1"]); 26 | }); 27 | 28 | it("should handle multi field object based sorting (POST)", () => { 29 | const input: SortObject[] = [ 30 | { field: "field1", direction: "asc" }, 31 | { field: "field2", direction: "desc" }, 32 | { field: "field3", direction: "asc" }, 33 | ]; 34 | expect(parseSortFields(input)).to.deep.equal(["field1", "-field2", "field3"]); 35 | }); 36 | 37 | it("should return an empty array for an empty array", () => { 38 | const parsedField = parseSortFields([]); 39 | 40 | expect(parsedField).to.deep.equal([]); 41 | }); 42 | 43 | it("should handle mixed array (treating non-strings as empty strings)", () => { 44 | const input: any[] = ["field1", { field: "field2", direction: "desc" }, "-field3"]; 45 | 46 | expect(parseSortFields(input)).to.deep.equal(["field1", "", "-field3"]); 47 | }); 48 | }); 49 | 50 | describe("mapIdSortKey", () => { 51 | it("should return a valid cmr sort value based on searchType", () => { 52 | const collectionMappedKey = mapIdSortKey("collection"); 53 | expect(collectionMappedKey).to.equal("entryId"); 54 | 55 | const itemMappedKey = mapIdSortKey("item"); 56 | expect(itemMappedKey).to.equal("readableGranuleName"); 57 | 58 | const unmappedKey = mapIdSortKey("anything"); 59 | expect(unmappedKey).to.equal(""); 60 | 61 | const emptyKey = mapIdSortKey(); 62 | expect(emptyKey).to.equal(""); 63 | }); 64 | }); 65 | -------------------------------------------------------------------------------- /src/utils/__tests__/utils.spec.ts: -------------------------------------------------------------------------------- 1 | import { Request } from "express"; 2 | import { expect } from "chai"; 3 | import { IncomingHttpHeaders } from "http"; 4 | 5 | import { buildRootUrl, mergeMaybe, scrubTokens, stacContext } from "../index"; 6 | 7 | describe("buildRootUrl", () => { 8 | describe("given request with HOST header set", () => { 9 | ["host", "x-forwarded-host"].forEach((hostHeader) => { 10 | it(`should handle ${hostHeader} and return a valid url`, () => { 11 | const headers: IncomingHttpHeaders = {}; 12 | headers[hostHeader] = "my-test-host"; 13 | 14 | expect(buildRootUrl({ headers } as Request)).to.deep.equal("http://my-test-host"); 15 | }); 16 | }); 17 | }); 18 | }); 19 | 20 | describe("scrub tokens", () => { 21 | it("anonymizes authorization headers", () => { 22 | expect(scrubTokens({ authorization: "Bearer zzzzzzzzzzzzzzzzzzzzzz" })).to.have.property( 23 | "authorization", 24 | "Bearer zzzzz... REDACTED" 25 | ); 26 | }); 27 | }); 28 | 29 | describe("mergeMaybe", () => { 30 | [ 31 | ["undefined", undefined], 32 | ["null", null], 33 | ["empty strings", ""], 34 | ["empty arrays", []], 35 | ["NaN", parseFloat("foo")], 36 | ].forEach(([label, badValue]) => { 37 | it(`filters ${label}`, () => { 38 | expect(mergeMaybe({ good: "value" }, { myKey: badValue })).to.deep.equal({ good: "value" }); 39 | }); 40 | }); 41 | 42 | it("does not filter out zero", () => { 43 | expect(mergeMaybe({ foo: "bar" }, { value: 0 })).to.deep.equal({ foo: "bar", value: 0 }); 44 | }); 45 | }); 46 | 47 | describe("stacContext", () => { 48 | describe("given a root catalog path", () => { 49 | it("returns an ID of STAC", () => { 50 | const mockRequest = { 51 | method: "GET", 52 | headers: { 53 | "cloudfront-forwarded-proto": "https", 54 | host: "example.api", 55 | } as IncomingHttpHeaders, 56 | originalUrl: "/stac", 57 | } as Request; 58 | expect(stacContext(mockRequest)).to.deep.equal({ 59 | id: "STAC", 60 | root: "https://example.api", 61 | stacRoot: "https://example.api/stac", 62 | path: "https://example.api/stac", 63 | self: "https://example.api/stac", 64 | }); 65 | }); 66 | }); 67 | 68 | describe("given a provider catalog path", () => { 69 | it("returns the provider path", () => { 70 | const mockRequest = { 71 | method: "GET", 72 | headers: { 73 | "cloudfront-forwarded-proto": "https", 74 | host: "example.api", 75 | } as IncomingHttpHeaders, 76 | originalUrl: "/stac/TEST_PROV", 77 | } as Request; 78 | expect(stacContext(mockRequest)).to.deep.equal({ 79 | id: "STAC", 80 | root: "https://example.api", 81 | stacRoot: "https://example.api/stac", 82 | path: "https://example.api/stac/TEST_PROV", 83 | self: "https://example.api/stac/TEST_PROV", 84 | }); 85 | }); 86 | }); 87 | 88 | describe("given a provider search path", () => { 89 | it("returns a self property including the query", () => { 90 | const mockRequest = { 91 | method: "GET", 92 | headers: { 93 | "cloudfront-forwarded-proto": "https", 94 | host: "example.api", 95 | } as IncomingHttpHeaders, 96 | originalUrl: "/stac/TEST_PROV/search?collections=ABC_123", 97 | } as Request; 98 | expect(stacContext(mockRequest)).to.deep.equal({ 99 | id: "STAC", 100 | root: "https://example.api", 101 | stacRoot: "https://example.api/stac", 102 | path: "https://example.api/stac/TEST_PROV/search", 103 | self: "https://example.api/stac/TEST_PROV/search?collections=ABC_123", 104 | }); 105 | }); 106 | }); 107 | 108 | describe("given a cloudstac provider search path", () => { 109 | it("returns a self property including the query", () => { 110 | const mockRequest = { 111 | method: "GET", 112 | headers: { 113 | "cloud-stac": "true", 114 | "cloudfront-forwarded-proto": "https", 115 | host: "example.api", 116 | } as IncomingHttpHeaders, 117 | originalUrl: "/cloudstac/TEST_PROV/search?collections=ABC_123", 118 | } as Request; 119 | expect(stacContext(mockRequest)).to.deep.equal({ 120 | id: "CLOUDSTAC", 121 | root: "https://example.api", 122 | stacRoot: "https://example.api/cloudstac", 123 | path: "https://example.api/cloudstac/TEST_PROV/search", 124 | self: "https://example.api/cloudstac/TEST_PROV/search?collections=ABC_123", 125 | }); 126 | }); 127 | }); 128 | 129 | describe("given a stac provider collections path ending in a trailing slash", () => { 130 | it("strips the trailing slash", () => { 131 | const mockRequest = { 132 | method: "GET", 133 | headers: { 134 | "cloud-stac": "true", 135 | "cloudfront-forwarded-proto": "https", 136 | host: "example.api", 137 | } as IncomingHttpHeaders, 138 | originalUrl: "/cloudstac/TEST_PROV/collections/", 139 | } as Request; 140 | expect(stacContext(mockRequest)).to.deep.equal({ 141 | id: "CLOUDSTAC", 142 | root: "https://example.api", 143 | stacRoot: "https://example.api/cloudstac", 144 | path: "https://example.api/cloudstac/TEST_PROV/collections", 145 | self: "https://example.api/cloudstac/TEST_PROV/collections", 146 | }); 147 | }); 148 | }); 149 | }); 150 | -------------------------------------------------------------------------------- /src/utils/datetime.ts: -------------------------------------------------------------------------------- 1 | const ISO_8601_DATE_RX = 2 | /^(\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d:[0-5]\d[,.]\d+([+-][0-2]\d:[0-5]\d|Z))|(\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d:[0-5]\d([+-][0-2]\d:[0-5]\d|Z))|(\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d([+-][0-2]\d:[0-5]\d|Z))/i; 3 | 4 | const RFC_3339_RX = /^\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d:[0-5]\d(\.\d+)Z$/i; 5 | 6 | const dateOnlyRx = /^\d{4}-\d{2}-\d{2}$/; 7 | 8 | const validTimeZoneRx = /[+-]?(\d{2}:\d{2})$|Z$/i; 9 | 10 | const splitOnDelimiters = (str: string, delimiters: string[]) => { 11 | const splits = delimiters 12 | .filter((delimiter) => str.indexOf(delimiter) !== -1) 13 | .flatMap((delimiter) => str.split(delimiter)); 14 | 15 | if (!splits.length) return [str]; 16 | return splits; 17 | }; 18 | 19 | const isValidDate = (date: string) => { 20 | if (date === "" || date === "..") return true; 21 | 22 | if (dateOnlyRx.test(date)) return false; 23 | if (!validTimeZoneRx.test(date)) return false; 24 | if (!(ISO_8601_DATE_RX.test(date) || RFC_3339_RX.test(date))) return false; 25 | 26 | return !Number.isNaN(new Date(date).getTime()); 27 | }; 28 | 29 | export const dateTimeToRange = (dateTime?: string) => { 30 | if (!dateTime) return; 31 | 32 | const dateTimeArray = splitOnDelimiters(dateTime, [",", "/"]); 33 | 34 | if (dateTimeArray.length === 1) { 35 | if (dateTimeArray[0].substring(dateTimeArray[0].length - 2, dateTimeArray[0].length) === "..") { 36 | return dateTimeArray[0].substring(0, dateTimeArray[0].length - 2); 37 | } else { 38 | return `${dateTimeArray[0]}/${dateTimeArray[0]}`; 39 | } 40 | } 41 | 42 | if (dateTimeArray.length === 2) { 43 | if (dateTimeArray[0] === "..") { 44 | return `0000-12-31T00:00:00.00Z/${dateTimeArray[1]}`; 45 | } 46 | if (dateTimeArray[1] === "..") { 47 | return `${dateTimeArray[0]}/9999-12-31T23:59:59.999Z`; 48 | } 49 | } 50 | 51 | return dateTime; 52 | }; 53 | 54 | const invalidUnboundRanges = ["/", "../..", "/..", "../"]; 55 | 56 | const isValidRange = (dates: string[]) => { 57 | const [start, end] = dates; 58 | 59 | if (!start || start === ".." || start === "") return true; 60 | if (!end || end === ".." || end === "") return true; 61 | 62 | return Number(new Date(end)) - Number(new Date(start)) > 0; 63 | }; 64 | 65 | export const validDateTime = (dateTimeString?: string) => { 66 | if (!dateTimeString) return; 67 | if (invalidUnboundRanges.find((invalid) => invalid === dateTimeString)) return false; 68 | 69 | const dates = splitOnDelimiters(dateTimeString, [",", "/"]); 70 | if (dates.length > 2) return false; 71 | 72 | return dates.reduce((validAcc, d) => validAcc && isValidDate(d), true) && isValidRange(dates); 73 | }; 74 | -------------------------------------------------------------------------------- /src/utils/index.ts: -------------------------------------------------------------------------------- 1 | import { Request, Response, NextFunction } from "express"; 2 | import { IncomingHttpHeaders } from "http"; 3 | import { isPlainObject } from "lodash"; 4 | 5 | export type OptionalString = string | null; 6 | 7 | export const WEEK_IN_MS = 7 * 24 * 60 * 60 * 1000; 8 | 9 | export const ERRORS = { 10 | internalServerError: 11 | "Oops! Something has gone wrong. We have been alerted and are working to resolve the problem. Please try your request again later.", 12 | serviceUnavailable: 13 | "Oops! A problem occurred upstream and we were unable to process your request. We have been alerted and are working to resolve the problem. Please try your request again later.", 14 | }; 15 | 16 | /** 17 | * Builds the root STAC url from the request. 18 | */ 19 | export const buildRootUrl = (req: Request): string => { 20 | const { headers } = req; 21 | 22 | const protocol = headers["cloudfront-forwarded-proto"] ?? headers["x-forwarded-proto"] ?? "http"; 23 | 24 | const host = headers["x-forwarded-host"] ?? headers["host"] ?? "localhost:3000"; 25 | 26 | return `${protocol}://${host}`; 27 | }; 28 | 29 | export const buildClientId = (clientId?: string): string => 30 | clientId ? `${clientId}-cmr-stac` : "cmr-stac"; 31 | 32 | /** 33 | * Wrap express handler with async error handling. 34 | */ 35 | export const wrapErrorHandler = (fn: (rq: Request, rs: Response) => Promise) => { 36 | return async (req: Request, res: Response, next: NextFunction) => { 37 | try { 38 | await fn(req, res); 39 | } catch (error) { 40 | next(error); 41 | } 42 | }; 43 | }; 44 | 45 | /** 46 | * Returns second to last value in a relatedUrl 47 | * to be used as a key for thumbnail assets link. 48 | * Defaults to string 'key' in the case of an unexpected URL format. 49 | */ 50 | export const extractAssetMapKey = (relatedUrl: string) => { 51 | const urlArray = relatedUrl.split("."); 52 | return urlArray[urlArray.length - 2] ? urlArray[urlArray.length - 2] : "asset_key"; 53 | }; 54 | 55 | export const stacContext = (req: Request) => { 56 | const { headers, originalUrl } = req; 57 | const isCloudStac = headers["cloud-stac"] === "true"; 58 | const root = buildRootUrl(req); 59 | const stac = isCloudStac ? "cloudstac" : "stac"; 60 | // default to empty string so `undefined` isn't printed 61 | const path = originalUrl.split("?")[0] ?? ""; 62 | 63 | return { 64 | id: isCloudStac ? "CLOUDSTAC" : "STAC", 65 | root, 66 | stacRoot: `${root}/${stac}`, 67 | path: `${root}${path}`.replace(/\/$/, ""), 68 | self: `${root}${originalUrl}`.replace(/\/$/, ""), 69 | }; 70 | }; 71 | 72 | /** 73 | * Merge non-trivial map entries. 74 | * Filters out 75 | * - null 76 | * - undefined 77 | * - NaN 78 | * - empty arrays 79 | * - empty strings 80 | */ 81 | export const mergeMaybe = (map: object, maybeMap?: unknown) => { 82 | const baseMap = map ?? {}; 83 | if (!maybeMap || !isPlainObject(maybeMap)) return baseMap; 84 | 85 | const coerced: { [key: string]: unknown } = { ...maybeMap }; 86 | 87 | return Object.keys(coerced).reduce( 88 | (nextMap, key) => { 89 | // JS safety 90 | if (!Object.prototype.hasOwnProperty.call(coerced, key)) return nextMap; 91 | 92 | // skip null or undefined, purposely not using === 93 | if (coerced[key] == null) return nextMap; 94 | 95 | // skip empty strings 96 | if (typeof coerced[key] === "string" && (coerced[key] as string).trim() === "") 97 | return nextMap; 98 | 99 | // skip NaNs 100 | if (Number.isNaN(coerced[key])) return nextMap; 101 | 102 | // don't bother with empty arrays 103 | if (Array.isArray(coerced[key]) && (coerced[key] as Array).length === 0) 104 | return nextMap; 105 | 106 | const keyPair: { [key: string]: unknown } = {}; 107 | keyPair[key] = coerced[key]; 108 | return { ...nextMap, ...keyPair }; 109 | }, 110 | { ...baseMap } 111 | ); 112 | }; 113 | 114 | /** 115 | * Scrub the `authorization` header when present and return updated. 116 | * This should only be used for logging purposes as it destroys the token. 117 | */ 118 | export const scrubTokens = (headers: IncomingHttpHeaders) => { 119 | if (!("authorization" in headers)) return headers; 120 | const { authorization } = headers; 121 | return { 122 | ...headers, 123 | authorization: `${(authorization as string).substring(0, 12)}... REDACTED`, 124 | }; 125 | }; 126 | 127 | /** 128 | * Return a JSON object as an array of nodes with a leaf value. 129 | * 130 | * @example 131 | * This tree has a single leaf 132 | * {a: {b: c}} => [ {key: [a,b]}, value: c} ] 133 | * 134 | * @example 135 | * This tree has 2 leaves 136 | * {a: {b: z, c: y}} => [ {key: [a,b]}, value: z}, 137 | * {key: [a,c]}, value: y} ] 138 | */ 139 | export const flattenTree = ( 140 | tree: { [key: string]: unknown }, 141 | nodes: string[] = [] 142 | ): { key: string[]; value: unknown }[] => { 143 | return Object.keys(tree).flatMap((key: string) => { 144 | if (isPlainObject(tree[key])) { 145 | return flattenTree(tree[key] as { [key: string]: unknown }, [...nodes, key]); 146 | } else { 147 | return { key: [...nodes, key], value: tree[key] }; 148 | } 149 | }); 150 | }; 151 | 152 | /** 153 | * Return all versions of an ID where a separator could be substituted. 154 | * 155 | * In the case of a collection ID containing the legacy separator we may be 156 | * incorrectly guessing which separator to replace when converting to entry_id. 157 | * Therefore we need to search using all possible variations. 158 | * 159 | * @example 160 | * ambiguateCollectionId("abc.v1.v2.1999", ".v", "_") => 161 | * ["abc_1.v2.1999", "abc.v1_2.1999", "abc.v1.v2.1999"] 162 | */ 163 | export const generatePossibleCollectionIds = (id: string, separator: string, replacement: string) => 164 | id.split(separator).map((currentToken, idx, tokens) => { 165 | if (idx + 1 >= tokens.length) { 166 | return tokens.join(separator); 167 | } 168 | 169 | const mergedToken = currentToken + replacement + tokens[idx + 1]; 170 | // splice mutates the original so use a copy 171 | const tokensCopy = [...tokens]; 172 | // splice returns the replaced objects, not the resulting array 173 | tokensCopy.splice(idx, 2, mergedToken); 174 | 175 | return tokensCopy.join(separator); 176 | }); 177 | 178 | /** 179 | * Returns the base URL 180 | * This is used to remove Query Parameters that may have been added to the URL. 181 | */ 182 | export const getBaseUrl = (url: string) => { 183 | return url.replace(/\?.*$/, ""); 184 | }; 185 | -------------------------------------------------------------------------------- /src/utils/sort.ts: -------------------------------------------------------------------------------- 1 | import { SortObject } from "../models/StacModels"; 2 | 3 | /** 4 | * Parses sortby value into a single array 5 | * This function handles three possible input formats: 6 | * 1. A string of comma-separated sort fields (used in GET requests) 7 | * - /collections?sortby=endDate 8 | * 2. An array of SortObject (used in POST requests) 9 | * { 10 | "sortby": [ 11 | { 12 | "field": "properties.endDate", 13 | "direction": "desc" 14 | } 15 | ] 16 | } 17 | * 3. Undefined or null (returns an empty array) 18 | * 19 | * @param sortBys - The sortby value 20 | * @returns An array of strings, each representing a sort field. 21 | * Fields for descending sort are prefixed with '-'. 22 | */ 23 | export const parseSortFields = (sortBys?: string | string[] | SortObject[]): string[] => { 24 | if (Array.isArray(sortBys)) { 25 | if (sortBys.length === 0) return []; 26 | 27 | if (typeof sortBys[0] === "object") { 28 | // Handle object-based sorting (POST) 29 | return (sortBys as SortObject[]).map( 30 | (sort) => `${sort.direction === "desc" ? "-" : ""}${sort.field}` 31 | ); 32 | } else { 33 | // Handle array of strings 34 | return sortBys.map((item) => (typeof item === "string" ? item.trim() : "")); 35 | } 36 | } else if (typeof sortBys === "string") { 37 | // Handle string-based sorting (GET) 38 | return sortBys.split(",").map((key) => key.trim()); 39 | } 40 | 41 | return []; 42 | }; 43 | 44 | export const mapIdSortKey = (searchType = ""): string => { 45 | if (searchType === "collection") { 46 | return "entryId"; 47 | } else if (searchType === "item") { 48 | return "readableGranuleName"; 49 | } 50 | 51 | return ""; 52 | }; 53 | -------------------------------------------------------------------------------- /src/utils/testUtils.ts: -------------------------------------------------------------------------------- 1 | import { faker } from "@faker-js/faker"; 2 | import { STACCollection } from "../@types/StacCollection"; 3 | import { STACItem } from "../@types/StacItem"; 4 | import { Granule, Collection, RelatedUrlType, UrlContentType } from "../models/GraphQLModels"; 5 | 6 | export const generateSTACCollections = (quantity: number) => { 7 | return Array(quantity) 8 | .fill(undefined) 9 | .map(() => { 10 | return { 11 | id: "TEST_COLLECTION", 12 | title: faker.animal.cat(), 13 | stac_version: "1.0.0", 14 | type: "Collection", 15 | description: faker.hacker.phrase(), 16 | license: "proprietary", 17 | providers: [ 18 | { 19 | name: "PROV1", 20 | roles: ["producer"], 21 | }, 22 | ], 23 | extent: { 24 | spatial: { 25 | bbox: [ 26 | [ 27 | faker.datatype.number({ min: -180, max: 0 }), 28 | faker.datatype.number({ min: -90, max: 0 }), 29 | faker.datatype.number({ min: 0, max: 180 }), 30 | faker.datatype.number({ min: 0, max: 90 }), 31 | ], 32 | ], 33 | }, 34 | temporal: { 35 | interval: [ 36 | [ 37 | faker.date.past().toISOString(), 38 | Math.random() > 0.5 ? null : faker.date.future().toISOString(), 39 | ], 40 | ], 41 | }, 42 | }, 43 | links: [], 44 | } as STACCollection; 45 | }); 46 | }; 47 | 48 | export const generateSTACItems = ( 49 | collection: string, 50 | quantity: number, 51 | opts: { 52 | provider?: string; 53 | root?: string; 54 | } = {} 55 | ): STACItem[] => { 56 | const provider = opts.provider ?? "TEST"; 57 | const root = opts.root ?? "https://localhost:3000/stac/"; 58 | 59 | return Array(quantity) 60 | .fill(undefined) 61 | .map(() => { 62 | const id = faker.random.words(5).replace(/\s+/gi, "_"); 63 | return { 64 | id, 65 | type: "Feature", 66 | stac_version: "1.0.0", 67 | collection, 68 | links: [ 69 | { 70 | rel: "self", 71 | href: `${root}${provider}/collections/${collection}/items/${id}`, 72 | }, 73 | { 74 | rel: "parent", 75 | href: `${root}${provider}/collections/${collection}`, 76 | }, 77 | ], 78 | assets: {}, 79 | properties: {}, 80 | geometry: { 81 | coordinates: [ 82 | [ 83 | [-15.982682390462173, 42.24843416342486], 84 | [-15.982682390462173, -2.914031510390572], 85 | [36.131315758569116, -2.914031510390572], 86 | [36.131315758569116, 42.24843416342486], 87 | [-15.982682390462173, 42.24843416342486], 88 | ], 89 | ], 90 | type: "Polygon", 91 | }, 92 | bbox: [-15.98, -2.91, 36.13, 42.2], 93 | } as STACItem; 94 | }); 95 | }; 96 | 97 | const baseGranule: Granule = { 98 | title: "test_title", 99 | conceptId: "G123456789-TEST_PROV", 100 | collection: null, 101 | cloudCover: null, 102 | lines: null, 103 | boxes: null, 104 | polygons: null, 105 | points: null, 106 | timeStart: "2009-09-14T00:00:00.000Z", 107 | timeEnd: "2010-09-14T00:00:00.000Z", 108 | relatedUrls: [ 109 | { 110 | urlContentType: UrlContentType.DISTRIBUTION_URL, 111 | url: "ftp://e4ftl014.cr.usgs.gov/MODIS_Composites/MOTA/.B09.tif", 112 | description: "Browse image for Earthdata Search", 113 | type: RelatedUrlType.GET_DATA, 114 | }, 115 | { 116 | urlContentType: UrlContentType.DISTRIBUTION_URL, 117 | url: "ftp://e4ftl015/ExampleBadUrl", 118 | description: "Example of bad url data", 119 | type: RelatedUrlType.GET_DATA, 120 | }, 121 | { 122 | urlContentType: UrlContentType.PUBLICATION_URL, 123 | url: "ftp://e4ftl01.cr.usgs.gov/MODIS_Composites/MOTA/MCD43A4.005/2009.09.14/MCD43A4.A2009257.h29v03.005.2009276045828.hdf.xml", 124 | description: "metadata", 125 | type: RelatedUrlType.DATA_SET_LANDING_PAGE, 126 | }, 127 | { 128 | urlContentType: UrlContentType.VISUALIZATION_URL, 129 | url: "ftp://e4ftl01.cr.usgs.gov/MODIS_Composites/MOTA/MCD43A4.005/2009.09.14/MCD43A4.A2009257.h29v03.005.2009276045828.vinr.img", 130 | description: "Browse image for Earthdata Search", 131 | type: RelatedUrlType.THUMBNAIL, 132 | }, 133 | { 134 | urlContentType: UrlContentType.VISUALIZATION_URL, 135 | url: "ftp://e4ftl012/ExampleBadUrl", 136 | description: "Browse image for Earthdata Search", 137 | type: RelatedUrlType.THUMBNAIL, 138 | }, 139 | ], 140 | }; 141 | 142 | export const generateGranules = ( 143 | quantity: number, 144 | opts: { 145 | collection?: { 146 | entryId: string; 147 | version: string; 148 | conceptId: string; 149 | }; 150 | provider?: string; 151 | } = {} 152 | ): Granule[] => { 153 | return Array(quantity) 154 | .fill(undefined) 155 | .map((_granule, idx) => { 156 | return { 157 | ...baseGranule, 158 | conceptId: `G00000000${idx}-${opts?.provider ?? "TEST_PROV"}`, 159 | collection: { 160 | conceptId: opts?.collection?.conceptId ?? "C123456789-TEST_PROV", 161 | entryId: "TEST_COLLECTION_1", 162 | }, 163 | title: faker.random.words(8).replace(/\s+/gi, "_"), 164 | } as Granule; 165 | }); 166 | }; 167 | 168 | /** 169 | * Generate some QUANTITY of collection responses. 170 | */ 171 | export const generateCollections = ( 172 | quantity: number, 173 | opts: { provider?: string } = {} 174 | ): Collection[] => { 175 | const provider = opts?.provider ?? "TEST_PROV"; 176 | 177 | return Array(quantity) 178 | .fill(undefined) 179 | .map((_collection, idx) => { 180 | return { 181 | conceptId: `C0000000${idx}-${provider}`, 182 | provider, 183 | summary: faker.lorem.paragraph(), 184 | description: "this is the abstract but aliased as description", 185 | title: "mock_coll", 186 | entryId: faker.random.words(4).replace(/\s+/, "_"), 187 | boxes: null, 188 | lines: null, 189 | polygons: null, 190 | points: null, 191 | timeStart: faker.date.past().toISOString(), 192 | timeEnd: faker.date.future().toISOString(), 193 | useConstraints: null, 194 | directDistributionInformation: null, 195 | relatedUrls: [], 196 | platforms: [ 197 | { 198 | type: faker.random.words(), 199 | shortName: faker.random.words(4), 200 | longName: faker.random.words(4), 201 | instruments: [ 202 | { 203 | shortName: faker.random.words(4), 204 | longName: faker.random.words(4), 205 | }, 206 | ], 207 | }, 208 | ], 209 | scienceKeywords: [ 210 | { 211 | category: "EARTH SCIENCE", 212 | topic: "LAND SURFACE", 213 | term: "TOPOGRAPHY", 214 | variableLevel1: "TERRAIN ELEVATION", 215 | }, 216 | ], 217 | } as Collection; 218 | }); 219 | }; 220 | -------------------------------------------------------------------------------- /start.js: -------------------------------------------------------------------------------- 1 | const concurrently = require('concurrently') 2 | 3 | concurrently([{ 4 | command: 'npm run watch', 5 | name: 'watch' 6 | }, { 7 | // Due to differences between deployed and local we use a slightly modified `cdk` template 8 | command: 'sam local start-api -t ./cdk/cmr-stac-dev/cdk.out/cmr-stac-dev.template.json --env-vars=sam_local_envs.json --warm-containers LAZY --port 3000 --docker-network host', 9 | name: 'api' 10 | }], { 11 | prefix: 'name', 12 | padPrefix: true, 13 | prefixColors: 'auto', 14 | handleInput: true 15 | }) 16 | -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "preserveConstEnums": true, 4 | "strictNullChecks": true, 5 | "sourceMap": true, 6 | "allowJs": true, 7 | "target": "es2020", 8 | "outDir": ".build", 9 | "moduleResolution": "node", 10 | "lib": ["es2020"], 11 | "rootDir": "./", 12 | "strict": true, 13 | "module": "commonjs", 14 | "esModuleInterop": true, 15 | "resolveJsonModule": true, 16 | "typeRoots": ["node_modules/@types", "src/@types"] 17 | }, 18 | "include": ["./**/*.ts"], 19 | "exclude": ["node_modules", "./**/*.spec.ts", "./cdk/*"] 20 | } 21 | --------------------------------------------------------------------------------