├── chat ├── dependencies │ └── .keep ├── src │ ├── __init__.py │ ├── agent │ │ ├── __init__.py │ │ └── callbacks │ │ │ └── __init__.py │ ├── core │ │ ├── __init__.py │ │ ├── websocket.py │ │ ├── secrets.py │ │ ├── apitoken.py │ │ ├── prompts.py │ │ ├── document.py │ │ └── setup.py │ ├── search │ │ └── __init__.py │ └── persistence │ │ ├── __init__.py │ │ ├── compressible_json_serializer.py │ │ └── selective_checkpointer.py ├── test │ ├── __init__.py │ ├── agent │ │ ├── __init__.py │ │ ├── callbacks │ │ │ └── __init__.py │ │ └── test_search_workflow.py │ ├── core │ │ ├── __init__.py │ │ ├── test_prompts.py │ │ ├── test_secrets.py │ │ └── test_apitoken.py │ ├── search │ │ ├── __init__.py │ │ └── test_hybrid_query.py │ ├── handlers │ │ └── __init__.py │ ├── persistence │ │ └── __init__.py │ └── fixtures │ │ ├── apitoken.py │ │ └── events.py ├── pytest.ini └── pyproject.toml ├── api ├── .npmignore ├── src │ ├── .eslintrc │ ├── handlers │ │ ├── options-request.js │ │ ├── search.js │ │ ├── get-work-auth.js │ │ ├── get-chat-endpoint.js │ │ ├── get-auth-whoami.js │ │ ├── get-file-set-by-id.js │ │ ├── oai │ │ │ ├── date-utils.js │ │ │ ├── xml-transformer.js │ │ │ └── search.js │ │ ├── get-file-set-auth.js │ │ ├── authorize-document.js │ │ ├── get-similar.js │ │ ├── get-work-by-id.js │ │ ├── get-file-set-annotations.js │ │ ├── get-collections.js │ │ ├── get-auth-logout.js │ │ ├── auth │ │ │ ├── nusso-login.js │ │ │ ├── magic-callback.js │ │ │ ├── magic-login.js │ │ │ ├── magic-link.js │ │ │ └── nusso-callback.js │ │ ├── get-shared-link-by-id.js │ │ ├── get-provider-capabilities.js │ │ ├── get-auth-token.js │ │ ├── get-auth-stage.js │ │ ├── transcode-templates.js │ │ ├── get-collection-by-id.js │ │ ├── get-annotation-by-id.js │ │ ├── oai.js │ │ └── get-thumbnail.js │ ├── api │ │ ├── response │ │ │ ├── error.js │ │ │ ├── transformer.js │ │ │ ├── iiif │ │ │ │ └── presentation-api │ │ │ │ │ ├── provider.js │ │ │ │ │ ├── placeholder-canvas.js │ │ │ │ │ └── metadata.js │ │ │ └── opensearch │ │ │ │ └── index.js │ │ ├── request │ │ │ ├── models.js │ │ │ └── pipeline.js │ │ ├── scopes.js │ │ └── pagination.js │ ├── honeybadger-setup.js │ ├── aws │ │ └── fetch.js │ ├── package.json │ └── environment.js ├── test │ ├── .eslintrc │ ├── fixtures │ │ └── mocks │ │ │ ├── thumbnail_full.jpg │ │ │ ├── thumbnail_square.jpg │ │ │ ├── missing-work-1234.json │ │ │ ├── missing-fileset-1234.json │ │ │ ├── missing-shared-link-5678.json │ │ │ ├── missing-collection-1234.json │ │ │ ├── work-1234-no-thumbnail.json │ │ │ ├── work-netid-1234.json │ │ │ ├── work-restricted-1234.json │ │ │ ├── fileset-baddata-1234.json │ │ │ ├── fileset-netid-1234.json │ │ │ ├── fileset-restricted-1234.json │ │ │ ├── fileset-unpublished-1234.json │ │ │ ├── collection-1234-no-thumbnail.json │ │ │ ├── fileset-1234.json │ │ │ ├── annotation-search-empty.json │ │ │ ├── shared-link-1234.json │ │ │ ├── work-restricted-unpublished-1234.json │ │ │ ├── fileset-restricted-unpublished-1234.json │ │ │ ├── expired-shared-link-9101112.json │ │ │ ├── fileset-audio-1234.json │ │ │ ├── fileset-video-1234.json │ │ │ ├── scroll-empty.json │ │ │ ├── private-work-1234.json │ │ │ ├── unpublished-work-1234.json │ │ │ ├── collection-1234.json │ │ │ ├── private-unpublished-work-1234.json │ │ │ ├── collection-1234-private-published.json │ │ │ ├── annotation-search-hit.json │ │ │ ├── missing-index.json │ │ │ ├── search-earliest-record.json │ │ │ ├── work-file-sets-access.json │ │ │ ├── scroll-missing.json │ │ │ ├── work-file-sets.json │ │ │ ├── real-search-event.json │ │ │ ├── oai-sets.json │ │ │ ├── fileset-annotated-1234.json │ │ │ └── oai-list-identifiers-sets.json │ ├── unit │ │ ├── package.test.js │ │ ├── api │ │ │ ├── response │ │ │ │ ├── error.test.js │ │ │ │ ├── iiif │ │ │ │ │ └── presentation-api │ │ │ │ │ │ ├── metadata.test.js │ │ │ │ │ │ ├── provider.test.js │ │ │ │ │ │ └── placeholder-canvas.test.js │ │ │ │ └── opensearch.test.js │ │ │ └── request │ │ │ │ └── models.test.js │ │ └── aws │ │ │ └── environment.test.js │ ├── integration │ │ ├── options-request.test.js │ │ ├── get-chat-endpoint.test.js │ │ ├── middleware.test.js │ │ ├── auth │ │ │ └── nusso-login.test.js │ │ ├── get-provider-capabilities.test.js │ │ ├── get-annotations.test.js │ │ ├── get-file-set-download.test.js │ │ ├── get-auth-logout.test.js │ │ └── get-auth-whoami.test.js │ └── test-helpers │ │ └── index.js ├── .mocharc.js ├── nyc.config.js ├── dependencies │ └── package.json └── package.json ├── .github ├── flags │ └── deploy-data-types ├── scripts │ └── honeybadger_deploy_notification.sh ├── PULL_REQUEST_TEMPLATE │ └── production.md └── workflows │ ├── test-node.yml │ ├── test-python.yml │ ├── validate-template.yml │ ├── next_version.yml │ ├── docs.yml │ └── staging-types.yml ├── loadtest ├── requirements.txt ├── README.md └── locustfile.py ├── docs ├── docs │ ├── spec.md │ ├── css │ │ ├── overrides.css │ │ └── fonts.css │ ├── spec │ │ └── openapi.html │ └── oai.md ├── redirect │ └── index.js ├── pyproject.toml ├── mkdocs.yml ├── template.yaml └── overrides │ └── .icons │ └── nul-logo.svg ├── .tool-versions ├── dev ├── samconfig.toml └── env.json ├── .husky └── pre-commit ├── openapitools.json ├── av-download └── lambdas │ ├── package.json │ ├── transcode-status.js │ ├── get-download-link.js │ ├── start-transcode.js │ ├── send-templated-email.js │ └── start-audio-transcode.js ├── chat-playground ├── opensearch.ipynb └── playground.ipynb ├── events └── event.json └── bin ├── make_env.sh └── make_deploy_config.sh /chat/dependencies/.keep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /chat/src/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /chat/test/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /api/.npmignore: -------------------------------------------------------------------------------- 1 | test/* 2 | -------------------------------------------------------------------------------- /chat/src/agent/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /chat/src/core/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /chat/src/search/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /chat/test/agent/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /chat/test/core/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /chat/test/search/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /chat/src/persistence/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /chat/test/handlers/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /chat/test/persistence/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /.github/flags/deploy-data-types: -------------------------------------------------------------------------------- 1 | 00004 -------------------------------------------------------------------------------- /chat/src/agent/callbacks/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /chat/test/agent/callbacks/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /loadtest/requirements.txt: -------------------------------------------------------------------------------- 1 | locust==2.32.2 -------------------------------------------------------------------------------- /docs/docs/spec.md: -------------------------------------------------------------------------------- 1 | !!swagger ./spec/openapi.yaml!! -------------------------------------------------------------------------------- /api/src/.eslintrc: -------------------------------------------------------------------------------- 1 | { 2 | "env": { 3 | "es2020": true, 4 | "node": true 5 | } 6 | } 7 | -------------------------------------------------------------------------------- /.tool-versions: -------------------------------------------------------------------------------- 1 | nodejs 20.15.0 2 | java corretto-19.0.1.10.1 3 | aws-sam-cli 1.148.0 4 | python 3.12.2 5 | uv 0.9.5 -------------------------------------------------------------------------------- /api/test/.eslintrc: -------------------------------------------------------------------------------- 1 | { 2 | "globals": { 3 | "helpers": true, 4 | "requireSource": true 5 | } 6 | } 7 | -------------------------------------------------------------------------------- /chat/pytest.ini: -------------------------------------------------------------------------------- 1 | [pytest] 2 | addopts = -m "not slow" 3 | markers = 4 | slow: marks tests as slow (deselect with '-m "not slow"') -------------------------------------------------------------------------------- /api/test/fixtures/mocks/thumbnail_full.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nulib/dc-api-v2/HEAD/api/test/fixtures/mocks/thumbnail_full.jpg -------------------------------------------------------------------------------- /api/test/fixtures/mocks/thumbnail_square.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nulib/dc-api-v2/HEAD/api/test/fixtures/mocks/thumbnail_square.jpg -------------------------------------------------------------------------------- /api/.mocharc.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | ignore: ["test/test-helpers/**/*"], 3 | recursive: true, 4 | require: ["test/test-helpers", "choma"] 5 | } -------------------------------------------------------------------------------- /api/test/fixtures/mocks/missing-work-1234.json: -------------------------------------------------------------------------------- 1 | { 2 | "_index": "dev-dc-v2-work", 3 | "_type": "_doc", 4 | "_id": "1234", 5 | "found": false 6 | } 7 | -------------------------------------------------------------------------------- /api/test/fixtures/mocks/missing-fileset-1234.json: -------------------------------------------------------------------------------- 1 | { 2 | "_index": "dev-dc-v2-file-set", 3 | "_type": "_doc", 4 | "_id": "1234", 5 | "found": false 6 | } 7 | -------------------------------------------------------------------------------- /api/test/fixtures/mocks/missing-shared-link-5678.json: -------------------------------------------------------------------------------- 1 | { 2 | "_index": "dev-shared_links", 3 | "_type": "_doc", 4 | "_id": "1234", 5 | "found": false 6 | } 7 | -------------------------------------------------------------------------------- /api/test/fixtures/mocks/missing-collection-1234.json: -------------------------------------------------------------------------------- 1 | { 2 | "_index": "dev-dc-v2-collection", 3 | "_type": "_doc", 4 | "_id": "1234", 5 | "found": false 6 | } 7 | -------------------------------------------------------------------------------- /api/src/handlers/options-request.js: -------------------------------------------------------------------------------- 1 | const { wrap } = require("./middleware"); 2 | 3 | module.exports.handler = wrap(async () => { 4 | return { statusCode: 200 }; 5 | }); 6 | -------------------------------------------------------------------------------- /dev/samconfig.toml: -------------------------------------------------------------------------------- 1 | version = 0.1 2 | [default] 3 | [default.deploy] 4 | [default.deploy.parameters] 5 | 6 | [default.local_start_api.parameters] 7 | env_vars = "env.json" 8 | -------------------------------------------------------------------------------- /.husky/pre-commit: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env sh 2 | . "$(dirname -- "$0")/_/husky.sh" 3 | cd api && npm run lint && npm run prettier && cd - 4 | cd chat/src && uv run ruff check . && cd - 5 | -------------------------------------------------------------------------------- /openapitools.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "./node_modules/@openapitools/openapi-generator-cli/config.schema.json", 3 | "spaces": 2, 4 | "generator-cli": { 5 | "version": "6.2.1" 6 | } 7 | } 8 | -------------------------------------------------------------------------------- /docs/docs/css/overrides.css: -------------------------------------------------------------------------------- 1 | body { 2 | font-family: "Akkurat Pro Regular", -apple-system, BlinkMacSystemFont, Helvetica, Arial, sans-serif 3 | } 4 | 5 | .md-header { 6 | background-color: rgb(64, 31, 104); 7 | } 8 | 9 | .md-header__topic { 10 | display: none; 11 | } -------------------------------------------------------------------------------- /.github/scripts/honeybadger_deploy_notification.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | curl \ 4 | --data "deploy[environment]=${DEPLOY_ENV}&deploy[local_username]=Github+Actions&deploy[revision]=${HONEYBADGER_REVISION}&api_key=${HONEYBADGER_API_KEY}" \ 5 | https://api.honeybadger.io/v1/deploys 6 | -------------------------------------------------------------------------------- /api/test/fixtures/mocks/work-1234-no-thumbnail.json: -------------------------------------------------------------------------------- 1 | { 2 | "_index": "dev-dc-v2-work", 3 | "_type": "_doc", 4 | "_id": "1234", 5 | "_version": 1, 6 | "found": true, 7 | "_source": { 8 | "id": "1234", 9 | "api_model": "Work", 10 | "published": true, 11 | "visibility": "Public" 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /api/test/fixtures/mocks/work-netid-1234.json: -------------------------------------------------------------------------------- 1 | { 2 | "_index": "dev-dc-v2-work", 3 | "_type": "_doc", 4 | "_id": "1234", 5 | "_version": 1, 6 | "found": true, 7 | "_source": { 8 | "id": "1234", 9 | "api_model": "Work", 10 | "visibility": "Institution", 11 | "published": true 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /api/test/fixtures/mocks/work-restricted-1234.json: -------------------------------------------------------------------------------- 1 | { 2 | "_index": "dev-dc-v2-work", 3 | "_type": "_doc", 4 | "_id": "1234", 5 | "_version": 1, 6 | "found": true, 7 | "_source": { 8 | "id": "1234", 9 | "api_model": "Work", 10 | "visibility": "Private", 11 | "published": true 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /api/test/fixtures/mocks/fileset-baddata-1234.json: -------------------------------------------------------------------------------- 1 | { 2 | "_index": "dev-dc-v2-file-set", 3 | "_type": "_doc", 4 | "_id": "1234", 5 | "_version": 1, 6 | "found": true, 7 | "_source": { 8 | "id": "1234", 9 | "api_model": "FileSet", 10 | "visibility": "Restricted", 11 | "published": true 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /api/test/fixtures/mocks/fileset-netid-1234.json: -------------------------------------------------------------------------------- 1 | { 2 | "_index": "dev-dc-v2-file-set", 3 | "_type": "_doc", 4 | "_id": "1234", 5 | "_version": 1, 6 | "found": true, 7 | "_source": { 8 | "id": "1234", 9 | "api_model": "FileSet", 10 | "visibility": "Institution", 11 | "published": true 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /api/test/fixtures/mocks/fileset-restricted-1234.json: -------------------------------------------------------------------------------- 1 | { 2 | "_index": "dev-dc-v2-file-set", 3 | "_type": "_doc", 4 | "_id": "1234", 5 | "_version": 1, 6 | "found": true, 7 | "_source": { 8 | "id": "1234", 9 | "api_model": "FileSet", 10 | "visibility": "Private", 11 | "published": true 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /api/test/fixtures/mocks/fileset-unpublished-1234.json: -------------------------------------------------------------------------------- 1 | { 2 | "_index": "dev-dc-v2-file-set", 3 | "_type": "_doc", 4 | "_id": "1234", 5 | "_version": 1, 6 | "found": true, 7 | "_source": { 8 | "id": "1234", 9 | "api_model": "FileSet", 10 | "visibility": "Public", 11 | "published": false 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /api/test/fixtures/mocks/collection-1234-no-thumbnail.json: -------------------------------------------------------------------------------- 1 | { 2 | "_index": "dev-dc-v2-collection", 3 | "_type": "_doc", 4 | "_id": "1234", 5 | "_version": 1, 6 | "found": true, 7 | "_source": { 8 | "id": "1234", 9 | "api_model": "Collection", 10 | "published": true, 11 | "representative_image": {} 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /docs/redirect/index.js: -------------------------------------------------------------------------------- 1 | exports.handler = async () => { 2 | const target = process.env.REDIRECT_TO; 3 | 4 | return { 5 | statusCode: 302, 6 | headers: { 7 | "content-type": "text/html", 8 | location: target 9 | }, 10 | body: `

Redirecting to API Documentation

` 11 | }; 12 | }; 13 | -------------------------------------------------------------------------------- /api/test/fixtures/mocks/fileset-1234.json: -------------------------------------------------------------------------------- 1 | { 2 | "_index": "dev-dc-v2-file-set", 3 | "_type": "_doc", 4 | "_id": "1234", 5 | "_version": 1, 6 | "found": true, 7 | "_source": { 8 | "id": "1234", 9 | "api_model": "FileSet", 10 | "visibility": "Public", 11 | "published": true, 12 | "mime_type": "image/tiff" 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /api/test/fixtures/mocks/annotation-search-empty.json: -------------------------------------------------------------------------------- 1 | { 2 | "took": 1, 3 | "timed_out": false, 4 | "_shards": { 5 | "total": 1, 6 | "successful": 1, 7 | "skipped": 0, 8 | "failed": 0 9 | }, 10 | "hits": { 11 | "total": { 12 | "value": 0, 13 | "relation": "eq" 14 | }, 15 | "max_score": null, 16 | "hits": [] 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /api/test/fixtures/mocks/shared-link-1234.json: -------------------------------------------------------------------------------- 1 | { 2 | "_index": "dev-shared_links", 3 | "_type": "_doc", 4 | "_id": "1234", 5 | "_version": 1, 6 | "found": true, 7 | "_source": { 8 | "target_index": "meadow", 9 | "target_id": "1234", 10 | "shared_link_id": "b47a0a1c-ca52-424f-877f-a2ab9f1c0735", 11 | "expires": "2028-10-26T16:47:06Z" 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /api/test/fixtures/mocks/work-restricted-unpublished-1234.json: -------------------------------------------------------------------------------- 1 | { 2 | "_index": "dev-dc-v2-work", 3 | "_type": "_doc", 4 | "_id": "756ea5b9-8ca1-4bd7-a70e-4b2082dd0440", 5 | "_version": 1, 6 | "found": true, 7 | "_source": { 8 | "id": "756ea5b9-8ca1-4bd7-a70e-4b2082dd0440", 9 | "api_model": "Work", 10 | "visibility": "Private", 11 | "published": false 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /api/test/fixtures/mocks/fileset-restricted-unpublished-1234.json: -------------------------------------------------------------------------------- 1 | { 2 | "_index": "dev-dc-v2-file-set", 3 | "_type": "_doc", 4 | "_id": "1234", 5 | "_version": 1, 6 | "found": true, 7 | "_source": { 8 | "id": "1234", 9 | "api_model": "FileSet", 10 | "visibility": "Private", 11 | "published": false, 12 | "work_id": "756ea5b9-8ca1-4bd7-a70e-4b2082dd0440" 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /api/test/fixtures/mocks/expired-shared-link-9101112.json: -------------------------------------------------------------------------------- 1 | { 2 | "_index": "dev-shared_links", 3 | "_type": "_doc", 4 | "_id": "1234", 5 | "_version": 1, 6 | "found": true, 7 | "_source": { 8 | "target_index": "meadow", 9 | "target_id": "23255308-53f4-4c96-a268-aefa596d9d21", 10 | "shared_link_id": "b47a0a1c-ca52-424f-877f-a2ab9f1c0735", 11 | "expires": "2021-10-26T16:47:06Z" 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /api/src/api/response/error.js: -------------------------------------------------------------------------------- 1 | const { getReasonPhrase } = require("http-status-codes"); 2 | 3 | function transformError(response) { 4 | const responseBody = { 5 | status: response.statusCode, 6 | error: getReasonPhrase(response.statusCode), 7 | }; 8 | 9 | return { 10 | statusCode: response.statusCode, 11 | body: JSON.stringify(responseBody), 12 | }; 13 | } 14 | 15 | module.exports = { transformError }; 16 | -------------------------------------------------------------------------------- /api/src/handlers/search.js: -------------------------------------------------------------------------------- 1 | const { doSearch } = require("./search-runner"); 2 | const { wrap } = require("./middleware"); 3 | 4 | const getSearch = wrap(async (event) => { 5 | const includeToken = !!event.queryStringParameters?.searchToken; 6 | return await doSearch(event, { includeToken }); 7 | }); 8 | 9 | const postSearch = wrap(async (event) => await doSearch(event)); 10 | 11 | module.exports = { postSearch, getSearch }; 12 | -------------------------------------------------------------------------------- /docs/pyproject.toml: -------------------------------------------------------------------------------- 1 | [project] 2 | name = "dc-api-v2-docs" 3 | version = "2.9.2" 4 | requires-python = ">=3.12" 5 | dependencies = [ 6 | "mkdocs>=1.1.2,<2.0.0", 7 | "mkdocs-macros-plugin @ git+https://github.com/fralau/mkdocs_macros_plugin.git@v0.5.12", 8 | "Pygments>=2.7.3,<3.0.0", 9 | "diagrams>=0.21.1,<1.0.0", 10 | "mkdocs-material>=9.0.0", 11 | "mkdocs-render-swagger-plugin>=0.1.2", 12 | "setuptools>=78.1.0", 13 | ] 14 | -------------------------------------------------------------------------------- /api/test/fixtures/mocks/fileset-audio-1234.json: -------------------------------------------------------------------------------- 1 | { 2 | "_index": "dev-dc-v2-file-set", 3 | "_type": "_doc", 4 | "_id": "1234", 5 | "_version": 1, 6 | "found": true, 7 | "_source": { 8 | "id": "1234", 9 | "api_model": "FileSet", 10 | "visibility": "Public", 11 | "published": true, 12 | "mime_type": "audio/mp3", 13 | "role": "Access", 14 | "streaming_url": "https://example.com/video.mp3" 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /api/test/fixtures/mocks/fileset-video-1234.json: -------------------------------------------------------------------------------- 1 | { 2 | "_index": "dev-dc-v2-file-set", 3 | "_type": "_doc", 4 | "_id": "1234", 5 | "_version": 1, 6 | "found": true, 7 | "_source": { 8 | "id": "1234", 9 | "api_model": "FileSet", 10 | "visibility": "Public", 11 | "published": true, 12 | "mime_type": "video/mp4", 13 | "role": "Access", 14 | "streaming_url": "https://example.com/video.mp4" 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /api/test/fixtures/mocks/scroll-empty.json: -------------------------------------------------------------------------------- 1 | { 2 | "_scroll_id": "FGluY2x1ZGVfY29udGV4dF91dWlkDXF1ZXJ5QW5kRmV0Y2gBFm1jN3ZCajdnUURpbUhad1hIYnNsQmcAAAAAAAB2DhZXbmtMZVF5Q1JsMi1ScGRsYUlHLUtB", 3 | "took": 3, 4 | "timed_out": false, 5 | "_shards": { "total": 1, "successful": 1, "skipped": 0, "failed": 0 }, 6 | "hits": { 7 | "total": { "value": 37345, "relation": "eq" }, 8 | "max_score": null, 9 | "hits": [] 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /api/test/fixtures/mocks/private-work-1234.json: -------------------------------------------------------------------------------- 1 | { 2 | "_index": "dev-dc-v2-work", 3 | "_type": "_doc", 4 | "_id": "1234", 5 | "_version": 1, 6 | "found": true, 7 | "_source": { 8 | "id": "1234", 9 | "api_model": "Work", 10 | "published": true, 11 | "visibility": "Private", 12 | "representative_file_set": { 13 | "fileSetId": "5678", 14 | "url": "https://index.test.library.northwestern.edu/iiif/2/mbk-dev/5678" 15 | } 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /api/test/fixtures/mocks/unpublished-work-1234.json: -------------------------------------------------------------------------------- 1 | { 2 | "_index": "dev-dc-v2-work", 3 | "_type": "_doc", 4 | "_id": "1234", 5 | "_version": 1, 6 | "found": true, 7 | "_source": { 8 | "id": "1234", 9 | "api_model": "Work", 10 | "published": false, 11 | "visibility": "Public", 12 | "representative_file_set": { 13 | "fileSetId": "5678", 14 | "url": "https://index.test.library.northwestern.edu/iiif/2/mbk-dev/5678" 15 | } 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /api/test/fixtures/mocks/collection-1234.json: -------------------------------------------------------------------------------- 1 | { 2 | "_index": "dev-dc-v2-collection", 3 | "_type": "_doc", 4 | "_id": "1234", 5 | "_version": 1, 6 | "found": true, 7 | "_source": { 8 | "id": "1234", 9 | "title": "Collection Title", 10 | "api_model": "Collection", 11 | "published": true, 12 | "representative_image": { 13 | "work_id": "1234", 14 | "url": "https://index.test.library.northwestern.edu/iiif/2/mbk-dev/5678" 15 | } 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /api/test/fixtures/mocks/private-unpublished-work-1234.json: -------------------------------------------------------------------------------- 1 | { 2 | "_index": "dev-dc-v2-work", 3 | "_type": "_doc", 4 | "_id": "1234", 5 | "_version": 1, 6 | "found": true, 7 | "_source": { 8 | "id": "1234", 9 | "api_model": "Work", 10 | "published": false, 11 | "visibility": "Private", 12 | "representative_file_set": { 13 | "fileSetId": "5678", 14 | "url": "https://index.test.library.northwestern.edu/iiif/2/mbk-dev/5678" 15 | } 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /api/nyc.config.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | const defaultExclude = require('@istanbuljs/schema/default-exclude'); 4 | const localExclude = [".aws-sam/**/*", "docs/**/*"]; 5 | module.exports = { 6 | all: true, 7 | branches: 80, 8 | lines: 80, 9 | functions: 80, 10 | statements: 80, 11 | watermarks: { 12 | lines: [80, 90], 13 | functions: [80, 90], 14 | branches: [80, 90], 15 | statements: [80, 90] 16 | }, 17 | "check-coverage": true, 18 | exclude: defaultExclude.concat(localExclude) 19 | }; 20 | -------------------------------------------------------------------------------- /api/src/handlers/get-work-auth.js: -------------------------------------------------------------------------------- 1 | const { getWork } = require("../api/opensearch"); 2 | const { authorizeDocument } = require("./authorize-document"); 3 | const { wrap } = require("./middleware"); 4 | 5 | /** 6 | * Authorizes a Work by id 7 | */ 8 | exports.handler = wrap(async (event) => { 9 | const id = event.pathParameters.id; 10 | 11 | const osResponse = await getWork(id, { 12 | allowPrivate: true, 13 | allowUnpublished: true, 14 | }); 15 | 16 | return authorizeDocument(event, osResponse); 17 | }); 18 | -------------------------------------------------------------------------------- /api/test/fixtures/mocks/collection-1234-private-published.json: -------------------------------------------------------------------------------- 1 | { 2 | "_index": "dev-dc-v2-collection", 3 | "_type": "_doc", 4 | "_id": "1234", 5 | "_version": 1, 6 | "found": true, 7 | "_source": { 8 | "id": "1234", 9 | "title": "Collection Title", 10 | "api_model": "Collection", 11 | "published": true, 12 | "visibility": "Private", 13 | "representative_image": { 14 | "work_id": "1234", 15 | "url": "https://index.test.library.northwestern.edu/iiif/2/mbk-dev/5678" 16 | } 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /api/test/unit/package.test.js: -------------------------------------------------------------------------------- 1 | const chai = require("chai"); 2 | const expect = chai.expect; 3 | 4 | describe("package.json", () => { 5 | const rootPackage = requireSource("../package.json"); 6 | const srcPackage = requireSource("./package.json"); 7 | 8 | it("root package has no external runtime dependencies", () => { 9 | expect(Object.keys(rootPackage.dependencies)).to.eql(["dc-api"]); 10 | }); 11 | 12 | it("root and src packages are the same version", () => { 13 | expect(rootPackage.version).to.eq(srcPackage.version); 14 | }); 15 | }); 16 | -------------------------------------------------------------------------------- /api/src/handlers/get-chat-endpoint.js: -------------------------------------------------------------------------------- 1 | const { wrap } = require("./middleware"); 2 | 3 | const handler = wrap(async (event) => { 4 | if (!event.userToken.can("chat")) { 5 | return { 6 | statusCode: 401, 7 | headers: { "Content-Type": "text/plain" }, 8 | body: "Authorization Required", 9 | }; 10 | } 11 | 12 | return { 13 | statusCode: 200, 14 | body: JSON.stringify({ 15 | endpoint: process.env.WEBSOCKET_URI, 16 | auth: event.userToken.sign(), 17 | }), 18 | }; 19 | }); 20 | 21 | module.exports = { handler }; 22 | -------------------------------------------------------------------------------- /api/test/fixtures/mocks/annotation-search-hit.json: -------------------------------------------------------------------------------- 1 | { 2 | "took": 1, 3 | "timed_out": false, 4 | "_shards": { 5 | "total": 1, 6 | "successful": 1, 7 | "skipped": 0, 8 | "failed": 0 9 | }, 10 | "hits": { 11 | "total": { 12 | "value": 1, 13 | "relation": "eq" 14 | }, 15 | "max_score": 1.0, 16 | "hits": [ 17 | { 18 | "_index": "dev-dc-v2-file-set", 19 | "_type": "_doc", 20 | "_id": "1234", 21 | "_score": 1.0, 22 | "_source": { 23 | "id": "1234" 24 | } 25 | } 26 | ] 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /av-download/lambdas/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "lambdas", 3 | "version": "2.9.2", 4 | "description": "Non-API handler lambdas", 5 | "scripts": { 6 | "test": "echo \"Error: no test specified\" && exit 1" 7 | }, 8 | "author": "nulib", 9 | "license": "Apache-2.0", 10 | "devDependencies": { 11 | "@aws-sdk/client-mediaconvert": "^3.410.0", 12 | "@aws-sdk/client-s3": "^3.410.0", 13 | "@aws-sdk/lib-storage": "^3.410.0", 14 | "@aws-sdk/s3-request-presigner": "^3.410.0", 15 | "@aws-sdk/signature-v4": "^3.130.0", 16 | "aws-sdk": "^2.1640.0" 17 | }, 18 | "dependencies": { 19 | "fluent-ffmpeg": "2.1.3" 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /.github/PULL_REQUEST_TEMPLATE/production.md: -------------------------------------------------------------------------------- 1 | # :open_book: Changelog 2 | 3 | - list changes 4 | - list changes 5 | 6 | # Version bump required by the PR 7 | 8 | See [Semantic Versioning 2.0.0](https://semver.org/) for help discerning which is required. 9 | 10 | - [ ] Patch 11 | - [ ] Minor 12 | - [ ] Major 13 | 14 | # :rocket: Deployment Notes 15 | 16 | - Backward compatible API changes 17 | - [ ] REST API 18 | - [ ] Chat Websocket API 19 | - Backwards-incompatible API changes 20 | - [ ] REST API 21 | - [ ] Chat Websocket API 22 | - [ ] Specific deployment synchronization instructions with other apps/API's 23 | - [ ] Other specific instructions/tasks 24 | -------------------------------------------------------------------------------- /api/src/handlers/get-auth-whoami.js: -------------------------------------------------------------------------------- 1 | const { wrap } = require("./middleware"); 2 | const Honeybadger = require("../honeybadger-setup"); 3 | 4 | /** 5 | * Whoami - validates JWT and returns user info, or issues an anonymous 6 | * token if none is present 7 | */ 8 | exports.handler = wrap(async (event) => { 9 | try { 10 | const token = event.userToken; 11 | 12 | return { 13 | statusCode: 200, 14 | body: JSON.stringify(token.userInfo()), 15 | }; 16 | } catch (error) { 17 | await Honeybadger.notifyAsync(error); 18 | return { 19 | statusCode: 401, 20 | body: "Error verifying API token: " + error.message, 21 | }; 22 | } 23 | }); 24 | -------------------------------------------------------------------------------- /api/src/handlers/get-file-set-by-id.js: -------------------------------------------------------------------------------- 1 | const { wrap } = require("./middleware"); 2 | const { getFileSet } = require("../api/opensearch"); 3 | const opensearchResponse = require("../api/response/opensearch"); 4 | 5 | /** 6 | * A simple function to get a FileSet by id 7 | */ 8 | exports.handler = wrap(async (event) => { 9 | const id = event.pathParameters.id; 10 | const allowPrivate = 11 | event.userToken.isSuperUser() || event.userToken.isReadingRoom(); 12 | const allowUnpublished = event.userToken.isSuperUser(); 13 | const esResponse = await getFileSet(id, { allowPrivate, allowUnpublished }); 14 | return await opensearchResponse.transform(esResponse); 15 | }); 16 | -------------------------------------------------------------------------------- /api/test/unit/api/response/error.test.js: -------------------------------------------------------------------------------- 1 | const chai = require("chai"); 2 | const expect = chai.expect; 3 | 4 | const errorTransformer = requireSource("api/response/error"); 5 | 6 | describe("The error response", () => { 7 | it("Transforms a missing work response", async () => { 8 | const response = { 9 | statusCode: 404, 10 | body: helpers.testFixture("mocks/missing-work-1234.json"), 11 | }; 12 | const result = errorTransformer.transformError(response); 13 | 14 | expect(result.statusCode).to.eq(404); 15 | expect(JSON.parse(result.body).status).to.eq(404); 16 | expect(JSON.parse(result.body).error).to.eq("Not Found"); 17 | }); 18 | }); 19 | -------------------------------------------------------------------------------- /api/test/fixtures/mocks/missing-index.json: -------------------------------------------------------------------------------- 1 | { 2 | "error": { 3 | "root_cause": [ 4 | { 5 | "type": "index_not_found_exception", 6 | "reason": "no such index [non-existent-index]", 7 | "index": "non-existent-index", 8 | "resource.id": "non-existent-index", 9 | "resource.type": "index_or_alias", 10 | "index_uuid": "_na_" 11 | } 12 | ], 13 | "type": "index_not_found_exception", 14 | "reason": "no such index [non-existent-index]", 15 | "index": "non-existent-index", 16 | "resource.id": "non-existent-index", 17 | "resource.type": "index_or_alias", 18 | "index_uuid": "_na_" 19 | }, 20 | "status": 404 21 | } 22 | -------------------------------------------------------------------------------- /chat/pyproject.toml: -------------------------------------------------------------------------------- 1 | [project] 2 | name = "dc-api-v2-chat" 3 | version = "2.9.2" 4 | requires-python = ">=3.12" 5 | dependencies = [ 6 | "boto3~=1.34", 7 | "honeybadger~=0.20", 8 | "langchain~=0.2", 9 | "langchain-aws~=0.2", 10 | "langchain-openai~=0.1", 11 | "langgraph~=0.2", 12 | "numpy==2.2.6", 13 | "openai~=1.35", 14 | "opensearch-py~=2.8", 15 | "pyjwt~=2.6.0", 16 | "python-dotenv~=1.0.0", 17 | "requests~=2.32", 18 | "requests-aws4auth~=1.3", 19 | "tiktoken~=0.7,<0.12", 20 | "wheel~=0.40" 21 | ] 22 | 23 | [dependency-groups] 24 | dev = [ 25 | "moto~=5.0", 26 | "pytest~=8.3", 27 | "ruff~=0.2", 28 | "coverage~=7.3" 29 | ] 30 | 31 | [tool.uv] 32 | default-groups = [] 33 | -------------------------------------------------------------------------------- /api/test/fixtures/mocks/search-earliest-record.json: -------------------------------------------------------------------------------- 1 | { 2 | "took": 3, 3 | "timed_out": false, 4 | "_shards": { 5 | "total": 1, 6 | "successful": 1, 7 | "skipped": 0, 8 | "failed": 0 9 | }, 10 | "hits": { 11 | "total": { 12 | "value": 10000, 13 | "relation": "gte" 14 | }, 15 | "max_score": null, 16 | "hits": [ 17 | { 18 | "_index": "dc-v2-work-1658438320350940", 19 | "_type": "_doc", 20 | "_id": "559ca7fb-55d1-45dc-9d8d-bd2ae2de6ae5", 21 | "_score": null, 22 | "_source": { 23 | "create_date": "2022-11-22T20:36:00.581418Z" 24 | }, 25 | "sort": [1669149360581] 26 | } 27 | ] 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /api/src/handlers/oai/date-utils.js: -------------------------------------------------------------------------------- 1 | function formatOaiDate(value) { 2 | if (!value) return value; 3 | 4 | if (typeof value === "string") { 5 | // Preserve day-level granularity values as-is. 6 | if (/^\d{4}-\d{2}-\d{2}$/.test(value)) return value; 7 | } 8 | 9 | const date = 10 | value instanceof Date 11 | ? value 12 | : typeof value === "string" 13 | ? new Date(value) 14 | : null; 15 | 16 | if (date && !Number.isNaN(date.getTime())) { 17 | return date.toISOString().replace(/\.\d+Z$/, "Z"); 18 | } 19 | 20 | if (typeof value === "string") { 21 | return value.replace(/\.\d+(?=Z$)/, ""); 22 | } 23 | 24 | return value; 25 | } 26 | 27 | module.exports = { formatOaiDate }; 28 | -------------------------------------------------------------------------------- /api/src/api/response/transformer.js: -------------------------------------------------------------------------------- 1 | const { transformError } = require("./error.js"); 2 | const iiifCollectionResponse = require("./iiif/collection.js"); 3 | const opensearchResponse = require("./opensearch"); 4 | 5 | async function transformSearchResult(response, pager) { 6 | if (response.statusCode === 200) { 7 | const responseBody = JSON.parse(response.body); 8 | const pageInfo = await pager.pageInfo(responseBody.hits.total.value); 9 | 10 | if (pageInfo.format === "iiif") { 11 | return await iiifCollectionResponse.transform(response, pager); 12 | } 13 | 14 | return await opensearchResponse.transform(response, { pager: pager }); 15 | } 16 | return transformError(response); 17 | } 18 | 19 | module.exports = { transformSearchResult }; 20 | -------------------------------------------------------------------------------- /loadtest/README.md: -------------------------------------------------------------------------------- 1 | # API Load Testing 2 | 3 | The API's load tests are written using the [Locust](https://locust.io/) load testing framework. 4 | 5 | ## Usage 6 | 7 | ### Set up dependencies 8 | ```shell 9 | python -m venv ./.venv 10 | . ./.venv/bin/activate 11 | pip install -r requirements.txt 12 | ``` 13 | 14 | ### Start server 15 | ```shell 16 | API_BASE_URL=https://dcapi.rdc-staging.library.northwestern.edu/api/v2 # or whatever 17 | locust -f locustfile.py --host=${API_BASE_URL} --processes -1 18 | ``` 19 | 20 | ### Run load tests 21 | 1. Open http://localhost:8089/ in a browser. 22 | 2. Customize test parameters (peak user count, ramp up, run time, etc.). 23 | 3. Click **Start**. 24 | 4. You can click around the UI while the test is running to see statistics, graphs, etc. 25 | -------------------------------------------------------------------------------- /api/test/fixtures/mocks/work-file-sets-access.json: -------------------------------------------------------------------------------- 1 | { 2 | "took": 2, 3 | "timed_out": false, 4 | "_shards": { 5 | "total": 5, 6 | "successful": 5, 7 | "skipped": 0, 8 | "failed": 0 9 | }, 10 | "hits": { 11 | "total": { 12 | "value": 1, 13 | "relation": "eq" 14 | }, 15 | "max_score": 1.0, 16 | "hits": [ 17 | { 18 | "_index": "dc-v2-file-set", 19 | "_type": "_doc", 20 | "_id": "fileset-access-1", 21 | "_score": 1.0, 22 | "_source": { 23 | "id": "fileset-access-1", 24 | "api_model": "FileSet", 25 | "work_id": "work-123", 26 | "role": "Access", 27 | "visibility": "Public", 28 | "published": true 29 | } 30 | } 31 | ] 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /api/src/handlers/get-file-set-auth.js: -------------------------------------------------------------------------------- 1 | const { getFileSet } = require("../api/opensearch"); 2 | const { authorizeDocument } = require("./authorize-document"); 3 | const { wrap } = require("./middleware"); 4 | 5 | const OPEN_DOCUMENT_NAMESPACE = /^0{8}-0{4}-0{4}-0{4}-0{9}[0-9A-Fa-f]{3}/; 6 | 7 | /** 8 | * Authorizes a FileSet by id 9 | */ 10 | exports.handler = wrap(async (event) => { 11 | const id = event.pathParameters.id; 12 | 13 | // Special namespace for entities that aren't actual entities 14 | // with indexed metadata (i.e., placeholder images) 15 | if (OPEN_DOCUMENT_NAMESPACE.test(id)) 16 | return { 17 | statusCode: 204, 18 | }; 19 | 20 | const osResponse = await getFileSet(id, { 21 | allowPrivate: true, 22 | allowUnpublished: true, 23 | }); 24 | 25 | return authorizeDocument(event, osResponse); 26 | }); 27 | -------------------------------------------------------------------------------- /api/test/fixtures/mocks/scroll-missing.json: -------------------------------------------------------------------------------- 1 | { 2 | "error": { 3 | "root_cause": [ 4 | { 5 | "type": "search_context_missing_exception", 6 | "reason": "No search context found for id [30222]" 7 | } 8 | ], 9 | "type": "search_phase_execution_exception", 10 | "reason": "all shards failed", 11 | "phase": "query", 12 | "grouped": true, 13 | "failed_shards": [ 14 | { 15 | "shard": -1, 16 | "index": null, 17 | "reason": { 18 | "type": "search_context_missing_exception", 19 | "reason": "No search context found for id [30222]" 20 | } 21 | } 22 | ], 23 | "caused_by": { 24 | "type": "search_context_missing_exception", 25 | "reason": "No search context found for id [30222]" 26 | } 27 | }, 28 | "status": 404 29 | } 30 | -------------------------------------------------------------------------------- /.github/workflows/test-node.yml: -------------------------------------------------------------------------------- 1 | name: Run NodeJS Tests 2 | on: 3 | push: 4 | paths: 5 | - ".github/workflows/test-node.yml" 6 | - "api/**" 7 | workflow_dispatch: 8 | defaults: 9 | run: 10 | working-directory: ./api 11 | jobs: 12 | test: 13 | runs-on: ubuntu-latest 14 | env: 15 | AWS_ACCESS_KEY_ID: ci 16 | AWS_SECRET_ACCESS_KEY: ci 17 | steps: 18 | - uses: actions/checkout@v3 19 | - uses: actions/setup-node@v3 20 | with: 21 | node-version: 20.x 22 | cache: "npm" 23 | cache-dependency-path: 'api/package-lock.json' 24 | - run: npm ci 25 | - name: Check code style 26 | run: npm run lint && npm run prettier 27 | - name: Run tests 28 | run: npm run test:coverage 29 | - name: Validate OpenAPI spec 30 | run: npm run validate-spec 31 | -------------------------------------------------------------------------------- /api/src/api/response/iiif/presentation-api/provider.js: -------------------------------------------------------------------------------- 1 | const nulLogo = { 2 | id: "https://iiif.dc.library.northwestern.edu/iiif/2/00000000-0000-0000-0000-000000000003/full/pct:50/0/default.webp", 3 | type: "Image", 4 | format: "image/webp", 5 | height: 139, 6 | width: 1190, 7 | }; 8 | 9 | const provider = { 10 | id: "https://www.library.northwestern.edu/", 11 | type: "Agent", 12 | label: { none: ["Northwestern University Libraries"] }, 13 | homepage: [ 14 | { 15 | id: "https://dc.library.northwestern.edu/", 16 | type: "Text", 17 | label: { 18 | none: [ 19 | "Northwestern University Libraries Digital Collections Homepage", 20 | ], 21 | }, 22 | format: "text/html", 23 | language: ["en"], 24 | }, 25 | ], 26 | logo: [nulLogo], 27 | }; 28 | 29 | module.exports = { nulLogo, provider }; 30 | -------------------------------------------------------------------------------- /av-download/lambdas/transcode-status.js: -------------------------------------------------------------------------------- 1 | /* istanbul ignore file */ 2 | const { GetJobCommand, MediaConvertClient } = require("@aws-sdk/client-mediaconvert"); 3 | 4 | module.exports.handler = async (event) => { 5 | if(!event.jobId) return {success: false} 6 | 7 | const status = await checkJobStatus(event.jobId) 8 | 9 | return {jobId: event.jobId, status: status, destination: event.destination} 10 | }; 11 | 12 | async function checkJobStatus(jobId){ 13 | const mediaConvertClient = new MediaConvertClient({endpoint: process.env.MEDIA_CONVERT_ENDPOINT}); 14 | 15 | // possible: "SUBMITTED" || "PROGRESSING" || "COMPLETE" || "CANCELED" || "ERROR" 16 | 17 | try { 18 | const data = await mediaConvertClient.send(new GetJobCommand({Id: jobId})); 19 | return data.Job.Status; 20 | } catch (err) { 21 | console.error("Error", err); 22 | return null; 23 | } 24 | 25 | } -------------------------------------------------------------------------------- /api/test/integration/options-request.test.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | 3 | const chai = require("chai"); 4 | const expect = chai.expect; 5 | 6 | const optionsHandler = requireSource("handlers/options-request"); 7 | 8 | describe("OPTIONS handler", async () => { 9 | helpers.saveEnvironment(); 10 | const event = helpers 11 | .mockEvent("OPTIONS", "/auth/whoami") 12 | .headers({ 13 | Origin: "https://dc.library.northwestern.edu/origin-test-path", 14 | }) 15 | .render(); 16 | 17 | it("sends the correct CORS headers", async () => { 18 | const response = await optionsHandler.handler(event); 19 | expect(response.headers).to.contain({ 20 | "Access-Control-Allow-Origin": 21 | "https://dc.library.northwestern.edu/origin-test-path", 22 | }); 23 | 24 | expect( 25 | response.headers["Access-Control-Allow-Headers"].split(/, /) 26 | ).to.include("Content-Type"); 27 | }); 28 | }); 29 | -------------------------------------------------------------------------------- /.github/workflows/test-python.yml: -------------------------------------------------------------------------------- 1 | name: Run Python Tests 2 | on: 3 | push: 4 | paths: 5 | - ".github/workflows/test-python.yml" 6 | - "chat/**" 7 | workflow_dispatch: 8 | defaults: 9 | run: 10 | working-directory: ./chat 11 | jobs: 12 | test: 13 | runs-on: ubuntu-latest 14 | env: 15 | AWS_ACCESS_KEY_ID: ci 16 | AWS_SECRET_ACCESS_KEY: ci 17 | steps: 18 | - uses: actions/checkout@v3 19 | - name: Install uv 20 | uses: astral-sh/setup-uv@v6 21 | with: 22 | enable-cache: true 23 | cache-dependency-glob: "**/pyproject.toml" 24 | - run: uv sync --group dev 25 | - name: Check code style 26 | run: uv run ruff check . 27 | - name: Run tests 28 | run: | 29 | uv run coverage run --include='src/**/*' -m pytest -m "" 30 | uv run coverage report 31 | env: 32 | AWS_REGION: us-east-1 33 | -------------------------------------------------------------------------------- /api/src/api/request/models.js: -------------------------------------------------------------------------------- 1 | const { prefix } = require("../../environment"); 2 | 3 | const mapTargets = { 4 | works: "dc-v2-work", 5 | "file-sets": "dc-v2-file-set", 6 | collections: "dc-v2-collection", 7 | }; 8 | 9 | function extractRequestedModels(requestedModels) { 10 | return requestedModels == null ? ["works"] : requestedModels.split(","); 11 | } 12 | 13 | function validModels(models, format) { 14 | if (format === "iiif") { 15 | return ( 16 | models.length == 1 && 17 | models.every((model) => model === "works" || "collections") 18 | ); 19 | } 20 | return models.every(isAllowed); 21 | } 22 | 23 | function isAllowed(model) { 24 | return Object.prototype.hasOwnProperty.call(mapTargets, model); 25 | } 26 | 27 | function modelsToTargets(models) { 28 | return String(models.map((model) => prefix(mapTargets[model]))); 29 | } 30 | 31 | module.exports = { extractRequestedModels, modelsToTargets, validModels }; 32 | -------------------------------------------------------------------------------- /docs/docs/spec/openapi.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 10 | SwaggerUI 11 | 12 | 13 | 14 |
15 | 16 | 17 | 27 | 28 | 29 | -------------------------------------------------------------------------------- /chat/src/core/websocket.py: -------------------------------------------------------------------------------- 1 | import json 2 | from core.setup import websocket_client 3 | 4 | 5 | class Websocket: 6 | def __init__(self, client=None, endpoint_url=None, connection_id=None, ref=None): 7 | self.client = client or websocket_client(endpoint_url) 8 | self.connection_id = connection_id 9 | self.ref = ref if ref else {} 10 | 11 | def send(self, data): 12 | if isinstance(data, str): 13 | data = {"message": data} 14 | data["ref"] = self.ref 15 | data_as_bytes = bytes(json.dumps(data), "utf-8") 16 | 17 | if self.connection_id == "debug": 18 | print(data) 19 | else: 20 | self.client.post_to_connection( 21 | Data=data_as_bytes, ConnectionId=self.connection_id 22 | ) 23 | return data 24 | 25 | def __str__(self): 26 | return f"Websocket({self.connection_id}, {self.ref})" 27 | 28 | def __repr__(self): 29 | return str(self) 30 | -------------------------------------------------------------------------------- /api/test/unit/api/request/models.test.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | 3 | const chai = require("chai"); 4 | const expect = chai.expect; 5 | 6 | const models = requireSource("api/request/models"); 7 | 8 | describe("models", () => { 9 | helpers.saveEnvironment(); 10 | 11 | it("knows valid models", () => { 12 | expect(models.validModels(["collections", "file-sets", "works"])).to.be 13 | .true; 14 | }); 15 | 16 | it("detects invalid models", () => { 17 | expect(models.validModels(["works", "foo"])).to.be.false; 18 | }); 19 | 20 | it("maps models to targets", () => { 21 | let result = models.modelsToTargets(["collections", "file-sets", "works"]); 22 | expect(result).to.eq("dc-v2-collection,dc-v2-file-set,dc-v2-work"); 23 | 24 | process.env.ENV_PREFIX = "pre"; 25 | result = models.modelsToTargets(["collections", "file-sets", "works"]); 26 | expect(result).to.eq( 27 | "pre-dc-v2-collection,pre-dc-v2-file-set,pre-dc-v2-work" 28 | ); 29 | }); 30 | }); 31 | -------------------------------------------------------------------------------- /api/src/api/scopes.js: -------------------------------------------------------------------------------- 1 | const { ProviderCapabilities } = require("../environment"); 2 | // Add all user scopes to the API Token's entitlements claim 3 | // The Scopes object maps scopes to functions that check 4 | // if the user has that scope. 5 | 6 | const Scopes = { 7 | "read:Public": () => true, 8 | "read:Published": () => true, 9 | "read:Institution": (user) => 10 | user.isSuperUser() || user.isInstitution() || user.isReadingRoom(), 11 | "read:Private": (user) => user.isSuperUser() || user.isReadingRoom(), 12 | "read:Unpublished": (user) => user.isSuperUser(), 13 | chat: (user) => 14 | (user.isLoggedIn() && 15 | ProviderCapabilities()[user.token.provider]?.includes("chat")) || 16 | user.isSuperUser(), 17 | }; 18 | 19 | const addScopes = (apiToken) => { 20 | for (const [scope, fn] of Object.entries(Scopes)) { 21 | if (fn(apiToken)) { 22 | apiToken.addScope(scope); 23 | } 24 | } 25 | return apiToken; 26 | }; 27 | 28 | module.exports = { addScopes }; 29 | -------------------------------------------------------------------------------- /av-download/lambdas/get-download-link.js: -------------------------------------------------------------------------------- 1 | /* istanbul ignore file */ 2 | const { S3Client, CopyObjectCommand, HeadObjectCommand } = require('@aws-sdk/client-s3'); 3 | 4 | module.exports.handler = async (event) => { 5 | const region = process.env.AWS_REGION || 'us-east-1'; 6 | const { bucket, key } = event; 7 | const s3Client = new S3Client({ region }); 8 | const head = await s3Client.send(new HeadObjectCommand({Bucket: bucket, Key: key})); 9 | if (!head.ContentDisposition || !head.ContentDisposition.includes('attachment')) { 10 | await s3Client.send( 11 | new CopyObjectCommand({ 12 | Bucket: bucket, 13 | Key: key, 14 | CopySource: `${bucket}/${key}`, 15 | ACL: "public-read", 16 | ContentDisposition: `attachment; filename=${event.disposition}`, 17 | MetadataDirective: "REPLACE" 18 | }) 19 | ); 20 | } 21 | const url = `https://${bucket}.s3.${region}.amazonaws.com/${key}`; 22 | 23 | return {downloadLink: url} 24 | }; 25 | 26 | 27 | -------------------------------------------------------------------------------- /api/src/handlers/authorize-document.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Authorizes a Document (Collection, FileSet or Work) by id 3 | */ 4 | const authorizeDocument = (event, osResponse) => { 5 | if (osResponse.statusCode != 200) { 6 | return sendResponse(osResponse.statusCode); 7 | } 8 | 9 | const body = JSON.parse(osResponse.body); 10 | const document = body._source; 11 | const token = event.userToken; 12 | 13 | const { published, visibility } = document; 14 | const workId = document.work_id || document.id; 15 | let allowed = token.hasEntitlement(workId); 16 | 17 | if (!allowed) { 18 | const publishedState = published ? "Published" : "Unpublished"; 19 | allowed = [`read:${visibility}`, `read:${publishedState}`].every((scope) => 20 | token.can(scope) 21 | ); 22 | } 23 | 24 | return sendResponse(allowed ? 204 : 403); 25 | }; 26 | 27 | function sendResponse(statusCode) { 28 | return { 29 | statusCode: statusCode, 30 | }; 31 | } 32 | 33 | module.exports = { authorizeDocument }; 34 | -------------------------------------------------------------------------------- /.github/workflows/validate-template.yml: -------------------------------------------------------------------------------- 1 | name: Validate Template 2 | on: 3 | push: 4 | paths: 5 | - ".github/workflows/validate-template.yml" 6 | - "./template.yml" 7 | workflow_dispatch: 8 | jobs: 9 | validate-template: 10 | runs-on: ubuntu-latest 11 | permissions: 12 | id-token: write 13 | contents: read 14 | environment: test 15 | steps: 16 | - uses: actions/setup-python@v4 17 | with: 18 | python-version: '3.12' 19 | - name: Install uv 20 | uses: astral-sh/setup-uv@v7 21 | - name: Install cfn-lint 22 | run: pip install cfn-lint 23 | - uses: aws-actions/setup-sam@v1 24 | - uses: aws-actions/configure-aws-credentials@master 25 | with: 26 | role-to-assume: arn:aws:iam::${{ secrets.AwsAccount }}:role/github-actions-role 27 | aws-region: us-east-1 28 | - uses: actions/checkout@v3 29 | - name: Validate template 30 | run: make build && make validate -------------------------------------------------------------------------------- /api/src/handlers/get-similar.js: -------------------------------------------------------------------------------- 1 | const { doSearch } = require("./search-runner"); 2 | const { wrap } = require("./middleware"); 3 | const { modelsToTargets } = require("../api/request/models"); 4 | 5 | /** 6 | * Get similar works via 'More Like This' query 7 | */ 8 | exports.handler = wrap(async (event) => { 9 | const id = event.pathParameters.id; 10 | const models = ["works"]; 11 | const workIndex = modelsToTargets(models); 12 | 13 | event.body = { 14 | query: { 15 | more_like_this: { 16 | fields: [ 17 | "title", 18 | "description", 19 | "subject.label", 20 | "genre.label", 21 | "contributor.label", 22 | "creator.label", 23 | ], 24 | like: [ 25 | { 26 | _index: workIndex, 27 | _id: id, 28 | }, 29 | ], 30 | max_query_terms: 10, 31 | min_doc_freq: 1, 32 | min_term_freq: 1, 33 | }, 34 | }, 35 | }; 36 | 37 | return doSearch(event, { includeToken: false }); 38 | }); 39 | -------------------------------------------------------------------------------- /api/dependencies/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "dc-api-dependencies", 3 | "version": "2.9.2", 4 | "description": "NUL Digital Collections API Dependencies", 5 | "repository": "https://github.com/nulib/dc-api-v2", 6 | "author": "nulib", 7 | "license": "Apache-2.0", 8 | "dependencies": { 9 | "@aws-crypto/sha256-browser": "^2.0.1", 10 | "@aws-sdk/client-sfn": "^3.563.0", 11 | "@aws-sdk/client-sns": "^3.606.0", 12 | "@aws-sdk/credential-provider-node": "^3.563.0", 13 | "@honeybadger-io/js": "^4.9.3", 14 | "@smithy/node-http-handler": "^2.5.0", 15 | "@smithy/protocol-http": "^3.3.0", 16 | "@smithy/signature-v4": "^2.3.0", 17 | "axios": ">=0.21.1", 18 | "cookie": "^0.5.0", 19 | "debug": "^4.3.4", 20 | "http-status-codes": "^2.2.0", 21 | "iiif-builder": "^1.0.7", 22 | "jsonschema": "^1.4.1", 23 | "jsonwebtoken": "^8.5.1", 24 | "lodash": "^4.17.21", 25 | "lz-string": "^1.4.4", 26 | "parse-http-header": "^1.0.1", 27 | "sort-json": "^2.0.1", 28 | "xml-js": "^1.6.11" 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /api/src/handlers/get-work-by-id.js: -------------------------------------------------------------------------------- 1 | const { getWork } = require("../api/opensearch"); 2 | const manifestResponse = require("../api/response/iiif/manifest"); 3 | const { wrap } = require("./middleware"); 4 | const opensearchResponse = require("../api/response/opensearch"); 5 | 6 | /** 7 | * A simple function to get a Work by id 8 | */ 9 | exports.handler = wrap(async (event) => { 10 | const id = event.pathParameters.id; 11 | 12 | const allowPrivate = 13 | event.userToken.isSuperUser() || 14 | event.userToken.isReadingRoom() || 15 | event.userToken.hasEntitlement(id); 16 | const allowUnpublished = 17 | event.userToken.isSuperUser() || event.userToken.hasEntitlement(id); 18 | 19 | const esResponse = await getWork(id, { allowPrivate, allowUnpublished }); 20 | 21 | const as = event.queryStringParameters.as; 22 | 23 | if (as && as === "iiif") { 24 | // Make it IIIFy 25 | return await manifestResponse.transform(esResponse, { 26 | allowPrivate, 27 | allowUnpublished, 28 | }); 29 | } 30 | 31 | return await opensearchResponse.transform(esResponse); 32 | }); 33 | -------------------------------------------------------------------------------- /dev/env.json: -------------------------------------------------------------------------------- 1 | { 2 | "Parameters": { 3 | "API_TOKEN_NAME": "", 4 | "API_TOKEN_SECRET": "", 5 | "AV_DOWNLOAD_EMAIL_TEMPLATE": "", 6 | "AV_DOWNLOAD_STATE_MACHINE_ARN": "", 7 | "AZURE_OPENAI_API_KEY": "", 8 | "AZURE_OPENAI_EMBEDDING_DEPLOYMENT_ID": "", 9 | "AZURE_OPENAI_LLM_DEPLOYMENT_ID": "", 10 | "AZURE_OPENAI_RESOURCE_NAME": "", 11 | "DC_API_ENDPOINT": "", 12 | "DC_URL": "", 13 | "DEBUG": "", 14 | "DEV_TEAM_NET_IDS": "", 15 | "ELASTICSEARCH_ENDPOINT": "", 16 | "OPENSEARCH_ENDPOINT": "", 17 | "ENV_PREFIX": "", 18 | "MEDIA_CONVERT_ENDPOINT": "", 19 | "MEDIA_CONVERT_JOB_QUEUE_ARN": "", 20 | "MEDIA_CONVERT_DESTINATION_BUCKET": "", 21 | "MEDIA_CONVERT_ROLE_ARN": "", 22 | "PYRAMID_BUCKET": "", 23 | "READING_ROOM_IPS": "", 24 | "STREAMING_BUCKET": "", 25 | "REPOSITORY_EMAIL": "", 26 | "START_AUDIO_TRANSCODE_FUNCTION": "", 27 | "START_TRANSCODE_FUNCTION": "", 28 | "TRANSCODE_STATUS_FUNCTION": "", 29 | "GET_DOWNLOAD_LINK_FUNCTION": "", 30 | "SEND_TEMPLATED_EMAIL_FUNCTION": "", 31 | "STEP_FUNCTION_ENDPOINT": "" 32 | } 33 | } -------------------------------------------------------------------------------- /chat/test/core/test_prompts.py: -------------------------------------------------------------------------------- 1 | # ruff: noqa: E402 2 | from core.prompts import prompt_template, document_template 3 | from unittest import TestCase 4 | 5 | 6 | class TestPromptTemplate(TestCase): 7 | def test_prompt_template(self): 8 | prompt = prompt_template() 9 | assert isinstance(prompt, str) 10 | assert len(prompt) > 0 11 | 12 | 13 | class TestDocumentTemplate(TestCase): 14 | def test_empty_attributes(self): 15 | self.assertEqual( 16 | document_template(), 17 | "Content: {title}\nMetadata:\nSource: {id}", 18 | ) 19 | 20 | def test_single_attribute(self): 21 | self.assertEqual( 22 | document_template(["title"]), 23 | "Content: {title}\nMetadata:\n title: {title}\nSource: {id}", 24 | ) 25 | 26 | def test_multiple_attributes(self): 27 | self.assertEqual( 28 | document_template(["title", "author", "subject", "description"]), 29 | "Content: {title}\nMetadata:\n title: {title}\n author: {author}\n subject: {subject}\n description: {description}\nSource: {id}", 30 | ) 31 | -------------------------------------------------------------------------------- /api/src/handlers/get-file-set-annotations.js: -------------------------------------------------------------------------------- 1 | const { wrap } = require("./middleware"); 2 | const { getFileSet } = require("../api/opensearch"); 3 | const { appInfo } = require("../environment"); 4 | const opensearchResponse = require("../api/response/opensearch"); 5 | 6 | /** 7 | * Returns annotations for a FileSet 8 | */ 9 | exports.handler = wrap(async (event) => { 10 | const id = event.pathParameters.id; 11 | const allowPrivate = 12 | event.userToken.isSuperUser() || event.userToken.isReadingRoom(); 13 | const allowUnpublished = event.userToken.isSuperUser(); 14 | 15 | const esResponse = await getFileSet(id, { allowPrivate, allowUnpublished }); 16 | if (esResponse.statusCode !== 200) { 17 | return await opensearchResponse.transform(esResponse); 18 | } 19 | 20 | const body = JSON.parse(esResponse.body); 21 | const annotations = body?._source?.annotations ?? null; 22 | 23 | return { 24 | statusCode: 200, 25 | headers: { 26 | "content-type": "application/json", 27 | }, 28 | body: JSON.stringify({ 29 | data: annotations, 30 | info: appInfo(), 31 | }), 32 | }; 33 | }); 34 | -------------------------------------------------------------------------------- /av-download/lambdas/start-transcode.js: -------------------------------------------------------------------------------- 1 | /* istanbul ignore file */ 2 | const { CreateJobCommand, MediaConvertClient } = require("@aws-sdk/client-mediaconvert"); 3 | 4 | 5 | module.exports.handler = async (event) => { 6 | const jobQueueArn = process.env.MEDIA_CONVERT_JOB_QUEUE_ARN; 7 | const iamRoleArn = process.env.MEDIA_CONVERT_ROLE_ARN; 8 | 9 | const params = { 10 | "Queue": jobQueueArn, 11 | "UserMetadata": {}, 12 | "Role": iamRoleArn, 13 | "Settings": event.settings, 14 | "AccelerationSettings": { 15 | "Mode": "DISABLED" 16 | }, 17 | "StatusUpdateInterval": "SECONDS_60", 18 | "Priority": 0 19 | } 20 | 21 | const job = await createJob(params); 22 | 23 | return {jobId: job.Job.Id, status: job.Job.Status} 24 | }; 25 | 26 | async function createJob(params){ 27 | const mediaConvertClient = new MediaConvertClient({endpoint: process.env.MEDIA_CONVERT_ENDPOINT}); 28 | 29 | try { 30 | const data = await mediaConvertClient.send(new CreateJobCommand(params)); 31 | return data 32 | } catch (err) { 33 | console.error("Error", err); 34 | return null; 35 | } 36 | 37 | } 38 | 39 | -------------------------------------------------------------------------------- /api/src/honeybadger-setup.js: -------------------------------------------------------------------------------- 1 | const Honeybadger = require("@honeybadger-io/js"); 2 | 3 | Honeybadger.configure({ 4 | apiKey: process.env.HONEYBADGER_API_KEY || "DEVELOPMENT_MODE", 5 | environment: process.env.HONEYBADGER_ENV || "development", 6 | revision: process.env.HONEYBADGER_REVISION, 7 | enableUncaught: !process.env.HONEYBADGER_DISABLED, 8 | enableUnhandledRejection: !process.env.HONEYBADGER_DISABLED, 9 | }); 10 | 11 | Honeybadger.beforeNotify((notice) => { 12 | if (notice.context?.event) { 13 | const event = notice.context?.event; 14 | notice.url = event.requestContext.http.path; 15 | notice.headers = event.headers; 16 | notice.cookies = event.cookieObject; 17 | notice.params = event.queryStringParameters; 18 | notice.cgiData = { 19 | REQUEST_METHOD: event.requestContext.http.method, 20 | SERVER_PROTOCOL: event.requestContext.http.protocol, 21 | QUERY_STRING: event.rawQueryString, 22 | REMOTE_ADDR: event.requestContext.http.sourceIp, 23 | REMOTE_USER: event.userToken?.sub, 24 | }; 25 | 26 | delete notice.context; 27 | } 28 | }); 29 | 30 | module.exports = Honeybadger; 31 | -------------------------------------------------------------------------------- /api/test/integration/get-chat-endpoint.test.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | 3 | const chai = require("chai"); 4 | const expect = chai.expect; 5 | 6 | const getChatEndpointHandler = requireSource("handlers/get-chat-endpoint"); 7 | const ApiToken = requireSource("api/api-token"); 8 | 9 | describe("GET /chat-endpoint", function () { 10 | helpers.saveEnvironment(); 11 | beforeEach(() => { 12 | process.env.PROVIDER_CAPABILITIES = '{"magic":[],"nusso":["chat"]}'; 13 | }); 14 | 15 | it("returns the websocket URI and token to a logged in user", async () => { 16 | let token = new ApiToken().user({ sub: "abc123" }).provider("nusso"); 17 | token = token.sign(); 18 | const event = helpers 19 | .mockEvent("GET", "/chat-endpoint") 20 | .headers({ 21 | Authorization: `Bearer ${token}`, 22 | }) 23 | .render(); 24 | 25 | const result = await getChatEndpointHandler.handler(event); 26 | expect(result.statusCode).to.eq(200); 27 | const response = JSON.parse(result.body); 28 | expect(response).to.contain({ 29 | endpoint: "wss://thisisafakewebsocketapiurl", 30 | auth: token, 31 | }); 32 | }); 33 | }); 34 | -------------------------------------------------------------------------------- /api/src/aws/fetch.js: -------------------------------------------------------------------------------- 1 | const { defaultProvider } = require("@aws-sdk/credential-provider-node"); 2 | const { SignatureV4 } = require("@smithy/signature-v4"); 3 | const { NodeHttpHandler } = require("@smithy/node-http-handler"); 4 | const { Sha256 } = require("@aws-crypto/sha256-browser"); 5 | const region = require("../environment").region(); 6 | 7 | async function awsFetch(request) { 8 | const signer = new SignatureV4({ 9 | credentials: defaultProvider(), 10 | region: region, 11 | service: "es", 12 | sha256: Sha256, 13 | }); 14 | 15 | const signedRequest = await signer.sign(request); 16 | 17 | const client = new NodeHttpHandler(); 18 | const { response } = await client.handle(signedRequest); 19 | 20 | return await new Promise((resolve, _reject) => { 21 | let returnValue = { 22 | statusCode: response.statusCode, 23 | }; 24 | let responseBody = ""; 25 | 26 | response.body.on("data", function (chunk) { 27 | responseBody += chunk; 28 | }); 29 | response.body.on("end", function (_chunk) { 30 | resolve({ ...returnValue, body: responseBody }); 31 | }); 32 | }); 33 | } 34 | 35 | module.exports = { awsFetch }; 36 | -------------------------------------------------------------------------------- /api/test/integration/middleware.test.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | 3 | const chai = require("chai"); 4 | const expect = chai.expect; 5 | chai.use(require("chai-http")); 6 | const sinon = require("sinon"); 7 | const { wrap, __Honeybadger } = requireSource("handlers/middleware.js"); 8 | 9 | describe("middleware", () => { 10 | helpers.saveEnvironment(); 11 | 12 | beforeEach(function () { 13 | __Honeybadger.configure({ enableUncaught: true }); 14 | }); 15 | 16 | afterEach(function () { 17 | __Honeybadger.configure({ enableUncaught: false }); 18 | sinon.restore(); 19 | }); 20 | 21 | it("reports uncaught errors to Honeybadger", async () => { 22 | const event = helpers.mockEvent("GET", "/error").render(); 23 | 24 | let fakeNotify = sinon.fake((error) => { 25 | expect(error.message).to.eq("Catch this!"); 26 | }); 27 | 28 | sinon.replace(__Honeybadger, "notifyAsync", fakeNotify); 29 | 30 | const handler = wrap(async (_event) => { 31 | throw new Error("Catch this!"); 32 | }); 33 | 34 | const result = await handler(event); 35 | expect(result.statusCode).to.eq(400); 36 | sinon.assert.calledOnce(fakeNotify); 37 | }); 38 | }); 39 | -------------------------------------------------------------------------------- /api/test/fixtures/mocks/work-file-sets.json: -------------------------------------------------------------------------------- 1 | { 2 | "took": 3, 3 | "timed_out": false, 4 | "_shards": { 5 | "total": 5, 6 | "successful": 5, 7 | "skipped": 0, 8 | "failed": 0 9 | }, 10 | "hits": { 11 | "total": { 12 | "value": 2, 13 | "relation": "eq" 14 | }, 15 | "max_score": 1.0, 16 | "hits": [ 17 | { 18 | "_index": "dc-v2-file-set", 19 | "_type": "_doc", 20 | "_id": "fileset-1", 21 | "_score": 1.0, 22 | "_source": { 23 | "id": "fileset-1", 24 | "api_model": "FileSet", 25 | "work_id": "work-123", 26 | "role": "Access", 27 | "visibility": "Public", 28 | "published": true 29 | } 30 | }, 31 | { 32 | "_index": "dc-v2-file-set", 33 | "_type": "_doc", 34 | "_id": "fileset-2", 35 | "_score": 1.0, 36 | "_source": { 37 | "id": "fileset-2", 38 | "api_model": "FileSet", 39 | "work_id": "work-123", 40 | "role": "Access", 41 | "visibility": "Public", 42 | "published": true 43 | } 44 | } 45 | ] 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /av-download/lambdas/send-templated-email.js: -------------------------------------------------------------------------------- 1 | /* istanbul ignore file */ 2 | const { SESClient, SendTemplatedEmailCommand } = require("@aws-sdk/client-ses"); 3 | 4 | module.exports.handler = async (event) => { 5 | 6 | const sesClient = new SESClient(); 7 | const templateName = event.template; 8 | const toAddress = event.to; 9 | const fromAddress = event.from; 10 | 11 | const params = event.params 12 | const sendTemplatedEmailCommand = createSendTemplatedEmailCommand( 13 | toAddress, 14 | fromAddress, 15 | templateName, 16 | params 17 | ); 18 | 19 | try { 20 | await sesClient.send(sendTemplatedEmailCommand); 21 | } catch (err) { 22 | console.error("Failed to send template email", err); 23 | return err; 24 | } 25 | return {success: true} 26 | }; 27 | 28 | 29 | const createSendTemplatedEmailCommand = ( 30 | toAddress, 31 | fromAddress, 32 | templateName, 33 | params 34 | ) => { 35 | return new SendTemplatedEmailCommand({ 36 | Destination: { ToAddresses: [toAddress] }, 37 | TemplateData: JSON.stringify(params), 38 | Source: `Northwestern University Libraries <${fromAddress}>`, 39 | Template: templateName, 40 | }); 41 | }; 42 | -------------------------------------------------------------------------------- /api/src/handlers/get-collections.js: -------------------------------------------------------------------------------- 1 | const { doSearch } = require("./search-runner"); 2 | const { wrap } = require("./middleware"); 3 | 4 | const getCollections = async (event) => { 5 | event.pathParameters.models = "collections"; 6 | event.body = { query: { match_all: {} } }; 7 | return doSearch(event, { includeToken: false }); 8 | }; 9 | 10 | const getCollectionsAsIiif = async (event) => { 11 | event.pathParameters.models = "collections"; 12 | event.body = { query: { match_all: {} } }; 13 | event.queryStringParameters.collectionLabel = 14 | "Northwestern University Libraries Digital Collections"; 15 | event.queryStringParameters.collectionSummary = 16 | "Explore digital resources from the Northwestern University Library collections – including letters, photographs, diaries, maps, and audiovisual materials."; 17 | 18 | return doSearch(event, { 19 | includeToken: false, 20 | parameterOverrides: { as: "iiif" }, 21 | }); 22 | }; 23 | 24 | /** 25 | * A simple function to get Collections 26 | */ 27 | exports.handler = wrap(async (event) => { 28 | return event.queryStringParameters?.as === "iiif" 29 | ? getCollectionsAsIiif(event) 30 | : getCollections(event); 31 | }); 32 | -------------------------------------------------------------------------------- /api/src/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "dc-api", 3 | "version": "2.9.2", 4 | "description": "NUL Digital Collections API", 5 | "repository": "https://github.com/nulib/dc-api-v2", 6 | "author": "nulib", 7 | "license": "Apache-2.0", 8 | "dependencies": { 9 | "@aws-crypto/sha256-browser": "^2.0.1", 10 | "@aws-sdk/client-s3": "^3.565.0", 11 | "@aws-sdk/client-secrets-manager": "^3.563.0", 12 | "@aws-sdk/client-ses": "^3.563.0", 13 | "@aws-sdk/client-sfn": "^3.563.0", 14 | "@aws-sdk/client-sns": "^3.606.0", 15 | "@aws-sdk/credential-provider-node": "^3.563.0", 16 | "@aws-sdk/s3-request-presigner": "^3.565.0", 17 | "@honeybadger-io/js": "^4.9.3", 18 | "@smithy/node-http-handler": "^2.5.0", 19 | "@smithy/protocol-http": "^3.3.0", 20 | "@smithy/signature-v4": "^2.3.0", 21 | "axios": ">=0.21.1", 22 | "cookie": "^0.5.0", 23 | "debug": "^4.3.4", 24 | "http-status-codes": "^2.2.0", 25 | "iiif-builder": "^1.0.7", 26 | "jsonschema": "^1.4.1", 27 | "jsonwebtoken": "^8.5.1", 28 | "lodash": "^4.17.21", 29 | "lz-string": "^1.4.4", 30 | "parse-http-header": "^1.0.1", 31 | "sort-json": "^2.0.1", 32 | "xml-js": "^1.6.11" 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /chat/src/core/secrets.py: -------------------------------------------------------------------------------- 1 | import boto3 2 | import json 3 | import os 4 | 5 | 6 | def load_secrets(): 7 | SecretsPath = os.getenv("SECRETS_PATH") 8 | ApiConfigPrefix = os.getenv("API_CONFIG_PREFIX") or SecretsPath 9 | EnvironmentMap = [ 10 | ["API_TOKEN_SECRET", "dcapi", "api_token_secret"], 11 | ["OPENSEARCH_ENDPOINT", "index", "endpoint"], 12 | ["OPENSEARCH_MODEL_ID", "index", "embedding_model"], 13 | ] 14 | 15 | client = boto3.client( 16 | "secretsmanager", region_name=os.getenv("AWS_REGION", "us-east-1") 17 | ) 18 | response = client.batch_get_secret_value( 19 | SecretIdList=[ 20 | f"{ApiConfigPrefix}/config/dcapi", 21 | f"{SecretsPath}/infrastructure/index", 22 | f"{SecretsPath}/infrastructure/azure_openai", 23 | ] 24 | ) 25 | 26 | secrets = { 27 | secret["Name"].split("/")[-1]: json.loads(secret["SecretString"]) 28 | for secret in response["SecretValues"] 29 | } 30 | 31 | for var, name, key in EnvironmentMap: 32 | value = secrets.get(name, {}).get(key) 33 | 34 | if var not in os.environ and value is not None: 35 | os.environ[var] = value 36 | -------------------------------------------------------------------------------- /api/test/unit/api/response/iiif/presentation-api/metadata.test.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | 3 | const chai = require("chai"); 4 | const expect = chai.expect; 5 | 6 | const { formatSingleValuedField, metadataLabelFields } = requireSource( 7 | "api/response/iiif/presentation-api/metadata" 8 | ); 9 | 10 | describe("IIIF response presentation API metadata helpers", () => { 11 | const response = { 12 | statusCode: 200, 13 | body: helpers.testFixture("mocks/work-1234.json"), 14 | }; 15 | const source = JSON.parse(response.body)._source; 16 | 17 | it("formatSingleValuedField(value)", () => { 18 | expect(formatSingleValuedField("This value.")) 19 | .to.be.an("array") 20 | .that.does.include("This value."); 21 | expect(formatSingleValuedField(null)).to.be.an("array").that.is.empty; 22 | }); 23 | 24 | it("metadataLabelFields(source)", () => { 25 | const metadata = metadataLabelFields(source); 26 | expect(Array.isArray(metadata)).to.be; 27 | expect(metadata.length).to.eq(30); 28 | metadata.forEach((item) => { 29 | expect(item.label).to.be.a("string"); 30 | expect(item.value).to.be.an("array"); 31 | expect(item.label).to.not.contain("Keyword"); 32 | }); 33 | }); 34 | }); 35 | -------------------------------------------------------------------------------- /api/src/handlers/get-auth-logout.js: -------------------------------------------------------------------------------- 1 | const axios = require("axios").default; 2 | const { dcUrl } = require("../environment"); 3 | const { wrap } = require("./middleware"); 4 | const ApiToken = require("../api/api-token"); 5 | const Honeybadger = require("../honeybadger-setup"); 6 | 7 | /** 8 | * Performs NUSSO logout 9 | */ 10 | exports.handler = wrap(async (event) => { 11 | try { 12 | let responseLocation = 13 | event.queryStringParameters?.goto || event.headers?.referer || dcUrl(); 14 | 15 | if (event.userToken && event.userToken.token.provider === "nusso") { 16 | const url = `${process.env.NUSSO_BASE_URL}logout`; 17 | const response = await axios.get(url, { 18 | headers: { apikey: process.env.NUSSO_API_KEY }, 19 | }); 20 | responseLocation = response.data.url; 21 | } 22 | 23 | event.userToken = new ApiToken().expire(); 24 | return { 25 | statusCode: 302, 26 | headers: { 27 | location: responseLocation, 28 | }, 29 | }; 30 | } catch (error) { 31 | await Honeybadger.notifyAsync(error, { tags: ["auth", "upstream"] }); 32 | console.error("NUSSO request error", error); 33 | return { 34 | statusCode: 401, 35 | }; 36 | } 37 | }); 38 | -------------------------------------------------------------------------------- /docs/docs/oai.md: -------------------------------------------------------------------------------- 1 | ## Introduction 2 | 3 | The Open Archives Initiative Protocol for Metadata Harvesting (OAI-PMH) is a protocol that enables the harvesting of metadata descriptions of records in a repository. The Digital Collections API is an OAI-PMH `Data Provider`, which means structured metadata is exposed according to the [OAI-PMH specification](http://www.openarchives.org/OAI/openarchivesprotocol.html). OAI-PMH is a set of verbs or services that are invoked using the HTTP protocol. The OAI-PMH endpoint is available at `https://api.dc.library.northwestern.edu/api/v2/oai`. 4 | 5 | Please see the [OpenAPI specification page](./spec.md) for more information about making requests using each OAI-PMH verb, including parameters and examples. 6 | 7 | ## Verbs 8 | 9 | All six verbs are supported: 10 | 11 | - `GetRecord`: Retrieves metadata records from a repository. 12 | - `Identify`: Provides information about the repository and its policies. 13 | - `ListIdentifiers`: Returns a list of headers for records, without the actual metadata records. 14 | - `ListMetadataFormats`: Lists the metadata formats supported by the repository. 15 | - `ListRecords`: Returns a list of records, along with metadata records. 16 | - `ListSets`: Lists the sets or collections available in the repository. 17 | -------------------------------------------------------------------------------- /api/src/handlers/oai/xml-transformer.js: -------------------------------------------------------------------------------- 1 | const convert = require("xml-js"); 2 | const { formatOaiDate } = require("./date-utils"); 3 | 4 | const json2xmlOptions = { compact: true, ignoreComment: true, spaces: 4 }; 5 | 6 | const declaration = { 7 | _declaration: { _attributes: { version: "1.0", encoding: "utf-8" } }, 8 | }; 9 | 10 | const invalidOaiRequest = (oaiCode, message, statusCode = 400) => { 11 | const obj = { 12 | "OAI-PMH": { 13 | _attributes: { 14 | xmlns: "http://www.openarchives.org/OAI/2.0/", 15 | "xmlns:xsi": "http://www.w3.org/2001/XMLSchema-instance", 16 | "xsi:schemaLocation": 17 | "http://www.openarchives.org/OAI/2.0/\nhttp://www.openarchives.org/OAI/2.0/OAI_PMH.xsd", 18 | }, 19 | responseDate: formatOaiDate(new Date()), 20 | error: { 21 | _attributes: { 22 | code: oaiCode, 23 | }, 24 | _text: message, 25 | }, 26 | }, 27 | }; 28 | return output(obj, statusCode); 29 | }; 30 | 31 | const output = (obj, statusCode = 200) => { 32 | return { 33 | statusCode: statusCode, 34 | headers: { "content-type": "application/xml" }, 35 | body: convert.js2xml({ ...declaration, ...obj }, json2xmlOptions), 36 | }; 37 | }; 38 | 39 | module.exports = { invalidOaiRequest, output }; 40 | -------------------------------------------------------------------------------- /chat/test/search/test_hybrid_query.py: -------------------------------------------------------------------------------- 1 | from search.hybrid_query import hybrid_query, filter 2 | 3 | 4 | class TestFunction: 5 | def test_hybrid_query(self): 6 | dsl = hybrid_query("Question?", "MODEL_ID", k=10) 7 | subject = dsl["query"]["hybrid"]["queries"] 8 | 9 | assert len(subject) == 2 10 | 11 | queries_first = subject[0]["bool"]["must"] 12 | assert queries_first[0]["query_string"]["query"] == "Question?" 13 | assert {"terms": {"visibility": ["Public", "Institution"]}} in queries_first 14 | assert {"term": {"published": True}} in queries_first 15 | 16 | queries_second = subject[1]["bool"]["must"] 17 | assert queries_second[0]["neural"]["embedding"]["model_id"] == "MODEL_ID" 18 | assert {"terms": {"visibility": ["Public", "Institution"]}} in queries_second 19 | assert {"term": {"published": True}} in queries_second 20 | 21 | def test_filter(self): 22 | dummy_query = {"match": {"title": "Hello World"}} 23 | result = filter(dummy_query) 24 | assert "bool" in result 25 | assert "must" in result["bool"] 26 | must_clause = result["bool"]["must"] 27 | assert must_clause[0] == dummy_query 28 | assert {"terms": {"visibility": ["Public", "Institution"]}} in must_clause 29 | assert {"term": {"published": True}} in must_clause 30 | -------------------------------------------------------------------------------- /api/test/fixtures/mocks/real-search-event.json: -------------------------------------------------------------------------------- 1 | { 2 | "version": "2.0", 3 | "routeKey": "GET /search/{models}", 4 | "rawPath": "/v2/search/collections", 5 | "rawQueryString": "", 6 | "cookies": [], 7 | "headers": { 8 | "accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 9 | "accept-encoding": "gzip, deflate, br", 10 | "accept-language": "en-US,en;q=0.9", 11 | "content-length": "0", 12 | "host": "api.test.library.northwestern.edu", 13 | "user-agent": "Mocha Test", 14 | "x-forwarded-for": "10.9.8.7", 15 | "x-forwarded-port": "443", 16 | "x-forwarded-proto": "https" 17 | }, 18 | "requestContext": { 19 | "accountId": "625046682746", 20 | "apiId": "pewpewpew", 21 | "domainName": "api.test.library.northwestern.edu", 22 | "domainPrefix": "dcapi", 23 | "http": { 24 | "method": "GET", 25 | "path": "/v2/search/collections", 26 | "protocol": "HTTP/1.1", 27 | "sourceIp": "10.9.8.7", 28 | "userAgent": "Mocha Test" 29 | }, 30 | "requestId": "aVmRZjmGoAMEVuw=", 31 | "routeKey": "GET /search/{models}", 32 | "stage": "v2", 33 | "time": "21/Oct/2022:04:18:42 +0000", 34 | "timeEpoch": 1666325922398 35 | }, 36 | "pathParameters": { 37 | "models": "collections" 38 | }, 39 | "stageVariables": { 40 | "basePath": "api/v2" 41 | }, 42 | "isBase64Encoded": false 43 | } 44 | -------------------------------------------------------------------------------- /api/src/handlers/auth/nusso-login.js: -------------------------------------------------------------------------------- 1 | const { dcApiEndpoint } = require("../../environment"); 2 | const axios = require("axios").default; 3 | const cookie = require("cookie"); 4 | const Honeybadger = require("../../honeybadger-setup"); 5 | 6 | /** 7 | * Performs NUSSO login 8 | */ 9 | exports.handler = async (event) => { 10 | const callbackUrl = `${dcApiEndpoint()}/auth/callback/nusso`; 11 | const url = `${process.env.NUSSO_BASE_URL}get-ldap-redirect-url`; 12 | const returnPath = 13 | event.queryStringParameters?.goto || 14 | event.headers?.referer || 15 | `${dcApiEndpoint()}/auth/whoami`; 16 | 17 | if (!returnPath) { 18 | return { 19 | statusCode: 400, 20 | }; 21 | } 22 | 23 | try { 24 | const response = await axios.get(url, { 25 | headers: { 26 | apikey: process.env.NUSSO_API_KEY, 27 | goto: callbackUrl, 28 | }, 29 | }); 30 | 31 | return { 32 | statusCode: 302, 33 | cookies: [ 34 | cookie.serialize( 35 | "redirectUrl", 36 | Buffer.from(returnPath, "utf8").toString("base64") 37 | ), 38 | ], 39 | headers: { 40 | location: response.data.redirecturl, 41 | }, 42 | }; 43 | } catch (error) { 44 | await Honeybadger.notifyAsync(error, { tags: ["auth", "upstream"] }); 45 | console.error("NUSSO request error", error); 46 | return { 47 | statusCode: 401, 48 | }; 49 | } 50 | }; 51 | -------------------------------------------------------------------------------- /chat/test/core/test_secrets.py: -------------------------------------------------------------------------------- 1 | import boto3 2 | import os 3 | import pytest 4 | from moto import mock_aws 5 | from unittest import TestCase 6 | 7 | from core.secrets import load_secrets 8 | 9 | 10 | @mock_aws 11 | @mock_aws 12 | @pytest.mark.filterwarnings("ignore::DeprecationWarning") 13 | class TestSecrets(TestCase): 14 | def setUp(self): 15 | client = boto3.client("secretsmanager", region_name="us-east-1") 16 | client.create_secret( 17 | Name="mock/infrastructure/index", 18 | SecretString='{"endpoint": "https://opensearch-endpoint", "embedding_model": "opensearch-model"}', 19 | ) 20 | client.create_secret( 21 | Name="mock/config/dcapi", SecretString='{"api_token_secret": "dcapi-token"}' 22 | ) 23 | 24 | def test_load_secrets(self): 25 | os.environ["SECRETS_PATH"] = "mock" 26 | self.assertNotEqual("dcapi-token", os.getenv("API_TOKEN_SECRET")) 27 | self.assertNotEqual( 28 | "https://opensearch-endpoint", os.getenv("OPENSEARCH_ENDPOINT") 29 | ) 30 | self.assertNotEqual("opensearch-model", os.getenv("OPENSEARCH_MODEL_ID")) 31 | load_secrets() 32 | self.assertEqual("dcapi-token", os.getenv("API_TOKEN_SECRET")) 33 | self.assertEqual( 34 | "https://opensearch-endpoint", os.getenv("OPENSEARCH_ENDPOINT") 35 | ) 36 | self.assertEqual("opensearch-model", os.getenv("OPENSEARCH_MODEL_ID")) 37 | -------------------------------------------------------------------------------- /api/src/handlers/auth/magic-callback.js: -------------------------------------------------------------------------------- 1 | const { verifyMagicToken } = require("./magic-link"); 2 | const ApiToken = require("../../api/api-token"); 3 | 4 | exports.handler = async (event) => { 5 | const token = event.queryStringParameters?.token; 6 | if (!token) { 7 | return { 8 | statusCode: 400, 9 | body: JSON.stringify({ error: "Missing token" }), 10 | headers: { 11 | "Content-Type": "application/json", 12 | }, 13 | }; 14 | } 15 | try { 16 | const { email, returnUrl } = verifyMagicToken(decodeURIComponent(token)); 17 | const user = { 18 | sub: email, 19 | name: email, 20 | }; 21 | console.info("User", user.sub, "logged in via magic link"); 22 | event.userToken = new ApiToken().user(user).provider("magic"); 23 | return { 24 | statusCode: 302, 25 | headers: { 26 | location: returnUrl, 27 | }, 28 | }; 29 | } catch (error) { 30 | const errorMessage = error.message; 31 | let statusCode = 500; 32 | switch (error.code) { 33 | case "INVALID_TOKEN_SIGNATURE": 34 | statusCode = 401; 35 | break; 36 | case "TOKEN_EXPIRED": 37 | statusCode = 401; 38 | break; 39 | default: 40 | console.error("Unknown error", error); 41 | } 42 | return { 43 | statusCode, 44 | body: JSON.stringify({ error: errorMessage }), 45 | headers: { 46 | "Content-Type": "application/json", 47 | }, 48 | }; 49 | } 50 | }; 51 | -------------------------------------------------------------------------------- /api/test/fixtures/mocks/oai-sets.json: -------------------------------------------------------------------------------- 1 | { 2 | "took": 1, 3 | "timed_out": false, 4 | "_shards": { "total": 1, "successful": 1, "skipped": 0, "failed": 0 }, 5 | "hits": { 6 | "total": { "value": 43, "relation": "eq" }, 7 | "max_score": null, 8 | "hits": [ 9 | { 10 | "_index": "dc-v2-collection-1673551116148", 11 | "_type": "_doc", 12 | "_id": "1c2e2200-c12d-4c7f-8b87-a935c349898a", 13 | "_score": null, 14 | "_source": { 15 | "id": "1c2e2200-c12d-4c7f-8b87-a935c349898a", 16 | "title": "16th-Early 20th Century Maps of Africa" 17 | }, 18 | "sort": ["16th-Early 20th Century Maps of Africa"] 19 | }, 20 | { 21 | "_index": "dc-v2-collection-1673551116148", 22 | "_type": "_doc", 23 | "_id": "c4f30015-88b5-4291-b3a6-8ac9b7c7069c", 24 | "_score": null, 25 | "_source": { 26 | "id": "c4f30015-88b5-4291-b3a6-8ac9b7c7069c", 27 | "title": "Alexander Hesler Photograph Collection" 28 | }, 29 | "sort": ["Alexander Hesler Photograph Collection"] 30 | }, 31 | { 32 | "_index": "dc-v2-collection-1673551116148", 33 | "_type": "_doc", 34 | "_id": "30d59215-8f2b-470d-967b-5a721fa366b7", 35 | "_score": null, 36 | "_source": { 37 | "id": "30d59215-8f2b-470d-967b-5a721fa366b7", 38 | "title": "Ann C. Gunter Slide Collection" 39 | }, 40 | "sort": ["Ann C. Gunter Slide Collection"] 41 | } 42 | ] 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /api/src/handlers/get-shared-link-by-id.js: -------------------------------------------------------------------------------- 1 | const { wrap } = require("./middleware"); 2 | const { getSharedLink, getWork } = require("../api/opensearch"); 3 | const opensearchResponse = require("../api/response/opensearch"); 4 | 5 | /** 6 | * Get a shared link document by id 7 | */ 8 | exports.handler = wrap(async (event) => { 9 | const id = event.pathParameters.id; 10 | const sharedLinkResponse = await getSharedLink(id); 11 | const sharedLinkResponseBody = JSON.parse(sharedLinkResponse.body); 12 | const expirationDate = new Date(sharedLinkResponseBody?._source?.expires); 13 | const workId = sharedLinkResponseBody?._source?.target_id; 14 | 15 | if (linkExpired(expirationDate) || !workId) 16 | return invalidRequest("Not Found"); 17 | 18 | const workResponse = await getWork(workId, { 19 | allowPrivate: true, 20 | allowUnpublished: true, 21 | }); 22 | if (workResponse.statusCode !== 200) return invalidRequest("Not Found"); 23 | 24 | event.userToken.addEntitlement(workId); 25 | return await opensearchResponse.transform(workResponse, { 26 | expires: expirationDate, 27 | }); 28 | }); 29 | 30 | const invalidRequest = (message) => { 31 | return { 32 | statusCode: 404, 33 | headers: { "content-type": "text/plain" }, 34 | body: JSON.stringify({ message: message }), 35 | }; 36 | }; 37 | 38 | const linkExpired = (expirationDate) => { 39 | return !isValid(expirationDate) || expirationDate <= new Date(); 40 | }; 41 | 42 | const isValid = (date) => { 43 | return date instanceof Date && !isNaN(date.getTime()); 44 | }; 45 | -------------------------------------------------------------------------------- /api/test/integration/auth/nusso-login.test.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | 3 | const chai = require("chai"); 4 | const expect = chai.expect; 5 | const nock = require("nock"); 6 | 7 | const getAuthLoginHandler = requireSource("handlers/get-auth-stage"); 8 | 9 | describe("auth login", function () { 10 | helpers.saveEnvironment(); 11 | 12 | beforeEach(() => { 13 | process.env.NUSSO_BASE_URL = "https://test-nusso.com/"; 14 | process.env.NUSSO_API_KEY = "abc123"; 15 | 16 | nock(process.env.NUSSO_BASE_URL) 17 | .get("/get-ldap-redirect-url") 18 | .reply(200, { redirecturl: "https://test-redirect.com" }); 19 | }); 20 | 21 | it("redirects to the NUSSO url", async () => { 22 | const event = helpers 23 | .mockEvent("GET", "/auth/login/nusso") 24 | .pathParams({ provider: "nusso", stage: "login" }) 25 | .queryParams({ goto: "https://test-goto.com" }) 26 | .render(); 27 | 28 | const result = await getAuthLoginHandler.handler(event); 29 | expect(result.statusCode).to.eq(302); 30 | expect(result.headers.location).to.eq("https://test-redirect.com"); 31 | }); 32 | 33 | it("defaults to the NUSSO url", async () => { 34 | const event = helpers 35 | .mockEvent("GET", "/auth/login") 36 | .pathParams({ stage: "login" }) 37 | .queryParams({ goto: "https://test-goto.com" }) 38 | .render(); 39 | 40 | const result = await getAuthLoginHandler.handler(event); 41 | expect(result.statusCode).to.eq(302); 42 | expect(result.headers.location).to.eq("https://test-redirect.com"); 43 | }); 44 | }); 45 | -------------------------------------------------------------------------------- /chat/test/fixtures/apitoken.py: -------------------------------------------------------------------------------- 1 | TEST_SECRET = "TEST_SECRET" 2 | TEST_TOKEN_NAME = "dcTestToken" 3 | SUPER_TOKEN = ('eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJleHAiOjQ4NDM1NzU2ODg5MTIsIm' 4 | 'lhdCI6MTY4Nzg4MDI0NywiaXNMb2dnZWRJbiI6dHJ1ZSwic3ViIjoiYXBpVGVzdFN1c' 5 | 'GVyVXNlciIsImlzU3VwZXJVc2VyIjp0cnVlLCJlbnRpdGxlbWVudHMiOltdLCJwcm92' 6 | 'aWRlciI6Im51c3NvIiwic2NvcGVzIjpbInJlYWQ6UHVibGljIiwicmVhZDpQdWJsaXN' 7 | 'oZWQiLCJyZWFkOlByaXZhdGUiLCJyZWFkOlVucHVibGlzaGVkIiwiY2hhdCJdLCJpc0' 8 | 'luc3RpdHV0aW9uIjp0cnVlfQ.TYuYUvAamOqvBnixTbOq7QiHvDbnycwLdAcPf_R4S2o') 9 | TEST_TOKEN = ('eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJleHAiOjQ4NDM1ODY2MDYxNjUsIml' 10 | 'hdCI6MTY4Nzg5MTM2OSwiZW50aXRsZW1lbnRzIjpbXSwiaXNMb2dnZWRJbiI6dHJ1ZSw' 11 | 'ic3ViIjoidGVzdFVzZXIiLCJwcm92aWRlciI6InRlc3QiLCJzY29wZXMiOlsicmVhZDp' 12 | 'QdWJsaWMiLCJyZWFkOlB1Ymxpc2hlZCIsImNoYXQiXSwiaXNJbnN0aXR1dGlvbiI6ZmF' 13 | 'sc2V9.WHHlVPVMs2OD4WvDpRYc8gNSDiOFJOibjKdds_fxGNA') 14 | DEV_TEAM_TOKEN = ('eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJleHAiOjI1MjY1OTQ2MDcsInN' 15 | '1YiI6ImFiYzEyMyIsImlzcyI6Im1lYWRvdyIsImlhdCI6MTcyNDk1OTUyNiwiZW5' 16 | '0aXRsZW1lbnRzIjpbXSwiaXNMb2dnZWRJbiI6dHJ1ZSwiaXNTdXBlclVzZXIiOmZ' 17 | 'hbHNlLCJpc0RldlRlYW0iOnRydWUsInByb3ZpZGVyIjoibnVzc28iLCJzY29wZXM' 18 | 'iOlsicmVhZDpQdWJsaWMiLCJyZWFkOlB1Ymxpc2hlZCIsImNoYXQiXSwiaXNJbnN' 19 | '0aXR1dGlvbiI6dHJ1ZX0.Ew3y9acVHjbI2tzx5lMiaWBUNVmX8-g9mF1LPXL0ytU') 20 | -------------------------------------------------------------------------------- /docs/mkdocs.yml: -------------------------------------------------------------------------------- 1 | site_name: "NUL Digital Collections API" 2 | site_url: https://api.dc.library.northwestern.edu/api/v2/ 3 | use_directory_urls: true 4 | nav: 5 | - Overview: "index.md" 6 | - Protocol for Metadata Harvesting (OAI-PMH): "oai.md" 7 | - OpenApi Specification: "spec.md" 8 | theme: 9 | name: material 10 | custom_dir: overrides 11 | icon: 12 | logo: nul-logo 13 | features: 14 | - navigation.sections 15 | - header.autohide 16 | - content.code.copy 17 | - content.tabs.link 18 | extra_css: 19 | - css/fonts.css 20 | - css/overrides.css 21 | repo_name: "nulib/dc-api-v2" 22 | repo_url: "https://github.com/nulib/dc-api-v2" 23 | edit_uri: blob/main/docs/docs/ 24 | plugins: 25 | - macros 26 | - search 27 | - render_swagger: 28 | allow_arbitrary_locations : true 29 | markdown_extensions: 30 | - admonition #adds scope to have custom highlight boxes with !!! 31 | - codehilite: 32 | guess_lang: false 33 | - def_list 34 | - footnotes 35 | - smarty 36 | - attr_list 37 | - md_in_html 38 | - pymdownx.superfences 39 | - pymdownx.details 40 | - pymdownx.inlinehilite 41 | - pymdownx.mark 42 | - pymdownx.superfences 43 | - pymdownx.tabbed: 44 | alternate_style: true 45 | - pymdownx.tilde 46 | - pymdownx.emoji: 47 | emoji_index: !!python/name:material.extensions.emoji.twemoji 48 | emoji_generator: !!python/name:material.extensions.emoji.to_svg 49 | options: 50 | custom_icons: 51 | - overrides/.icons 52 | - toc: 53 | permalink: true 54 | baselevel: 2 55 | -------------------------------------------------------------------------------- /api/src/handlers/get-provider-capabilities.js: -------------------------------------------------------------------------------- 1 | const { wrap } = require("./middleware"); 2 | const { ProviderCapabilities } = require("../environment"); 3 | 4 | const handler = wrap(async (event) => { 5 | try { 6 | const provider = event.pathParameters?.provider; 7 | const feature = event.pathParameters?.feature; 8 | 9 | if (!provider || !feature) { 10 | return { 11 | statusCode: 400, 12 | body: JSON.stringify({ 13 | error: "Missing required path parameters: provider and feature", 14 | }), 15 | }; 16 | } 17 | 18 | if ( 19 | !Object.prototype.hasOwnProperty.call(ProviderCapabilities(), provider) 20 | ) { 21 | return { 22 | statusCode: 404, 23 | body: JSON.stringify({ 24 | error: `Provider '${provider}' not found`, 25 | enabled: false, 26 | }), 27 | }; 28 | } 29 | 30 | const isFeatureEnabled = 31 | Array.isArray(ProviderCapabilities()[provider]) && 32 | ProviderCapabilities()[provider].includes(feature); 33 | 34 | return { 35 | statusCode: 200, 36 | body: JSON.stringify({ 37 | enabled: isFeatureEnabled, 38 | provider, 39 | feature, 40 | }), 41 | }; 42 | } catch (error) { 43 | console.error("Error processing request:", error); 44 | 45 | return { 46 | statusCode: 500, 47 | body: JSON.stringify({ 48 | error: "Internal server error", 49 | message: error.message, 50 | }), 51 | }; 52 | } 53 | }); 54 | 55 | module.exports = { handler }; 56 | -------------------------------------------------------------------------------- /api/test/fixtures/mocks/fileset-annotated-1234.json: -------------------------------------------------------------------------------- 1 | { 2 | "_index": "dev-dc-v2-file-set", 3 | "_type": "_doc", 4 | "_id": "1234", 5 | "_version": 1, 6 | "found": true, 7 | "_source": { 8 | "id": "1234", 9 | "api_model": "FileSet", 10 | "visibility": "Public", 11 | "published": true, 12 | "mime_type": "image/tiff", 13 | "annotations": [ 14 | { 15 | "id": "36a47020-5410-4dda-a7ca-967fe3885bcd", 16 | "type": "transcription", 17 | "language": ["lg", "en"], 18 | "content": "Lorem ipsum dolor sit amet, consectetur adipiscing elit. Integer vitae nisl a leo faucibus consectetur a vel ante. Vivamus imperdiet, arcu a luctus mollis, libero lectus porta ex, quis dapibus quam lectus id urna. Pellentesque nec eros non dolor pharetra rutrum nec sit amet velit. Ut a nisl augue. Pellentesque fermentum odio risus, eget placerat sem vehicula sodales. Quisque pulvinar urna sit amet mi hendrerit faucibus. Phasellus a maximus est. Fusce bibendum pulvinar ipsum, nec blandit nulla feugiat non. Pellentesque est odio, ornare porta pulvinar sit amet, posuere congue nisi. Nam finibus felis metus, id dignissim nisl condimentum in. Proin convallis, leo ac imperdiet luctus, leo velit pulvinar dolor, ut lacinia massa est eu felis. Phasellus porta efficitur ex eu commodo. In fermentum neque sit amet porttitor pharetra. Sed sit amet pellentesque erat, sit amet accumsan risus. Sed varius condimentum nunc, sed luctus metus pretium nec.", 19 | "model": "us.anthropic.claude-sonnet-4-5-20250929-v1:0" 20 | } 21 | ] 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /api/test/unit/aws/environment.test.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | 3 | const chai = require("chai"); 4 | const expect = chai.expect; 5 | 6 | const environment = requireSource("environment"); 7 | 8 | describe("environment", function () { 9 | helpers.saveEnvironment(); 10 | 11 | it("returns the index endpoint", function () { 12 | process.env.OPENSEARCH_ENDPOINT = "index.test.library.northwestern.edu"; 13 | expect(environment.openSearchEndpoint()).to.eq( 14 | "index.test.library.northwestern.edu" 15 | ); 16 | }); 17 | 18 | it("correctly handles an environment prefix", function () { 19 | process.env.ENV_PREFIX = "test-env"; 20 | expect(environment.prefix()).to.eq("test-env"); 21 | expect(environment.prefix("name")).to.eq("test-env-name"); 22 | }); 23 | 24 | it("correctly handles an empty environment prefix", function () { 25 | process.env.ENV_PREFIX = ""; 26 | expect(environment.prefix()).to.eq(""); 27 | expect(environment.prefix("name")).to.eq("name"); 28 | }); 29 | 30 | it("correctly handles a missing environment prefix", function () { 31 | delete process.env.ENV_PREFIX; 32 | expect(environment.prefix()).to.eq(""); 33 | expect(environment.prefix("name")).to.eq("name"); 34 | }); 35 | 36 | it("returns the AWS region", function () { 37 | process.env.AWS_REGION = "my-region-1"; 38 | expect(environment.region()).to.eq("my-region-1"); 39 | }); 40 | 41 | it("returns the default region", function () { 42 | delete process.env.AWS_REGION; 43 | expect(environment.region()).to.eq("us-east-1"); 44 | }); 45 | }); 46 | -------------------------------------------------------------------------------- /av-download/lambdas/start-audio-transcode.js: -------------------------------------------------------------------------------- 1 | const { S3Client, PutObjectCommand } = require('@aws-sdk/client-s3'); 2 | const { Upload } = require('@aws-sdk/lib-storage'); 3 | const ffmpeg = require('fluent-ffmpeg'); 4 | const stream = require('stream'); 5 | const path = require('path'); 6 | 7 | module.exports.handler = async (event) => { 8 | const s3Client = new S3Client(); 9 | const url = event.streamingUrl; 10 | const referer = event.referer || null; 11 | const inputOptions = referer ? ["-referer", referer] : []; 12 | const bucket = event.destinationBucket; 13 | const key = event.destinationKey; 14 | const pass = new stream.PassThrough(); 15 | const upload = new Upload({ 16 | client: s3Client, 17 | params: { 18 | Bucket: bucket, 19 | Key: key, 20 | Body: pass, 21 | ContentType: 'audio/mp3', 22 | ContentDisposition: `attachment; filename=${path.basename(key)}`, 23 | ACL: 'public-read' 24 | } 25 | }); 26 | 27 | const ffmpegPromise = new Promise((resolve, reject) => { 28 | try { 29 | ffmpeg() 30 | .input(url) 31 | .inputOptions(inputOptions) 32 | .format("mp3") 33 | .output(pass, { end: true }) 34 | .on("error", (error) => { 35 | console.error("ffmpeg error", error); 36 | reject(error); 37 | }) 38 | .run(); 39 | resolve(); 40 | } catch (error) { 41 | console.error("ffmpeg error", error); 42 | reject(error); 43 | } 44 | }); 45 | 46 | await ffmpegPromise; 47 | await upload.done(); 48 | return { status: 200, message: 'done' }; 49 | } -------------------------------------------------------------------------------- /api/src/handlers/get-auth-token.js: -------------------------------------------------------------------------------- 1 | const { wrap } = require("./middleware"); 2 | const Honeybadger = require("../honeybadger-setup"); 3 | 4 | const DEFAULT_TTL = 86400; // one day 5 | const MAX_TTL = DEFAULT_TTL * 7; // one week; 6 | 7 | const makeError = (code, message) => { 8 | return { 9 | statusCode: code, 10 | headers: { 11 | "Content-Type": "text/plain", 12 | }, 13 | body: message, 14 | }; 15 | }; 16 | 17 | const present = (value) => 18 | value !== undefined && value !== null && value !== ""; 19 | 20 | /** 21 | * Token - Returns auth token for current user with requested expiration 22 | * 23 | */ 24 | exports.handler = wrap(async (event) => { 25 | try { 26 | const ttl = event.queryStringParameters?.ttl; 27 | if (present(ttl) && ttl.match(/\D/)) { 28 | return makeError(400, `'${ttl}' is not a valid value for ttl`); 29 | } 30 | const ttl_in_seconds = Number(ttl) || DEFAULT_TTL; 31 | if (ttl_in_seconds > MAX_TTL) { 32 | return makeError(400, `ttl cannot exceed ${MAX_TTL} seconds`); 33 | } 34 | 35 | const token = event.userToken; 36 | const expiration = new Date(new Date().getTime() + ttl_in_seconds * 1000); 37 | expiration.setMilliseconds(0); 38 | token.expireAt(expiration); 39 | 40 | return { 41 | statusCode: 200, 42 | body: JSON.stringify({ 43 | token: token.sign(), 44 | expires: expiration.toISOString(), 45 | }), 46 | }; 47 | } catch (error) { 48 | await Honeybadger.notifyAsync(error); 49 | return makeError(401, "Error verifying API token: " + error.message); 50 | } 51 | }); 52 | -------------------------------------------------------------------------------- /chat/src/core/apitoken.py: -------------------------------------------------------------------------------- 1 | from datetime import datetime 2 | import jwt 3 | import os 4 | 5 | class ApiToken: 6 | @classmethod 7 | def empty_token(cls): 8 | time = int(datetime.now().timestamp()) 9 | return { 10 | "iss": os.getenv("DC_API_ENDPOINT"), 11 | "exp": datetime.fromtimestamp(time + 12 * 60 * 60).timestamp(), # 12 hours 12 | "iat": time, 13 | "scopes": [], 14 | "entitlements": [], 15 | "isLoggedIn": False, 16 | "isDevTeam": False, 17 | } 18 | 19 | def __init__(self, signed_token=None): 20 | if signed_token is None: 21 | self.token = ApiToken.empty_token() 22 | else: 23 | try: 24 | secret = os.getenv("API_TOKEN_SECRET") 25 | self.token = jwt.decode(signed_token, secret, algorithms=["HS256"]) 26 | except Exception: 27 | self.token = ApiToken.empty_token() 28 | 29 | def __str__(self): 30 | return f"ApiToken(token={self.token})" 31 | 32 | def can(self, scope): 33 | if self.is_superuser(): 34 | return True 35 | else: 36 | return scope in self.token.get("scopes", []) 37 | 38 | def is_logged_in(self): 39 | return self.token.get("isLoggedIn", False) 40 | 41 | def is_superuser(self): 42 | return self.token.get("isSuperUser", False) 43 | 44 | def is_dev_team(self): 45 | return self.token.get("isDevTeam", False) 46 | 47 | def is_institution(self): 48 | return self.token.get("isInstitution", False) 49 | -------------------------------------------------------------------------------- /api/src/api/response/opensearch/index.js: -------------------------------------------------------------------------------- 1 | const { appInfo } = require("../../../environment"); 2 | const { transformError } = require("../error"); 3 | 4 | async function transform(response, options = {}) { 5 | if (response.statusCode === 200) { 6 | const responseBody = JSON.parse(response.body); 7 | return await (responseBody?.hits?.hits 8 | ? transformMany(responseBody, options) 9 | : transformOne(responseBody, options)); 10 | } 11 | return transformError(response); 12 | } 13 | 14 | async function transformOne(responseBody, options = {}) { 15 | return { 16 | statusCode: 200, 17 | headers: { 18 | "content-type": "application/json", 19 | }, 20 | body: JSON.stringify({ 21 | data: responseBody._source, 22 | info: appInfo(options), 23 | }), 24 | }; 25 | } 26 | 27 | async function transformMany(responseBody, options) { 28 | return { 29 | statusCode: 200, 30 | headers: { 31 | "content-type": "application/json", 32 | }, 33 | body: JSON.stringify({ 34 | data: extractSource(responseBody.hits.hits), 35 | pagination: await paginationInfo(responseBody, options?.pager), 36 | info: appInfo(), 37 | aggregations: responseBody.aggregations, 38 | }), 39 | }; 40 | } 41 | 42 | async function paginationInfo(responseBody, pager) { 43 | let { ...pageInfo } = await pager.pageInfo(responseBody.hits.total.value); 44 | 45 | return pageInfo; 46 | } 47 | 48 | function extractSource(hits) { 49 | return hits.map((hit) => extractSingle(hit)); 50 | } 51 | 52 | function extractSingle(hit) { 53 | return hit._source; 54 | } 55 | 56 | module.exports = { transform }; 57 | -------------------------------------------------------------------------------- /api/src/handlers/get-auth-stage.js: -------------------------------------------------------------------------------- 1 | const { wrap } = require("./middleware"); 2 | const { ProviderCapabilities } = require("../environment"); 3 | 4 | const DEFAULT_PROVIDER = "nusso"; 5 | 6 | exports.handler = wrap(async (event, context) => { 7 | const provider = event.pathParameters.provider || DEFAULT_PROVIDER; 8 | const stage = event.pathParameters.stage || "login"; 9 | 10 | const capabilities = ProviderCapabilities(); 11 | if (!capabilities[provider]) { 12 | return { 13 | statusCode: 404, 14 | body: JSON.stringify({ 15 | error: `Unknown provider: '${provider}'`, 16 | }), 17 | }; 18 | } 19 | 20 | if (!capabilities[provider].includes("login")) { 21 | return { 22 | statusCode: 404, 23 | body: JSON.stringify({ 24 | error: `Login not enabled for provider '${provider}'`, 25 | }), 26 | }; 27 | } 28 | 29 | try { 30 | const providerModule = `./auth/${provider}-${stage}`; 31 | console.info("Delegating to provider module:", providerModule); 32 | const providerHandler = require(providerModule).handler; 33 | const result = await providerHandler(event, context); 34 | return result; 35 | } catch (error) { 36 | if (error.code === "MODULE_NOT_FOUND") { 37 | console.error("Module not found:", error); 38 | return { 39 | statusCode: 404, 40 | body: JSON.stringify({ 41 | error: `Provider module not found: ${provider}`, 42 | }), 43 | }; 44 | } 45 | console.error(error); 46 | return { 47 | statusCode: 500, 48 | body: JSON.stringify({ 49 | error: error.message, 50 | }), 51 | }; 52 | } 53 | }); 54 | -------------------------------------------------------------------------------- /api/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "dc-api-build", 3 | "version": "2.9.2", 4 | "description": "NUL Digital Collections API Build Environment", 5 | "repository": "https://github.com/nulib/dc-api-v2", 6 | "author": "nulib", 7 | "license": "Apache-2.0", 8 | "dependencies": { 9 | "dc-api": "file:./src" 10 | }, 11 | "scripts": { 12 | "lint": "eslint src/**/*.js test/**/*.js", 13 | "preinstall": "cd src && npm i && cd - && cd ../av-download/lambdas && npm i && cd -", 14 | "prettier": "prettier -c src test", 15 | "prettier:fix": "prettier -cw src test", 16 | "test": "mocha", 17 | "test:coverage": "nyc npm test", 18 | "validate-spec": "openapi-generator-cli validate -i ../docs/docs/spec/openapi.yaml" 19 | }, 20 | "devDependencies": { 21 | "@openapitools/openapi-generator-cli": "^2.5.2", 22 | "aws-sdk-client-mock": "^4.0.1", 23 | "chai": "^4.2.0", 24 | "chai-http": "^4.3.0", 25 | "choma": "^1.2.1", 26 | "deep-equal-in-any-order": "^2.0.6", 27 | "eslint": "^8.32.0", 28 | "eslint-plugin-json": "^3.1.0", 29 | "husky": "^8.0.3", 30 | "mocha": "^9.1.4", 31 | "nock": "^13.2.9", 32 | "nyc": "^15.1.0", 33 | "prettier": "^2.7.1", 34 | "sinon": "^16.1.1" 35 | }, 36 | "eslintConfig": { 37 | "extends": "eslint:recommended", 38 | "env": { 39 | "es6": true, 40 | "mocha": true, 41 | "node": true 42 | }, 43 | "parserOptions": { 44 | "ecmaVersion": "latest" 45 | }, 46 | "root": true, 47 | "rules": { 48 | "no-unused-vars": [ 49 | "error", 50 | { 51 | "argsIgnorePattern": "^_" 52 | } 53 | ] 54 | } 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /.github/workflows/next_version.yml: -------------------------------------------------------------------------------- 1 | --- 2 | name: Increment Version & Create Draft PR 3 | on: 4 | workflow_dispatch 5 | concurrency: 6 | group: ${{ github.workflow }}-${{ github.ref }} 7 | cancel-in-progress: true 8 | permissions: 9 | contents: write 10 | pull-requests: write 11 | jobs: 12 | increment: 13 | runs-on: ubuntu-latest 14 | permissions: 15 | contents: write 16 | pull-requests: write 17 | steps: 18 | - uses: actions/checkout@v2 19 | with: 20 | ref: deploy/staging 21 | - name: Install uv 22 | uses: astral-sh/setup-uv@v7 23 | - name: Bump Version 24 | id: increment 25 | run: | 26 | NEXT_VERSION=$(make version BUMP=patch) 27 | git config --global user.name 'github-actions[bot]' 28 | git config --global user.email '41898282+github-actions[bot]@users.noreply.github.com' 29 | git commit -am "Bump version to ${NEXT_VERSION}" 30 | - name: Push changes 31 | uses: ad-m/github-push-action@master 32 | with: 33 | github_token: ${{ secrets.GITHUB_TOKEN }} 34 | branch: deploy/staging 35 | - name: Read PR Template 36 | id: template 37 | uses: jaywcjlove/github-action-read-file@main 38 | with: 39 | path: .github/PULL_REQUEST_TEMPLATE/production.md 40 | - name: Create New Production PR 41 | uses: repo-sync/pull-request@v2 42 | with: 43 | source_branch: deploy/staging 44 | destination_branch: main 45 | pr_label: "release" 46 | pr_title: Deploy vX.X.X to production 47 | pr_body: | 48 | ${{ steps.template.outputs.content }} 49 | pr_draft: true 50 | -------------------------------------------------------------------------------- /api/test/unit/api/response/iiif/presentation-api/provider.test.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | 3 | const chai = require("chai"); 4 | const expect = chai.expect; 5 | 6 | const { provider, nulLogo } = requireSource( 7 | "api/response/iiif/presentation-api/provider" 8 | ); 9 | 10 | describe("IIIF response presentation API provider and logo", () => { 11 | it("outputs a IIIF provider property", async () => { 12 | expect(provider.id).to.contain("https://www.library.northwestern.edu"); 13 | expect(provider.type).to.eq("Agent"); 14 | expect(provider.label.none[0]).to.eq("Northwestern University Libraries"); 15 | expect(provider.homepage[0].id).to.contain( 16 | "https://dc.library.northwestern.edu" 17 | ); 18 | expect(provider.homepage[0].label.none[0]).to.eq( 19 | "Northwestern University Libraries Digital Collections Homepage" 20 | ); 21 | expect(provider.logo).to.be.an("array"); 22 | expect(provider.logo[0].id).to.contain( 23 | "https://iiif.dc.library.northwestern.edu/iiif/2/00000000-0000-0000-0000-000000000003/full/pct:50/0/default.webp" 24 | ); 25 | expect(provider.logo[0].type).to.eq("Image"); 26 | expect(provider.logo[0].format).to.eq("image/webp"); 27 | expect(provider.logo[0].height).to.be.a("number"); 28 | expect(provider.logo[0].width).to.be.a("number"); 29 | }); 30 | 31 | it("outputs a IIIF logo property", async () => { 32 | expect(nulLogo.id).to.contain( 33 | "https://iiif.dc.library.northwestern.edu/iiif/2/00000000-0000-0000-0000-000000000003/full/pct:50/0/default.webp" 34 | ); 35 | expect(nulLogo.type).to.eq("Image"); 36 | expect(nulLogo.format).to.eq("image/webp"); 37 | expect(nulLogo.height).to.be.a("number"); 38 | expect(nulLogo.width).to.be.a("number"); 39 | }); 40 | }); 41 | -------------------------------------------------------------------------------- /api/test/integration/get-provider-capabilities.test.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | 3 | const chai = require("chai"); 4 | const expect = chai.expect; 5 | 6 | const { handler } = requireSource("handlers/get-provider-capabilities"); 7 | 8 | describe("Provider status check", () => { 9 | helpers.saveEnvironment(); 10 | 11 | beforeEach(() => { 12 | process.env.PROVIDER_CAPABILITIES = '{"magic":[],"nusso":["chat"]}'; 13 | }); 14 | 15 | it("should return enabled=true for enabled provider", async () => { 16 | const event = helpers 17 | .mockEvent("GET", "/status/nusso/chat") 18 | .pathParams({ provider: "nusso", feature: "chat" }) 19 | .render(); 20 | const response = await handler(event); 21 | expect(response.statusCode).to.equal(200); 22 | const body = JSON.parse(response.body); 23 | expect(body).to.have.property("enabled", true); 24 | }); 25 | 26 | it("should return enabled=false for disabled provider", async () => { 27 | const event = helpers 28 | .mockEvent("GET", "/status/magic/chat") 29 | .pathParams({ provider: "magic", feature: "chat" }) 30 | .render(); 31 | const response = await handler(event); 32 | expect(response.statusCode).to.equal(200); 33 | const body = JSON.parse(response.body); 34 | expect(body).to.have.property("enabled", false); 35 | }); 36 | 37 | it("should return not found for unknown provider", async () => { 38 | const event = helpers 39 | .mockEvent("GET", "/status/google/chat") 40 | .pathParams({ provider: "google", feature: "chat" }) 41 | .render(); 42 | const response = await handler(event); 43 | expect(response.statusCode).to.equal(404); 44 | const body = JSON.parse(response.body); 45 | expect(body).to.have.property("error", "Provider 'google' not found"); 46 | }); 47 | }); 48 | -------------------------------------------------------------------------------- /api/src/handlers/transcode-templates.js: -------------------------------------------------------------------------------- 1 | function videoTranscodeSettings(fileInput, fileOutput) { 2 | return { 3 | TimecodeConfig: { 4 | Source: "ZEROBASED", 5 | }, 6 | OutputGroups: [ 7 | { 8 | CustomName: "priv-s3", 9 | Name: "File Group", 10 | Outputs: [ 11 | { 12 | ContainerSettings: { 13 | Container: "MP4", 14 | Mp4Settings: {}, 15 | }, 16 | VideoDescription: { 17 | CodecSettings: { 18 | Codec: "H_264", 19 | H264Settings: { 20 | MaxBitrate: 5000000, 21 | RateControlMode: "QVBR", 22 | SceneChangeDetect: "TRANSITION_DETECTION", 23 | }, 24 | }, 25 | }, 26 | AudioDescriptions: [ 27 | { 28 | CodecSettings: { 29 | Codec: "AAC", 30 | AacSettings: { 31 | Bitrate: 96000, 32 | CodingMode: "CODING_MODE_2_0", 33 | SampleRate: 48000, 34 | }, 35 | }, 36 | }, 37 | ], 38 | }, 39 | ], 40 | OutputGroupSettings: { 41 | Type: "FILE_GROUP_SETTINGS", 42 | FileGroupSettings: { 43 | Destination: fileOutput, 44 | }, 45 | }, 46 | }, 47 | ], 48 | Inputs: [ 49 | { 50 | AudioSelectors: { 51 | "Audio Selector 1": { 52 | DefaultSelection: "DEFAULT", 53 | }, 54 | }, 55 | VideoSelector: {}, 56 | TimecodeSource: "ZEROBASED", 57 | FileInput: fileInput, 58 | }, 59 | ], 60 | }; 61 | } 62 | 63 | module.exports = { videoTranscodeSettings }; 64 | -------------------------------------------------------------------------------- /api/src/api/request/pipeline.js: -------------------------------------------------------------------------------- 1 | const sortJson = require("sort-json"); 2 | const { defaultSearchSize } = require("../../environment"); 3 | 4 | function filterFor(query, event) { 5 | const matchTheQuery = query; 6 | const beUnpublished = { term: { published: false } }; 7 | const beRestricted = { term: { visibility: "Private" } }; 8 | 9 | let filter = { must: [matchTheQuery], must_not: [] }; 10 | 11 | if (!event.userToken.can("read:Unpublished")) { 12 | filter.must_not.push(beUnpublished); 13 | } 14 | 15 | if (!event.userToken.can("read:Private")) { 16 | filter.must_not.push(beRestricted); 17 | } 18 | 19 | return { bool: filter }; 20 | } 21 | 22 | module.exports = class RequestPipeline { 23 | constructor(searchContext) { 24 | this.searchContext = { ...searchContext }; 25 | if (this.searchContext.size === undefined) 26 | this.searchContext.size = defaultSearchSize(); 27 | if (!this.searchContext.from) this.searchContext.from = 0; 28 | } 29 | 30 | // Things tranformer needs to do: 31 | // - not allow unpuplished or restricted items 32 | // - Reading room/IP (not in first iteration) 33 | // - Add `track_total_hits` to search context (so we can get accurate hits.total.value) 34 | 35 | authFilter(event) { 36 | if (this.searchContext.query?.hybrid?.queries) { 37 | this.searchContext.query = { 38 | hybrid: { 39 | queries: this.searchContext.query.hybrid.queries.map((query) => 40 | filterFor(query, event) 41 | ), 42 | }, 43 | }; 44 | } else { 45 | this.searchContext.query = filterFor(this.searchContext.query, event); 46 | } 47 | this.searchContext.track_total_hits = true; 48 | 49 | return this; 50 | } 51 | 52 | toJson() { 53 | return JSON.stringify(sortJson(this.searchContext)); 54 | } 55 | }; 56 | -------------------------------------------------------------------------------- /chat/test/fixtures/events.py: -------------------------------------------------------------------------------- 1 | from copy import deepcopy 2 | from test.fixtures.apitoken import TEST_TOKEN_NAME, TEST_TOKEN 3 | 4 | POST_EVENT = { 5 | "version": "2.0", 6 | "routeKey": "$default", 7 | "rawPath": "/chat", 8 | "cookies": [ 9 | "cookie_1=cookie_value_1", 10 | "cookie_2=cookie_value_2", 11 | ], 12 | "headers": { 13 | "Authorization": f"Bearer {TEST_TOKEN}", 14 | "origin": "https://example.edu", 15 | }, 16 | "queryStringParameters": { 17 | "param1": "value1", 18 | "param2": "value2", 19 | }, 20 | "requestContext": { 21 | "accountId": "123456789012", 22 | "apiId": "api-id", 23 | "domainName": "id.execute-api.us-east-1.amazonaws.com", 24 | "domainPrefix": "id", 25 | "http": { 26 | "method": "POST", 27 | "path": "/chat", 28 | "protocol": "HTTP/1.1", 29 | "sourceIp": "192.168.0.1/32", 30 | "userAgent": "agent", 31 | }, 32 | "requestId": "id", 33 | "routeKey": "$default", 34 | "stage": "$default", 35 | "time": "12/Mar/2020:19:03:58 +0000", 36 | "timeEpoch": 1583348638390, 37 | }, 38 | "body": "UE9TVGVkIENvbnRlbnQ=", 39 | "pathParameters": {}, 40 | "isBase64Encoded": True, 41 | "stageVariables": {}, 42 | } 43 | 44 | PLAIN_BODY_EVENT = deepcopy(POST_EVENT) 45 | PLAIN_BODY_EVENT["isBase64Encoded"] = False 46 | PLAIN_BODY_EVENT["body"] = "POSTed Content" 47 | 48 | NO_BODY_EVENT = deepcopy(POST_EVENT) 49 | NO_BODY_EVENT["isBase64Encoded"] = False 50 | NO_BODY_EVENT["body"] = "" 51 | 52 | NO_TOKEN_EVENT = deepcopy(POST_EVENT) 53 | del NO_TOKEN_EVENT["headers"]["Authorization"] 54 | 55 | COOKIE_TOKEN_EVENT = deepcopy(NO_TOKEN_EVENT) 56 | COOKIE_TOKEN_EVENT["cookies"].append(f"{TEST_TOKEN_NAME}={TEST_TOKEN}") 57 | -------------------------------------------------------------------------------- /api/src/api/response/iiif/presentation-api/placeholder-canvas.js: -------------------------------------------------------------------------------- 1 | function buildPlaceholderCanvas(id, fileSet, size = 640) { 2 | const { representative_image_url } = fileSet; 3 | const { placeholderWidth, placeholderHeight } = getPlaceholderSizes( 4 | fileSet, 5 | size 6 | ); 7 | 8 | return { 9 | id: `${id}/placeholder`, 10 | type: "Canvas", 11 | width: placeholderWidth, 12 | height: placeholderHeight, 13 | items: [ 14 | { 15 | id: `${id}/placeholder/annotation-page/0`, 16 | type: "AnnotationPage", 17 | items: [ 18 | { 19 | id: `${id}/placeholder/annotation/0`, 20 | type: "Annotation", 21 | motivation: "painting", 22 | body: { 23 | id: `${representative_image_url}/full/!${placeholderWidth},${placeholderHeight}/0/default.jpg`, 24 | type: "Image", 25 | format: fileSet.mime_type, 26 | width: placeholderWidth, 27 | height: placeholderHeight, 28 | service: [ 29 | { 30 | ["@id"]: representative_image_url, 31 | ["@type"]: "ImageService2", 32 | profile: "http://iiif.io/api/image/2/level2.json", 33 | }, 34 | ], 35 | }, 36 | target: `${id}/placeholder`, 37 | }, 38 | ], 39 | }, 40 | ], 41 | }; 42 | } 43 | 44 | function getPlaceholderSizes(fileset, size) { 45 | const width = fileset?.width ?? 100; 46 | const height = fileset?.height ?? 100; 47 | const placeholderWidth = width > size ? size : width; 48 | const placeholderHeight = Math.floor((placeholderWidth / width) * height); 49 | return { placeholderWidth, placeholderHeight }; 50 | } 51 | 52 | module.exports = { 53 | buildPlaceholderCanvas, 54 | getPlaceholderSizes, 55 | }; 56 | -------------------------------------------------------------------------------- /chat/src/core/prompts.py: -------------------------------------------------------------------------------- 1 | from typing import List, Optional 2 | 3 | 4 | def prompt_template() -> str: 5 | return """Please provide a brief answer to the question based on the documents provided. Include specific details from the documents that support your answer. Keep your answer concise and keep reading time under 45 seconds. Each document is identified by a 'title' and a unique 'source' UUID. 6 | 7 | Documents: 8 | {context} 9 | Answer in raw markdown, but not within a code block. When referencing a document by title, link to it using its UUID like this: [title](https://dc.library.northwestern.edu/items/UUID). For example: [Judy Collins, Jackson Hole Folk Festival](https://dc.library.northwestern.edu/items/f1ca513b-7d13-4af6-ad7b-8c7ffd1d3a37). Suggest keyword searches using this format: [keyword](https://dc.library.northwestern.edu/search?q=keyword). Offer a variety of search terms that cover different aspects of the topic. Include as many direct links to Digital Collections searches as necessary for a thorough study. The `collection` field contains information about the collection the document belongs to. In the summary, mention the top 1 or 2 collections, explain why they are relevant and link to them using the collection title and id: [collection['title']](https://dc.library.northwestern.edu/collections/collection['id']), for example [World War II Poster Collection](https://dc.library.northwestern.edu/collections/faf4f60e-78e0-4fbf-96ce-4ca8b4df597a): 10 | 11 | Question: 12 | {question} 13 | """ 14 | 15 | 16 | def document_template(attributes: Optional[List[str]] = None) -> str: 17 | if attributes is None: 18 | attributes = [] 19 | lines = ( 20 | ["Content: {title}", "Metadata:"] 21 | + [f" {attribute}: {{{attribute}}}" for attribute in attributes] 22 | + ["Source: {id}"] 23 | ) 24 | return "\n".join(lines) 25 | -------------------------------------------------------------------------------- /.github/workflows/docs.yml: -------------------------------------------------------------------------------- 1 | name: Build and deploy API documentation 2 | on: 3 | push: 4 | branches: 5 | - deploy/staging 6 | - main 7 | paths: 8 | - .github/workflows/docs.yaml 9 | - docs/* 10 | workflow_dispatch: 11 | concurrency: 12 | group: ${{ github.workflow }}-${{ github.ref }} 13 | jobs: 14 | publish-docs: 15 | runs-on: ubuntu-latest 16 | permissions: 17 | id-token: write 18 | contents: read 19 | environment: ${{ github.ref == 'refs/heads/main' && 'production' || 'staging' }} 20 | steps: 21 | - name: Configure AWS Credentials 22 | uses: aws-actions/configure-aws-credentials@master 23 | with: 24 | role-to-assume: arn:aws:iam::${{ secrets.AwsAccount }}:role/github-actions-role 25 | aws-region: us-east-1 26 | - uses: actions/checkout@v2 27 | - uses: actions/setup-python@v2 28 | with: 29 | python-version: '3.12' 30 | - name: Install uv 31 | uses: astral-sh/setup-uv@v7 32 | - name: Install dependencies 33 | run: uv sync 34 | working-directory: ./docs 35 | - name: Build docs 36 | run: uv run mkdocs build --clean 37 | working-directory: ./docs 38 | - name: Determine correct deploy domain for environment 39 | run: sed -i s/API_HOST/${HOSTNAME}/g docs/site/spec/openapi.* 40 | env: 41 | HOSTNAME: ${{ secrets.Hostname }}.${{ secrets.HostedZone }} 42 | - name: Generate JSON API 43 | uses: openapi-generators/openapitools-generator-action@v1 44 | with: 45 | generator: openapi 46 | openapi-file: docs/site/spec/openapi.yaml 47 | command-args: -o docs/site/spec 48 | - name: Copy to S3 49 | run: aws s3 sync --delete docs/site/ s3://${HOST}-docs.${ZONE}/ 50 | env: 51 | HOST: ${{ secrets.Hostname }} 52 | ZONE: ${{ secrets.HostedZone }} 53 | -------------------------------------------------------------------------------- /chat/src/core/document.py: -------------------------------------------------------------------------------- 1 | def minimize_documents(docs): 2 | return [minimize_document(doc) for doc in docs] 3 | 4 | 5 | def minimize_document(doc): 6 | return { 7 | "id": doc.get("id"), 8 | "title": minimize(doc.get("title")), 9 | "alternate_title": minimize(doc.get("alternate_title")), 10 | "description": minimize(doc.get("description")), 11 | "abstract": minimize(doc.get("abstract")), 12 | "subject": labels_only(doc.get("subject")), 13 | "date_created": minimize(doc.get("date_created")), 14 | "provenance": minimize(doc.get("provenance")), 15 | "collection": minimize(doc.get("collection", {}).get("title")), 16 | "creator": labels_only(doc.get("creator")), 17 | "contributor": labels_only(doc.get("contributor")), 18 | "work_type": minimize(doc.get("work_type")), 19 | "genre": labels_only(doc.get("genre")), 20 | "scope_and_contents": minimize(doc.get("scope_and_contents")), 21 | "table_of_contents": minimize(doc.get("table_of_contents")), 22 | "cultural_context": minimize(doc.get("cultural_context")), 23 | "notes": minimize(doc.get("notes")), 24 | "keywords": minimize(doc.get("keywords")), 25 | "visibility": minimize(doc.get("visibility")), 26 | "canonical_link": minimize(doc.get("canonical_link")), 27 | "rights_statement": label_only(doc.get("rights_statement")), 28 | } 29 | 30 | 31 | def labels_only(list_of_fields): 32 | return minimize([label_only(field) for field in list_of_fields]) 33 | 34 | 35 | def label_only(field): 36 | if field is None: 37 | return None 38 | return field.get("label_with_role", field.get("label", None)) 39 | 40 | 41 | def minimize(field): 42 | try: 43 | if field is None: 44 | return None 45 | if len(field) == 0: 46 | return None 47 | return field 48 | except TypeError: 49 | return field 50 | -------------------------------------------------------------------------------- /api/src/environment.js: -------------------------------------------------------------------------------- 1 | const fs = require("fs"); 2 | const jwt = require("jsonwebtoken"); 3 | const path = require("path"); 4 | const PackageInfo = JSON.parse( 5 | fs.readFileSync(path.join(__dirname, "package.json")) 6 | ); 7 | 8 | function apiToken() { 9 | const token = { 10 | displayName: ["Digital Collection API v2"], 11 | iat: Math.floor(Number(new Date()) / 1000), 12 | }; 13 | 14 | return jwt.sign(token, apiTokenSecret()); 15 | } 16 | 17 | function apiTokenName() { 18 | return process.env.API_TOKEN_NAME; 19 | } 20 | 21 | function apiTokenSecret() { 22 | return process.env.API_TOKEN_SECRET; 23 | } 24 | 25 | function appInfo(options = {}) { 26 | return { 27 | name: PackageInfo.name, 28 | description: PackageInfo.description, 29 | version: PackageInfo.version, 30 | link_expiration: options.expires || null, 31 | }; 32 | } 33 | 34 | function dcApiEndpoint() { 35 | return process.env.DC_API_ENDPOINT; 36 | } 37 | 38 | function dcUrl() { 39 | return process.env.DC_URL; 40 | } 41 | 42 | function defaultSearchSize() { 43 | return Number(process.env.DEFAULT_SEARCH_SIZE || "10"); 44 | } 45 | 46 | function devTeamNetIds() { 47 | return process.env.DEV_TEAM_NET_IDS?.split(",") || []; 48 | } 49 | 50 | function openSearchEndpoint() { 51 | return process.env.OPENSEARCH_ENDPOINT; 52 | } 53 | 54 | function prefix(value) { 55 | const envPrefix = 56 | process.env.ENV_PREFIX === "" ? undefined : process.env.ENV_PREFIX; 57 | return [envPrefix, value].filter((val) => !!val).join("-"); 58 | } 59 | 60 | function ProviderCapabilities() { 61 | return JSON.parse(process.env.PROVIDER_CAPABILITIES); 62 | } 63 | 64 | function region() { 65 | return process.env.AWS_REGION || "us-east-1"; 66 | } 67 | 68 | module.exports = { 69 | apiToken, 70 | apiTokenName, 71 | apiTokenSecret, 72 | appInfo, 73 | dcApiEndpoint, 74 | dcUrl, 75 | defaultSearchSize, 76 | devTeamNetIds, 77 | openSearchEndpoint, 78 | prefix, 79 | ProviderCapabilities, 80 | region, 81 | }; 82 | -------------------------------------------------------------------------------- /api/test/integration/get-annotations.test.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | 3 | const chai = require("chai"); 4 | const expect = chai.expect; 5 | chai.use(require("chai-http")); 6 | 7 | describe("Annotation routes", () => { 8 | helpers.saveEnvironment(); 9 | const mock = helpers.mockIndex(); 10 | 11 | describe("GET /file-sets/{id}/annotations", () => { 12 | const { handler } = requireSource("handlers/get-file-set-annotations"); 13 | 14 | it("returns annotations for a file set", async () => { 15 | mock 16 | .get("/dc-v2-file-set/_doc/1234") 17 | .reply(200, helpers.testFixture("mocks/fileset-annotated-1234.json")); 18 | 19 | const event = helpers 20 | .mockEvent("GET", "/file-sets/{id}/annotations") 21 | .pathParams({ id: 1234 }) 22 | .render(); 23 | const result = await handler(event); 24 | expect(result.statusCode).to.eq(200); 25 | 26 | const body = JSON.parse(result.body); 27 | expect(body.data).to.be.an("array").with.lengthOf(1); 28 | expect(body.data[0].type).to.eq("transcription"); 29 | }); 30 | }); 31 | 32 | describe("GET /annotations/{id}", () => { 33 | const { handler } = requireSource("handlers/get-annotation-by-id"); 34 | 35 | it("returns a single annotation", async () => { 36 | mock 37 | .post("/dc-v2-file-set/_search", () => true) 38 | .reply(200, helpers.testFixture("mocks/annotation-search-hit.json")); 39 | 40 | mock 41 | .get("/dc-v2-file-set/_doc/1234") 42 | .reply(200, helpers.testFixture("mocks/fileset-annotated-1234.json")); 43 | 44 | const event = helpers 45 | .mockEvent("GET", "/annotations/{id}") 46 | .pathParams({ id: "36a47020-5410-4dda-a7ca-967fe3885bcd" }) 47 | .render(); 48 | const result = await handler(event); 49 | expect(result.statusCode).to.eq(200); 50 | 51 | const body = JSON.parse(result.body); 52 | expect(body.data.id).to.eq("36a47020-5410-4dda-a7ca-967fe3885bcd"); 53 | }); 54 | }); 55 | }); 56 | -------------------------------------------------------------------------------- /api/src/handlers/auth/magic-login.js: -------------------------------------------------------------------------------- 1 | const { dcApiEndpoint } = require("../../environment"); 2 | const { createMagicToken } = require("./magic-link"); 3 | const { SESClient, SendTemplatedEmailCommand } = require("@aws-sdk/client-ses"); 4 | 5 | const { MAGIC_LINK_EMAIL_TEMPLATE, REPOSITORY_EMAIL } = process.env; 6 | 7 | exports.handler = async (event, context) => { 8 | const callbackUrl = new URL(`${dcApiEndpoint()}/auth/callback/magic`); 9 | 10 | const returnUrl = 11 | event.queryStringParameters?.goto || 12 | event.headers?.referer || 13 | `${dcApiEndpoint()}/auth/whoami`; 14 | 15 | const email = event.queryStringParameters?.email; 16 | if (!email) { 17 | return { 18 | statusCode: 400, 19 | body: JSON.stringify({ error: "Email is required" }), 20 | headers: { 21 | "Content-Type": "application/json", 22 | }, 23 | }; 24 | } 25 | 26 | const { token, expiration } = createMagicToken(email, returnUrl); 27 | callbackUrl.searchParams.set("token", token); 28 | const magicLink = callbackUrl.toString(); 29 | 30 | const sesClient = context?.injections?.sesClient || new SESClient({}); 31 | 32 | const cmd = new SendTemplatedEmailCommand({ 33 | Destination: { ToAddresses: [email] }, 34 | TemplateData: JSON.stringify({ magicLink }), 35 | Source: `Northwestern University Libraries <${REPOSITORY_EMAIL}>`, 36 | Template: MAGIC_LINK_EMAIL_TEMPLATE, 37 | }); 38 | 39 | try { 40 | await sesClient.send(cmd); 41 | console.info("Magic link sent to <%s>", email); 42 | } catch (err) { 43 | console.error("Failed to send template email", err); 44 | return { 45 | statusCode: 500, 46 | body: JSON.stringify({ 47 | error: "Failed to send email", 48 | reason: err.message, 49 | }), 50 | headers: { 51 | "Content-Type": "application/json", 52 | }, 53 | }; 54 | } 55 | 56 | return { 57 | statusCode: 200, 58 | body: JSON.stringify({ 59 | message: "Magic link sent", 60 | email, 61 | expires: new Date(expiration), 62 | }), 63 | }; 64 | }; 65 | -------------------------------------------------------------------------------- /api/src/handlers/get-collection-by-id.js: -------------------------------------------------------------------------------- 1 | const { baseUrl } = require("../helpers"); 2 | const { doSearch } = require("./search-runner"); 3 | const { getCollection } = require("../api/opensearch"); 4 | const { wrap } = require("./middleware"); 5 | const opensearchResponse = require("../api/response/opensearch"); 6 | 7 | const getOpts = (event) => { 8 | const id = event.pathParameters.id; 9 | 10 | const allowPrivate = 11 | event.userToken.isSuperUser() || 12 | event.userToken.isReadingRoom() || 13 | event.userToken.hasEntitlement(id); 14 | const allowUnpublished = 15 | event.userToken.isSuperUser() || event.userToken.hasEntitlement(id); 16 | return { allowPrivate, allowUnpublished }; 17 | }; 18 | 19 | const getCollectionById = async (event) => { 20 | const esResponse = await getCollection( 21 | event.pathParameters.id, 22 | getOpts(event) 23 | ); 24 | return await opensearchResponse.transform(esResponse); 25 | }; 26 | 27 | const getIiifCollectionById = async (event) => { 28 | const id = event.pathParameters.id; 29 | const esResponse = await getCollection(id, getOpts(event)); 30 | const collection = JSON.parse(esResponse.body)?._source; 31 | if (!collection) return { statusCode: 404, body: "Not Found" }; 32 | const parameterOverrides = { ...event.queryStringParameters }; 33 | 34 | event.queryStringParameters.query = `collection.id:${id}`; 35 | event.queryStringParameters.collectionLabel = collection?.title; 36 | event.queryStringParameters.collectionSummary = collection?.description; 37 | return await doSearch(event, { 38 | includeToken: false, 39 | parameterOverrides, 40 | }); 41 | }; 42 | 43 | const isEmpty = (string) => { 44 | return string === undefined || string === null || string === ""; 45 | }; 46 | 47 | /** 48 | * Get a colletion by id 49 | */ 50 | exports.handler = wrap(async (event) => { 51 | if (isEmpty(event.pathParameters.id)) { 52 | return { 53 | statusCode: 301, 54 | headers: { 55 | location: baseUrl(event) + "collections", 56 | }, 57 | }; 58 | } 59 | 60 | return event.queryStringParameters?.as === "iiif" 61 | ? getIiifCollectionById(event) 62 | : getCollectionById(event); 63 | }); 64 | -------------------------------------------------------------------------------- /api/src/handlers/get-annotation-by-id.js: -------------------------------------------------------------------------------- 1 | const { wrap } = require("./middleware"); 2 | const { search, getFileSet } = require("../api/opensearch"); 3 | const { prefix, appInfo } = require("../environment"); 4 | const { transformError } = require("../api/response/error"); 5 | 6 | /** 7 | * Retrieves a single annotation by id 8 | */ 9 | exports.handler = wrap(async (event) => { 10 | const annotationId = event.pathParameters.id; 11 | 12 | const searchBody = { 13 | size: 1, 14 | _source: ["id"], 15 | query: { 16 | bool: { 17 | should: [ 18 | { term: { "annotations.id.keyword": annotationId } }, 19 | { term: { "annotations.id": annotationId } }, 20 | ], 21 | minimum_should_match: 1, 22 | }, 23 | }, 24 | }; 25 | 26 | const searchResponse = await search( 27 | prefix("dc-v2-file-set"), 28 | JSON.stringify(searchBody) 29 | ); 30 | 31 | if (searchResponse.statusCode !== 200) { 32 | return transformError(searchResponse); 33 | } 34 | 35 | const searchPayload = JSON.parse(searchResponse.body); 36 | const hit = searchPayload?.hits?.hits?.[0]; 37 | if (!hit) return transformError({ statusCode: 404 }); 38 | 39 | const fileSetId = hit?._source?.id || hit?._id; 40 | if (!fileSetId) return transformError({ statusCode: 404 }); 41 | 42 | const allowPrivate = 43 | event.userToken.isSuperUser() || event.userToken.isReadingRoom(); 44 | const allowUnpublished = event.userToken.isSuperUser(); 45 | const fileSetResponse = await getFileSet(fileSetId, { 46 | allowPrivate, 47 | allowUnpublished, 48 | }); 49 | 50 | if (fileSetResponse.statusCode !== 200) { 51 | return transformError(fileSetResponse); 52 | } 53 | 54 | const fileSetPayload = JSON.parse(fileSetResponse.body); 55 | const annotation = fileSetPayload?._source?.annotations?.find( 56 | (item) => item.id === annotationId 57 | ); 58 | 59 | if (!annotation) return transformError({ statusCode: 404 }); 60 | 61 | return { 62 | statusCode: 200, 63 | headers: { 64 | "content-type": "application/json", 65 | }, 66 | body: JSON.stringify({ 67 | data: annotation, 68 | info: appInfo(), 69 | }), 70 | }; 71 | }); 72 | -------------------------------------------------------------------------------- /chat-playground/opensearch.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": null, 6 | "metadata": {}, 7 | "outputs": [], 8 | "source": [ 9 | "from dotenv import load_dotenv\n", 10 | "import sys\n", 11 | "import os\n", 12 | "\n", 13 | "load_dotenv(override=True)\n", 14 | "try:\n", 15 | " del os.environ[\"DEV_PREFIX\"]\n", 16 | " del os.environ[\"DEV_ENV\"]\n", 17 | "except:\n", 18 | " pass\n", 19 | "\n", 20 | "sys.path.insert(0, os.path.join(os.curdir, \"../chat/src\"))\n", 21 | "\n", 22 | "from core.setup import opensearch_vector_store\n", 23 | "from core.secrets import load_secrets\n", 24 | "\n", 25 | "load_secrets()\n", 26 | "opensearch = opensearch_vector_store()" 27 | ] 28 | }, 29 | { 30 | "cell_type": "markdown", 31 | "metadata": {}, 32 | "source": [ 33 | "## Aggregations" 34 | ] 35 | }, 36 | { 37 | "cell_type": "code", 38 | "execution_count": null, 39 | "metadata": {}, 40 | "outputs": [], 41 | "source": [ 42 | "opensearch.aggregations_search(agg_field=\"visibility\")" 43 | ] 44 | }, 45 | { 46 | "cell_type": "code", 47 | "execution_count": null, 48 | "metadata": {}, 49 | "outputs": [], 50 | "source": [ 51 | "opensearch.aggregations_search(agg_field=\"published\")" 52 | ] 53 | }, 54 | { 55 | "cell_type": "markdown", 56 | "metadata": {}, 57 | "source": [ 58 | "## Similiarity Search" 59 | ] 60 | }, 61 | { 62 | "cell_type": "code", 63 | "execution_count": null, 64 | "metadata": {}, 65 | "outputs": [], 66 | "source": [ 67 | "opensearch.similarity_search(query=\"Football stadiums\", k=80, size=3)" 68 | ] 69 | } 70 | ], 71 | "metadata": { 72 | "kernelspec": { 73 | "display_name": ".venv", 74 | "language": "python", 75 | "name": "python3" 76 | }, 77 | "language_info": { 78 | "codemirror_mode": { 79 | "name": "ipython", 80 | "version": 3 81 | }, 82 | "file_extension": ".py", 83 | "mimetype": "text/x-python", 84 | "name": "python", 85 | "nbconvert_exporter": "python", 86 | "pygments_lexer": "ipython3", 87 | "version": "3.12.2" 88 | } 89 | }, 90 | "nbformat": 4, 91 | "nbformat_minor": 2 92 | } 93 | -------------------------------------------------------------------------------- /events/event.json: -------------------------------------------------------------------------------- 1 | { 2 | "body": "{\"message\": \"hello world\"}", 3 | "resource": "/works/{id}", 4 | "path": "/works/aa974591-d540-43a5-aff4-b8a1ff4e9c40", 5 | "httpMethod": "GET", 6 | "isBase64Encoded": false, 7 | "queryStringParameters": { 8 | "foo": "bar" 9 | }, 10 | "pathParameters": { 11 | "id": "aa974591-d540-43a5-aff4-b8a1ff4e9c40" 12 | }, 13 | "stageVariables": { 14 | "baz": "qux" 15 | }, 16 | "headers": { 17 | "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8", 18 | "Accept-Encoding": "gzip, deflate, sdch", 19 | "Accept-Language": "en-US,en;q=0.8", 20 | "Cache-Control": "max-age=0", 21 | "CloudFront-Forwarded-Proto": "https", 22 | "CloudFront-Is-Desktop-Viewer": "true", 23 | "CloudFront-Is-Mobile-Viewer": "false", 24 | "CloudFront-Is-SmartTV-Viewer": "false", 25 | "CloudFront-Is-Tablet-Viewer": "false", 26 | "CloudFront-Viewer-Country": "US", 27 | "Host": "1234567890.execute-api.us-east-1.amazonaws.com", 28 | "Upgrade-Insecure-Requests": "1", 29 | "User-Agent": "Custom User Agent String", 30 | "Via": "1.1 08f323deadbeefa7af34d5feb414ce27.cloudfront.net (CloudFront)", 31 | "X-Amz-Cf-Id": "cDehVQoZnx43VYQb9j2-nvCh-9z396Uhbp027Y2JvkCPNLmGJHqlaA==", 32 | "X-Forwarded-For": "127.0.0.1, 127.0.0.2", 33 | "X-Forwarded-Port": "443", 34 | "X-Forwarded-Proto": "https" 35 | }, 36 | "requestContext": { 37 | "accountId": "123456789012", 38 | "resourceId": "123456", 39 | "stage": "prod", 40 | "requestId": "c6af9ac6-7b61-11e6-9a41-93e8deadbeef", 41 | "requestTime": "09/Apr/2015:12:34:56 +0000", 42 | "requestTimeEpoch": 1428582896000, 43 | "identity": { 44 | "cognitoIdentityPoolId": null, 45 | "accountId": null, 46 | "cognitoIdentityId": null, 47 | "caller": null, 48 | "accessKey": null, 49 | "sourceIp": "127.0.0.1", 50 | "cognitoAuthenticationType": null, 51 | "cognitoAuthenticationProvider": null, 52 | "userArn": null, 53 | "userAgent": "Custom User Agent String", 54 | "user": null 55 | }, 56 | "path": "/prod/path/to/resource", 57 | "resourcePath": "/{proxy+}", 58 | "httpMethod": "GET", 59 | "apiId": "1234567890", 60 | "protocol": "HTTP/1.1" 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /api/test/unit/api/response/opensearch.test.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | 3 | const chai = require("chai"); 4 | const expect = chai.expect; 5 | 6 | const transformer = requireSource("api/response/opensearch"); 7 | const { Paginator } = requireSource("api/pagination"); 8 | 9 | describe("OpenSearch response transformer", () => { 10 | helpers.saveEnvironment(); 11 | 12 | let pager; 13 | beforeEach(() => { 14 | pager = new Paginator( 15 | "http://dcapi.library.northwestern.edu/v2/", 16 | "search", 17 | ["works"], 18 | { query: { match_all: {} } } 19 | ); 20 | }); 21 | 22 | it("transforms a doc response", async () => { 23 | const response = { 24 | statusCode: 200, 25 | body: helpers.testFixture("mocks/work-1234.json"), 26 | }; 27 | const result = await transformer.transform(response, { pager: pager }); 28 | expect(result.statusCode).to.eq(200); 29 | 30 | const body = JSON.parse(result.body); 31 | expect(body.data).to.be.an("object"); 32 | expect(body.info).to.include.key("version"); 33 | expect(body).not.to.include.key("pagination"); 34 | }); 35 | 36 | it("transforms a search response", async () => { 37 | const response = { 38 | statusCode: 200, 39 | body: helpers.testFixture("mocks/search.json"), 40 | }; 41 | const result = await transformer.transform(response, { pager: pager }); 42 | expect(result.statusCode).to.eq(200); 43 | 44 | const body = JSON.parse(result.body); 45 | expect(body.data).to.be.an("array"); 46 | expect(body.info).to.include.key("version"); 47 | expect(body).to.include.key("pagination"); 48 | expect(body.pagination).to.include.keys([ 49 | "query_url", 50 | "current_page", 51 | "limit", 52 | "next_url", 53 | "offset", 54 | "total_hits", 55 | "total_pages", 56 | ]); 57 | }); 58 | 59 | it("transforms an error response", async () => { 60 | const response = { 61 | statusCode: 404, 62 | body: helpers.testFixture("mocks/missing-index.json"), 63 | }; 64 | 65 | const result = await transformer.transform(response, { pager: pager }); 66 | expect(result.statusCode).to.eq(404); 67 | 68 | const body = JSON.parse(result.body); 69 | expect(body.status).to.eq(404); 70 | expect(body.error).to.be.a("string"); 71 | }); 72 | }); 73 | -------------------------------------------------------------------------------- /chat/test/core/test_apitoken.py: -------------------------------------------------------------------------------- 1 | # ruff: noqa: E402 2 | import os 3 | 4 | from core.apitoken import ApiToken 5 | from test.fixtures.apitoken import DEV_TEAM_TOKEN, SUPER_TOKEN, TEST_SECRET, TEST_TOKEN 6 | from unittest import mock, TestCase 7 | 8 | 9 | @mock.patch.dict(os.environ, {"DEV_TEAM_NET_IDS": "abc123"}) 10 | @mock.patch.dict(os.environ, {"API_TOKEN_SECRET": TEST_SECRET}) 11 | class TestFunction(TestCase): 12 | def test_empty_token(self): 13 | subject = ApiToken() 14 | self.assertIsInstance(subject, ApiToken) 15 | self.assertFalse(subject.is_logged_in()) 16 | self.assertFalse(subject.is_institution()) 17 | 18 | def test_valid_token(self): 19 | subject = ApiToken(TEST_TOKEN) 20 | self.assertIsInstance(subject, ApiToken) 21 | self.assertTrue(subject.is_logged_in()) 22 | self.assertFalse(subject.is_superuser()) 23 | self.assertFalse(subject.is_institution()) 24 | self.assertTrue(subject.can("read:Public")) 25 | self.assertFalse(subject.can("read:Private")) 26 | 27 | def test_superuser_token(self): 28 | subject = ApiToken(SUPER_TOKEN) 29 | self.assertIsInstance(subject, ApiToken) 30 | self.assertTrue(subject.is_logged_in()) 31 | self.assertTrue(subject.is_superuser()) 32 | self.assertTrue(subject.is_institution()) 33 | self.assertTrue(subject.can("read:Public")) 34 | self.assertTrue(subject.can("read:Private")) 35 | 36 | def test_devteam_token(self): 37 | subject = ApiToken(DEV_TEAM_TOKEN) 38 | self.assertIsInstance(subject, ApiToken) 39 | self.assertTrue(subject.is_dev_team()) 40 | self.assertTrue(subject.is_institution()) 41 | 42 | def test_invalid_token(self): 43 | subject = ApiToken("INVALID_TOKEN") 44 | self.assertIsInstance(subject, ApiToken) 45 | self.assertFalse(subject.is_logged_in()) 46 | self.assertFalse(subject.is_institution()) 47 | 48 | def test_empty_token_class_method(self): 49 | empty_token = ApiToken.empty_token() 50 | self.assertIsInstance(empty_token, dict) 51 | self.assertFalse(empty_token["isLoggedIn"]) 52 | 53 | def test_str_method(self): 54 | subject = ApiToken(TEST_TOKEN) 55 | self.assertEqual(str(subject), f"ApiToken(token={subject.token})") 56 | -------------------------------------------------------------------------------- /api/src/handlers/auth/magic-link.js: -------------------------------------------------------------------------------- 1 | const { apiTokenSecret } = require("../../environment"); 2 | const crypto = require("crypto"); 3 | 4 | const TIMESTAMP_SIZE = 6; // 48 bits 5 | const LINK_EXPIRATION = 15 * 60 * 1000; // 15 minutes 6 | 7 | const sign = (data) => 8 | crypto 9 | .createHmac("sha256", apiTokenSecret()) 10 | .update(data) 11 | .digest() 12 | .subarray(0, 16); 13 | 14 | const error = (message, code) => { 15 | const err = new Error(message); 16 | err.code = code; 17 | return err; 18 | }; 19 | 20 | exports.createMagicToken = (email, returnUrl, expiration) => { 21 | expiration = expiration || Date.now() + LINK_EXPIRATION; 22 | const expirationBytes = Buffer.alloc(6); 23 | expirationBytes.writeUIntLE(expiration, 0, TIMESTAMP_SIZE); 24 | const payloadBytes = Buffer.from([email, returnUrl].join("|"), "utf-8"); 25 | 26 | const payload = Buffer.concat([payloadBytes, expirationBytes]); 27 | const signature = sign(payload); 28 | 29 | const encodedPayload = payload.toString("base64").replace(/=+$/g, ""); 30 | const encodedSignature = signature.toString("base64").replace(/=+$/g, ""); 31 | 32 | const token = encodedPayload + encodedSignature; 33 | return { token, expiration }; 34 | }; 35 | 36 | exports.verifyMagicToken = (token) => { 37 | const signatureLength = Math.ceil((16 * 8) / 6); // 16 bytes = 22 encoded chars 38 | const encodedPayload = token.slice(0, token.length - signatureLength); 39 | const encodedSignature = token.slice(token.length - signatureLength); 40 | 41 | const payload = Buffer.from(encodedPayload, "base64"); 42 | const expectedSignature = sign(payload); 43 | const signature = Buffer.from(encodedSignature, "base64"); 44 | let verified; 45 | try { 46 | verified = crypto.timingSafeEqual(signature, expectedSignature); 47 | } catch (e) { 48 | verified = false; 49 | } 50 | 51 | if (!verified) { 52 | throw error("Invalid token signature", "INVALID_TOKEN_SIGNATURE"); 53 | } 54 | 55 | const [email, returnUrl] = payload 56 | .subarray(0, -TIMESTAMP_SIZE) 57 | .toString("utf-8") 58 | .split("|"); 59 | const expirationBytes = payload.subarray(-TIMESTAMP_SIZE); 60 | const expiration = expirationBytes.readUIntLE(0, TIMESTAMP_SIZE); 61 | 62 | if (Date.now() > expiration) { 63 | throw error("Token expired", "TOKEN_EXPIRED"); 64 | } 65 | 66 | return { email, returnUrl }; 67 | }; 68 | -------------------------------------------------------------------------------- /api/src/handlers/oai/search.js: -------------------------------------------------------------------------------- 1 | const { search } = require("../../api/opensearch"); 2 | const { 3 | extractRequestedModels, 4 | modelsToTargets, 5 | } = require("../../api/request/models"); 6 | 7 | async function earliestRecord() { 8 | const body = { 9 | size: 1, 10 | _source: "create_date", 11 | query: { 12 | bool: { 13 | must: [ 14 | { term: { api_model: "Work" } }, 15 | { term: { published: true } }, 16 | { term: { visibility: "Public" } }, 17 | ], 18 | }, 19 | }, 20 | sort: [{ create_date: "asc" }], 21 | }; 22 | const esResponse = await search( 23 | modelsToTargets(extractRequestedModels()), 24 | JSON.stringify(body) 25 | ); 26 | const responseBody = JSON.parse(esResponse.body); 27 | return responseBody?.hits?.hits[0]?._source?.create_date; 28 | } 29 | 30 | async function oaiSearch(dates, set, size = 250) { 31 | const range = { 32 | range: { 33 | modified_date: { 34 | ...(dates.from && { gt: dates.from }), 35 | ...(dates.until && { lt: dates.until }), 36 | }, 37 | }, 38 | }; 39 | const query = { 40 | bool: { 41 | must: [ 42 | { term: { api_model: "Work" } }, 43 | { term: { published: true } }, 44 | { term: { visibility: "Public" } }, 45 | range, 46 | ], 47 | }, 48 | }; 49 | if (set) query.bool.must.push({ term: { "collection.id": set } }); 50 | 51 | const body = { 52 | size, 53 | query, 54 | sort: [{ modified_date: "asc" }], 55 | }; 56 | 57 | const esResponse = await search( 58 | modelsToTargets(extractRequestedModels()), 59 | JSON.stringify(body), 60 | { scroll: "2m" } 61 | ); 62 | 63 | return { 64 | ...esResponse, 65 | expiration: new Date(new Date().getTime() + 2 * 60000).toISOString(), 66 | }; 67 | } 68 | 69 | async function oaiSets() { 70 | const body = { 71 | size: 10000, 72 | _source: ["id", "title"], 73 | query: { 74 | bool: { 75 | must: [ 76 | { term: { api_model: "Collection" } }, 77 | { term: { published: true } }, 78 | { term: { visibility: "Public" } }, 79 | ], 80 | }, 81 | }, 82 | sort: [{ title: "asc" }], 83 | }; 84 | 85 | const esResponse = await search( 86 | modelsToTargets(["collections"]), 87 | JSON.stringify(body) 88 | ); 89 | return esResponse; 90 | } 91 | 92 | module.exports = { earliestRecord, oaiSearch, oaiSets }; 93 | -------------------------------------------------------------------------------- /.github/workflows/staging-types.yml: -------------------------------------------------------------------------------- 1 | name: Generate typescript and push to nulib/dcapi-types@staging 2 | on: 3 | pull_request: 4 | types: 5 | - closed 6 | branches: 7 | - "deploy/staging" 8 | paths: 9 | - "docs/docs/spec/data-types.yaml" 10 | - ".github/flags/deploy-data-types" 11 | workflow_dispatch: 12 | jobs: 13 | Push-To-Types-Repo-Staging: 14 | if: (github.event_name == 'pull_request' && github.event.pull_request.merged == true) || github.event_name == 'workflow_dispatch' 15 | runs-on: ubuntu-latest 16 | permissions: 17 | id-token: write 18 | contents: read 19 | environment: staging 20 | env: 21 | TARGET_BRANCH: "staging" 22 | TMP_SRCDIR: "./source" 23 | TMP_DSTDIR: "./dest" 24 | TYPES_REPO: "nulib/dcapi-types" 25 | steps: 26 | - run: | 27 | echo PR with changes to datatypes.yml was merged into staging 28 | - name: Checkout dc-api-v2 29 | uses: actions/checkout@v3 30 | with: 31 | path: "${{ env.TMP_SRCDIR }}" 32 | fetch-depth: 1 33 | - uses: actions/setup-node@v4 34 | with: 35 | cache-dependency-path: "${{ env.TMP_SRCDIR }}/api/package-lock.json" 36 | node-version: 20.x 37 | cache: "npm" 38 | - name: get-npm-version 39 | id: package-version 40 | uses: martinbeentjes/npm-get-version-action@main 41 | with: 42 | path: "${{ env.TMP_SRCDIR }}/api" 43 | - name: Generate Typescript file 44 | working-directory: "${{ env.TMP_SRCDIR }}/api" 45 | run: | 46 | npm ci 47 | npx openapi-typescript ../docs/docs/spec/data-types.yaml --output schemas.ts 48 | - name: Checkout dcapi-types 49 | uses: actions/checkout@v3 50 | with: 51 | path: ${{ env.TMP_DSTDIR }} 52 | ref: ${{ env.TARGET_BRANCH }} 53 | repository: ${{env.TYPES_REPO}} 54 | - run: mv -f "${{ env.TMP_SRCDIR }}/api/schemas.ts" "${{ env.TMP_DSTDIR }}/schemas.ts" 55 | shell: bash 56 | - name: Set GitHub Deploy Key 57 | uses: webfactory/ssh-agent@v0.5.3 58 | with: 59 | ssh-private-key: ${{ secrets.DCAPI_TYPES_DEPLOY_KEY }} 60 | - working-directory: "${{ env.TMP_DSTDIR }}" 61 | run: | 62 | git config --global user.email "github-actions[bot]@users.noreply.github.com" 63 | git config --global user.name "github-actions[bot]" 64 | git add . 65 | git commit -m "nulib/dc-api-v2 commit: $GITHUB_SHA, package.json version: ${{ steps.package-version.outputs.current-version}}" 66 | git push origin HEAD 67 | shell: bash 68 | -------------------------------------------------------------------------------- /docs/docs/css/fonts.css: -------------------------------------------------------------------------------- 1 | @font-face { 2 | font-family: "Akkurat Pro Light"; 3 | src: url("https://common.northwestern.edu/v8/css/fonts/AkkuratProLight.woff") format("woff"); 4 | font-weight: normal; 5 | font-style:normal 6 | } 7 | 8 | @font-face { 9 | font-family: "Akkurat Pro Light Italic"; 10 | src: url("https://common.northwestern.edu/v8/css/fonts/AkkuratProLightItalic.woff") format("woff"); 11 | font-weight: normal; 12 | font-style:normal 13 | } 14 | 15 | @font-face { 16 | font-family: "Akkurat Pro Regular"; 17 | src: url("https://common.northwestern.edu/v8/css/fonts/AkkuratProRegular.woff") format("woff"); 18 | font-weight: normal; 19 | font-style:normal 20 | } 21 | 22 | @font-face { 23 | font-family: "Akkurat Pro Italic"; 24 | src: url("https://common.northwestern.edu/v8/css/fonts/AkkuratProItalic.woff") format("woff"); 25 | font-weight: normal; 26 | font-style:normal 27 | } 28 | 29 | @font-face { 30 | font-family: "Akkurat Pro Bold"; 31 | src: url("https://common.northwestern.edu/v8/css/fonts/AkkuratProBold.woff") format("woff"); 32 | font-weight: normal; 33 | font-style:normal 34 | } 35 | 36 | @font-face { 37 | font-family: "Akkurat Pro Bold Italic"; 38 | src: url("https://common.northwestern.edu/v8/css/fonts/AkkuratProBoldItalic.woff") format("woff"); 39 | font-weight: normal; 40 | font-style:normal 41 | } 42 | 43 | @font-face { 44 | font-family: "Campton Book"; 45 | src: url("https://common.northwestern.edu/v8/css/fonts/CamptonBook.woff") format("woff"); 46 | font-weight: normal; 47 | font-style:normal 48 | } 49 | 50 | @font-face { 51 | font-family: "Campton Bold"; 52 | src: url("https://common.northwestern.edu/v8/css/fonts/CamptonBold.woff") format("woff"); 53 | font-weight: normal; 54 | font-style:normal 55 | } 56 | 57 | @font-face { 58 | font-family: "Campton Extra Bold"; 59 | src: url("https://common.northwestern.edu/v8/css/fonts/CamptonExtraBold.woff") format("woff"); 60 | font-weight: normal; 61 | font-style:normal 62 | } 63 | 64 | @font-face { 65 | font-family: "Campton Extra Light"; 66 | src: url("https://common.northwestern.edu/v8/css/fonts/CamptonExtraLight.woff") format("woff"); 67 | font-weight: normal; 68 | font-style:normal 69 | } 70 | 71 | @font-face { 72 | font-family: "Noto Serif Italic"; 73 | font-style: italic; 74 | font-weight: 400; 75 | src: url("https://common.northwestern.edu/v8/css/fonts/noto-serif-v16-latin-italic.woff") format("woff") 76 | } 77 | 78 | @font-face { 79 | font-family: "Noto Serif Bold Italic"; 80 | font-style: italic; 81 | font-weight: 700; 82 | src: url("https://common.northwestern.edu/v8/css/fonts/noto-serif-v16-latin-700italic.woff") format("woff") 83 | } -------------------------------------------------------------------------------- /bin/make_env.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | get_secret() { 4 | local secret_name=$1 5 | local secret_value=$(aws secretsmanager get-secret-value --secret-id $secret_name --query SecretString --output text) 6 | 7 | local key_name=$2 8 | if [ -n "$key_name" ]; then 9 | jq -r ".$key_name" <<< $secret_value 10 | else 11 | echo $secret_value 12 | fi 13 | } 14 | 15 | aws_account_id=$(aws sts get-caller-identity --query 'Account' --output text) 16 | media_convert_endpoint=$(aws mediaconvert describe-endpoints --query 'Endpoints[0].Url' --output text) 17 | media_convert_queue=$(aws mediaconvert get-queue --name Default --query Queue.Name --output text) 18 | 19 | cat < env.json 20 | { 21 | "Parameters": { 22 | "AWS_REGION": "us-east-1", 23 | "API_TOKEN_NAME": "dcApiLocal", 24 | "API_TOKEN_SECRET": "$(get_secret staging/config/dcapi api_token_secret)", 25 | "DC_API_ENDPOINT": "https://${DEV_PREFIX}.dev.rdc.library.northwestern.edu:3002", 26 | "DC_URL": "https://${DEV_PREFIX}.dev.rdc.library.northwestern.edu:3001", 27 | "DEFAULT_SEARCH_SIZE": "10", 28 | "DEV_TEAM_NET_IDS": "$(aws ec2 describe-tags --filters "Name=resource-id,Values=$(curl -s http://169.254.169.254/latest/meta-data/instance-id)" "Name=key,Values=NetID" --query 'Tags[0].Value' --output text)", 29 | "ENV_PREFIX": "${DEV_PREFIX}-${DEV_ENV}", 30 | "READING_ROOM_IPS": "", 31 | "SECRETS_PATH": "dev-environment", 32 | "AV_DOWNLOAD_EMAIL_TEMPLATE": "av-download-template", 33 | "AV_DOWNLOAD_STATE_MACHINE_ARN": "arn:aws:states:us-east-1:123456789012:stateMachine:hlsStitcherStepFunction", 34 | "GET_DOWNLOAD_LINK_FUNCTION": "arn:aws:lambda:us-east-1:123456789012:function:getDownloadLinkFunction", 35 | "MEDIA_CONVERT_DESTINATION_BUCKET": "${DEV_PREFIX}-${DEV_ENV}-streaming", 36 | "MEDIA_CONVERT_ENDPOINT": "${media_convert_endpoint}", 37 | "MEDIA_CONVERT_JOB_QUEUE_ARN": "${media_convert_queue}", 38 | "MEDIA_CONVERT_ROLE_ARN": "arn:aws:iam::${aws_account_id}:role/service-role/MediaConvert_Default_Role", 39 | "PYRAMID_BUCKET": "${DEV_PREFIX}-${DEV_ENV}-pyramids", 40 | "REPOSITORY_EMAIL": "repository@northwestern.edu", 41 | "SEND_TEMPLATED_EMAIL_FUNCTION": "arn:aws:lambda:us-east-1:123456789012:function:sendTemplatedEmailFunction", 42 | "START_AUDIO_TRANSCODE_FUNCTION": "arn:aws:lambda:us-east-1:123456789012:function:startAudioTranscodeFunction", 43 | "START_TRANSCODE_FUNCTION": "arn:aws:lambda:us-east-1:123456789012:function:startTranscodeFunction", 44 | "STEP_FUNCTION_ENDPOINT": "http://172.17.0.1:8083", 45 | "STREAMING_BUCKET": "${DEV_PREFIX}-${DEV_ENV}-streaming", 46 | "TRANSCODE_STATUS_FUNCTION": "arn:aws:lambda:us-east-1:123456789012:function:transcodeStatusFunction" 47 | } 48 | } 49 | EOF 50 | -------------------------------------------------------------------------------- /chat/src/persistence/compressible_json_serializer.py: -------------------------------------------------------------------------------- 1 | from typing import Any, Optional, Tuple 2 | 3 | import base64 4 | import bz2 5 | import gzip 6 | import json 7 | import langchain_core.messages as langchain_messages 8 | from langchain_core.messages import BaseMessage 9 | from langgraph.checkpoint.serde.jsonplus import JsonPlusSerializer 10 | 11 | 12 | class CompressibleJsonSerializer(JsonPlusSerializer): 13 | def __init__(self, compression: Optional[str] = None): 14 | self.compression = compression 15 | 16 | def dumps_typed(self, obj: Any) -> Tuple[str, Any]: 17 | def default(o): 18 | if isinstance(o, BaseMessage): 19 | return { 20 | "__type__": o.__class__.__name__, 21 | "data": o.model_dump(), 22 | } 23 | raise TypeError( 24 | f"Object of type {o.__class__.__name__} is not JSON serializable" 25 | ) 26 | 27 | json_str = json.dumps(obj, default=default) 28 | 29 | if self.compression is None: 30 | return "json", json_str 31 | elif self.compression == "bz2": 32 | compressed_str = base64.b64encode( 33 | bz2.compress(json_str.encode("utf-8")) 34 | ).decode("utf-8") 35 | return "bz2_json", compressed_str 36 | elif self.compression == "gzip": 37 | compressed_str = base64.b64encode( 38 | gzip.compress(json_str.encode("utf-8")) 39 | ).decode("utf-8") 40 | return "gzip_json", compressed_str 41 | else: 42 | raise ValueError(f"Unsupported compression type: {self.compression}") 43 | 44 | def loads_typed(self, data: Tuple[str, Any]) -> Any: 45 | type_, payload = data 46 | 47 | if type_ == "json": 48 | json_str = payload 49 | elif type_ == "bz2_json": 50 | json_str = bz2.decompress(base64.b64decode(payload)).decode("utf-8") 51 | elif type_ == "gzip_json": 52 | json_str = gzip.decompress(base64.b64decode(payload)).decode("utf-8") 53 | else: 54 | raise ValueError(f"Unknown data type: {type_}") 55 | 56 | def object_hook(dct): 57 | if "__type__" in dct: 58 | type_name = dct["__type__"] 59 | data = dct["data"] 60 | cls = getattr(langchain_messages, type_name, None) 61 | if cls and issubclass(cls, BaseMessage): 62 | return cls.model_construct(**data) 63 | else: 64 | raise ValueError(f"Unknown type: {type_name}") 65 | return dct 66 | 67 | obj = json.loads(json_str, object_hook=object_hook) 68 | return obj 69 | -------------------------------------------------------------------------------- /chat/src/core/setup.py: -------------------------------------------------------------------------------- 1 | from persistence.selective_checkpointer import SelectiveCheckpointer 2 | from search.opensearch_neural_search import OpenSearchNeuralSearch 3 | from langchain_aws import ChatBedrock 4 | from langchain_core.language_models.base import BaseModel 5 | from langgraph.checkpoint.base import BaseCheckpointSaver 6 | from opensearchpy import OpenSearch, RequestsHttpConnection 7 | from requests_aws4auth import AWS4Auth 8 | from urllib.parse import urlparse 9 | import os 10 | import boto3 11 | 12 | 13 | def chat_model(**kwargs) -> BaseModel: 14 | return ChatBedrock(**kwargs) 15 | 16 | 17 | def checkpoint_saver(**kwargs) -> BaseCheckpointSaver: 18 | checkpoint_bucket: str = os.getenv("CHECKPOINT_BUCKET_NAME") 19 | return SelectiveCheckpointer( 20 | bucket_name=checkpoint_bucket, retain_history=False, **kwargs 21 | ) 22 | 23 | 24 | def prefix(value): 25 | env_prefix = os.getenv("ENV_PREFIX") 26 | env_prefix = None if env_prefix == "" else env_prefix 27 | return "-".join(filter(None, [env_prefix, value])) 28 | 29 | 30 | def opensearch_endpoint(): 31 | endpoint = os.getenv("OPENSEARCH_ENDPOINT") 32 | parsed = urlparse(endpoint) 33 | if parsed.netloc != "": 34 | return parsed.netloc 35 | else: 36 | return endpoint 37 | 38 | 39 | def opensearch_client(region_name=None): 40 | region_name = region_name or os.getenv("AWS_REGION") # Evaluate at runtime 41 | session = boto3.Session(region_name=region_name) 42 | awsauth = AWS4Auth( 43 | region=region_name, 44 | service="es", 45 | refreshable_credentials=session.get_credentials(), 46 | ) 47 | endpoint = opensearch_endpoint() 48 | 49 | return OpenSearch( 50 | hosts=[{"host": endpoint, "port": 443}], 51 | use_ssl=True, 52 | connection_class=RequestsHttpConnection, 53 | http_auth=awsauth, 54 | ) 55 | 56 | 57 | def opensearch_vector_store(region_name=None): 58 | region_name = region_name or os.getenv("AWS_REGION") # Evaluate at runtime 59 | session = boto3.Session(region_name=region_name) 60 | awsauth = AWS4Auth( 61 | region=region_name, 62 | service="es", 63 | refreshable_credentials=session.get_credentials(), 64 | ) 65 | 66 | docsearch = OpenSearchNeuralSearch( 67 | index=prefix("dc-v2-work"), 68 | model_id=os.getenv("OPENSEARCH_MODEL_ID"), 69 | endpoint=opensearch_endpoint(), 70 | connection_class=RequestsHttpConnection, 71 | http_auth=awsauth, 72 | text_field="id", 73 | ) 74 | return docsearch 75 | 76 | 77 | def websocket_client(endpoint_url: str): 78 | endpoint_url = endpoint_url or os.getenv("APIGATEWAY_URL") 79 | try: 80 | client = boto3.client("apigatewaymanagementapi", endpoint_url=endpoint_url) 81 | return client 82 | except Exception as e: 83 | raise e 84 | -------------------------------------------------------------------------------- /loadtest/locustfile.py: -------------------------------------------------------------------------------- 1 | from locust import HttpUser, task, between 2 | import random 3 | 4 | 5 | class DcApiUser(HttpUser): 6 | wait_time = between(1, 5) 7 | 8 | @task(1) 9 | def page_through_works(self): 10 | with self.client.rename_request("/search/works (paged)"): 11 | response = self.client.get("/search/works").json() 12 | for i in range(4): 13 | next_url = response["pagination"]["next_url"] 14 | response = self.client.get(next_url).json() 15 | 16 | @task(1) 17 | def search_works(self): 18 | with self.client.rename_request("/search/works with query (paged)"): 19 | query = {"query": {"term": {"title": "baez"}}} 20 | response = self.client.post("/search/works", json=query).json() 21 | for i in range(4): 22 | next_url = response["pagination"]["next_url"] 23 | response = self.client.get(next_url).json() 24 | 25 | @task(3) 26 | def load_collection_as_json(self): 27 | id = random.choice(self.collection_ids) 28 | self.client.get(f"/collections/{id}", name="/collections/:id") 29 | 30 | @task(3) 31 | def load_collection_as_iiif(self): 32 | id = random.choice(self.collection_ids) 33 | self.client.get(f"/collections/{id}?as=iiif", name="/collections/:id?as=iiif") 34 | 35 | @task(3) 36 | def load_work_as_json(self): 37 | id = random.choice(self.work_ids) 38 | self.client.get(f"/works/{id}", name="/works/:id") 39 | 40 | @task(3) 41 | def load_work_as_iiif(self): 42 | id = random.choice(self.work_ids) 43 | self.client.get(f"/works/{id}?as=iiif", name="/works/:id?as=iiif") 44 | 45 | @task(3) 46 | def load_work_thumbnail(self): 47 | id = random.choice(self.work_ids) 48 | self.client.get(f"/works/{id}/thumbnail", name="/works/:id/thumbnail") 49 | 50 | @task(3) 51 | def load_file_set(self): 52 | id = random.choice(self.file_set_ids) 53 | self.client.get(f"/file-sets/{id}", name="/file-sets/:id") 54 | 55 | def on_start(self): 56 | response = self.random_docs("works", include="id,file_sets.id") 57 | self.work_ids = [doc["id"] for doc in response["data"]] 58 | self.file_set_ids = [ 59 | file_set["id"] 60 | for item in response["data"] 61 | for file_set in item["file_sets"] 62 | ] 63 | self.collection_ids = [ 64 | doc["id"] for doc in self.random_docs("collections")["data"] 65 | ] 66 | 67 | def random_docs(self, type, count=100, include="id"): 68 | query = { 69 | "size": count, 70 | "query": { 71 | "function_score": {"query": {"match_all": {}}, "random_score": {}} 72 | }, 73 | } 74 | 75 | response = self.client.post( 76 | f"/search/{type}?_source_includes={include}", 77 | json=query, 78 | name=f"Load {count} random {type}", 79 | ) 80 | json = response.json() 81 | return json 82 | -------------------------------------------------------------------------------- /chat/test/agent/test_search_workflow.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | from langchain_core.messages.base import BaseMessage 3 | from langchain_core.messages.system import SystemMessage 4 | from langgraph.graph import END 5 | 6 | from agent.search_agent import SearchWorkflow, SearchAgentState 7 | 8 | 9 | class FakeMessage(BaseMessage): 10 | type: str = "fake" # specify a default type 11 | content: str 12 | tool_calls: list = [] 13 | 14 | 15 | class FakeModel: 16 | def invoke(self, messages): 17 | # Just return a mock response 18 | return SystemMessage(content="Mock Response") 19 | 20 | 21 | class TestSearchWorkflow(unittest.TestCase): 22 | def setUp(self): 23 | self.model = FakeModel() 24 | self.workflow = SearchWorkflow( 25 | model=self.model, system_message="Test system message" 26 | ) 27 | 28 | def test_should_continue_with_tool_calls(self): 29 | state = SearchAgentState(messages=[ 30 | FakeMessage(content="Hello"), 31 | FakeMessage(content="Calling tool", tool_calls=["test_tool"]), 32 | ]) 33 | result = self.workflow.should_continue(state) 34 | self.assertEqual(result, "tools") 35 | 36 | def test_should_continue_without_tool_calls(self): 37 | state = SearchAgentState(messages=[ 38 | FakeMessage(content="Hello"), 39 | FakeMessage(content="No tool calls here"), 40 | ]) 41 | result = self.workflow.should_continue(state) 42 | self.assertEqual(result, END) 43 | 44 | def test_call_model(self): 45 | state = SearchAgentState(messages=[FakeMessage(content="User input")]) 46 | result = self.workflow.call_model(state) 47 | self.assertIn("messages", result) 48 | self.assertEqual(len(result["messages"]), 1) 49 | self.assertEqual(result["messages"][0].content, "Mock Response") 50 | 51 | def test_call_model_with_facets(self): 52 | facets = [{"subject.label": "Nigeria"}, {"collection.title.keyword": "Test Collection"}] 53 | state = SearchAgentState( 54 | messages=[FakeMessage(content="User input")], 55 | facets=facets 56 | ) 57 | result = self.workflow.call_model(state) 58 | self.assertIn("messages", result) 59 | self.assertEqual(len(result["messages"]), 1) 60 | self.assertEqual(result["messages"][0].content, "Mock Response") 61 | 62 | def test_create_facets_context(self): 63 | facets = [ 64 | {"subject.label": ["Nigeria", "Ghana"]}, 65 | {"collection.title.keyword": "E. H. Duckworth Photograph Collection"}, 66 | {"work_type.keyword": "Image"} 67 | ] 68 | 69 | context = self.workflow._create_facets_context(facets) 70 | 71 | self.assertIn("IMPORTANT CONTEXT", context) 72 | self.assertIn("Subject: Nigeria, Ghana", context) 73 | self.assertIn("Collection.Title: E. H. Duckworth Photograph Collection", context) 74 | self.assertIn("Work Type: Image", context) 75 | self.assertIn("Do NOT attempt to broaden", context) 76 | -------------------------------------------------------------------------------- /api/test/integration/get-file-set-download.test.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | 3 | const chai = require("chai"); 4 | const expect = chai.expect; 5 | chai.use(require("chai-http")); 6 | 7 | const ApiToken = requireSource("api/api-token"); 8 | 9 | describe("Download file set", () => { 10 | helpers.saveEnvironment(); 11 | const mock = helpers.mockIndex(); 12 | 13 | beforeEach(() => { 14 | process.env.API_TOKEN_SECRET = "abc123"; 15 | process.env.API_TOKEN_NAME = "dcapiTEST"; 16 | }); 17 | 18 | describe("GET /file-sets/{id}/download", () => { 19 | const { handler } = requireSource("handlers/get-file-set-download"); 20 | 21 | it("returns unauthorized for a video without a superuser token", async () => { 22 | mock 23 | .get("/dc-v2-file-set/_doc/1234") 24 | .reply(200, helpers.testFixture("mocks/fileset-video-1234.json")); 25 | 26 | const event = helpers 27 | .mockEvent("GET", "/file-sets/{id}/download") 28 | .pathParams({ id: 1234 }) 29 | .queryParams({ email: "example@example.com" }) 30 | .render(); 31 | const result = await handler(event); 32 | expect(result.statusCode).to.eq(401); 33 | }); 34 | 35 | it("returns unauthorized for an audio without a superuser token", async () => { 36 | mock 37 | .get("/dc-v2-file-set/_doc/1234") 38 | .reply(200, helpers.testFixture("mocks/fileset-audio-1234.json")); 39 | 40 | const event = helpers 41 | .mockEvent("GET", "/file-sets/{id}/download") 42 | .pathParams({ id: 1234 }) 43 | .queryParams({ email: "example@example.com" }) 44 | .render(); 45 | const result = await handler(event); 46 | expect(result.statusCode).to.eq(401); 47 | }); 48 | 49 | it("returns an error for video if it does not contain an email query string parameters", async () => { 50 | mock 51 | .get("/dc-v2-file-set/_doc/1234") 52 | .reply(200, helpers.testFixture("mocks/fileset-video-1234.json")); 53 | 54 | const token = new ApiToken().superUser().sign(); 55 | 56 | const event = helpers 57 | .mockEvent("GET", "/file-sets/{id}/download") 58 | .pathParams({ id: 1234 }) 59 | .headers({ 60 | Cookie: `${process.env.API_TOKEN_NAME}=${token};`, 61 | }) 62 | .render(); 63 | 64 | const result = await handler(event); 65 | expect(result.statusCode).to.eq(400); 66 | }); 67 | 68 | it("returns an error for audio if it does not contain an email query string parameters", async () => { 69 | mock 70 | .get("/dc-v2-file-set/_doc/1234") 71 | .reply(200, helpers.testFixture("mocks/fileset-audio-1234.json")); 72 | 73 | const token = new ApiToken().superUser().sign(); 74 | 75 | const event = helpers 76 | .mockEvent("GET", "/file-sets/{id}/download") 77 | .pathParams({ id: 1234 }) 78 | .headers({ 79 | Cookie: `${process.env.API_TOKEN_NAME}=${token};`, 80 | }) 81 | .render(); 82 | 83 | const result = await handler(event); 84 | expect(result.statusCode).to.eq(400); 85 | }); 86 | }); 87 | }); 88 | -------------------------------------------------------------------------------- /chat/src/persistence/selective_checkpointer.py: -------------------------------------------------------------------------------- 1 | import os 2 | from typing import Optional 3 | from langchain_core.messages import AIMessage, HumanMessage, ToolMessage 4 | from langchain_core.runnables import RunnableConfig 5 | from langgraph.checkpoint.base import ChannelVersions, Checkpoint, CheckpointMetadata 6 | from persistence.s3_checkpointer import S3Checkpointer 7 | 8 | 9 | # Split messages into interactions, each one starting with a HumanMessage 10 | def _split_interactions(messages): 11 | if messages is None: 12 | return [] 13 | 14 | interactions = [] 15 | current_interaction = [] 16 | 17 | for message in messages: 18 | if isinstance(message, HumanMessage) and current_interaction: 19 | interactions.append(current_interaction) 20 | current_interaction = [] 21 | current_interaction.append(message) 22 | 23 | if current_interaction: 24 | interactions.append(current_interaction) 25 | 26 | return interactions 27 | 28 | 29 | def _is_tool_message(message): 30 | if isinstance(message, ToolMessage): 31 | return True 32 | if ( 33 | isinstance(message, AIMessage) 34 | and message.response_metadata.get("stop_reason", "") == "tool_use" 35 | ): 36 | return True 37 | return False 38 | 39 | 40 | def _prune_messages(messages): 41 | interactions = _split_interactions(messages) 42 | # Remove all tool-related messages except those related to the most recent interaction 43 | for i, interaction in enumerate(interactions[:-1]): 44 | interactions[i] = [ 45 | message for message in interaction if not _is_tool_message(message) 46 | ] 47 | 48 | # Return the flattened list of messages 49 | return [message for interaction in interactions for message in interaction] 50 | 51 | 52 | class SelectiveCheckpointer(S3Checkpointer): 53 | """S3 Checkpointer that discards ToolMessages from previous checkpoints.""" 54 | 55 | def __init__( 56 | self, 57 | bucket_name: str, 58 | region_name: str = os.getenv("AWS_REGION"), 59 | endpoint_url: Optional[str] = None, 60 | compression: Optional[str] = None, 61 | retain_history: Optional[bool] = True, 62 | ) -> None: 63 | super().__init__(bucket_name, region_name, endpoint_url, compression) 64 | self.retain_history = retain_history 65 | 66 | def put( 67 | self, 68 | config: RunnableConfig, 69 | checkpoint: Checkpoint, 70 | metadata: CheckpointMetadata, 71 | new_versions: ChannelVersions, 72 | ) -> RunnableConfig: 73 | # Remove previous checkpoints 74 | thread_id = config["configurable"]["thread_id"] 75 | if not self.retain_history: 76 | self.delete_checkpoints(thread_id) 77 | 78 | # Remove all ToolMessages except those related to the most 79 | # recent question (HumanMessage) 80 | messages = checkpoint.get("channel_values", {}).get("messages", []) 81 | checkpoint["channel_values"]["messages"] = _prune_messages(messages) 82 | 83 | return super().put(config, checkpoint, metadata, new_versions) 84 | -------------------------------------------------------------------------------- /api/test/unit/api/response/iiif/presentation-api/placeholder-canvas.test.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | 3 | const chai = require("chai"); 4 | const expect = chai.expect; 5 | const transformer = requireSource("api/response/iiif/manifest"); 6 | const { buildPlaceholderCanvas, getPlaceholderSizes } = requireSource( 7 | "api/response/iiif/presentation-api/placeholder-canvas" 8 | ); 9 | 10 | describe("IIIF response presentation API placeholderCanvas helpers", () => { 11 | async function setup() { 12 | const response = { 13 | statusCode: 200, 14 | body: helpers.testFixture("mocks/work-1234.json"), 15 | }; 16 | const source = JSON.parse(response.body)._source; 17 | 18 | const result = await transformer.transform(response); 19 | expect(result.statusCode).to.eq(200); 20 | 21 | return { source, manifest: JSON.parse(result.body) }; 22 | } 23 | 24 | it("buildPlaceholderCanvas(value)", async () => { 25 | const { source, manifest } = await setup(); 26 | const id = manifest.items[0].id; 27 | const fileSet = source.file_sets[0]; 28 | const placeholder = buildPlaceholderCanvas(id, fileSet, 640); 29 | 30 | expect(placeholder.id).to.eq(`${id}/placeholder`); 31 | expect(placeholder.type).to.eq("Canvas"); 32 | expect(placeholder.width).to.eq(640); 33 | expect(placeholder.height).to.eq(480); 34 | expect(placeholder.items[0].id).to.eq( 35 | `${id}/placeholder/annotation-page/0` 36 | ); 37 | expect(placeholder.items[0].type).to.eq("AnnotationPage"); 38 | expect(placeholder.items[0].items[0].type).to.eq("Annotation"); 39 | expect(placeholder.items[0].items[0].motivation).to.eq("painting"); 40 | expect(placeholder.items[0].items[0].body.id).to.eq( 41 | `${fileSet.representative_image_url}/full/!640,480/0/default.jpg` 42 | ); 43 | expect(placeholder.items[0].items[0].body.type).to.eq("Image"); 44 | expect(placeholder.items[0].items[0].body.format).to.eq(fileSet.mime_type); 45 | expect(placeholder.items[0].items[0].body.width).to.eq(640); 46 | expect(placeholder.items[0].items[0].body.height).to.eq(480); 47 | expect(placeholder.items[0].items[0].body.service[0]["@id"]).to.eq( 48 | fileSet.representative_image_url 49 | ); 50 | }); 51 | 52 | it("getPlaceholderSizes(fileSet, size)", () => { 53 | const fileSets = [ 54 | { 55 | width: 3125, 56 | height: 2240, 57 | }, 58 | { 59 | width: 500, 60 | height: 300, 61 | }, 62 | { 63 | width: null, 64 | height: null, 65 | }, 66 | ]; 67 | 68 | const expected = [ 69 | { 70 | placeholderWidth: 1000, 71 | placeholderHeight: 716, 72 | }, 73 | { 74 | placeholderWidth: 500, 75 | placeholderHeight: 300, 76 | }, 77 | { 78 | placeholderWidth: 100, 79 | placeholderHeight: 100, 80 | }, 81 | ]; 82 | 83 | fileSets.forEach(function (fileSet, index) { 84 | const { placeholderHeight, placeholderWidth } = getPlaceholderSizes( 85 | fileSet, 86 | 1000 87 | ); 88 | 89 | expect(placeholderWidth).to.eq(expected[index].placeholderWidth); 90 | expect(placeholderHeight).to.eq(expected[index].placeholderHeight); 91 | }); 92 | }); 93 | }); 94 | -------------------------------------------------------------------------------- /api/test/test-helpers/index.js: -------------------------------------------------------------------------------- 1 | const fs = require("fs"); 2 | const nock = require("nock"); 3 | const path = require("path"); 4 | const EventBuilder = require("./event-builder.js"); 5 | 6 | process.env.HONEYBADGER_DISABLED = "true"; 7 | process.env.HONEYBADGER_ENV = "test"; 8 | process.env.__SKIP_SECRETS__ = "true"; 9 | 10 | const TestEnvironment = { 11 | API_TOKEN_SECRET: "abc123", 12 | API_TOKEN_NAME: "dcapiTEST", 13 | DC_URL: "https://thisisafakedcurl", 14 | DC_API_ENDPOINT: "https://thisisafakeapiurl", 15 | DEV_TEAM_NET_IDS: "abc123,def456", 16 | NUSSO_BASE_URL: "https://nusso-base.com/", 17 | NUSSO_API_KEY: "abc123", 18 | WEBSOCKET_URI: "wss://thisisafakewebsocketapiurl", 19 | CHAT_FEEDBACK_BUCKET: "test-chat-feedback-bucket", 20 | DEFAULT_SEARCH_SIZE: "10", 21 | PROVIDER_CAPABILITIES: 22 | '{"magic":["chat", "login"],"nusso":["chat", "login"]}', 23 | }; 24 | 25 | for (const v in TestEnvironment) delete process.env[v]; 26 | 27 | let SavedEnvironments = []; 28 | 29 | function requireSource(module) { 30 | const absolute = path.resolve(__dirname, "../../src", module); 31 | return require(absolute); 32 | } 33 | 34 | const { __processRequest } = requireSource("handlers/middleware"); 35 | 36 | function saveEnvironment() { 37 | beforeEach(function () { 38 | SavedEnvironments.push({ ...process.env }); 39 | for (const v in TestEnvironment) process.env[v] = TestEnvironment[v]; 40 | }); 41 | 42 | afterEach(function () { 43 | process.env = { ...SavedEnvironments.pop() }; 44 | }); 45 | } 46 | 47 | function mockEvent(method, route) { 48 | return new EventBuilder(method, route); 49 | } 50 | 51 | function mockIndex() { 52 | const mock = nock("https://index.test.library.northwestern.edu"); 53 | 54 | beforeEach(function () { 55 | process.env.OPENSEARCH_ENDPOINT = "index.test.library.northwestern.edu"; 56 | }); 57 | 58 | afterEach(function () { 59 | nock.cleanAll(); 60 | mock.removeAllListeners(); 61 | }); 62 | 63 | return mock; 64 | } 65 | 66 | function encodedFixture(file) { 67 | const content = testFixture(file); 68 | return new Buffer.from(content).toString("base64"); 69 | } 70 | 71 | function testFixture(file) { 72 | const fixtureFile = path.join("test/fixtures", file); 73 | return fs.readFileSync(fixtureFile); 74 | } 75 | 76 | function cookieValue(cookies, cookieName) { 77 | if (!Array.isArray(cookies)) return undefined; 78 | 79 | let cookieValue = { value: "" }; 80 | const regex = new RegExp(`^${cookieName}=(?[^;]+)(?.+)?$`); 81 | for (const c of cookies) { 82 | const match = regex.exec(c); 83 | if (match) { 84 | const { value, props } = match.groups; 85 | cookieValue.value = value; 86 | if (props) { 87 | for (const prop of props.split(/;\s+/)) { 88 | const [propKey, propValue] = prop.split(/=/); 89 | if (propKey != "") cookieValue[propKey] = propValue; 90 | } 91 | } 92 | } 93 | } 94 | return cookieValue; 95 | } 96 | 97 | global.helpers = { 98 | saveEnvironment, 99 | mockEvent, 100 | mockIndex, 101 | encodedFixture, 102 | testFixture, 103 | cookieValue, 104 | preprocess: __processRequest, 105 | }; 106 | 107 | global.requireSource = requireSource; 108 | -------------------------------------------------------------------------------- /api/test/integration/get-auth-logout.test.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | 3 | const chai = require("chai"); 4 | const expect = chai.expect; 5 | const nock = require("nock"); 6 | 7 | const getAuthLogoutHandler = requireSource("handlers/get-auth-logout"); 8 | const ApiToken = requireSource("api/api-token"); 9 | 10 | describe("auth logout", function () { 11 | helpers.saveEnvironment(); 12 | 13 | it("logs a user out of NU WebSSO and expires the DC API Token", async () => { 14 | process.env.NUSSO_BASE_URL = "https://nusso-base.com/"; 15 | process.env.NUSSO_API_KEY = "abc123"; 16 | 17 | const url = "https://test.com/northwestern#logout"; 18 | nock(process.env.NUSSO_BASE_URL).get("/logout").reply(200, { 19 | url: url, 20 | }); 21 | 22 | const token = new ApiToken().provider("nusso").sign(); 23 | const event = helpers 24 | .mockEvent("GET", "/auth/logout") 25 | .headers({ 26 | Cookie: `${process.env.API_TOKEN_NAME}=${token};`, 27 | }) 28 | .render(); 29 | 30 | const result = await getAuthLogoutHandler.handler(event); 31 | 32 | expect(result.statusCode).to.eq(302); 33 | expect(result.headers.location).to.eq(url); 34 | 35 | const dcApiCookie = helpers.cookieValue( 36 | result.cookies, 37 | process.env.API_TOKEN_NAME 38 | ); 39 | 40 | const apiToken = new ApiToken(dcApiCookie.value); 41 | expect(apiToken.token.sub).to.not.exist; 42 | expect(apiToken.token.isLoggedIn).to.be.false; 43 | expect(dcApiCookie.Expires).to.eq("Thu, 01 Jan 1970 00:00:00 GMT"); 44 | }); 45 | 46 | describe("non-NUSSO Logout", () => { 47 | let event; 48 | 49 | beforeEach(() => { 50 | const token = new ApiToken().provider("test-provider").sign(); 51 | event = helpers.mockEvent("GET", "/auth/logout").headers({ 52 | Cookie: `${process.env.API_TOKEN_NAME}=${token};`, 53 | }); 54 | }); 55 | 56 | it("expires the DC API Token", async () => { 57 | const result = await getAuthLogoutHandler.handler(event.render()); 58 | const dcApiCookie = helpers.cookieValue( 59 | result.cookies, 60 | process.env.API_TOKEN_NAME 61 | ); 62 | 63 | const apiToken = new ApiToken(dcApiCookie.value); 64 | expect(apiToken.token.sub).to.not.exist; 65 | expect(apiToken.token.isLoggedIn).to.be.false; 66 | expect(dcApiCookie.Expires).to.eq("Thu, 01 Jan 1970 00:00:00 GMT"); 67 | }); 68 | 69 | it("redirects to the goto URL", async () => { 70 | event = event.queryParams({ goto: "http://example.edu/logged-out" }); 71 | const result = await getAuthLogoutHandler.handler(event.render()); 72 | expect(result.statusCode).to.eq(302); 73 | expect(result.headers.location).to.eq("http://example.edu/logged-out"); 74 | }); 75 | 76 | it("redirects to the referer", async () => { 77 | event = event.headers({ Referer: "http://example.edu/logged-out" }); 78 | const result = await getAuthLogoutHandler.handler(event.render()); 79 | expect(result.statusCode).to.eq(302); 80 | expect(result.headers.location).to.eq("http://example.edu/logged-out"); 81 | }); 82 | 83 | it("redirects to the default location", async () => { 84 | const result = await getAuthLogoutHandler.handler(event.render()); 85 | expect(result.statusCode).to.eq(302); 86 | expect(result.headers.location).to.eq("https://thisisafakedcurl"); 87 | }); 88 | }); 89 | }); 90 | -------------------------------------------------------------------------------- /api/src/handlers/oai.js: -------------------------------------------------------------------------------- 1 | const { baseUrl } = require("../helpers"); 2 | const { 3 | getRecord, 4 | identify, 5 | listIdentifiers, 6 | listMetadataFormats, 7 | listRecords, 8 | listSets, 9 | } = require("./oai/verbs"); 10 | const { invalidOaiRequest } = require("./oai/xml-transformer"); 11 | const { wrap } = require("./middleware"); 12 | 13 | function invalidDateParameters(verb, dates) { 14 | if (!["ListRecords", "ListIdentifiers"].includes(verb)) return []; 15 | 16 | // OAI-PMH spec allows three date formats: 17 | // 1. YYYY-MM-DD (date only) 18 | // 2. YYYY-MM-DDThh:mm:ssZ (no fractional seconds) 19 | // 3. YYYY-MM-DDThh:mm:ssZ (seconds granularity, no fractional seconds) 20 | const dateOnlyRegex = /^\d{4}-\d{2}-\d{2}$/; 21 | const dateTimeRegex = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z$/; 22 | let invalidDates = []; 23 | 24 | for (const [dateParameter, dateValue] of Object.entries(dates)) { 25 | if ( 26 | dateValue && 27 | !dateOnlyRegex.test(dateValue) && 28 | !dateTimeRegex.test(dateValue) 29 | ) { 30 | invalidDates.push(dateParameter); 31 | } else { 32 | continue; 33 | } 34 | } 35 | 36 | return invalidDates; 37 | } 38 | 39 | /** 40 | * A function to support the "OAI-PMH" harvesting specfication 41 | */ 42 | exports.handler = wrap(async (event) => { 43 | const url = `${baseUrl(event)}oai`; 44 | let verb, identifier, metadataPrefix, resumptionToken, from, until, set; 45 | if (event.requestContext.http.method === "GET") { 46 | verb = event.queryStringParameters?.verb; 47 | identifier = event.queryStringParameters?.identifier; 48 | metadataPrefix = event.queryStringParameters?.metadataPrefix; 49 | resumptionToken = event.queryStringParameters?.resumptionToken; 50 | from = event.queryStringParameters?.from; 51 | until = event.queryStringParameters?.until; 52 | set = event.queryStringParameters?.set; 53 | } else { 54 | const body = new URLSearchParams(event.body); 55 | verb = body.get("verb"); 56 | identifier = body.get("identifier"); 57 | metadataPrefix = body.get("metadataPrefix"); 58 | resumptionToken = body.get("resumptionToken"); 59 | from = body.get("from"); 60 | until = body.get("until"); 61 | set = body.get("set"); 62 | } 63 | 64 | const dates = { from, until }; 65 | if (invalidDateParameters(verb, dates).length > 0) 66 | return invalidOaiRequest( 67 | "badArgument", 68 | "Invalid date -- make sure that 'from' or 'until' parameters are formatted as: 'YYYY-MM-DD' or 'YYYY-MM-DDThh:mm:ssZ'" 69 | ); 70 | if (!verb) return invalidOaiRequest("badArgument", "Missing required verb"); 71 | 72 | switch (verb) { 73 | case "GetRecord": 74 | return await getRecord(url, identifier, metadataPrefix); 75 | case "Identify": 76 | return await identify(url); 77 | case "ListIdentifiers": 78 | return await listIdentifiers( 79 | url, 80 | metadataPrefix, 81 | dates, 82 | set, 83 | resumptionToken 84 | ); 85 | case "ListMetadataFormats": 86 | return await listMetadataFormats(url); 87 | case "ListRecords": 88 | return await listRecords( 89 | url, 90 | metadataPrefix, 91 | dates, 92 | set, 93 | resumptionToken 94 | ); 95 | case "ListSets": 96 | return await listSets(url, resumptionToken); 97 | default: 98 | return invalidOaiRequest("badVerb", "Illegal OAI verb"); 99 | } 100 | }); 101 | -------------------------------------------------------------------------------- /docs/template.yaml: -------------------------------------------------------------------------------- 1 | # Build and Deploy Template for DC API 2 | # 3 | # Note: Any comment starting with `#*` will be removed 4 | # at build time. This allows us to run without the 5 | # dependency layer in development without removing the 6 | # layer from the build. 7 | 8 | AWSTemplateFormatVersion: "2010-09-09" 9 | Transform: 10 | - AWS::Serverless-2016-10-31 11 | - AWS::LanguageExtensions 12 | Description: dc-api-v2 Docs 13 | Parameters: 14 | CustomDomainHost: 15 | Type: String 16 | Description: Hostname within Custom Domain Zone 17 | CustomDomainZone: 18 | Type: String 19 | Description: Hosted Zone Name for Custom Domain 20 | RootApiID: 21 | Type: String 22 | Description: ID of the root API 23 | Resources: 24 | rootRedirect: 25 | Type: AWS::Serverless::Function 26 | Properties: 27 | Runtime: nodejs20.x 28 | CodeUri: ./redirect 29 | Handler: index.handler 30 | Timeout: 1 31 | Description: Redirects to latest version of docs 32 | Environment: 33 | Variables: 34 | REDIRECT_TO: /docs/v2/index.html 35 | rootRedirectIntegration: 36 | Type: AWS::ApiGatewayV2::Integration 37 | Properties: 38 | ApiId: !Ref RootApiID 39 | IntegrationType: AWS_PROXY 40 | IntegrationUri: !GetAtt rootRedirect.Arn 41 | PayloadFormatVersion: "2.0" 42 | rootRedirectRouteGet: 43 | Type: AWS::ApiGatewayV2::Route 44 | Properties: 45 | ApiId: !Ref RootApiID 46 | RouteKey: GET / 47 | Target: !Sub integrations/${rootRedirectIntegration} 48 | rootRedirectRouteHead: 49 | Type: AWS::ApiGatewayV2::Route 50 | Properties: 51 | ApiId: !Ref RootApiID 52 | RouteKey: HEAD / 53 | Target: !Sub integrations/${rootRedirectIntegration} 54 | rootRedirectPermission: 55 | Type: AWS::Lambda::Permission 56 | Properties: 57 | Action: lambda:InvokeFunction 58 | FunctionName: !Ref rootRedirect 59 | Principal: apigateway.amazonaws.com 60 | SourceArn: !Sub arn:aws:execute-api:${AWS::Region}:${AWS::AccountId}:${RootApiID}/*/*/ 61 | docsBucket: 62 | Type: AWS::S3::Bucket 63 | Properties: 64 | BucketName: !Sub "${CustomDomainHost}-docs.${CustomDomainZone}" 65 | PublicAccessBlockConfiguration: 66 | BlockPublicAcls: false 67 | BlockPublicPolicy: false 68 | IgnorePublicAcls: false 69 | RestrictPublicBuckets: false 70 | WebsiteConfiguration: 71 | IndexDocument: index.html 72 | ErrorDocument: index.html 73 | docsBucketPolicy: 74 | Type: AWS::S3::BucketPolicy 75 | Properties: 76 | PolicyDocument: 77 | Id: MyPolicy 78 | Version: 2012-10-17 79 | Statement: 80 | - Sid: PublicReadForGetBucketObjects 81 | Effect: Allow 82 | Principal: "*" 83 | Action: "s3:GetObject" 84 | Resource: !Sub "arn:aws:s3:::${docsBucket}/*" 85 | Bucket: !Ref docsBucket 86 | docsIntegration: 87 | Type: AWS::ApiGatewayV2::Integration 88 | Properties: 89 | ApiId: !Ref RootApiID 90 | IntegrationMethod: GET 91 | IntegrationType: HTTP_PROXY 92 | IntegrationUri: !Sub "http://${docsBucket}.s3-website-us-east-1.amazonaws.com/{proxy}" 93 | PayloadFormatVersion: "1.0" 94 | docsRoute: 95 | Type: AWS::ApiGatewayV2::Route 96 | Properties: 97 | ApiId: !Ref RootApiID 98 | AuthorizationType: NONE 99 | RouteKey: GET /docs/v2/{proxy+} 100 | Target: !Sub "integrations/${docsIntegration}" 101 | -------------------------------------------------------------------------------- /docs/overrides/.icons/nul-logo.svg: -------------------------------------------------------------------------------- 1 | Northwestern University -------------------------------------------------------------------------------- /api/src/api/pagination.js: -------------------------------------------------------------------------------- 1 | const { 2 | decompressFromEncodedURIComponent: decompress, 3 | compressToEncodedURIComponent: compress, 4 | } = require("lz-string"); 5 | const { defaultSearchSize } = require("../environment"); 6 | 7 | const encodeFields = ["query", "size", "sort", "fields", "_source"]; 8 | 9 | async function decodeSearchToken(token) { 10 | return JSON.parse(await decompress(token)); 11 | } 12 | 13 | async function encodeSearchToken(models, body, format, options) { 14 | let token = { body: { size: 10 }, models, format, options }; 15 | for (const field in body) { 16 | if (encodeFields.includes(field)) { 17 | token.body[field] = body[field]; 18 | } 19 | } 20 | return await compress(JSON.stringify(token)); 21 | } 22 | 23 | function from(body) { 24 | return body?.from || 0; 25 | } 26 | 27 | function size(body) { 28 | return body?.size || defaultSearchSize(); 29 | } 30 | 31 | function maxPage(body, count) { 32 | return Math.ceil(count / size(body)); 33 | } 34 | 35 | function nextPage(body, count) { 36 | const current = thisPage(body); 37 | return maxPage(body, count) > current ? current + 1 : null; 38 | } 39 | 40 | function prevPage(body, _count) { 41 | return body.from > 0 ? thisPage(body) - 1 : null; 42 | } 43 | 44 | function thisPage(body) { 45 | return Math.floor(from(body) / size(body) + 1); 46 | } 47 | 48 | class Paginator { 49 | constructor(baseUrl, route, models, body, format, options) { 50 | this.baseUrl = baseUrl; 51 | this.route = route; 52 | this.models = models; 53 | this.body = { ...body }; 54 | this.format = format; 55 | this.options = options; 56 | } 57 | 58 | async pageInfo(count, opts = {}) { 59 | let url = new URL(this.route, this.baseUrl); 60 | let searchToken; 61 | 62 | if (this.options?.includeToken != false) { 63 | searchToken = 64 | this.options?.parameterOverrides?.searchToken || 65 | this.options?.queryStringParameters?.searchToken || 66 | (await encodeSearchToken( 67 | this.models, 68 | this.body, 69 | this.format, 70 | this.options 71 | )); 72 | 73 | url.searchParams.set("searchToken", searchToken); 74 | } 75 | 76 | const queryStringParameters = 77 | this.options?.parameterOverrides || this.options?.queryStringParameters; 78 | if (typeof queryStringParameters === "object") { 79 | for (const param in queryStringParameters) { 80 | url.searchParams.set(param, queryStringParameters[param]); 81 | } 82 | } 83 | 84 | const prev = prevPage(this.body, count); 85 | const next = nextPage(this.body, count); 86 | url.searchParams.delete("from"); 87 | 88 | let result = { 89 | query_url: url.toString(), 90 | current_page: thisPage(this.body), 91 | limit: size(this.body), 92 | offset: from(this.body), 93 | total_hits: count, 94 | total_pages: maxPage(this.body, count), 95 | format: this.format, 96 | }; 97 | if (opts.includeOptions) { 98 | result.options = this.options; 99 | } 100 | if (prev) { 101 | url.searchParams.set("page", prev); 102 | result.prev_url = url.toString(); 103 | } 104 | if (next) { 105 | url.searchParams.set("page", next); 106 | result.next_url = url.toString(); 107 | } 108 | if (searchToken) { 109 | result.search_token = searchToken; 110 | } 111 | 112 | return result; 113 | } 114 | } 115 | 116 | module.exports = { decodeSearchToken, encodeSearchToken, Paginator }; 117 | -------------------------------------------------------------------------------- /api/src/api/response/iiif/presentation-api/metadata.js: -------------------------------------------------------------------------------- 1 | /** Build manifest metadata */ 2 | function formatSingleValuedField(value) { 3 | return value ? [value] : []; 4 | } 5 | 6 | function metadataLabelFields(source) { 7 | return [ 8 | { 9 | label: "Alternate Title", 10 | value: source.alternate_title, 11 | }, 12 | { 13 | label: "Abstract", 14 | value: source.abstract, 15 | }, 16 | { 17 | label: "Caption", 18 | value: source.caption, 19 | }, 20 | { 21 | label: "Contributor", 22 | value: source.contributor.map((item) => item.label_with_role), 23 | }, 24 | { 25 | label: "Creator", 26 | value: source.creator.map((item) => item.label), 27 | }, 28 | { 29 | label: "Cultural Context", 30 | value: source.cultural_context, 31 | }, 32 | { 33 | label: "Date", 34 | value: source.date_created, 35 | }, 36 | { 37 | label: "Department", 38 | value: formatSingleValuedField(source.library_unit), 39 | }, 40 | { 41 | label: "Dimensions", 42 | value: source.physical_description_size, 43 | }, 44 | { 45 | label: "Genre", 46 | value: source.genre.map((item) => item.label), 47 | }, 48 | { 49 | label: "Identifier", 50 | value: source.identifier, 51 | }, 52 | { 53 | label: "Last Modified", 54 | value: formatSingleValuedField(source.modified_date), 55 | }, 56 | { 57 | label: "Language", 58 | value: source.language.map((item) => item.label), 59 | }, 60 | { 61 | label: "License", 62 | value: formatSingleValuedField(source.license?.label), 63 | }, 64 | { 65 | label: "Location", 66 | value: source.location?.map((item) => item.label), 67 | }, 68 | { 69 | label: "Materials", 70 | value: source.physical_description_material, 71 | }, 72 | { 73 | label: "Notes", 74 | value: source.notes.map((item) => `${item.note} (${item.type})`), 75 | }, 76 | { 77 | label: "Provenance", 78 | value: source.provenance, 79 | }, 80 | { 81 | label: "Publisher", 82 | value: source.publisher, 83 | }, 84 | { 85 | label: "Related Material", 86 | value: source.related_material, 87 | }, 88 | { 89 | label: "Related URL", 90 | value: source.related_url.map( 91 | (item) => `${item.label}` 92 | ), 93 | }, 94 | { 95 | label: "Rights Holder", 96 | value: source.rights_holder, 97 | }, 98 | { 99 | label: "Rights Statement", 100 | value: formatSingleValuedField(source.rights_statement?.label), 101 | }, 102 | { 103 | label: "Scope and Contents", 104 | value: source.scope_and_contents, 105 | }, 106 | { 107 | label: "Series", 108 | value: source.series, 109 | }, 110 | { 111 | label: "Source", 112 | value: source.source, 113 | }, 114 | { 115 | label: "Style Period", 116 | value: source.style_period.map((item) => item.label), 117 | }, 118 | { 119 | label: "Subject", 120 | value: source.subject.map((item) => item.label), 121 | }, 122 | { 123 | label: "Table of Contents", 124 | value: source.table_of_contents, 125 | }, 126 | { 127 | label: "Technique", 128 | value: source.technique.map((item) => item.label), 129 | }, 130 | ]; 131 | } 132 | 133 | module.exports = { formatSingleValuedField, metadataLabelFields }; 134 | -------------------------------------------------------------------------------- /chat-playground/playground.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": null, 6 | "metadata": {}, 7 | "outputs": [], 8 | "source": [ 9 | "from dotenv import load_dotenv\n", 10 | "import sys\n", 11 | "import os\n", 12 | "import json\n", 13 | "\n", 14 | "load_dotenv(override=True)\n", 15 | "try:\n", 16 | " del os.environ[\"DEV_PREFIX\"]\n", 17 | " del os.environ[\"DEV_ENV\"]\n", 18 | "except:\n", 19 | " pass\n", 20 | "\n", 21 | "sys.path.insert(0, os.path.join(os.curdir, \"../chat/src\"))\n", 22 | "import core.secrets # noqa" 23 | ] 24 | }, 25 | { 26 | "cell_type": "code", 27 | "execution_count": null, 28 | "metadata": {}, 29 | "outputs": [], 30 | "source": [ 31 | "from IPython.display import display\n", 32 | "from typing import Any, Dict, Optional\n", 33 | "from langchain_core.callbacks import BaseCallbackHandler\n", 34 | "from langchain_core.messages.tool import ToolMessage\n", 35 | "from langchain_core.outputs import LLMResult\n", 36 | "\n", 37 | "\n", 38 | "class DebugHandler(BaseCallbackHandler):\n", 39 | " def on_llm_start(\n", 40 | " self,\n", 41 | " serialized: dict[str, Any],\n", 42 | " prompts: list[str],\n", 43 | " metadata: Optional[dict[str, Any]] = None,\n", 44 | " **kwargs: Dict[str, Any],\n", 45 | " ):\n", 46 | " print(\"on_llm_start:\")\n", 47 | " display({\"serialized\": serialized, \"metadata\": metadata, \"kwargs\": kwargs})\n", 48 | "\n", 49 | " def on_llm_end(self, response: LLMResult, **kwargs: Dict[str, Any]):\n", 50 | " print(\"on_llm_end:\")\n", 51 | " display({\"response\": response, \"kwargs\": kwargs})\n", 52 | "\n", 53 | " def on_tool_start(\n", 54 | " self, serialized: Dict[str, Any], input_str: str, **kwargs: Dict[str, Any]\n", 55 | " ):\n", 56 | " print(\"on_tool_start:\")\n", 57 | " display({\"serialized\": serialized, \"kwargs\": kwargs})\n", 58 | "\n", 59 | " def on_tool_end(self, output: ToolMessage, **kwargs: Dict[str, Any]):\n", 60 | " print(\"on_tool_end:\")\n", 61 | " display({\"output\": output, \"kwargs\": kwargs})" 62 | ] 63 | }, 64 | { 65 | "cell_type": "code", 66 | "execution_count": null, 67 | "metadata": {}, 68 | "outputs": [], 69 | "source": [ 70 | "import agent.search_agent\n", 71 | "from agent.search_agent import SearchAgent\n", 72 | "from core.setup import chat_model\n", 73 | "\n", 74 | "model = chat_model(\n", 75 | " model=\"us.anthropic.claude-3-5-sonnet-20241022-v2:0\", streaming=False\n", 76 | ")\n", 77 | "agent = SearchAgent(model=model)\n", 78 | "agent.invoke(\n", 79 | " \"What works in the collection pertain to Iranian film?\",\n", 80 | " ref=\"abc123\",\n", 81 | " callbacks=[DebugHandler()],\n", 82 | " forget=True,\n", 83 | ")" 84 | ] 85 | } 86 | ], 87 | "metadata": { 88 | "kernelspec": { 89 | "display_name": ".venv", 90 | "language": "python", 91 | "name": "python3" 92 | }, 93 | "language_info": { 94 | "codemirror_mode": { 95 | "name": "ipython", 96 | "version": 3 97 | }, 98 | "file_extension": ".py", 99 | "mimetype": "text/x-python", 100 | "name": "python", 101 | "nbconvert_exporter": "python", 102 | "pygments_lexer": "ipython3", 103 | "version": "3.12.2" 104 | } 105 | }, 106 | "nbformat": 4, 107 | "nbformat_minor": 2 108 | } 109 | -------------------------------------------------------------------------------- /api/src/handlers/auth/nusso-callback.js: -------------------------------------------------------------------------------- 1 | const axios = require("axios").default; 2 | const cookie = require("cookie"); 3 | const { dcApiEndpoint } = require("../../environment"); 4 | const ApiToken = require("../../api/api-token"); 5 | const Honeybadger = require("../../honeybadger-setup"); 6 | 7 | const BAD_DIRECTORY_SEARCH_FAULT = 8 | /Reason: ResponseCode 404 is treated as error/; 9 | 10 | /** 11 | * NUSSO auth callback 12 | */ 13 | exports.handler = async (event) => { 14 | let returnPath = `${dcApiEndpoint()}/auth/whoami`; 15 | const redirectUrl = event.cookieObject.redirectUrl; 16 | if (redirectUrl) { 17 | returnPath = Buffer.from( 18 | decodeURIComponent(event.cookieObject.redirectUrl), 19 | "base64" 20 | ).toString("utf8"); 21 | } 22 | 23 | const user = await redeemSsoToken(event); 24 | if (user) { 25 | console.info("User", user.sub, "logged in via nusso"); 26 | event.userToken = new ApiToken().user(user).provider("nusso"); 27 | return { 28 | statusCode: 302, 29 | cookies: [ 30 | cookie.serialize("redirectUrl", null, { 31 | expires: new Date(1), 32 | }), 33 | ], 34 | headers: { 35 | location: returnPath, 36 | }, 37 | }; 38 | } 39 | return { statusCode: 400 }; 40 | }; 41 | 42 | async function invokeNuApi(path, headers) { 43 | const url = new URL(process.env.NUSSO_BASE_URL); 44 | url.pathname = path; 45 | return await axios.get(url.toString(), { 46 | headers: { apikey: process.env.NUSSO_API_KEY, ...headers }, 47 | }); 48 | } 49 | 50 | async function getNetIdFromToken(nusso) { 51 | const response = await invokeNuApi("/agentless-websso/validateWebSSOToken", { 52 | webssotoken: nusso, 53 | }); 54 | return response?.data?.netid; 55 | } 56 | 57 | function transform(user) { 58 | return { 59 | sub: user?.uid, 60 | name: user?.displayName?.[0], 61 | email: user?.mail, 62 | primaryAffiliation: user?.primaryAffiliation, 63 | }; 64 | } 65 | 66 | async function redeemSsoToken(event) { 67 | const nusso = event.cookieObject.nusso; 68 | const netid = await getNetIdFromToken(nusso); 69 | if (netid) { 70 | try { 71 | const response = await invokeNuApi( 72 | `/directory-search/res/netid/bas/${netid}` 73 | ); 74 | const user = fillInBlanks({ ...response.data.results[0], uid: netid }); 75 | return transform(user); 76 | } catch (err) { 77 | if ( 78 | BAD_DIRECTORY_SEARCH_FAULT.test(err?.response?.data?.fault?.faultstring) 79 | ) { 80 | return transform(fillInBlanks({ uid: netid })); 81 | } 82 | await Honeybadger.notifyAsync(err, { tags: ["auth", "upstream"] }); 83 | console.error(err.response.data); 84 | return null; 85 | } 86 | } else { 87 | console.warn("NUSSO token could not be redeemed"); 88 | return null; 89 | } 90 | } 91 | 92 | function fillInBlanks(response) { 93 | const { uid, displayName, eduPersonPrimaryAffiliation, givenName, mail } = 94 | response; 95 | return { 96 | uid, 97 | givenName, 98 | displayName: ifEmpty(displayName, [uid]), 99 | mail: ifEmpty(mail, `${uid}@e.northwestern.edu`), 100 | primaryAffiliation: eduPersonPrimaryAffiliation, 101 | }; 102 | } 103 | 104 | function ifEmpty(val, replacement) { 105 | return isEmpty(val) ? replacement : val; 106 | } 107 | 108 | function isEmpty(val) { 109 | if (val === null || val === undefined) { 110 | return true; 111 | } 112 | 113 | if (Array.isArray(val)) { 114 | return val.every(isEmpty); 115 | } 116 | 117 | return val.length == 0; 118 | } 119 | -------------------------------------------------------------------------------- /api/test/fixtures/mocks/oai-list-identifiers-sets.json: -------------------------------------------------------------------------------- 1 | { 2 | "took": 29, 3 | "timed_out": false, 4 | "_shards": { 5 | "total": 1, 6 | "successful": 1, 7 | "skipped": 0, 8 | "failed": 0 9 | }, 10 | "hits": { 11 | "total": { 12 | "value": 1, 13 | "relation": "eq" 14 | }, 15 | "max_score": null, 16 | "hits": [ 17 | { 18 | "_index": "dc-v2-work-1673550300442", 19 | "_type": "_doc", 20 | "_id": "c6b8a744-aa8a-4437-b885-175f3816f9cd", 21 | "_score": null, 22 | "_source": { 23 | "folder_names": [], 24 | "catalog_key": [], 25 | "related_url": [], 26 | "batch_ids": [], 27 | "legacy_identifier": [], 28 | "publisher": [], 29 | "creator": [], 30 | "work_type": "Image", 31 | "accession_number": "Accession:inu-test", 32 | "date_created": [], 33 | "genre": [], 34 | "box_name": [], 35 | "abstract": [], 36 | "modified_date": "2022-11-22T06:16:12.323005Z", 37 | "keywords": [], 38 | "description": [], 39 | "location": [], 40 | "preservation_level": "Level 1", 41 | "box_number": ["2"], 42 | "cultural_context": [], 43 | "caption": [], 44 | "rights_holder": [], 45 | "physical_description_material": [], 46 | "title": "Test Collection", 47 | "collection": { 48 | "description": "Test Description", 49 | "id": "c4f30015-88b5-4291-b3a6-8ac9b7c7069c", 50 | "title": "Test Collection" 51 | }, 52 | "source": [], 53 | "scope_and_contents": [], 54 | "ingest_project": {}, 55 | "thumbnail": "https://dcapi.test.library.northwestern.edu/works/c6b8a744-aa8a-4437-b885-175f3816f9cd/thumbnail", 56 | "identifier": [], 57 | "physical_description_size": [], 58 | "ark": "ark:/test/test", 59 | "file_sets": [], 60 | "related_material": [], 61 | "language": [], 62 | "api_link": "https://dcapi.test.library.northwestern.edu/works/c6b8a744-aa8a-4437-b885-175f3816f9cd", 63 | "alternate_title": [], 64 | "series": [], 65 | "api_model": "Work", 66 | "id": "c6b8a744-aa8a-4437-b885-175f3816f9cd", 67 | "project": {}, 68 | "published": true, 69 | "rights_statement": { 70 | "id": "http://rightsstatements.org/vocab/NoC-US/1.0/", 71 | "label": "No Copyright - United States" 72 | }, 73 | "notes": [], 74 | "visibility": "Public", 75 | "table_of_contents": [], 76 | "indexed_at": "2023-01-12T19:05:00.626875", 77 | "representative_file_set": { 78 | "aspect_ratio": 1.59402, 79 | "id": "a04e4a28-340f-46cb-aca2-9e7f5713ae84", 80 | "url": "https://iiif.test.rdc.library.northwestern.edu/iiif/2/dev/a04e4a28-340f-46cb-aca2-9e7f5713ae84" 81 | }, 82 | "library_unit": "", 83 | "subject": [], 84 | "status": "Done", 85 | "ingest_sheet": {}, 86 | "technique": [], 87 | "folder_numbers": [], 88 | "license": null, 89 | "terms_of_use": "", 90 | "provenance": [], 91 | "iiif_manifest": "https://dcapi.rdc-staging.library.northwestern.edu/works/c6b8a744-aa8a-4437-b885-175f3816f9cd?as=iiif", 92 | "style_period": [], 93 | "create_date": "2021-03-16T16:22:09.679863Z", 94 | "csv_metadata_update_jobs": [] 95 | }, 96 | "sort": [1669097772323] 97 | } 98 | ] 99 | } 100 | } 101 | -------------------------------------------------------------------------------- /bin/make_deploy_config.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | CMD_WITH=$WITH 4 | 5 | get_secret() { 6 | local secret_name=$1 7 | local secret_value=$(aws secretsmanager get-secret-value --secret-id $secret_name --query SecretString --output text) 8 | 9 | local key_name=$2 10 | if [ -n "$key_name" ]; then 11 | jq -r ".$key_name" <<< $secret_value 12 | else 13 | echo $secret_value 14 | fi 15 | } 16 | 17 | with() { 18 | local feature=$1 19 | if [[ ",$WITH," == *",$feature,"* ]]; then 20 | echo "true" 21 | else 22 | echo "false" 23 | fi 24 | } 25 | 26 | aws_account_id=$(aws sts get-caller-identity --query 'Account' --output text) 27 | net_id=$(aws ec2 describe-tags \ 28 | --filters "Name=resource-id,Values=$(curl -s http://169.254.169.254/latest/meta-data/instance-id)" \ 29 | "Name=key,Values=NetID" \ 30 | --query 'Tags[0].Value' \ 31 | --output text) 32 | nusso_config=$(get_secret dev-environment/infrastructure/nusso) 33 | cat < samconfig.${DEV_PREFIX}.yaml 34 | --- 35 | version: 1.0 36 | default: 37 | EOF 38 | 39 | for section in deploy sync; do 40 | case $section in 41 | deploy) 42 | WITH=${CMD_WITH:-"API,AV_DOWNLOAD,CHAT"} 43 | ;; 44 | sync) 45 | WITH=${CMD_WITH:-"CHAT"} 46 | ;; 47 | esac 48 | cat <> samconfig.${DEV_PREFIX}.yaml 49 | ${section}: 50 | parameters: 51 | stack_name: dc-api-${DEV_PREFIX} 52 | s3_bucket: $(aws s3api list-buckets --query "Buckets[?starts_with(Name, 'aws-sam-cli-managed')].{Name:Name, CreationDate:CreationDate}" --output json | jq -r 'sort_by(.CreationDate) | .[0].Name') 53 | s3_prefix: dc-api-${DEV_PREFIX} 54 | region: us-east-1 55 | confirm_changeset: true 56 | capabilities: 57 | - CAPABILITY_IAM 58 | - CAPABILITY_AUTO_EXPAND 59 | image_repositories: [] 60 | parameter_overrides: > 61 | ApiConfigPrefix="dev-environment-${DEV_PREFIX}" 62 | ApiTokenName="dcapi$(openssl rand -hex 4 | cut -c1-7)" 63 | ApiTokenSecret="$(get_secret staging/config/dcapi api_token_secret)" 64 | CustomDomainCertificateArn="$(aws acm list-certificates --query "CertificateSummaryList[?DomainName=='*.rdc-staging.library.northwestern.edu'].CertificateArn" --output text)" 65 | CustomDomainHost="dcapi-${DEV_PREFIX}" 66 | CustomDomainZone="rdc-staging.library.northwestern.edu" 67 | DcApiEndpoint="https://dcapi-${DEV_PREFIX}.rdc-staging.library.northwestern.edu/api/v2" 68 | DcUrl="https://dc.rdc-staging.library.northwestern.edu" 69 | DeployAPI="$(with API)" 70 | DeployAVDownload="$(with AV_DOWNLOAD)" 71 | DeployChat="$(with CHAT)" 72 | DeployDocs="$(with DOCS)" 73 | DevTeamNetIds="${net_id}" 74 | ElasticsearchEndpoint="$(get_secret dev-environment/infrastructure/index | jq -r '.endpoint | ltrimstr("https://")')" 75 | EnvironmentPrefix="${DEV_PREFIX}-${DEV_ENV}" 76 | MediaConvertDestinationBucket="${DEV_PREFIX}-${DEV_ENV}-streaming" 77 | MediaConvertEndpoint="$(aws mediaconvert describe-endpoints --query 'Endpoints[0].Url' --output text)" 78 | MediaConvertJobQueueArn="arn:aws:mediaconvert:us-east-1:${aws_account_id}:queues/Default" 79 | MediaConvertRoleArn="arn:aws:iam::${aws_account_id}:role/service-role/MediaConvert_Default_Role" 80 | NussoApiKey="$(jq -r '.api_key' <<< $nusso_config)" 81 | NussoBaseUrl="$(jq -r '.base_url' <<< $nusso_config)" 82 | ProviderCapabilities='{"magic":["chat", "login"],"nusso":["chat", "login"]}' 83 | PyramidBucket="${DEV_PREFIX}-${DEV_ENV}-pyramids" 84 | ReadingRoomIPs="" 85 | RepositoryEmail="repository@northwestern.edu" 86 | SecretsPath="dev-environment" 87 | StreamingBucket="${DEV_PREFIX}-${DEV_ENV}-streaming" 88 | EOF 89 | done -------------------------------------------------------------------------------- /api/src/handlers/get-thumbnail.js: -------------------------------------------------------------------------------- 1 | const ApiToken = require("../api/api-token"); 2 | const axios = require("axios").default; 3 | const cookie = require("cookie"); 4 | const opensearchResponse = require("../api/response/opensearch"); 5 | const { apiTokenName } = require("../environment"); 6 | const { getCollection, getWork } = require("../api/opensearch"); 7 | const { wrap } = require("./middleware"); 8 | 9 | function getAxiosResponse(url, config) { 10 | return new Promise((resolve) => { 11 | axios 12 | .get(url, config) 13 | .then((response) => resolve(response)) 14 | .catch((error) => resolve(error.response)); 15 | }); 16 | } 17 | 18 | function validateRequest(event) { 19 | const id = event.pathParameters.id; 20 | const aspect = event?.queryStringParameters?.aspect || "full"; 21 | const sizeParam = event?.queryStringParameters?.size || 300; 22 | const size = Number(sizeParam); 23 | 24 | if (!["full", "square"].includes(aspect)) 25 | throw new Error(`Unknown aspect ratio: ${aspect}`); 26 | if (isNaN(size)) throw new Error(`${sizeParam} is not a valid size`); 27 | if (size > 300) 28 | throw new Error(`Requested size of ${size}px exceeds maximum of 300px`); 29 | 30 | return { id, aspect, size }; 31 | } 32 | 33 | const getThumbnail = async (id, aspect, size, event) => { 34 | const allowUnpublished = 35 | event.userToken.isSuperUser() || event.userToken.hasEntitlement(id); 36 | const allowPrivate = allowUnpublished || event.userToken.isReadingRoom(); 37 | 38 | let esResponse; 39 | let body; 40 | let iiif_base; 41 | if (event.rawPath.match(/\/collections\//)) { 42 | esResponse = await getCollection(id, { 43 | allowPrivate, 44 | allowUnpublished, 45 | }); 46 | if (esResponse.statusCode != 200) 47 | return { error: await opensearchResponse.transform(esResponse) }; 48 | body = JSON.parse(esResponse.body); 49 | iiif_base = body?._source?.representative_image?.url; 50 | } else { 51 | esResponse = await getWork(id, { 52 | allowPrivate, 53 | allowUnpublished, 54 | }); 55 | if (esResponse.statusCode != 200) 56 | return { error: await opensearchResponse.transform(esResponse) }; 57 | body = JSON.parse(esResponse.body); 58 | iiif_base = body?._source?.representative_file_set?.url; 59 | } 60 | 61 | if (!iiif_base) { 62 | return { 63 | statusCode: 404, 64 | headers: { "content-type": "text/plain" }, 65 | body: "Not Found", 66 | }; 67 | } 68 | 69 | const thumbnail = `${iiif_base}/${aspect}/!${size},${size}/0/default.jpg`; 70 | 71 | const { status, headers, data } = await getAxiosResponse(thumbnail, { 72 | headers: { 73 | cookie: cookie.serialize( 74 | apiTokenName(), 75 | new ApiToken().superUser().sign(), 76 | { 77 | domain: "library.northwestern.edu", 78 | path: "/", 79 | secure: true, 80 | } 81 | ), 82 | }, 83 | responseType: "arraybuffer", 84 | }); 85 | 86 | if (status != 200) { 87 | return { 88 | statusCode: status, 89 | body: data.toString(), 90 | headers: headers, 91 | }; 92 | } 93 | 94 | return { 95 | statusCode: status, 96 | isBase64Encoded: true, 97 | body: data.toString("base64"), 98 | headers: { 99 | "content-type": headers["content-type"], 100 | }, 101 | }; 102 | }; 103 | 104 | /** 105 | * A simple function to proxy a Collection or Work thumbnail from the IIIF server 106 | */ 107 | exports.handler = wrap(async (event) => { 108 | try { 109 | const { id, aspect, size } = validateRequest(event); 110 | return await getThumbnail(id, aspect, size, event); 111 | } catch (err) { 112 | return { 113 | statusCode: 400, 114 | headers: { "content-type": "text/plain" }, 115 | body: err.message, 116 | }; 117 | } 118 | }); 119 | -------------------------------------------------------------------------------- /api/test/integration/get-auth-whoami.test.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | 3 | const chai = require("chai"); 4 | const expect = chai.expect; 5 | const jwt = require("jsonwebtoken"); 6 | 7 | const getAuthWhoamiHandler = requireSource("handlers/get-auth-whoami"); 8 | 9 | describe("auth whoami", function () { 10 | helpers.saveEnvironment(); 11 | 12 | let payload; 13 | 14 | beforeEach(() => { 15 | payload = { 16 | iss: "https://thisisafakeapiurl", 17 | sub: "user123", 18 | name: "Some One", 19 | exp: Math.floor(Number(new Date()) / 1000) + 12 * 60 * 60, 20 | iat: Math.floor(Number(new Date()) / 1000), 21 | email: "user@example.com", 22 | }; 23 | }); 24 | 25 | it("returns user info", async () => { 26 | const token = jwt.sign(payload, process.env.API_TOKEN_SECRET); 27 | 28 | const event = helpers 29 | .mockEvent("GET", "/auth/whoami") 30 | .headers({ 31 | Cookie: `${process.env.API_TOKEN_NAME}=${token};`, 32 | }) 33 | .render(); 34 | 35 | const result = await getAuthWhoamiHandler.handler(event); 36 | expect(result.statusCode).to.eq(200); 37 | const response = JSON.parse(result.body); 38 | expect(response).to.contain({ name: "Some One" }); 39 | expect(response).to.have.property("scopes"); 40 | }); 41 | 42 | it("Doesn't set a new cookie if the token is not updated", async () => { 43 | const token = jwt.sign(payload, process.env.API_TOKEN_SECRET); 44 | 45 | let event = helpers 46 | .mockEvent("GET", "/auth/whoami") 47 | .headers({ 48 | Cookie: `${process.env.API_TOKEN_NAME}=${token};`, 49 | }) 50 | .render(); 51 | 52 | let result = await getAuthWhoamiHandler.handler(event); 53 | let dcApiCookie = helpers.cookieValue( 54 | result.cookies, 55 | process.env.API_TOKEN_NAME 56 | ); 57 | expect(dcApiCookie).to.have.property("value"); 58 | 59 | event = helpers 60 | .mockEvent("GET", "/auth/whoami") 61 | .headers({ 62 | Cookie: `${process.env.API_TOKEN_NAME}=${dcApiCookie.value};`, 63 | }) 64 | .render(); 65 | result = await getAuthWhoamiHandler.handler(event); 66 | dcApiCookie = helpers.cookieValue( 67 | result.cookies, 68 | process.env.API_TOKEN_NAME 69 | ); 70 | expect(dcApiCookie).to.be.undefined; 71 | }); 72 | 73 | it("Expires the DC API Token and appears anonymous when an expired token is present", async () => { 74 | payload.exp = Math.floor(Number(new Date()) / 1000); 75 | const token = jwt.sign(payload, process.env.API_TOKEN_SECRET); 76 | 77 | const event = helpers 78 | .mockEvent("GET", "/auth/whoami") 79 | .headers({ 80 | Cookie: `${process.env.API_TOKEN_NAME}=${token};`, 81 | }) 82 | .render(); 83 | 84 | const result = await getAuthWhoamiHandler.handler(event); 85 | const body = JSON.parse(result.body); 86 | 87 | expect(result.statusCode).to.eq(200); 88 | expect(body).to.contain({ 89 | iss: process.env.DC_API_ENDPOINT, 90 | isLoggedIn: false, 91 | }); 92 | expect(body).not.to.contain({ sub: "user123" }); 93 | 94 | const dcApiCookie = helpers.cookieValue( 95 | result.cookies, 96 | process.env.API_TOKEN_NAME 97 | ); 98 | expect(dcApiCookie.Expires).to.eq("Thu, 01 Jan 1970 00:00:00 GMT"); 99 | }); 100 | 101 | it("Issues an anonymous API token if no token is present", async () => { 102 | const event = helpers.mockEvent("GET", "/auth/whoami").render(); 103 | const result = await getAuthWhoamiHandler.handler(event); 104 | const body = JSON.parse(result.body); 105 | 106 | expect(result.statusCode).to.eq(200); 107 | expect(body).to.contain({ 108 | iss: process.env.DC_API_ENDPOINT, 109 | isLoggedIn: false, 110 | }); 111 | expect(body).not.to.contain({ sub: "user123" }); 112 | }); 113 | }); 114 | --------------------------------------------------------------------------------