├── .cfnlintrc ├── .env ├── .gitignore ├── .vscode ├── launch.json └── settings.json ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── LICENSE ├── NOTICE ├── README.md ├── buildspec.yml ├── deploy_edge_auth.sh ├── docs ├── api_authentication.md ├── api_entry_points.md ├── auto_context.md ├── caching.md ├── configuration.md ├── experimentation.md └── item_metadata.md ├── images ├── architecture-apikey.png ├── architecture-noauth.png ├── architecture-oauth2.png ├── config-create.png ├── configuration-concepts.png ├── personalization-apis-value-prop.png ├── postman.png ├── swagger_editor.png └── swagger_ui.png ├── run_tests.sh ├── samconfig-edge.toml ├── samconfig.toml ├── samples ├── config_auto_context.json ├── config_experiment.json ├── config_simple.json └── lambdas │ └── custom_recommender_lambda.py ├── src ├── __init__.py ├── config_validator_env_function │ ├── __init__.py │ ├── main.py │ └── requirements.txt ├── config_validator_function │ ├── README.md │ ├── __init__.py │ ├── main.py │ ├── openapi.py │ ├── openapi_template.json │ ├── personalization_apis_config_schema.json │ └── requirements.txt ├── copy_swagger_ui_assets_function │ ├── __init__.py │ ├── main.py │ └── requirements.txt ├── edge_auth_function │ ├── README.md │ ├── index.js │ └── package.json ├── edge_update_function │ ├── README.md │ ├── main.py │ └── requirements.txt ├── generate_config_function │ ├── README.md │ ├── __init__.py │ ├── main.py │ └── requirements.txt ├── layer │ ├── README.md │ ├── __init__.py │ ├── personalization_config.py │ ├── personalization_constants.py │ └── requirements.txt ├── load_item_metadata_function │ ├── README.md │ ├── __init__.py │ ├── main.py │ └── requirements.txt ├── personalization_api_function │ ├── README.md │ ├── __init__.py │ ├── auto_values.py │ ├── background_tasks.py │ ├── event_targets.py │ ├── evidently.py │ ├── lambda_resolver.py │ ├── main.py │ ├── personalization_error.py │ ├── personalize_resolver.py │ ├── requirements.txt │ ├── response_decorator.py │ ├── response_post_process.py │ ├── sagemaker_resolver.py │ └── util.py ├── statemachine │ ├── README.md │ └── sync_resources.asl.json ├── swagger-ui │ ├── favicon-16x16.png │ ├── favicon-32x32.png │ └── index.html ├── sync_cache_settings_function │ ├── README.md │ ├── __init__.py │ ├── main.py │ └── requirements.txt └── sync_dynamodb_tables_function │ ├── README.md │ ├── __init__.py │ ├── main.py │ └── requirements.txt ├── stage.sh ├── template-edge.yaml ├── template.yaml └── tests ├── __init__.py ├── requirements.txt └── unit ├── __init__.py ├── test_auto_values.py ├── test_config.py └── test_openapi.py /.cfnlintrc: -------------------------------------------------------------------------------- 1 | ignore_checks: 2 | - W3005 # Disable rule W3005 - obsolete DependsOn (needed for API stage timing) 3 | -------------------------------------------------------------------------------- /.env: -------------------------------------------------------------------------------- 1 | PYTHONPATH=./src -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | __pycache__/ 2 | .DS_Store 3 | .aws-sam 4 | .venv 5 | **/jj_* 6 | items.csv 7 | users.csv 8 | .coverage 9 | pytestdebug.log -------------------------------------------------------------------------------- /.vscode/launch.json: -------------------------------------------------------------------------------- 1 | { 2 | "version": "0.2.0", 3 | "configurations": [ 4 | { 5 | "type": "aws-sam", 6 | "request": "direct-invoke", 7 | "name": "amazon-personalize-api-middleware:LoadProductsFunction", 8 | "invokeTarget": { 9 | "target": "template", 10 | "templatePath": "${workspaceFolder}/template.yaml", 11 | "logicalId": "LoadProductsFunction" 12 | }, 13 | "lambda": { 14 | "payload": {}, 15 | "environmentVariables": {} 16 | } 17 | }, 18 | { 19 | "name": "Python: Run Current File", 20 | "type": "python", 21 | "request": "launch", 22 | "program": "${file}", 23 | "console": "integratedTerminal", 24 | "env": { 25 | "PYTHONPATH": "./src" 26 | } 27 | }, 28 | { 29 | "name": "Python: pytest", 30 | "type": "python", 31 | "request": "launch", 32 | "module": "pytest", 33 | "cwd": "${workspaceRoot}", 34 | "env": { 35 | "PYTHONPATH": "./src" 36 | }, 37 | "envFile": "${workspaceRoot}/.env", 38 | "console": "integratedTerminal", 39 | } 40 | ] 41 | } -------------------------------------------------------------------------------- /.vscode/settings.json: -------------------------------------------------------------------------------- 1 | { 2 | "python.testing.pytestArgs": [ 3 | "tests" 4 | ], 5 | "python.testing.unittestEnabled": false, 6 | "python.testing.pytestEnabled": true 7 | } -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | ## Code of Conduct 2 | This project has adopted the [Amazon Open Source Code of Conduct](https://aws.github.io/code-of-conduct). 3 | For more information see the [Code of Conduct FAQ](https://aws.github.io/code-of-conduct-faq) or contact 4 | opensource-codeofconduct@amazon.com with any additional questions or comments. 5 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing Guidelines 2 | 3 | Thank you for your interest in contributing to our project. Whether it's a bug report, new feature, correction, or additional 4 | documentation, we greatly value feedback and contributions from our community. 5 | 6 | Please read through this document before submitting any issues or pull requests to ensure we have all the necessary 7 | information to effectively respond to your bug report or contribution. 8 | 9 | 10 | ## Reporting Bugs/Feature Requests 11 | 12 | We welcome you to use the GitHub issue tracker to report bugs or suggest features. 13 | 14 | When filing an issue, please check existing open, or recently closed, issues to make sure somebody else hasn't already 15 | reported the issue. Please try to include as much information as you can. Details like these are incredibly useful: 16 | 17 | * A reproducible test case or series of steps 18 | * The version of our code being used 19 | * Any modifications you've made relevant to the bug 20 | * Anything unusual about your environment or deployment 21 | 22 | 23 | ## Contributing via Pull Requests 24 | Contributions via pull requests are much appreciated. Before sending us a pull request, please ensure that: 25 | 26 | 1. You are working against the latest source on the *main* branch. 27 | 2. You check existing open, and recently merged, pull requests to make sure someone else hasn't addressed the problem already. 28 | 3. You open an issue to discuss any significant work - we would hate for your time to be wasted. 29 | 30 | To send us a pull request, please: 31 | 32 | 1. Fork the repository. 33 | 2. Modify the source; please focus on the specific change you are contributing. If you also reformat all the code, it will be hard for us to focus on your change. 34 | 3. Ensure local tests pass. 35 | 4. Commit to your fork using clear commit messages. 36 | 5. Send us a pull request, answering any default questions in the pull request interface. 37 | 6. Pay attention to any automated CI failures reported in the pull request, and stay involved in the conversation. 38 | 39 | GitHub provides additional document on [forking a repository](https://help.github.com/articles/fork-a-repo/) and 40 | [creating a pull request](https://help.github.com/articles/creating-a-pull-request/). 41 | 42 | 43 | ## Finding contributions to work on 44 | Looking at the existing issues is a great way to find something to contribute on. As our projects, by default, use the default GitHub issue labels (enhancement/bug/duplicate/help wanted/invalid/question/wontfix), looking at any 'help wanted' issues is a great place to start. 45 | 46 | 47 | ## Code of Conduct 48 | This project has adopted the [Amazon Open Source Code of Conduct](https://aws.github.io/code-of-conduct). 49 | For more information see the [Code of Conduct FAQ](https://aws.github.io/code-of-conduct-faq) or contact 50 | opensource-codeofconduct@amazon.com with any additional questions or comments. 51 | 52 | 53 | ## Security issue notifications 54 | If you discover a potential security issue in this project we ask that you notify AWS/Amazon Security via our [vulnerability reporting page](http://aws.amazon.com/security/vulnerability-reporting/). Please do **not** create a public github issue. 55 | 56 | 57 | ## Licensing 58 | 59 | See the [LICENSE](LICENSE) file for our project's licensing. We will ask you to confirm the licensing of your contribution. 60 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy of 4 | this software and associated documentation files (the "Software"), to deal in 5 | the Software without restriction, including without limitation the rights to 6 | use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of 7 | the Software, and to permit persons to whom the Software is furnished to do so. 8 | 9 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 10 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS 11 | FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 12 | COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER 13 | IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 14 | CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 15 | 16 | -------------------------------------------------------------------------------- /NOTICE: -------------------------------------------------------------------------------- 1 | Personalization APIs 2 | Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -------------------------------------------------------------------------------- /buildspec.yml: -------------------------------------------------------------------------------- 1 | # 2 | # AWS Codebuild specifications 3 | # This file is only used for internal deployment of the code on each public commit 4 | # It is NOT needed if you're demoing or developping locally 5 | # 6 | 7 | version: 0.2 8 | env: 9 | shell: bash 10 | phases: 11 | install: 12 | commands: 13 | - echo Entered the install phase... 14 | - yum -y install git 15 | build: 16 | commands: 17 | - echo Entered the build phase... 18 | - chmod +x stage.sh 19 | - ./stage.sh template.yaml personalize-solution-staging-us-east-1 personalization-apis/ 20 | - ./stage.sh template.yaml personalize-solution-staging-us-east-2 personalization-apis/ 21 | - ./stage.sh template.yaml personalize-solution-staging-us-west-2 personalization-apis/ 22 | - ./stage.sh template.yaml personalize-solution-staging-eu-west-1 personalization-apis/ 23 | - ./stage.sh template.yaml personalize-solution-staging-ap-southeast-2 personalization-apis/ 24 | - ./stage.sh template-edge.yaml personalize-solution-staging-us-east-1 personalization-apis-edge/ -------------------------------------------------------------------------------- /deploy_edge_auth.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # Edge auth resources are optional depending on the configuration of the Personalization API deployment. 4 | # 5 | # When to deploy edge auth: 6 | # - You are using OAuth2 with Cognito to authenticate your API calls. 7 | # Why deploy edge auth: 8 | # - To maximize the cache hit rate and reduce latency of serving cached resources. 9 | # Requirements: 10 | # - Amazon Cognito must be used as the JWT token provider for API authentication. 11 | # - Edge auth resources must be deployed in the us-east-1 region. This is a Lambda@Edge requirement. 12 | # However, you can deploy your Personalization APIs in any region where the dependent AWS services 13 | # (i.e. Personalize) are available. 14 | 15 | sam build --use-container --cached --template-file template-edge.yaml && sam deploy --region us-east-1 --config-file samconfig-edge.toml --guided -------------------------------------------------------------------------------- /docs/api_authentication.md: -------------------------------------------------------------------------------- 1 | # Personalization APIs Authentication 2 | 3 | There are three authentication schemes supported by the Personalization APIs solution. 4 | 5 | - **OAuth2 edge (Amazon Cognito)**: requests from your client application(s) to the Personalization APIs must include a JWT token provided by Amazon Cognito in the `Authorization` request header 6 | - **API Key**: requests from your client applications to the Personalization APIs must include an Amazon API Gateway API Key in the `X-Api-Key` request header 7 | - **None**: requests from your client applications to the Personalization APIs require no authentication (**this also means anyone can call your APIs as well!**) 8 | 9 | You select your desired authentication scheme when deploying the Personalization APIs solution. Changing your scheme requires redeploying the solution with a different authentication scheme option. The sections below go into more detail on each scheme. 10 | 11 | ## OAuth2 edge authentication 12 | 13 | **You must deploy the Personalization APIs with an Authentication Scheme of `OAuth2-Cognito` to use this authentication method AND you must deploy the edge authentication resources (see the installation instructions).** 14 | 15 | ![Deployment Architecture OAuth2 Auth](../images/architecture-oauth2.png) 16 | 17 | For testing purposes, you can create a JWT token from Cognito for an existing user in the Cognito user pool using the following command. 18 | 19 | ```bash 20 | aws cognito-idp initiate-auth \ 21 | --client-id [YOUR_USER_POOL_CLIENT_APP_ID] \ 22 | --auth-flow USER_PASSWORD_AUTH \ 23 | --auth-parameters USERNAME=[COGNITO_USERNAME],PASSWORD=[COGNITO_USER_PASSWORD] \ 24 | --region [REGION] \ 25 | --query AuthenticationResult.AccessToken \ 26 | --output text 27 | ``` 28 | 29 | Where... 30 | 31 | - `[YOUR_USER_POOL_CLIENT_APP_ID]` is the Amazon Cognito user pool client application ID. If you deployed the Personalization APIs with `CreateCognitoResources` set to `Yes`, you can find the client app ID in the `CognitoUserPoolClientId` CloudFormation output parameter. Otherwise, you will need to create a Cognito client app and use its ID. 32 | - `[COGNITO_USERNAME]` is the username for a Cognito user in your Cognito user pool. 33 | - `[COGNITO_USER_PASSWORD]` is the password for the Cognito user identified by `[USERNAME]`. 34 | - `[REGION]` is the region where your Cognito user pool and client are deployed. 35 | 36 | The returned value is the token that should be specified in the `Authorization` header as a `Bearer` token. Here is an example of how this would be done using `curl`: 37 | 38 | ```bash 39 | curl -H "Authorization: Bearer [TOKEN]" https://[HOST]/recommend-items/[NAMESPACE]/[RECOMMENDER/[USER_ID] 40 | ``` 41 | 42 | ### Protecting your origin API from direct access 43 | 44 | Since authentication is done in CloudFront and not in API Gateway, callers can potentially bypass CloudFront and call API Gateway directly if they know your API Gateway endpoint URL. You can protect the origin API endpoint (API Gateway) using AWS WAF. This will deny requests that attempt to call API Gateway directly. Details [here](https://www.wellarchitectedlabs.com/security/300_labs/300_multilayered_api_security_with_cognito_and_waf/3_prevent_requests_from_accessing_api_directly/). 45 | 46 | ## API Key authentication 47 | 48 | **You must deploy the Personalization APIs with an Authentication Scheme of `ApiKey` to use this authentication method.** 49 | 50 | When you deployed the Personaliation APIs, an API Key was automatically created for you. The key name can be found in the `RestApiKey` CloudFormation output parameter value when the solution was deployed or in the AWS console for API Gateway. The API Key value for the key name can be found in the AWS console for API Gateway. 51 | 52 | ![Deployment Architecture API Key Auth](../images/architecture-apikey.png) 53 | 54 | Once you have the API Key value, you use it as the value of the `X-Api-Key` request header when making requests to the Personalization APIs. Here is an example of how this would be done using `curl`. 55 | 56 | ```bash 57 | curl -H "X-Api-Key: [API-KEY-VALUE]" https://[HOST]/recommend-items/[NAMESPACE]/[RECOMMENDER/[USER_ID] 58 | ``` 59 | 60 | ## No authentication 61 | 62 | **You must deploy the Personalization APIs with an Authentication Scheme of `None` to use this authentication method.** 63 | 64 | There is no authentication required when the authentication scheme is none. **This means that there are no authentication protections deployed with your APIs and anyone that knows your API endpoint URL and path layout can call your APIs.** 65 | 66 | This authentication scheme is useful if want to layer your own custom authentication approach, such as an [API Gateway Lambda authorizer](https://docs.aws.amazon.com/apigateway/latest/developerguide/apigateway-use-lambda-authorizer.html), on top of the Personalization APIs after the solution is deployed. 67 | 68 | ![Deployment Architecture No-Auth](../images/architecture-noauth.png) 69 | -------------------------------------------------------------------------------- /docs/caching.md: -------------------------------------------------------------------------------- 1 | # Personalization API Caching 2 | 3 | Caching API responses provides key benefits such as greatly reduced response time latency and reduced load on origin recommenders. Reducing the load on origin recommenders such as Amazon Personalize can also reduce the cost of operating the recommender and smoothing out auto-scaling spikes since fewer requests are served by the recommender itself. 4 | 5 | The Personalization APIs solution provides the ability to deploy caching at multiple layers: 6 | 7 | - **Device cache** (private): Caching layer on the user's physical device (i.e., laptop, tablet, phone, etc). Provides the lowest possible latency and most responsive user experience since cached responses are served directly from the user's device. 8 | - Network latency: _eliminated!_ 9 | - Origin latency: _eliminated!_ 10 | - **Edge caches** (shared): Distributed caching layer deployed through a CDN (CloudFront) that caches responses geographically near end users. Provides the next lowest latency since round-trip requests to edge caches are as short as possible. Edge caches are particularly effective when the user base is geographically distributed across multiple regions or not close to the origin recommender's region. 11 | - Network latency: optimized 12 | - Origin latency: _eliminated!_ 13 | - **Gateway cache** (shared): Centralized caching layer co-located with the origin recommender. Reduces latency by the amount of time that it would take the origin recommender to generate a response but does not eliminate network latency significantly. 14 | - Network latency: negligible improvement 15 | - Origin latency: _eliminated!_ 16 | 17 | Caching responses effectively, particularly authenticated and personalized API responses, does present some challenges when it comes to optimizing the impact and effectiveness of caches. For example, when authentication is deployed with an API (recommended!), every request must be authenticated before a cached response can be returned. This means that authentication must be done at the same layer as each cache. For the device (private) caching layer, the client application and/or device authentication is used. However, for the edge caching layer to be effective, authentication has to be done at the edge as well. For the OAuth2 authentication method, the Personalization APIs solution provides a Lambda@Edge function that is deployed with CloudFront that authenticates requests with regional edge caches. Similarly, API Key authentication at the gateway is handled by API Gateway before it checks its cache for responses. 18 | 19 | ## Cache keys and cache validation 20 | 21 | Cache keys are used to uniquely identify responses in the cache. Therefore, it's vital that each key accurately reflects the inputs that went into generating the response but does not include superflous inputs that aren't specific to the response. An overly specific cache key can significantly reduce cache hit rates. Typically generating the cache key involves hashing a combination of the request URI, query string parameters, and specific request headers that control how the response is generated. Both CloudFront and API Gateway allow the cache key components to be customized. By default, the Personalization APIs solution can be directed to automatically synchronize settings in the configuration to CloudFront Caching Policy and API Gateway cache settings. 22 | 23 | Cache validation is the process that client and shared caches follow when a cached resource expires to ask an origin whether the cached resource is still valid. This process is supported by the origin returning an `ETag` response header that contains an opaque value (which can be like a cache key) that the client sends to the origin in subsequent requests with the `If-None-Match` request header. This allows the origin to either generate and return a fresh response or return an HTTP 304 status code that tells the client that the cached resource is still valid to use. This does not eliminate the latency of round trip network call but can eliminate the time needed to generate a new response. The Personalization APIs transparently generates the `ETag` response header and implements cache valiation when the `If-None-Match` request header is present. 24 | 25 | ## CloudFront caching / OAuth2 (Cognito) Authentication 26 | 27 | ![Deployment Architecture OAuth2 Auth](../images/architecture-oauth2.png) 28 | 29 | ## API Gateway caching / API Key Authentication 30 | 31 | ![Deployment Architecture API Key Auth](../images/architecture-apikey.png) 32 | 33 | ## CloudFront caching / No Authentication 34 | 35 | ![Deployment Architecture No-Auth](../images/architecture-noauth.png) 36 | -------------------------------------------------------------------------------- /docs/experimentation.md: -------------------------------------------------------------------------------- 1 | # Personalization APIs experimentation: A/B testing 2 | 3 | Support for A/B testing recommender strategies is built-in to the Personalization APIs solution. All that is needed to run an A/B test is to setup an experiment in [Amazon CloudWatch Evidently](https://aws.amazon.com/blogs/aws/cloudwatch-evidently/), configure your recommender variations in your Personalize APIs configuration, and take your experiment live. These steps are outlined in more detail below. 4 | 5 | ## Step 1: Setup features and experiment in [Amazon CloudWatch Evidently](https://aws.amazon.com/blogs/aws/cloudwatch-evidently/) 6 | 7 | - [Create a project](https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/CloudWatch-Evidently-newproject.html) in Evidently for your application. 8 | - [Add a feature](https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/CloudWatch-Evidently-newfeature.html) to your project representing the personalization use-case in your application. This can be a UI control or widget where you're displaying personalized recommendations. 9 | - Your feature can have one or more variations. To test different personalization recommenders for this feature, create a string type variation for each recommender that you want to test. For the variation value, enter a unique name for each variation for the feature. The name should be alphanumeric. 10 | - You will use the variation values from the Evidently feature configuration in the Personalization APIs configuration. This is how the Personalization APIs will map the output of Evidently's [EvaluateFeature](https://docs.aws.amazon.com/cloudwatchevidently/latest/APIReference/API_EvaluateFeature.html) API response to the appropriate variation in the API configuration. 11 | - [Create an experiment](https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/CloudWatch-Evidently-newexperiment.html) for the feature. 12 | - For the Evidently experiment's metrics, you will map the metric name as well as it's `entityIdKey` and `valueKey` in the Personalization APIs configuration (see below). 13 | 14 | For example, a metric definition for tracking product clicks for click-through-rate for an experiment might look like this in Evidently. 15 | 16 | ```json 17 | { 18 | "entityIdKey": "userDetails.userId", 19 | "valueKey": "details.productClicked", 20 | "eventPattern": { 21 | "userDetails.userId": [ 22 | { 23 | "exists": true 24 | } 25 | ], 26 | "details.productClicked": [ 27 | { 28 | "exists": true 29 | } 30 | ] 31 | } 32 | } 33 | ``` 34 | 35 | ## Step 2: Setup the recommender variations in your Personalization APIs configuration 36 | 37 | Next, take the details of the Evidently project, feature, variations, and experiment created above to your Personalization APIs configuration. Below is an example configuration that defines an experiment with two variations that tests a related items recommender trained with the Personalize SIMS recipe against a recommender trained with the Similar-Items recipe. Under `experiments` you can define one or more features where the key is the feature name from your Evidently experiment (`product-detail-view` in this case). Under `metrics`, the metric name (`productDetailRecClicked`) must match the metric name you setup in Evidently for your experiment and the `entityIdKey` and `valueKey` matches the corresponding metric settings for your experiment (see metric definition above). The `trackExposures` field instructs the Personalization APIs endpoint to call the Evidently `PutProjectEvents` API to track variation exposures (this is needed for CTR tracking). The variation names are `sims` and `similar-items` and these must match the variation string values for the feature of the experiment in Evidently. 38 | 39 | ```json 40 | { 41 | "namespaces": { 42 | "my-store": { 43 | "recommenders": { 44 | "related-items": { 45 | "similar": { 46 | "experiments": { 47 | "product-detail-view": { 48 | "method": "evidently", 49 | "project": "my-store", 50 | "metrics": { 51 | "productDetailRecClicked": { 52 | "trackExposures": true, 53 | "entityIdKey": "userDetails.userId", 54 | "valueKey": "details.productClicked" 55 | } 56 | } 57 | } 58 | }, 59 | "variations": { 60 | "sims": { 61 | "type": "personalize-campaign", 62 | "arn": "arn:aws:personalize:[REGION:[ACCOUNT]:campaign/[CAMPAIGN_NAME]" 63 | }, 64 | "similar-items": { 65 | "type": "personalize-campaign", 66 | "arn": "arn:aws:personalize:[REGION:[ACCOUNT]:campaign/[CAMPAIGN_NAME]" 67 | } 68 | } 69 | } 70 | } 71 | } 72 | } 73 | } 74 | } 75 | ``` 76 | 77 | Once your configuration is updated in AppConfig and deployed, the Personalization APIs will automatically pick it up. 78 | 79 | ## Step 3: Take your experiment live 80 | 81 | Within Evidently, take your experiement live. The Personalization APIs solution will automatically call Evidently to evaluate the feature for the recommender to determine which variation to use for the current user and record an exposure event back to Evidently to indicate that the user receiving a variation. When an experiment is active, the API response will include details on the experiment. This will allow you to include this information back in the `/events` API to indicate the user converted (i.e. clicked on a recommendation, eventually purchased a recommended product or watched a recommended video). 82 | 83 | ### Exposure events 84 | 85 | When an exeriment is active for a recommender and a user is provided recommendations from a recommender's variation, a exposure event is automatically posted to Evidently's PutProjectEvents API. This event indicates that the user has been presented with a variation and is part of the experiment. 86 | 87 | ### Conversion events 88 | 89 | When a user converts for an experiment variation, you can include details on the experiment when you send events back to the Personalization APIs. The `experimentConversions` array can include one or more conversion events for the user. You must include the recommender name, feature name, metric name, and (optionally) a conversion value. If a value is not specified, the Personalization APIs solution will use `1.0` as the metric value. 90 | 91 | `POST /events/{namespace}` 92 | 93 | ```json 94 | { 95 | "userId": "12", 96 | "experimentConversions": [ 97 | { 98 | "recommender": "similar", 99 | "feature": "product-detail-view", 100 | "metric": "productDetailRecClicked" 101 | } 102 | ] 103 | } 104 | ``` 105 | -------------------------------------------------------------------------------- /docs/item_metadata.md: -------------------------------------------------------------------------------- 1 | # Personalization APIs Inference Item Metadata 2 | 3 | Having access to item metadata (item name, price, description, category, brand, genre, etc) directly in recommender responses allows applications to more easily render recommendations in their UI. However, many primitive recommendation systems, such as Amazon Personalize, provide only item IDs and not item metadata in their responses. Although Amazon Personalize campaigns and recommenders can be configured to return item metadata, there are limits on the number of columns (10) and items (50) that can be returned when item metadata for responses is enabled. There is also an additional charge to have item metadata returned in Personalize responses (see the [pricing page](https://aws.amazon.com/personalize/pricing/) for details). 4 | 5 | This project provides the ability to leverage the native capability of underlying recommenders from Amazon Personalize to provide item metadata as well as an alternative sidecar item metadata storage and retrievel mechanism that injects item metadata in responses before they are returned by the API. See the detailed instructions below on each mechanism. 6 | 7 | ** Keep in mind that item metadata in responses can be controlled by the `decorateItems` query string parameter to API requests. By default item metadata is enabled (if configured as described below) but it can be disabled at the request level by setting `decorateItems` to 0/false/no in your requests. 8 | 9 | ## Personalize item metadata 10 | 11 | If you're using Amazon Personalize campaigns and/or recommenders with a deployment of this project and your use case is within the 10 metadata returned column and 50 item limit, then using the native Amazon Personalize item metadata return feature may be the best option. Some configuration is still required to use this approach. 12 | 13 | First, you will have to enable metadata to be returned from your Amazon Personalize campaigns and/or recommenders. This can be done in the Amazon Personalize console or API when creating your campaigns or recommenders (for the API/SDK, see the `campaignConfig.enableMetadataWithRecommendations` parameter for the [CreateCampaign](https://docs.aws.amazon.com/personalize/latest/dg/API_CreateCampaign.html) and [UpdateCampaign](https://docs.aws.amazon.com/personalize/latest/dg/API_UpdateCampaign.html) APIs for campaigns and the `recommenderConfig.enableMetadataWithRecommendations` parameter for the [CreateRecommender](https://docs.aws.amazon.com/personalize/latest/dg/API_CreateRecommender.html) and [UpdateRecommender](https://docs.aws.amazon.com/personalize/latest/dg/API_UpdateRecommender.html) APIs for recommenders). 14 | 15 | Once metadata has been enabled for your Amazon Personalize campaigns/recommenders, you can then configure the Personalization APIs to request item metadata when making inference calls. This is done with the `inferenceMetadata` section in the Personalization APIs configuration. Below is an example of using item metadata provided by Amazon Personalize (`type` of `personalize`) and specifying that the columns `NAME`, `DESCRIPTION`, `PRICE`, and `CATEGORY` should be requested from Personalize and returned by the API response. The column names must match columns in your Amazon Personalize items dataset schema. 16 | 17 | ```json 18 | { 19 | "namespaces": { 20 | "my-app-1": { 21 | "inferenceItemMetadata": { 22 | "type": "personalize", 23 | "itemColumns": [ 24 | "NAME", 25 | "DESCRIPTION", 26 | "PRICE", 27 | "CATEGORY" 28 | ] 29 | }, 30 | "recommenders": { 31 | } 32 | } 33 | } 34 | } 35 | ``` 36 | 37 | - `namespaces.{NAMESPACE_KEY}.inferenceItemMetadata.type`: Must be `"personalize"` (required to use Amazon Personalize provided item metadata). 38 | - `namespaces.{NAMESPACE_KEY}.inferenceItemMetadata.itemColumns`: Array of column names to request Personalize to return in responses (required). 39 | 40 | Although the example above shows the `inferenceItemMetadata` being specified at the namespace level, it can also be specified at the recommender or variation level instead. This allows you override item metadata configurations at different levels of the configuration. 41 | 42 | An API response that includes item metadata from Amazon Personalize would look something like this where the `metadata` dictionary in the response includes the item metadata columns that were configured in `inferenceItemMetadata.itemColumns`: 43 | 44 | 45 | ```json 46 | { 47 | "itemList": [ 48 | { 49 | "itemId": "f6231107-7050-44ea-ac6a-dcb09f4a0b33", 50 | "score": 0.298052, 51 | "metadata": { 52 | "name": "Camping Lamp", 53 | "category": "outdoors", 54 | "description": "Be sure to bring this camping lamp with you to the outdoors", 55 | "price": 19.99 56 | } 57 | } 58 | ] 59 | } 60 | ``` 61 | 62 | Note that Amazon Personalize will convert snake case column names to camel case in the response. For example, the schema column `BRAND_NAME` will be converted to `brandName` in the response. In addition, categorical field values will be returned as formatted when they were ingested (i.e., `ONE|TWO|THREE`) into Amazon Personalize rather than being returned as an array of values (i.e., `["ONE","TWO","THREE"]`) 63 | 64 | ## Sidecar item metadata storage, retrieval, and injection 65 | 66 | As mentioned above, the Personalization APIs project also supports an item metadata sidecar feature whereby item metadata is injected into recommender responses before they are returned from the API layer. There are currently two sidecar implementations supported by the project. 67 | 68 | To take advantage of the sidecar item metadata capability, you upload your inference item metadata to the S3 staging bucket created by the Personalization APIs deployment. The name of this bucket can be found in the CloudFormation output parameters (look for the `StagingBucket` output parameter). When you upload your inference item metadata (described in detail below) to the appropriate folder in the staging bucket (the folder name is based on the namespace key), an AWS Lambda function is invoked that automatically updates the appropriate sidecar datastore(s) based on the configuration described below. **Therefore, it's vital that you update your configuration with inference item metadata configuration before uploading your item metadata to the staging bucket.** 69 | 70 | ### Local DBM datastore 71 | 72 | Declares that item metadata should be managed in a local DBM datastore for a namespace that is automatically downloaded from S3 and stored on the local Lambda volume of the API origin function. This option provides the lowest possible latency for item metadata decoration (~1-3ms) but is not suitable for very large item catalogs for when a large number of namespaces are served by the same Personalization APIs deployment. 73 | 74 | ```json 75 | { 76 | "namespaces": { 77 | "my-app-1": { 78 | "inferenceItemMetadata": { 79 | "type": "localdb", 80 | "syncInterval": 300 81 | }, 82 | "recommenders": { 83 | } 84 | } 85 | } 86 | } 87 | ``` 88 | 89 | - `namespaces.{NAMESPACE_KEY}.inferenceItemMetadata.type`: Must be `"localdb"` (required). 90 | - `namespaces.{NAMESPACE_KEY}.inferenceItemMetadata.syncInterval`: How often to sync DBM files from the S3 staging bucket in seconds (optional, default is 300 seconds). 91 | 92 | ### Amazon DynamoDB tables 93 | 94 | Declares that a Amazon DynamoDB table should be used to query for item metadata for a particular namespace. The table can optionally be automatically provisioned by the Personalization APIs solution (when the configuration changes) or you can create the table directly (see `autoProvision` field). The table name is derived based on a concatenation of `PersonalizationApiItemMetadata_` and the namespace key. So for the example configuration fragment below, the table name would be `PersonalizationApiItemMetadata_my-app1`. Therefore, if you create the DynamoDB table yourself, you must use the apppropriate table name. 95 | 96 | ```json 97 | { 98 | "namespaces": { 99 | "my-app-1": { 100 | "inferenceItemMetadata": { 101 | "autoProvision": true, 102 | "type": "dynamodb", 103 | "billingMode": "PROVISIONED", 104 | "provisionedThroughput": { 105 | "readCapacityUnits": 10, 106 | "writeCapacityUnits": 2 107 | } 108 | }, 109 | "recommenders": { 110 | } 111 | } 112 | } 113 | } 114 | ``` 115 | 116 | - `namespaces.{NAMESPACE_KEY}.inferenceItemMetadata.autoProvision`: boolean that controls whether the DynamoDB table should be created automatically on-the-fly and its billing mode and provisioned throughput updated based on the configuration (optional). The default is `true`. 117 | - `namespaces.{NAMESPACE_KEY}.inferenceItemMetadata.type`: Must be `"dynamodb"` (required). 118 | - `namespaces.{NAMESPACE_KEY}.inferenceItemMetadata.billingMode`: Valid values are `"PROVISIONED"` or `"PAY_PER_REQUEST"` (optional). Default is `"PAY_PER_REQUEST"`. 119 | - `namespaces.{NAMESPACE_KEY}.inferenceItemMetadata.provisionedThroughput.readCapacityUnits`: Read capacity units (required if `billingMode` is `"PROVISIONED"`). 120 | - `namespaces.{NAMESPACE_KEY}.inferenceItemMetadata.provisionedThroughput.writeCapacityUnits`: Write capacity units (required if `billingMode` is `"PROVISIONED"`). 121 | 122 | ### Preparing and Uploading Inference Item Metadata to S3 123 | 124 | When the Personalization APIs solution is deployed, an S3 bucket is created that is used as a staging area for uploading item metadata. The bucket name can be determined from the CloudFormation output parameter named `StagingBucket`. To provide inference item metadata for your items to the Personalization APIs solution, create a [JSON Lines](https://jsonlines.org/) file for each namespace where each file contains metadata for every item that could be recommended by recommenders for that namespace. All of the fields for an item's ID will be used to decorate the response for the item. For example, the following JSONL fragment includes metadata for 6 products for a sample e-commerce item catalog. 125 | 126 | ```json 127 | {"id": "6579c22f-be2b-444c-a52b-0116dd82df6c", "current_stock": 15, "name": "Tan Backpack", "category": "accessories", "style": "backpack", "description": "This tan backpack is nifty for traveling", "price": 90.99, "image": "6579c22f-be2b-444c-a52b-0116dd82df6c.jpg", "gender_affinity": "F", "where_visible": "UI", "image_url": "https://d22kv7nk938ern.cloudfront.net/images/accessories/6579c22f-be2b-444c-a52b-0116dd82df6c.jpg"} 128 | {"id": "2e852905-c6f4-47db-802c-654013571922", "current_stock": 15, "name": "Pale Pink Backpack", "category": "accessories", "style": "backpack", "description": "Pale pink backpack for women", "price": 123.99, "image": "2e852905-c6f4-47db-802c-654013571922.jpg", "gender_affinity": "F", "where_visible": "UI", "image_url": "https://d22kv7nk938ern.cloudfront.net/images/accessories/2e852905-c6f4-47db-802c-654013571922.jpg"} 129 | {"id": "4ec7ff5c-f70f-4984-b6c4-c7ef37cc0c09", "current_stock": 17, "name": "Gainsboro Backpack", "category": "accessories", "style": "backpack", "description": "This gainsboro backpack for women is first-rate for the season", "price": 87.99, "image": "4ec7ff5c-f70f-4984-b6c4-c7ef37cc0c09.jpg", "gender_affinity": "F", "where_visible": "UI", "image_url": "https://d22kv7nk938ern.cloudfront.net/images/accessories/4ec7ff5c-f70f-4984-b6c4-c7ef37cc0c09.jpg"} 130 | {"id": "7977f680-2cf7-457d-8f4d-afa0aa168cb9", "current_stock": 17, "name": "Gray Backpack", "category": "accessories", "style": "backpack", "description": "This gray backpack for women is first-rate for the season", "price": 125.99, "image": "7977f680-2cf7-457d-8f4d-afa0aa168cb9.jpg", "gender_affinity": "F", "where_visible": "UI", "image_url": "https://d22kv7nk938ern.cloudfront.net/images/accessories/7977f680-2cf7-457d-8f4d-afa0aa168cb9.jpg"} 131 | {"id": "b5649d7c-4651-458d-a07f-912f253784ce", "current_stock": 13, "name": "Peru-Orange Backpack", "category": "accessories", "style": "backpack", "description": "Peru-orange backpack for women", "price": 141.99, "image": "b5649d7c-4651-458d-a07f-912f253784ce.jpg", "gender_affinity": "F", "where_visible": "UI", "image_url": "https://d22kv7nk938ern.cloudfront.net/images/accessories/b5649d7c-4651-458d-a07f-912f253784ce.jpg"} 132 | {"id": "296d144e-7f86-464b-9c5a-f545257f1700", "current_stock": 11, "name": "Black Backpack", "category": "accessories", "style": "backpack", "description": "This black backpack for women is first-class for the season", "price": 144.99, "image": "296d144e-7f86-464b-9c5a-f545257f1700.jpg", "gender_affinity": "F", "where_visible": "UI", "image_url": "https://d22kv7nk938ern.cloudfront.net/images/accessories/296d144e-7f86-464b-9c5a-f545257f1700.jpg"} 133 | ``` 134 | 135 | The only requirements for this file is that every line represent a single item as a complete JSON document and that the document has an `"id"` field that represents the item's ID. The item ID should be the same ID that will be returned by recommenders for the namespace. You can optionally gzip the file before uploading it to the S3 staging bucket. The file must be uploaded into a folder in the staging bucket with the following format. 136 | 137 | ``` 138 | import/{NAMESPACE_KEY}/ 139 | ``` 140 | 141 | Where `{NAMESPACE_KEY}` is the namespace key in the configuration. For the example configuration fragments above for the `my-app-1` namespace, if you put your metadata in a file named `item-metadata.jsonl` and gzipped the file, the file would be uploaded as: 142 | 143 | ``` 144 | import/my-app-1/item-metadata.jsonl.gz 145 | ``` 146 | 147 | Once the file has been uploaded for a namespace, the file is loaded into the datastore based on the configuration. For example, if DynamoDB is configured as the item metadata datastore type for a namespace, the contents of the uploaded file will be loaded into a DynamoDB table for the namespace. Any items that already existed in the table that were not included in the uploaded file be automatically deleted. If the datastore type is `localdb`, a DBM file will be built and staged in the S3 staging bucket in the `localdbs/` folder. This staged DBM file is automatically downloaded by the Personalization APIs Lambda function to the Lambda instance's local volume. 148 | 149 | A decorated API response that includes item metadata would look something like this where the `metadata` dictionary includes the item metadata that was uploaded to the S3 staging bucket: 150 | 151 | ```json 152 | { 153 | "itemList": [ 154 | { 155 | "itemId": "f6231107-7050-44ea-ac6a-dcb09f4a0b33", 156 | "score": 0.298052, 157 | "metadata": { 158 | "current_stock": 16, 159 | "name": "Camping Lamp", 160 | "category": "outdoors", 161 | "style": "camping", 162 | "description": "Be sure to bring this camping lamp with you to the outdoors", 163 | "price": 19.99, 164 | "image": "f6231107-7050-44ea-ac6a-dcb09f4a0b33.jpg", 165 | "where_visible": "UI", 166 | "image_url": "https://d22kv7nk938ern.cloudfront.net/images/outdoors/f6231107-7050-44ea-ac6a-dcb09f4a0b33.jpg" 167 | } 168 | } 169 | ] 170 | } 171 | ``` 172 | -------------------------------------------------------------------------------- /images/architecture-apikey.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/personalization-apis/2593e0f0a8f22a25440be1a2e6c52fcefd2ab65f/images/architecture-apikey.png -------------------------------------------------------------------------------- /images/architecture-noauth.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/personalization-apis/2593e0f0a8f22a25440be1a2e6c52fcefd2ab65f/images/architecture-noauth.png -------------------------------------------------------------------------------- /images/architecture-oauth2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/personalization-apis/2593e0f0a8f22a25440be1a2e6c52fcefd2ab65f/images/architecture-oauth2.png -------------------------------------------------------------------------------- /images/config-create.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/personalization-apis/2593e0f0a8f22a25440be1a2e6c52fcefd2ab65f/images/config-create.png -------------------------------------------------------------------------------- /images/configuration-concepts.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/personalization-apis/2593e0f0a8f22a25440be1a2e6c52fcefd2ab65f/images/configuration-concepts.png -------------------------------------------------------------------------------- /images/personalization-apis-value-prop.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/personalization-apis/2593e0f0a8f22a25440be1a2e6c52fcefd2ab65f/images/personalization-apis-value-prop.png -------------------------------------------------------------------------------- /images/postman.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/personalization-apis/2593e0f0a8f22a25440be1a2e6c52fcefd2ab65f/images/postman.png -------------------------------------------------------------------------------- /images/swagger_editor.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/personalization-apis/2593e0f0a8f22a25440be1a2e6c52fcefd2ab65f/images/swagger_editor.png -------------------------------------------------------------------------------- /images/swagger_ui.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/personalization-apis/2593e0f0a8f22a25440be1a2e6c52fcefd2ab65f/images/swagger_ui.png -------------------------------------------------------------------------------- /run_tests.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | PYTHONPATH=./src pytest --cov=src --cov-branch --cov-report term-missing -------------------------------------------------------------------------------- /samconfig-edge.toml: -------------------------------------------------------------------------------- 1 | version = 0.1 2 | [default] 3 | [default.deploy] 4 | [default.deploy.parameters] 5 | stack_name = "personalization-apis-edge" 6 | region = "us-east-1" 7 | confirm_changeset = true 8 | capabilities = "CAPABILITY_IAM" 9 | image_repositories = [] 10 | -------------------------------------------------------------------------------- /samconfig.toml: -------------------------------------------------------------------------------- 1 | version = 0.1 2 | [oauth2] 3 | [oauth2.deploy] 4 | [oauth2.deploy.parameters] 5 | stack_name = "p13n-apis-oauth2" 6 | confirm_changeset = true 7 | capabilities = "CAPABILITY_IAM" 8 | parameter_overrides = "ApplicationName=\"P13n-APIs-oauth2\" EnvironmentName=\"prod\" TimeZone=\"UTC\" AuthenticationScheme=\"OAuth2-Cognito\" CreateCognitoResources=\"Yes\" ApiEntryPointType=\"API-Gateway-HTTP\" CacheScheme=\"CloudFront\"" 9 | image_repositories = [] 10 | 11 | [apikey] 12 | [apikey.deploy] 13 | [apikey.deploy.parameters] 14 | stack_name = "p13n-apis-apikey" 15 | region = "us-east-1" 16 | confirm_changeset = true 17 | capabilities = "CAPABILITY_IAM" 18 | parameter_overrides = "ApplicationName=\"P13n-APIs-apikey\" EnvironmentName=\"prod\" TimeZone=\"UTC\" AuthenticationScheme=\"ApiKey\" CreateCognitoResources=\"No\" ApiEntryPointType=\"API-Gateway-REST\" CacheScheme=\"API-Gateway-Cache\"" 19 | image_repositories = [] 20 | 21 | [noauth] 22 | [noauth.deploy] 23 | [noauth.deploy.parameters] 24 | stack_name = "p13n-apis-noauth" 25 | region = "us-east-1" 26 | confirm_changeset = true 27 | capabilities = "CAPABILITY_IAM" 28 | parameter_overrides = "ApplicationName=\"P13n-APIs-noauth\" EnvironmentName=\"prod\" TimeZone=\"UTC\" AuthenticationScheme=\"None\" CreateCognitoResources=\"No\" ApiEntryPointType=\"API-Gateway-HTTP\" CacheScheme=\"CloudFront\" GenerateConfigDatasetGroupNames=\"retaildemostore-products,personalize-poc-movielens-20m\" CreateSwaggerUI=\"Yes\"" 29 | image_repositories = [] 30 | resolve_s3 = true 31 | s3_prefix = "p13n-apis-noauth" 32 | -------------------------------------------------------------------------------- /samples/config_auto_context.json: -------------------------------------------------------------------------------- 1 | { 2 | "version": "1", 3 | "namespaces": { 4 | "my-store": { 5 | "inferenceItemMetadata": { 6 | "type": "localdb", 7 | "syncInterval": 300 8 | }, 9 | "autoContext": { 10 | "deviceType": { 11 | "type": "string", 12 | "default": "Desktop", 13 | "evaluateAll": true, 14 | "rules": [ 15 | { 16 | "type": "header-value", 17 | "header": "cloudfront-is-desktop-viewer", 18 | "valueMappings": [ 19 | { 20 | "operator": "equals", 21 | "value": "true", 22 | "mapTo": "Desktop" 23 | } 24 | ] 25 | }, 26 | { 27 | "type": "header-value", 28 | "header": "cloudfront-is-mobile-viewer", 29 | "valueMappings": [ 30 | { 31 | "operator": "equals", 32 | "value": "true", 33 | "mapTo": "Phone" 34 | } 35 | ] 36 | }, 37 | { 38 | "type": "header-value", 39 | "header": "cloudfront-is-smarttv-viewer", 40 | "valueMappings": [ 41 | { 42 | "operator": "equals", 43 | "value": "true", 44 | "mapTo": "TV" 45 | } 46 | ] 47 | }, 48 | { 49 | "type": "header-value", 50 | "header": "cloudfront-is-tablet-viewer", 51 | "valueMappings": [ 52 | { 53 | "operator": "equals", 54 | "value": "true", 55 | "mapTo": "Tablet" 56 | } 57 | ] 58 | } 59 | ] 60 | }, 61 | "timeOfDay": { 62 | "type": "string", 63 | "evaluateAll": false, 64 | "rules": [ 65 | { 66 | "type": "hour-of-day", 67 | "valueMappings": [ 68 | { 69 | "operator": "less-than", 70 | "value": 4, 71 | "mapTo": "Night" 72 | }, 73 | { 74 | "operator": "less-than", 75 | "value": 11, 76 | "mapTo": "Morning" 77 | }, 78 | { 79 | "operator": "less-than", 80 | "value": 18, 81 | "mapTo": "Afternoon" 82 | }, 83 | { 84 | "operator": "less-than", 85 | "value": 22, 86 | "mapTo": "Evening" 87 | }, 88 | { 89 | "operator": "greater-than", 90 | "value": 21, 91 | "mapTo": "Night" 92 | } 93 | ] 94 | } 95 | ] 96 | }, 97 | "city": { 98 | "type": "string", 99 | "rules": [ 100 | { 101 | "type": "header-value", 102 | "header": "cloudfront-viewer-city" 103 | } 104 | ] 105 | } 106 | }, 107 | "recommenders": { 108 | "recommend-items": { 109 | "product-recommender": { 110 | "variations": { 111 | "product-personalization": { 112 | "type": "personalize-campaign", 113 | "arn": "arn:aws:personalize:[REGION]:[ACCOUNT]:campaign/[CAMPAIGN_NAME]", 114 | "filters": [{ 115 | "arn": "arn:aws:personalize:[REGION]:[ACCOUNT]:filter/[FILTER_NAME]", 116 | "autoDynamicFilterValues": { 117 | "METRO_CODE": { 118 | "type": "string", 119 | "rules": [ 120 | { 121 | "type": "header-value", 122 | "header": "cloudfront-viewer-metro-code" 123 | } 124 | ] 125 | } 126 | } 127 | }] 128 | } 129 | } 130 | } 131 | } 132 | }, 133 | "eventTargets": [ 134 | { 135 | "type": "personalize-event-tracker", 136 | "trackingId": "[TRACKING_ID]" 137 | } 138 | ] 139 | } 140 | } 141 | } -------------------------------------------------------------------------------- /samples/config_experiment.json: -------------------------------------------------------------------------------- 1 | { 2 | "version": "1", 3 | "namespaces": { 4 | "my-store": { 5 | "inferenceItemMetadata": { 6 | "type": "localdb", 7 | "syncInterval": 300 8 | }, 9 | "recommenders": { 10 | "related-items": { 11 | "similar": { 12 | "experiments": { 13 | "product-detail-view": { 14 | "method": "evidently", 15 | "project": "my-store", 16 | "metrics": { 17 | "productDetailRecClicked": { 18 | "trackExposures": true, 19 | "entityIdKey": "userDetails.userId", 20 | "valueKey": "details.productClicked" 21 | } 22 | } 23 | } 24 | }, 25 | "variations": { 26 | "sims": { 27 | "type": "personalize-campaign", 28 | "arn": "arn:aws:personalize:[REGION:[ACCOUNT]:campaign/[CAMPAIGN_NAME]" 29 | }, 30 | "similar-items": { 31 | "type": "personalize-campaign", 32 | "arn": "arn:aws:personalize:[REGION:[ACCOUNT]:campaign/[CAMPAIGN_NAME]" 33 | } 34 | } 35 | } 36 | } 37 | } 38 | } 39 | } 40 | } -------------------------------------------------------------------------------- /samples/config_simple.json: -------------------------------------------------------------------------------- 1 | { 2 | "version": "1", 3 | "cacheControl": { 4 | "autoProvision": true, 5 | "userSpecified": { 6 | "maxAge": 10, 7 | "directives": "private" 8 | }, 9 | "syntheticUserSpecified": { 10 | "maxAge": 300, 11 | "directives": "public" 12 | }, 13 | "noUserSpecified": { 14 | "maxAge": 1200, 15 | "directives": "public" 16 | } 17 | }, 18 | "namespaces": { 19 | "my-store": { 20 | "name": "My Store", 21 | "inferenceItemMetadata": { 22 | "type": "localdb", 23 | "syncInterval": 300 24 | }, 25 | "recommenders": { 26 | "recommend-items": { 27 | "recommended-for-you": { 28 | "variations": { 29 | "user-personalization": { 30 | "type": "personalize-campaign", 31 | "arn": "arn:aws:personalize:[REGION]:[ACCOUNT]:campaign/[CAMPAIGN_NAME]", 32 | "filters": [{ 33 | "arn": "arn:aws:personalize:[REGION]:[ACCOUNT]:filter/[FILTER_NAME]" 34 | }] 35 | } 36 | } 37 | } 38 | }, 39 | "related-items": { 40 | "similar": { 41 | "variations": { 42 | "sims": { 43 | "type": "personalize-campaign", 44 | "arn": "arn:aws:personalize:[REGION]:[ACCOUNT]:campaign/[CAMPAIGN_NAME]" 45 | } 46 | } 47 | } 48 | }, 49 | "rerank-items": { 50 | "personalized-ranking": { 51 | "variations": { 52 | "ranking": { 53 | "type": "personalize-campaign", 54 | "arn": "arn:aws:personalize:[REGION]:[ACCOUNT]:campaign/[CAMPAIGN_NAME]" 55 | } 56 | } 57 | } 58 | } 59 | }, 60 | "eventTargets": [ 61 | { 62 | "type": "personalize-event-tracker", 63 | "trackingId": "[TRACKING_ID]" 64 | } 65 | ] 66 | } 67 | } 68 | } -------------------------------------------------------------------------------- /samples/lambdas/custom_recommender_lambda.py: -------------------------------------------------------------------------------- 1 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # SPDX-License-Identifier: MIT-0 3 | 4 | ''' 5 | This is a very simple function that provides a sample of how a custom 6 | recommender can be implemented using a Lambda function. A real custom 7 | recommender would make the appropriate calls to a custom model or 8 | rule-based approach to item recommendations. 9 | 10 | To wire up a custom recommender, add the recommender as a "lambda" 11 | type with the Lambda function ARN like the following. 12 | 13 | { 14 | "namespaces": { 15 | "my-namespace": { 16 | "recommenders": { 17 | "recommend-items": { 18 | "lambda-recs": { 19 | "variations": { 20 | "lambda-rfy": { 21 | "type": "lambda", 22 | "arn": "arn:aws:lambda:us-east-1:999999999999:function:My-Custom-Function" 23 | } 24 | } 25 | } 26 | } 27 | } 28 | } 29 | } 30 | } 31 | 32 | You will also need to modify the IAM role for the PersonalizationHttpApiFunction or 33 | PersonalizationRestApiFunction function (PersonalizationApiExecutionRole) to add a policy 34 | that allows "lambda:InvokeFunction" for the same function ARN in the configuration. 35 | 36 | { 37 | "Action": [ 38 | "lambda:InvokeFunction" 39 | ], 40 | "Effect": "Allow", 41 | "Resource": "arn:aws:lambda:us-east-1:999999999999:function:My-Custom-Function" 42 | } 43 | ''' 44 | 45 | import json 46 | import logging 47 | 48 | logger = logging.getLogger() 49 | logger.setLevel(logging.INFO) 50 | 51 | def lambda_handler(event, _): 52 | logger.info(json.dumps(event, indent=2, default=str)) 53 | 54 | recs_to_generate = event.get('numResults', 10) 55 | 56 | recs = [] 57 | for i in range(recs_to_generate): 58 | recs.append({'itemId': f'item-{i+1}'}) 59 | 60 | return { 'itemList': recs } -------------------------------------------------------------------------------- /src/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/personalization-apis/2593e0f0a8f22a25440be1a2e6c52fcefd2ab65f/src/__init__.py -------------------------------------------------------------------------------- /src/config_validator_env_function/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/personalization-apis/2593e0f0a8f22a25440be1a2e6c52fcefd2ab65f/src/config_validator_env_function/__init__.py -------------------------------------------------------------------------------- /src/config_validator_env_function/main.py: -------------------------------------------------------------------------------- 1 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # SPDX-License-Identifier: MIT-0 3 | 4 | """ 5 | This custom resource function is responsible for updating the environment 6 | variables for the ConfigValidatorFunction with the generated values for 7 | the API Gateway host/URL and CloudFront host/URL. A custom resource had 8 | to be used to workaround dependency recursion in the CloudFormation template. 9 | """ 10 | 11 | import os 12 | import boto3 13 | import json 14 | import logging 15 | 16 | from crhelper import CfnResource 17 | 18 | logger = logging.getLogger() 19 | logger.setLevel(logging.DEBUG) 20 | 21 | helper = CfnResource() 22 | 23 | function_arn = os.environ['ConfigValidatorFunctionArn'] 24 | apigw_host = os.environ['ApiGatewayHost'] 25 | cloudfront_host = os.environ['CloudFrontHost'] 26 | 27 | lambda_client = boto3.client('lambda') 28 | 29 | def update_function(): 30 | """ Updates the configuration validator function's environment variables""" 31 | logger.info('Updating function configuration: %s', function_arn) 32 | 33 | logger.info('Getting current function configuration/environment variables') 34 | response = lambda_client.get_function_configuration(FunctionName = function_arn) 35 | logger.debug(response) 36 | 37 | env = response.get('Environment', {}) 38 | env_vars = env.setdefault('Variables', {}) 39 | env_vars['ApiGatewayHost'] = apigw_host 40 | env_vars['CloudFrontHost'] = cloudfront_host 41 | 42 | logger.info('Updating function environment variables') 43 | logger.debug(env) 44 | response = lambda_client.update_function_configuration( 45 | FunctionName = function_arn, 46 | Environment = env 47 | ) 48 | logger.debug(response) 49 | 50 | @helper.create 51 | @helper.update 52 | def create_or_update_resource(event, _): 53 | update_function() 54 | 55 | def lambda_handler(event, context): 56 | logger.info(os.environ) 57 | logger.info(json.dumps(event, indent = 2, default = str)) 58 | 59 | # If the event has a RequestType, we're being called by CFN as custom resource 60 | if event.get('RequestType'): 61 | logger.info('Function called from CloudFormation as custom resource') 62 | helper(event, context) 63 | else: 64 | logger.info('Function called outside of CloudFormation') 65 | # Call function directly (i.e. testing in Lambda console or called directly) 66 | update_function() 67 | -------------------------------------------------------------------------------- /src/config_validator_env_function/requirements.txt: -------------------------------------------------------------------------------- 1 | crhelper -------------------------------------------------------------------------------- /src/config_validator_function/README.md: -------------------------------------------------------------------------------- 1 | # Cofiguration validator function 2 | 3 | This Lambda function is called automatically by [AWS AppConfig](https://aws.amazon.com/systems-manager/features/appconfig/) as a [configuration validator](https://docs.aws.amazon.com/appconfig/latest/userguide/appconfig-creating-configuration-and-profile-validators.html) when you create/update your configuration. This function will validate your configuration against the [JSON schema](./personalization_apis_config_schema.json) to ensure it's formatted correctly and syntactically accurate. 4 | 5 | If the configuration passes validation, an event will be fired to [Amazon EventBridge](https://aws.amazon.com/eventbridge/) that will trigger the execution of an [AWS Step Functions](https://aws.amazon.com/step-functions/?step-functions.sort-by=item.additionalFields.postDateTime&step-functions.sort-order=desc) [state machine](../statemachine/sync_resources.asl.json). This state machine will [synchronize cache settings](../sync_cache_settings_function/) in your configuration to CloudFront or API Gateway and [prepare item metadata datastores](../sync_dynamodb_tables_function/). -------------------------------------------------------------------------------- /src/config_validator_function/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/personalization-apis/2593e0f0a8f22a25440be1a2e6c52fcefd2ab65f/src/config_validator_function/__init__.py -------------------------------------------------------------------------------- /src/config_validator_function/main.py: -------------------------------------------------------------------------------- 1 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # SPDX-License-Identifier: MIT-0 3 | 4 | import boto3 5 | import json 6 | import jsonschema 7 | import os 8 | import base64 9 | 10 | from typing import List 11 | from aws_lambda_powertools import Logger, Tracer 12 | from openapi import OpenApiGenerator 13 | 14 | tracer = Tracer() 15 | logger = Logger() 16 | 17 | staging_bucket = os.environ['StagingBucket'] 18 | openapi_output_key = 'openapi/openapi.json' 19 | 20 | apigw_host = os.environ['ApiGatewayHost'] 21 | cloudfront_host = os.environ['CloudFrontHost'] 22 | api_auth_scheme = os.environ['AuthenticationScheme'] 23 | 24 | s3 = boto3.resource('s3') 25 | event_bridge = boto3.client('events') 26 | 27 | ''' 28 | Event structure: 29 | { 30 | "applicationId": "The application Id of the configuration profile being validated", 31 | "configurationProfileId": "The configuration profile Id of the configuration profile being validated", 32 | "configurationVersion": "The configuration version of the configuration profile being validated", 33 | "content": "Base64EncodedByteString", 34 | "uri": "The uri of the configuration (e.g. 'hosted')" 35 | } 36 | ''' 37 | 38 | SCHEMA_FILE = 'personalization_apis_config_schema.json' 39 | 40 | logger.info('Loading schema from %s', SCHEMA_FILE) 41 | with open(SCHEMA_FILE) as file: 42 | schema = json.loads(file.read()) 43 | 44 | @logger.inject_lambda_context 45 | @tracer.capture_lambda_handler 46 | def lambda_handler(event, _): 47 | # Step 1: validate configuration against the JSON Schema 48 | personalization_config = json.loads(base64.b64decode(event['content'])) 49 | 50 | try: 51 | jsonschema.validate(instance = personalization_config, schema = schema) 52 | except jsonschema.exceptions.ValidationError as e: 53 | # Log all details of the exception 54 | logger.exception(e) 55 | # Return a user-friendly error message 56 | raise Exception(e.message) 57 | 58 | # Step 2: do some logical validation against the configuration 59 | 60 | errors: List[str] = [] 61 | # TODO 62 | if errors: 63 | raise ValueError('; '.join(errors)) 64 | 65 | # Step 3: generate an updated OpenAPI spec file and save to the staging bucket. 66 | openapi_generator = OpenApiGenerator() 67 | openapi_spec = openapi_generator.generate( 68 | apis_config = personalization_config, 69 | apigw_host = apigw_host, 70 | cloudfront_host = cloudfront_host, 71 | auth_scheme = api_auth_scheme 72 | ) 73 | 74 | openapi_uri = None 75 | if 'paths' in openapi_spec and len(openapi_spec['paths']) > 0: 76 | object = s3.Object(staging_bucket, openapi_output_key) 77 | result = object.put(Body=json.dumps(openapi_spec, indent = 4)) 78 | logger.debug(result) 79 | openapi_uri = f's3://{staging_bucket}/{openapi_output_key}' 80 | 81 | # Step 4: post an event to EventBridge to trigger the resource synchronization step functions state machine. 82 | 83 | # Set the decoded config into event so that it's accessible as JSON in targets (i.e., step function state machine). 84 | event['content'] = personalization_config 85 | if openapi_uri: 86 | event['openApiSpecUri'] = openapi_uri 87 | 88 | region = os.environ['AWS_REGION'] 89 | account_id = boto3.client('sts').get_caller_identity()['Account'] 90 | resource = f'arn::appconfig:{region}:{account_id}:application/{event["applicationId"]}/configurationprofile/{event["configurationProfileId"]}' 91 | 92 | event_bridge.put_events( 93 | Entries=[ 94 | { 95 | 'Source': 'personalization.apis', 96 | 'Resources': [ resource ], 97 | 'DetailType': 'PersonalizationApisConfigurationChange', 98 | 'Detail': json.dumps(event) 99 | } 100 | ] 101 | ) 102 | -------------------------------------------------------------------------------- /src/config_validator_function/requirements.txt: -------------------------------------------------------------------------------- 1 | # If jsonschema version is changed, be sure to test against sample schemas to make sure recursion errors in validator have been fixed. 2 | jsonschema==3.2.0 -------------------------------------------------------------------------------- /src/copy_swagger_ui_assets_function/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/personalization-apis/2593e0f0a8f22a25440be1a2e6c52fcefd2ab65f/src/copy_swagger_ui_assets_function/__init__.py -------------------------------------------------------------------------------- /src/copy_swagger_ui_assets_function/main.py: -------------------------------------------------------------------------------- 1 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # SPDX-License-Identifier: MIT-0 3 | 4 | import os 5 | import json 6 | import boto3 7 | import logging 8 | import requests 9 | 10 | from typing import Dict 11 | from crhelper import CfnResource 12 | 13 | logger = logging.getLogger() 14 | logger.setLevel(logging.DEBUG) 15 | 16 | helper = CfnResource() 17 | 18 | s3 = boto3.resource('s3') 19 | 20 | assets = [ 21 | { 22 | 'source': 'https://raw.githubusercontent.com/aws-samples/personalization-apis/main/src/swagger-ui/index.html', 23 | 'target': 'index.html' 24 | }, 25 | { 26 | 'source': 'https://raw.githubusercontent.com/aws-samples/personalization-apis/main/src/swagger-ui/favicon-32x32.png', 27 | 'target': 'favicon-32x32.png' 28 | }, 29 | { 30 | 'source': 'https://raw.githubusercontent.com/aws-samples/personalization-apis/main/src/swagger-ui/favicon-16x16.png', 31 | 'target': 'favicon-16x16.png' 32 | } 33 | ] 34 | 35 | def copy_assets(event: Dict): 36 | target_bucket_name = event['ResourceProperties']['TargetBucket'] 37 | logger.info('Copying Swagger UI assets to %s', target_bucket_name) 38 | 39 | bucket = s3.Bucket(target_bucket_name) 40 | 41 | for asset in assets: 42 | r = requests.get(asset['source']) 43 | if r.ok: 44 | logger.info('Saving %s to %s', asset['target'], target_bucket_name) 45 | content_type = r.headers['Content-Type'] 46 | if asset['target'].endswith('.html'): 47 | content_type = 'text/html' 48 | response = bucket.put_object(Key = asset['target'], Body = r.content, ContentType = content_type) 49 | logger.debug(response) 50 | else: 51 | logger.error('Error retrieving asset %s: %s', asset['source'], r.status_code) 52 | 53 | logger.info('Successfully copied static assets to target bucket') 54 | 55 | @helper.create 56 | @helper.update 57 | def create_or_update_resource(event, _): 58 | copy_assets(event) 59 | 60 | def lambda_handler(event, context): 61 | logger.info(os.environ) 62 | logger.info(json.dumps(event, indent = 2, default = str)) 63 | 64 | # If the event has a RequestType, we're being called by CFN as custom resource 65 | if event.get('RequestType'): 66 | logger.info('Function called from CloudFormation as custom resource') 67 | helper(event, context) 68 | else: 69 | logger.info('Function called outside of CloudFormation') 70 | # Call function directly (i.e. testing in Lambda console or called directly) 71 | copy_assets(event) 72 | -------------------------------------------------------------------------------- /src/copy_swagger_ui_assets_function/requirements.txt: -------------------------------------------------------------------------------- 1 | crhelper 2 | requests -------------------------------------------------------------------------------- /src/edge_auth_function/README.md: -------------------------------------------------------------------------------- 1 | # Edge authentication function (OAuth2) 2 | 3 | This [Lambda@Edge](https://aws.amazon.com/lambda/edge/) function performs OAuth2 token verification within CloudFront. Therefore it is only needed when the authentication mode specified at deployment is `OAuth2-Cognito`. 4 | 5 | The reason for performing authentication in CloudFront using this function rather just letting API Gateway handle OAuth2 authentication is to allow CloudFront caches to be used to cache responses closer to users (and therefore further reduce latency). If API Gateway was used for authentication, every request would have to travel to API Gateway. 6 | 7 | You can install the OAuth2 edge resources into the us-east-1 region using the "Launch Stack" button below. 8 | 9 | [![Launch Stack](https://cdn.rawgit.com/buildkite/cloudformation-launch-stack-button-svg/master/launch-stack.svg)](https://console.aws.amazon.com/cloudformation/home?region=us-east-1#/stacks/create/review?templateURL=https://s3.amazonaws.com/personalize-solution-staging-us-east-1/personalization-apis-edge/template.yaml&stackName=personalization-apis-edge) 10 | 11 | If you'd rather install the OAuth2 edge resources manually using AWS SAM, execute the following commands at the command line (the `deploy_edge_auth.sh` shell script can also be used as a shortcut). 12 | 13 | ```bash 14 | sam build --use-container --cached --template-file template-edge.yaml && sam deploy --region us-east-1 --config-file samconfig-edge.toml --guided 15 | ``` 16 | 17 | Once deployment finishes successfully, sign in to the AWS console, switch to the `N. Virginia - us-east-1` region, browse to the Lambda service page, find the `EdgeAuthFunction`, and deploy it to the CloudFront distribution created when you installed the solution (check the CloudFormation output parameters to determine the distribution URL and ID) as a **Viewer Request**. See the [API authentication documentation](../../docs/api_authentication.md) for details. This does not mean that the Personalization APIs must be deployed in "us-east-1" as well--those resources should be deployed in the AWS region where your recommenders are deployed. 18 | 19 | **API requests will not be authenticated by this function until the edge function is successfully deployed to CloudFront.** -------------------------------------------------------------------------------- /src/edge_auth_function/index.js: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // SPDX-License-Identifier: MIT-0 3 | 4 | 'use strict'; 5 | var jwt = require('jsonwebtoken'); 6 | var jwkToPem = require('jwk-to-pem'); 7 | 8 | var USERPOOLID = '##USERPOOLID##'; 9 | var JWKS = '##JWKS##'; 10 | var COGNITOREGION = '##COGNITOREGION##'; 11 | 12 | var iss = 'https://cognito-idp.' + COGNITOREGION + '.amazonaws.com/' + USERPOOLID; 13 | var pems; 14 | 15 | var NO_AUTH_METHODS = [ 'OPTIONS' ] 16 | 17 | pems = {}; 18 | var keys = JSON.parse(JWKS).keys; 19 | for(var i = 0; i < keys.length; i++) { 20 | //Convert each key to PEM 21 | var key_id = keys[i].kid; 22 | var modulus = keys[i].n; 23 | var exponent = keys[i].e; 24 | var key_type = keys[i].kty; 25 | var jwk = { kty: key_type, n: modulus, e: exponent}; 26 | var pem = jwkToPem(jwk); 27 | pems[key_id] = pem; 28 | } 29 | 30 | const response401 = { 31 | status: '401', 32 | statusDescription: 'Unauthorized' 33 | }; 34 | 35 | exports.handler = (event, context, callback) => { 36 | const cfrequest = event.Records[0].cf.request; 37 | const headers = cfrequest.headers; 38 | console.log('getting started'); 39 | console.log('pems=' + pems); 40 | 41 | // If origin header is missing, set it equal to the host header. 42 | if (!headers.origin && headers.host) { 43 | console.log('Request is missing Origin header; adding header'); 44 | var host = headers.host[0].value; 45 | headers.origin = [ {key: 'Origin', value:`https://${host}`} ]; 46 | } 47 | 48 | // Fail if no authorization header found 49 | if (!headers.authorization) { 50 | if (NO_AUTH_METHODS.indexOf(cfrequest.method.toUpperCase()) != -1) { 51 | console.log('Request is missing authorization header but method is allowed to pass-through; sending request through'); 52 | callback(null, cfrequest); 53 | return true; 54 | } 55 | console.log("no auth header"); 56 | callback(null, response401); 57 | return false; 58 | } 59 | 60 | // Strip "Bearer " from header value to extract JWT token only 61 | var jwtToken = headers.authorization[0].value.slice(7); 62 | console.log('jwtToken=' + jwtToken); 63 | 64 | //Fail if the token is not jwt 65 | var decodedJwt = jwt.decode(jwtToken, {complete: true}); 66 | if (!decodedJwt) { 67 | console.log("Not a valid JWT token"); 68 | callback(null, response401); 69 | return false; 70 | } 71 | 72 | // Fail if token is not from your UserPool 73 | if (decodedJwt.payload.iss != iss) { 74 | console.log("invalid issuer"); 75 | callback(null, response401); 76 | return false; 77 | } 78 | 79 | //Reject the jwt if it's not an 'Access Token' 80 | if (decodedJwt.payload.token_use != 'access') { 81 | console.log("Not an access token"); 82 | callback(null, response401); 83 | return false; 84 | } 85 | 86 | //Get the kid from the token and retrieve corresponding PEM 87 | var kid = decodedJwt.header.kid; 88 | var pem = pems[kid]; 89 | if (!pem) { 90 | console.log('Invalid access token'); 91 | callback(null, response401); 92 | return false; 93 | } 94 | 95 | console.log('Start verify token'); 96 | 97 | //Verify the signature of the JWT token to ensure it's really coming from your User Pool 98 | jwt.verify(jwtToken, pem, { issuer: iss }, function(err, payload) { 99 | if(err) { 100 | console.log('Token failed verification'); 101 | callback(null, response401); 102 | return false; 103 | } else { 104 | //Valid token. 105 | console.log('Successful verification'); 106 | //remove authorization header 107 | delete cfrequest.headers.authorization; 108 | //CloudFront can proceed to fetch the content from origin 109 | callback(null, cfrequest); 110 | return true; 111 | } 112 | }); 113 | }; 114 | -------------------------------------------------------------------------------- /src/edge_auth_function/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "edge_auth_function", 3 | "version": "1.0.0", 4 | "description": "Edge authentication with Cognito", 5 | "main": "index.js", 6 | "license": "MIT-0", 7 | "dependencies": { 8 | "jsonwebtoken": "^8.5.1", 9 | "jwk-to-pem": "^2.0.5" 10 | } 11 | } -------------------------------------------------------------------------------- /src/edge_update_function/README.md: -------------------------------------------------------------------------------- 1 | # Edge authentication function updater (OAuth2) 2 | 3 | This AWS Lambda function is CloudFormation custom resource function and is a companion to the [edge_auth_function](../edge_auth_function/) Lambda@Edge function. It is responsible for injecting Cognito and JWT details into the [edge_auth_function/index.js](../edge_auth_function/index.js) source file. This is required since Lambda@Edge functions do no support Lambda environment variables. 4 | 5 | **This function is NOT a Lambda@Edge function! It's a regular Lambda function that is called by CloudFormation when the [template-edge.yaml](../../template-edge.yaml) template is deployed.** You can also call it directly to trigger the [edge_auth_function](../edge_auth_function/) to be updated when, say, you want to switch Cognito user pool settings. -------------------------------------------------------------------------------- /src/edge_update_function/main.py: -------------------------------------------------------------------------------- 1 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # SPDX-License-Identifier: MIT-0 3 | 4 | import os 5 | import io 6 | import json 7 | import boto3 8 | import logging 9 | import zipfile 10 | 11 | from urllib.request import urlopen 12 | from crhelper import CfnResource 13 | 14 | logger = logging.getLogger() 15 | logger.setLevel(logging.INFO) 16 | 17 | helper = CfnResource() 18 | lambda_client = boto3.client('lambda') 19 | 20 | def update_function(event): 21 | user_pool_id = event['ResourceProperties']['UserPoolId'] 22 | cognito_region = event['ResourceProperties']['CognitoRegion'] 23 | source_url = event['ResourceProperties'].get('SourceUrl') 24 | edge_function_arn = event['ResourceProperties']['EdgeFunctionArn'] 25 | function_filename = event['ResourceProperties'].get('FunctionFilename', 'index.js') 26 | 27 | logger.info("Downloading well-known jwks.json from Cognito") 28 | jwks_url = f'https://cognito-idp.{cognito_region}.amazonaws.com/{user_pool_id}/.well-known/jwks.json' 29 | with urlopen(jwks_url) as http_response: 30 | jwks = str(http_response.read()) 31 | 32 | jwks = jwks.replace('b\'{', '{') 33 | jwks = jwks.replace('}\'', '}') 34 | logger.debug(json.dumps(jwks, indent = 2, default = str)) 35 | 36 | if not source_url: 37 | logger.info('SourceUrl not specified so determining code location from Lambda for "Templated" alias') 38 | # The "Templated" alias is created when the edge auth function is deployed and represents the original 39 | # version of the function that is templated with replacement variables. 40 | response = lambda_client.get_function( 41 | FunctionName = f'{edge_function_arn}:Templated' 42 | ) 43 | 44 | source_url = response['Code']['Location'] 45 | 46 | logger.info("Building updated function zip archive") 47 | js = None 48 | with urlopen(source_url) as zip_resp: 49 | with zipfile.ZipFile(io.BytesIO(zip_resp.read())) as zin: 50 | with zipfile.ZipFile('/tmp/edge-code.zip', 'w') as zout: 51 | zout.comment = zin.comment 52 | for item in zin.infolist(): 53 | if item.filename == function_filename: 54 | js = io.TextIOWrapper(io.BytesIO(zin.read(item.filename))).read() 55 | else: 56 | zout.writestr(item, zin.read(item.filename)) 57 | 58 | if not js: 59 | raise Exception(f'Function code archive does not contain the file "{function_filename}"') 60 | 61 | js = js.replace('##JWKS##', jwks) 62 | js = js.replace('##USERPOOLID##', user_pool_id) 63 | js = js.replace('##COGNITOREGION##', cognito_region) 64 | 65 | logger.info('Writing updated js file %s to archive', function_filename) 66 | with zipfile.ZipFile('/tmp/edge-code.zip', mode='a', compression=zipfile.ZIP_DEFLATED) as zf: 67 | zf.writestr(function_filename, js) 68 | 69 | # Load file into memory 70 | with open('/tmp/edge-code.zip', 'rb') as file_data: 71 | bytes_content = file_data.read() 72 | 73 | logger.info('Updating lambda function with updated code archive') 74 | response = lambda_client.update_function_code( 75 | FunctionName = edge_function_arn, 76 | ZipFile = bytes_content 77 | ) 78 | 79 | logger.debug(response) 80 | 81 | @helper.create 82 | @helper.update 83 | def create_or_update_resource(event, _): 84 | update_function(event) 85 | 86 | def lambda_handler(event, context): 87 | logger.info(os.environ) 88 | logger.info(json.dumps(event, indent = 2, default = str)) 89 | 90 | # If the event has a RequestType, we're being called by CFN as custom resource 91 | if event.get('RequestType'): 92 | logger.info('Function called from CloudFormation as custom resource') 93 | helper(event, context) 94 | else: 95 | logger.info('Function called outside of CloudFormation') 96 | # Call function directly (i.e. testing in Lambda console or called directly) 97 | update_function(event) 98 | -------------------------------------------------------------------------------- /src/edge_update_function/requirements.txt: -------------------------------------------------------------------------------- 1 | crhelper -------------------------------------------------------------------------------- /src/generate_config_function/README.md: -------------------------------------------------------------------------------- 1 | # Generate APIs configuration function 2 | 3 | This AWS Lambda function is invoked when you deploy the project and when it is deleted from your account. When the project is deployed, this function will automatically create a personalization APIs configuration in AWS AppConfig based on the Amazon Personalize recommenders, campaigns, and event trackers in the dataset groups that you specify. When this project is deleted from your AWS account, this function is also called to delete the hosted configurations in AppConfig. This is necessary so that all AppConfig dependent resources used in the project can be cleanly deleted. -------------------------------------------------------------------------------- /src/generate_config_function/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/personalization-apis/2593e0f0a8f22a25440be1a2e6c52fcefd2ab65f/src/generate_config_function/__init__.py -------------------------------------------------------------------------------- /src/generate_config_function/main.py: -------------------------------------------------------------------------------- 1 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # SPDX-License-Identifier: MIT-0 3 | 4 | import json 5 | import boto3 6 | import os 7 | import logging 8 | from typing import Dict 9 | from crhelper import CfnResource 10 | 11 | logger = logging.getLogger() 12 | logger.setLevel(logging.DEBUG) 13 | 14 | personalize = boto3.client('personalize') 15 | appconfig = boto3.client('appconfig') 16 | helper = CfnResource() 17 | 18 | appconfig_application_id = os.environ['AppConfigApplicationId'] 19 | appconfig_config_profile_id = os.environ['AppConfigConfigurationProfileId'] 20 | appconfig_environment_id = os.environ['AppConfigEnvironmentId'] 21 | appconfig_deployment_strategy_id = os.environ['AppConfigDeploymentStrategyId'] 22 | 23 | recipe_arn_type_mapping = { 24 | "arn:aws:personalize:::recipe/aws-ecomm-customers-who-viewed-x-also-viewed": "related-items", 25 | "arn:aws:personalize:::recipe/aws-ecomm-frequently-bought-together": "related-items", 26 | "arn:aws:personalize:::recipe/aws-ecomm-popular-items-by-purchases": "recommend-items", 27 | "arn:aws:personalize:::recipe/aws-ecomm-popular-items-by-views": "recommend-items", 28 | "arn:aws:personalize:::recipe/aws-ecomm-recommended-for-you": "recommend-items", 29 | "arn:aws:personalize:::recipe/aws-vod-because-you-watched-x": "related-items", 30 | "arn:aws:personalize:::recipe/aws-vod-more-like-x": "related-items", 31 | "arn:aws:personalize:::recipe/aws-vod-most-popular": "recommend-items", 32 | "arn:aws:personalize:::recipe/aws-vod-top-picks": "recommend-items", 33 | "arn:aws:personalize:::recipe/aws-hrnn": "recommend-items", 34 | "arn:aws:personalize:::recipe/aws-hrnn-coldstart": "recommend-items", 35 | "arn:aws:personalize:::recipe/aws-hrnn-metadata": "recommend-items", 36 | "arn:aws:personalize:::recipe/aws-personalized-ranking": "rerank-items", 37 | "arn:aws:personalize:::recipe/aws-popularity-count": "recommend-items", 38 | "arn:aws:personalize:::recipe/aws-similar-items": "related-items", 39 | "arn:aws:personalize:::recipe/aws-sims": "related-items", 40 | "arn:aws:personalize:::recipe/aws-user-personalization": "recommend-items" 41 | } 42 | 43 | def generate_api_config(dataset_group_names_prop: str) -> Dict: 44 | """ Generates personalization APIs app config based on recommenders, campaigns, and event trackers for the specified dataset groups 45 | 46 | Arguments: 47 | dataset_group_names_prop (string) - comma separated list of Personalize dataset group names to check for recommenders, campaigns, 48 | and event trackers or "all" to check all dataset groups in the current account & region 49 | """ 50 | 51 | # Start with an empty base configuration that implements some general caching. 52 | config = { 53 | "version": "2", 54 | "description": "This configuration was automatically generated based on the active recommenders/campaigns for a supplied list of dataset groups", 55 | "cacheControl": { 56 | "autoProvision": True, 57 | "userSpecified": { 58 | "maxAge": 10, 59 | "directives": "private" 60 | }, 61 | "syntheticUserSpecified": { 62 | "maxAge": 300, 63 | "directives": "public" 64 | }, 65 | "noUserSpecified": { 66 | "maxAge": 1200, 67 | "directives": "public" 68 | } 69 | }, 70 | "namespaces": {} 71 | } 72 | 73 | dataset_group_names = [dsg.strip() for dsg in dataset_group_names_prop.split(',')] 74 | all_dsgs = len(dataset_group_names) == 1 and dataset_group_names[0].lower() == 'all' 75 | 76 | logger.info('Dataset group names: %s', dataset_group_names) 77 | logger.info('Matching all dataset groups in current region for account: %s', all_dsgs) 78 | 79 | logger.info('Looking up recommenders and matching to dataset group(s)') 80 | paginator = personalize.get_paginator('list_recommenders') 81 | for recommender_page in paginator.paginate(): 82 | for recommender in recommender_page['recommenders']: 83 | dataset_group_name = recommender['datasetGroupArn'].split('/')[-1] 84 | 85 | if all_dsgs or dataset_group_name in dataset_group_names: 86 | action_type = recipe_arn_type_mapping.get(recommender['recipeArn']) 87 | if not action_type: 88 | # Perhaps a new recipe? 89 | logger.error('Unable to determine action type for recipe %s for recommender %s; skipping recommender', recommender['recipeArn'], recommender['recommenderArn']) 90 | continue 91 | 92 | variation_name = recommender['recipeArn'].split('/')[-1].replace('aws-', 'personalize-') 93 | 94 | variation_config = (config['namespaces'] 95 | .setdefault(dataset_group_name, {}) 96 | .setdefault('recommenders', {}) 97 | .setdefault(action_type, {}) 98 | .setdefault(recommender['name'], {}) 99 | .setdefault('variations', {}) 100 | .setdefault(variation_name, {}) 101 | ) 102 | 103 | variation_config['type'] = 'personalize-recommender' 104 | variation_config['arn'] = recommender['recommenderArn'] 105 | 106 | logger.info('Looking up campaigns and matching to dataset group(s)') 107 | paginator = personalize.get_paginator('list_campaigns') 108 | for campaign_page in paginator.paginate(): 109 | for campaign in campaign_page['campaigns']: 110 | response = personalize.describe_campaign(campaignArn = campaign['campaignArn']) 111 | sv_arn = response['campaign']['solutionVersionArn'] 112 | 113 | response = personalize.describe_solution_version(solutionVersionArn = sv_arn) 114 | dataset_group_name = response['solutionVersion']['datasetGroupArn'].split('/')[-1] 115 | 116 | if all_dsgs or dataset_group_name in dataset_group_names: 117 | recipe_arn = response['solutionVersion']['recipeArn'] 118 | action_type = recipe_arn_type_mapping.get(recipe_arn) 119 | if not action_type: 120 | # Perhaps a new recipe? 121 | logger.error('Unable to determine action type for recipe %s for campaign %s; skipping campaign', recipe_arn, campaign['campaignArn']) 122 | continue 123 | 124 | variation_name = recipe_arn.split('/')[-1].replace('aws-', 'personalize-') 125 | 126 | variation_config = (config['namespaces'] 127 | .setdefault(dataset_group_name, {}) 128 | .setdefault('recommenders', {}) 129 | .setdefault(action_type, {}) 130 | .setdefault(campaign['name'], {}) 131 | .setdefault('variations', {}) 132 | .setdefault(variation_name, {}) 133 | ) 134 | 135 | variation_config['type'] = 'personalize-campaign' 136 | variation_config['arn'] = campaign['campaignArn'] 137 | 138 | logger.info('Looking up event trackers and matching to dataset group(s)') 139 | paginator = personalize.get_paginator('list_event_trackers') 140 | for event_tracker_page in paginator.paginate(): 141 | for event_tracker in event_tracker_page['eventTrackers']: 142 | response = personalize.describe_event_tracker(eventTrackerArn = event_tracker['eventTrackerArn']) 143 | 144 | dataset_group_name = response['eventTracker']['datasetGroupArn'].split('/')[-1] 145 | if all_dsgs or dataset_group_name in dataset_group_names: 146 | targets = (config['namespaces'] 147 | .setdefault(dataset_group_name, {}) 148 | .setdefault('eventTargets', []) 149 | ) 150 | 151 | targets.append({ 152 | 'type': 'personalize-event-tracker', 153 | 'trackingId': response['eventTracker']['trackingId'] 154 | }) 155 | 156 | return config 157 | 158 | def create_and_deploy_hosted_config(config: Dict): 159 | """ Creates and deploys a configuration to AppConfig as a hosted configuration version """ 160 | 161 | logger.info('Creating hosted configuration...') 162 | logger.debug(config) 163 | response = appconfig.create_hosted_configuration_version( 164 | ApplicationId = appconfig_application_id, 165 | ConfigurationProfileId = appconfig_config_profile_id, 166 | Description = 'Generated configuration based on supplied list of dataset groups', 167 | ContentType = 'application/json', 168 | Content = json.dumps(config, indent = 4) 169 | ) 170 | 171 | logger.debug(json.dumps(response, indent = 2, default = str)) 172 | config_version = response['VersionNumber'] 173 | 174 | logger.info('Starting deployment...') 175 | response = appconfig.start_deployment( 176 | ApplicationId = appconfig_application_id, 177 | EnvironmentId = appconfig_environment_id, 178 | DeploymentStrategyId = appconfig_deployment_strategy_id, 179 | ConfigurationProfileId = appconfig_config_profile_id, 180 | ConfigurationVersion = str(config_version), 181 | Description = 'Automatic configuration deployment after generating configuration', 182 | Tags={ 183 | 'CreatedBy': 'Personalization-APIs-Solution' 184 | } 185 | ) 186 | logger.debug(json.dumps(response, indent = 2, default = str)) 187 | 188 | def generate_and_deploy_config(dataset_group_names_prop: str): 189 | if dataset_group_names_prop.strip(): 190 | config = generate_api_config(dataset_group_names_prop) 191 | if len(config['namespaces']) > 0: 192 | create_and_deploy_hosted_config(config) 193 | else: 194 | logger.warning('No namespaces discovered in current acccount/region') 195 | else: 196 | logger.info('Dataset group name(s) not specified; skipping generation of configuration') 197 | 198 | @helper.create 199 | def create_resource(event, _): 200 | generate_and_deploy_config(event['ResourceProperties']['DatasetGroupNames']) 201 | 202 | @helper.delete 203 | def delete_resource(event, _): 204 | """ Delete hosted configuration versions 205 | 206 | This is necessary here since hosted configurations are created outside of CloudFormation 207 | and therefore need to cleaned up before depedent AppConfig resources can be deleted by 208 | CloudFormation when the project is deleted. 209 | """ 210 | logger.info('Deleting all hosted configuration versions for application %s and config profile %s', appconfig_application_id, appconfig_config_profile_id) 211 | 212 | page_count = 0 213 | while page_count < 10: 214 | response = appconfig.list_hosted_configuration_versions( 215 | ApplicationId = appconfig_application_id, 216 | ConfigurationProfileId = appconfig_config_profile_id, 217 | MaxResults = 50 # no paginator and max is 50 218 | ) 219 | 220 | if len(response['Items']) == 0: 221 | break 222 | 223 | for config_version in response['Items']: 224 | logger.info('Deleting hosted configuration version %s', config_version["VersionNumber"]) 225 | response = appconfig.delete_hosted_configuration_version( 226 | ApplicationId = appconfig_application_id, 227 | ConfigurationProfileId = appconfig_config_profile_id, 228 | VersionNumber = config_version['VersionNumber'] 229 | ) 230 | 231 | page_count += 1 232 | 233 | def lambda_handler(event, context): 234 | """ Entry point of function called from either CloudFormation or directly under test 235 | """ 236 | logger.debug('## ENVIRONMENT VARIABLES') 237 | logger.debug(os.environ) 238 | logger.debug('## EVENT') 239 | logger.debug(event) 240 | 241 | # If the event has a RequestType, we're being called by CFN as custom resource 242 | if event.get('RequestType'): 243 | logger.info('Function called from CloudFormation as custom resource') 244 | helper(event, context) 245 | else: 246 | logger.info('Function called outside of CloudFormation') 247 | # Called function directly (i.e. testing in Lambda console or called directly) 248 | generate_and_deploy_config(event['ResourceProperties']['DatasetGroupNames']) -------------------------------------------------------------------------------- /src/generate_config_function/requirements.txt: -------------------------------------------------------------------------------- 1 | crhelper -------------------------------------------------------------------------------- /src/layer/README.md: -------------------------------------------------------------------------------- 1 | # Lambda layer - shared code 2 | 3 | This AWS Lambda layer provides shared code that is used across some of the functions in this project. For example, the Personalization APIs configuration helper class is located in this layer. -------------------------------------------------------------------------------- /src/layer/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/personalization-apis/2593e0f0a8f22a25440be1a2e6c52fcefd2ab65f/src/layer/__init__.py -------------------------------------------------------------------------------- /src/layer/personalization_config.py: -------------------------------------------------------------------------------- 1 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # SPDX-License-Identifier: MIT-0 3 | 4 | import json 5 | import os 6 | import urllib.request 7 | import copy 8 | 9 | from typing import Any, Dict 10 | from abc import ABC, abstractmethod 11 | from datetime import datetime, timedelta 12 | from aws_lambda_powertools import Logger, Tracer 13 | 14 | logger = Logger(child=True) 15 | tracer = Tracer() 16 | 17 | LOCAL_DB_FILENAME = 'p13n-item-metadata.db' 18 | LOCAL_DB_GZIP_FILENAME = LOCAL_DB_FILENAME + '.gz' 19 | 20 | class PersonalizationConfig(ABC): 21 | def __init__(self): 22 | pass 23 | 24 | def get_namespace_config(self, namespace_path: str) -> Dict: 25 | config = None 26 | root_config = self.get_config() 27 | if root_config: 28 | namespaces = root_config.get('namespaces') 29 | if namespaces: 30 | config = self.inherit_config(root_config, namespaces.get(namespace_path)) 31 | 32 | return config 33 | 34 | def get_recommender_config(self, namespace_path: str, recommender_path: str, api_action: str = None) -> Dict: 35 | config = None 36 | ns_config = self.get_namespace_config(namespace_path) 37 | if ns_config: 38 | recommenders = ns_config.get('recommenders') 39 | if recommenders: 40 | if api_action: 41 | config = recommenders.get(api_action) 42 | if config: 43 | config = self.inherit_config(ns_config, config.get(recommender_path)) 44 | else: 45 | for action_config in recommenders.values(): 46 | if action_config.get(recommender_path): 47 | config = action_config.get(recommender_path) 48 | 49 | return config 50 | 51 | def get_version(self, default: str = None) -> str: 52 | return self.get_config().get('version', default) 53 | 54 | def inherit_config(self, parent: Dict, config: Dict) -> Dict: 55 | if parent is not None and config is not None: 56 | inherited = ['autoContext', 'filters', 'cacheControl', 'inferenceItemMetadata'] 57 | for inherit in inherited: 58 | if config.get(inherit) is None and parent.get(inherit) is not None: 59 | config[inherit] = copy.copy(parent.get(inherit)) 60 | 61 | return config 62 | 63 | @abstractmethod 64 | def get_config(self, max_age: int = 60) -> Dict: 65 | pass 66 | 67 | @staticmethod 68 | def get_instance(type: str = 'AppConfig') -> Any: 69 | """ Creates a config based on the type """ 70 | if type != 'AppConfig': 71 | raise ValueError('Invalid personalization API config type') 72 | return AppConfigPersonalizationConfig() 73 | 74 | class AppConfigPersonalizationConfig(PersonalizationConfig): 75 | def __init__(self): 76 | super().__init__() 77 | self.config = None 78 | self.ttl = 0 79 | 80 | def get_config(self, max_age: int = 10) -> Dict: 81 | if self.config and self.ttl > datetime.now(): 82 | return self.config 83 | 84 | return self._get_from_app_config(max_age) 85 | 86 | @tracer.capture_method(capture_response=False) 87 | def _get_from_app_config(self, max_age: int = 10) -> Dict: 88 | logger.debug('Fetching configuration from AppConfig Lambda extension') 89 | 90 | url = f'http://localhost:2772{os.environ["AWS_APPCONFIG_EXTENSION_PREFETCH_LIST"]}' 91 | resp = urllib.request.urlopen(url).read() 92 | self.ttl = datetime.now() + timedelta(seconds=max_age) 93 | self.config = json.loads(resp) if resp else {} 94 | return self.config 95 | -------------------------------------------------------------------------------- /src/layer/personalization_constants.py: -------------------------------------------------------------------------------- 1 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # SPDX-License-Identifier: MIT-0 3 | 4 | ACTION_RECOMMEND_ITEMS = 'recommend-items' 5 | ACTION_RELATED_ITEMS = 'related-items' 6 | ACTION_RERANK_ITEMS = 'rerank-items' 7 | 8 | LOCAL_DB_FILENAME = 'p13n_item_metadata.db' 9 | LOCAL_DB_GZIP_FILENAME = LOCAL_DB_FILENAME + '.gz' -------------------------------------------------------------------------------- /src/layer/requirements.txt: -------------------------------------------------------------------------------- 1 | # Runtime requirements: 2 | # Note: the following dependency must be provided at runtime as Lambda layer: 3 | # - AWS Lambda Power Tools as a Lambda layer. 4 | # Note: If the PersonalizationConfig class is going to be used, the following Lambda extension must also be configured for the function: 5 | # - AWS AppConfig Lambda extension (https://docs.aws.amazon.com/appconfig/latest/userguide/appconfig-integration-lambda-extensions.html) 6 | -------------------------------------------------------------------------------- /src/load_item_metadata_function/README.md: -------------------------------------------------------------------------------- 1 | # Load item metadata function 2 | 3 | This AWS Lambda function is invoked when you upload [item metadata files](../../docs/item_metadata.md) to the Amazon S3 item metadata staging bucket. It will parse the contents of the uploaded item metadata file and either write the metadata to a DynamoDB table or prepare a DBM file containing the metadata and write that file back to the same S3 bucket. The prepared DBM files for each namespace are downloaded by the [personalization_api_function](../personalization_api_function/) and saved locally on the Lambda instance volume. -------------------------------------------------------------------------------- /src/load_item_metadata_function/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/personalization-apis/2593e0f0a8f22a25440be1a2e6c52fcefd2ab65f/src/load_item_metadata_function/__init__.py -------------------------------------------------------------------------------- /src/load_item_metadata_function/main.py: -------------------------------------------------------------------------------- 1 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # SPDX-License-Identifier: MIT-0 3 | 4 | import boto3 5 | import jsonlines 6 | import os 7 | import copy 8 | import urllib 9 | import json 10 | import dbm 11 | import gzip 12 | 13 | from typing import Tuple 14 | from io import TextIOWrapper, RawIOBase 15 | from decimal import Decimal 16 | from boto3.dynamodb.conditions import Attr 17 | from aws_lambda_powertools import Logger, Tracer 18 | from personalization_config import PersonalizationConfig 19 | from personalization_constants import LOCAL_DB_FILENAME, LOCAL_DB_GZIP_FILENAME 20 | 21 | tracer = Tracer() 22 | logger = Logger() 23 | 24 | s3 = boto3.client('s3') 25 | dynamodb = boto3.resource('dynamodb') 26 | config = PersonalizationConfig.get_instance() 27 | primary_key_name = os.environ.get('ItemsTablePrimaryKeyFieldName', 'id') 28 | table_name_prefix = os.environ.get('ItemsTableNamePrefix', 'PersonalizationApiItemMetadata_') 29 | 30 | class StreamingBodyIO(RawIOBase): 31 | """Wrap a boto StreamingBody in the IOBase API.""" 32 | def __init__(self, body): 33 | self.body = body 34 | 35 | def readable(self): 36 | return True 37 | 38 | def read(self, n: int = -1): 39 | n = None if n < 0 else n 40 | return self.body.read(n) 41 | 42 | @tracer.capture_method 43 | def bulk_load_datastore(bucket: str, key: str, namespace: str): 44 | ns_config = config.get_namespace_config(namespace) 45 | if not ns_config: 46 | raise Exception(f'Namespace ("{namespace}") in S3 object key ("{key}") not found in solution configuration') 47 | 48 | metadata_config = ns_config.get('inferenceItemMetadata') 49 | if not metadata_config: 50 | raise Exception(f'Namespace ("{namespace}") is missing inferenceItemMetadata configuration') 51 | 52 | type = metadata_config.get('type') 53 | if type == 'dynamodb': 54 | table_name, timestamp = bulk_write_ddb_table(bucket, key, namespace) 55 | purge_obsolete_ddb_items(table_name, timestamp) 56 | elif type == 'localdb': 57 | build_dbm_file(bucket, key, namespace) 58 | 59 | def build_dbm_file(bucket: str, key: str, namespace: str): 60 | logger.info('Downloading object from S3') 61 | response = s3.get_object(Bucket = bucket, Key = key) 62 | 63 | if key.endswith('.gz') or key.endswith('.gzip'): 64 | stream = gzip.GzipFile(None, 'rb', fileobj = response['Body']) 65 | else: 66 | stream = StreamingBodyIO(response['Body']) 67 | 68 | lines_read = 0 69 | 70 | logger.info('Building local DBM file') 71 | with dbm.open(f'/tmp/{LOCAL_DB_FILENAME}', 'c') as db: 72 | reader = jsonlines.Reader(TextIOWrapper(stream)) 73 | for item in reader: 74 | lines_read += 1 75 | if not primary_key_name in item: 76 | raise KeyError(f'Item ({lines_read}) is missing required field "{primary_key_name}"') 77 | 78 | id = item.pop(primary_key_name) 79 | db[id] = json.dumps(item, default=str) 80 | 81 | logger.info('Gzipping local DBM file') 82 | with open(f'/tmp/{LOCAL_DB_FILENAME}', 'rb') as src, gzip.open(f'/tmp/{LOCAL_DB_GZIP_FILENAME}', 'wb') as dst: 83 | dst.writelines(src) 84 | 85 | output_key = f'localdbs/{namespace}/{LOCAL_DB_GZIP_FILENAME}' 86 | 87 | logger.info('Uploading DBM file to S3') 88 | response = s3.upload_file(f'/tmp/{LOCAL_DB_GZIP_FILENAME}', bucket, output_key) 89 | 90 | def bulk_write_ddb_table(bucket: str, key: str, namespace: str) -> Tuple[str, str]: 91 | table_name = table_name_prefix + namespace 92 | 93 | logger.info('Downloading object from S3') 94 | response = s3.get_object(Bucket = bucket, Key = key) 95 | 96 | # Use the last modified date from the file as each record's version/timestamp in the metadata table. 97 | last_modified = response['LastModified'] 98 | timestamp = '{:%Y-%m-%dT%H:%M:%S}.{:03d}'.format(last_modified, int(last_modified.microsecond/1000)) 99 | 100 | if key.endswith('.gz') or key.endswith('.gzip'): 101 | stream = gzip.GzipFile(None, 'rb', fileobj = response['Body']) 102 | else: 103 | stream = StreamingBodyIO(response['Body']) 104 | 105 | logger.info('Loading items into table %s', table_name) 106 | table = dynamodb.Table(table_name) 107 | 108 | lines_read = 0 109 | 110 | with table.batch_writer() as batch: 111 | reader = jsonlines.Reader(TextIOWrapper(stream)) 112 | 113 | for item in reader: 114 | lines_read += 1 115 | if not primary_key_name in item: 116 | raise KeyError(f'Item ({lines_read}) is missing required field "{primary_key_name}"') 117 | 118 | attribs = copy.copy(item) 119 | attribs.pop(primary_key_name) 120 | ddb_item = { 121 | primary_key_name: item[primary_key_name], 122 | 'version': timestamp, 123 | 'put_via': 'bulk', 124 | 'attributes': attribs 125 | } 126 | batch.put_item(Item=json.loads(json.dumps(ddb_item), parse_float=Decimal)) 127 | 128 | logger.info('Items loaded: %d', lines_read) 129 | 130 | return table_name, timestamp 131 | 132 | @tracer.capture_method 133 | def purge_obsolete_ddb_items(table_name: str, timestamp: str): 134 | logger.info('Purging items from table with a version prior to %s', timestamp) 135 | table = dynamodb.Table(table_name) 136 | 137 | scan_kwargs = { 138 | 'FilterExpression': Attr('version').lt(timestamp), 139 | 'ProjectionExpression': primary_key_name 140 | } 141 | 142 | with table.batch_writer() as batch: 143 | deleted = 0 144 | done = False 145 | start_key = None 146 | while not done: 147 | if start_key: 148 | scan_kwargs['ExclusiveStartKey'] = start_key 149 | response = table.scan(**scan_kwargs) 150 | for item in response.get('Items', []): 151 | batch.delete_item(Key = item) 152 | deleted += 1 153 | 154 | start_key = response.get('LastEvaluatedKey', None) 155 | done = start_key is None 156 | 157 | logger.info('Purged %d items with a version prior to %s', deleted, timestamp) 158 | 159 | @tracer.capture_method 160 | def process_event_record(record): 161 | bucket = record['s3']['bucket']['name'] 162 | key = urllib.parse.unquote_plus(record['s3']['object']['key']) 163 | 164 | # Key format should be "import/[namespace]/file.jsonl[.gz|.gzip]" 165 | path_bits = key.split('/') 166 | 167 | if path_bits[0] != 'import': 168 | raise Exception('Key does not start with expected folder ("import/"); ignoring event record') 169 | 170 | if len(path_bits) < 3: 171 | logger.warn('Key does not conform to expected path (not enough elements in path); ignoring event record') 172 | return 173 | 174 | if record['s3']['object']['size'] == 0: 175 | logger.warn('Object is empty (size is zero); ignoring event record') 176 | return 177 | 178 | namespace = path_bits[1] 179 | bulk_load_datastore(bucket, key, namespace) 180 | 181 | @logger.inject_lambda_context 182 | @tracer.capture_lambda_handler 183 | def lambda_handler(event, _): 184 | if event.get('Records'): 185 | for record in event['Records']: 186 | if record.get('s3'): 187 | try: 188 | process_event_record(record) 189 | except Exception as e: 190 | logger.exception(e) 191 | else: 192 | logger.error('Event Record does not appear to be for an S3 event; missing "s3" details') 193 | else: 194 | logger.error('Invalid/unsupported event; missing "Records"') -------------------------------------------------------------------------------- /src/load_item_metadata_function/requirements.txt: -------------------------------------------------------------------------------- 1 | # Note: AWS Lambda Power Tools is required but is satisfied by a Lambda layer at runtime. 2 | jsonlines -------------------------------------------------------------------------------- /src/personalization_api_function/README.md: -------------------------------------------------------------------------------- 1 | # Personalization API function 2 | 3 | This AWS Lambda function provides the core functionality of the Personalization APIs project. This is where the implementation of the [API entry points](../../docs/api_entry_points.md) can be found (in [main.py](./main.py)). -------------------------------------------------------------------------------- /src/personalization_api_function/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/personalization-apis/2593e0f0a8f22a25440be1a2e6c52fcefd2ab65f/src/personalization_api_function/__init__.py -------------------------------------------------------------------------------- /src/personalization_api_function/auto_values.py: -------------------------------------------------------------------------------- 1 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # SPDX-License-Identifier: MIT-0 3 | 4 | import pytz 5 | 6 | from datetime import datetime 7 | from typing import Any, Dict 8 | from aws_lambda_powertools import Logger, Tracer 9 | 10 | tracer = Tracer() 11 | logger = Logger(child=True) 12 | 13 | def get_season(date: datetime, latitude: float = None) -> int: 14 | """ Determines season index (0-3) based on datetime and latitude 15 | 16 | Season indeces: 17 | 0 = Spring 18 | 1 = Summer 19 | 2 = Fall 20 | 3 = Winter 21 | Latitude is used to determine the Northern/Southern hemisphere 22 | """ 23 | md = date.month * 100 + date.day 24 | 25 | if ((md > 320) and (md < 621)): 26 | s = 0 # Spring 27 | elif ((md > 620) and (md < 923)): 28 | s = 1 # Summer 29 | elif ((md > 922) and (md < 1223)): 30 | s = 2 # Fall 31 | else: 32 | s = 3 # Winter 33 | 34 | if latitude and latitude < 0: 35 | if s < 2: 36 | s += 2 37 | else: 38 | s -= 2 39 | 40 | return s 41 | 42 | def resolve_auto_values(context_config: Dict, headers: Dict[str,str]) -> Dict[str,Dict[str,Any]]: 43 | """ Resolves automated context based on the specified config and headers 44 | 45 | Returns a dictionary where the keys are the field names and the values are 46 | a dict with "values" and "type" fields. 47 | """ 48 | resolved_values = {} 49 | 50 | if not context_config: 51 | return resolved_values 52 | 53 | if headers.get('cloudfront-viewer-time-zone'): 54 | tz = pytz.timezone(headers.get('cloudfront-viewer-time-zone')) 55 | now = tz.localize(datetime.now()) 56 | else: 57 | now = datetime.now() 58 | 59 | for field, auto_ctx in context_config.items(): 60 | values = set() 61 | 62 | eval_all = auto_ctx.get('evaluateAll', False) 63 | 64 | for rule in auto_ctx.get('rules'): 65 | resolved = None 66 | 67 | if rule.get('type') == 'header-value': 68 | header_value = headers.get(rule.get('header')) 69 | resolved = _resolve(rule, header_value) 70 | elif rule.get('type') == 'hour-of-day': 71 | resolved = _resolve(rule, now.hour) 72 | elif rule.get('type') == 'day-of-week': 73 | resolved = _resolve(rule, now.weekday()) 74 | elif rule.get('type') == 'season-of-year': 75 | season = get_season(now, headers.get('cloudfront-viewer-latitude')) 76 | resolved = _resolve(rule, season) 77 | 78 | if resolved: 79 | values.add(resolved) 80 | if not eval_all: 81 | break 82 | 83 | if len(values) == 0 and auto_ctx.get('default'): 84 | values.add(auto_ctx['default']) 85 | 86 | if len(values) > 0: 87 | resolved_values[field] = { 88 | 'values': list(values) 89 | } 90 | if auto_ctx.get('type'): 91 | resolved_values[field]['type'] = auto_ctx['type'] 92 | 93 | return resolved_values 94 | 95 | def _resolve(rule: Dict, value: Any) -> Any: 96 | resolved_value = None 97 | 98 | if value is not None: 99 | if rule.get('valueMappings'): 100 | for value_mapping in rule.get('valueMappings'): 101 | operator = value_mapping['operator'] 102 | mapping_value = value_mapping['value'] 103 | map_to = value_mapping['mapTo'] 104 | 105 | if operator == 'equals' and value == mapping_value: 106 | resolved_value = map_to 107 | elif operator == 'less-than' and value < mapping_value: 108 | resolved_value = map_to 109 | elif operator == 'greater-than' and value > mapping_value: 110 | resolved_value = map_to 111 | elif operator == 'contains' and str(mapping_value) in str(value): 112 | resolved_value = map_to 113 | elif operator == 'start-with' and str(value).startswith(str(mapping_value)): 114 | resolved_value = map_to 115 | elif operator == 'ends-with' and str(value).endswith(str(mapping_value)): 116 | resolved_value = map_to 117 | 118 | if resolved_value: 119 | break 120 | else: 121 | resolved_value = value 122 | 123 | return resolved_value 124 | -------------------------------------------------------------------------------- /src/personalization_api_function/background_tasks.py: -------------------------------------------------------------------------------- 1 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # SPDX-License-Identifier: MIT-0 3 | 4 | import time 5 | 6 | from typing import List 7 | from concurrent.futures import ThreadPoolExecutor, Future 8 | from aws_lambda_powertools import Logger, Tracer 9 | 10 | logger = Logger(child=True) 11 | tracer = Tracer() 12 | 13 | class BackgroundTasks(): 14 | def __init__(self): 15 | self.pool: ThreadPoolExecutor = None 16 | self.futures: List[Future] = [] 17 | self.pool_init = 0 18 | self.task_count = 0 19 | 20 | def submit(self, fn, /, *args, **kwargs): 21 | if not self.pool: 22 | self.pool_init = time.time() 23 | self.pool = ThreadPoolExecutor() 24 | 25 | self.futures.append(self.pool.submit(fn, *args, **kwargs)) 26 | self.task_count += 1 27 | 28 | def __enter__(self): 29 | return self 30 | 31 | def __exit__(self, exc_type, exc_value, exc_traceback): 32 | if self.pool: 33 | logger.info('Waiting for background tasks to complete') 34 | self.pool.shutdown(True) 35 | logger.info('%s background tasks completed in %0.2fms', self.task_count, time.time() - self.pool_init) 36 | -------------------------------------------------------------------------------- /src/personalization_api_function/event_targets.py: -------------------------------------------------------------------------------- 1 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # SPDX-License-Identifier: MIT-0 3 | 4 | import boto3 5 | import json 6 | 7 | from copy import copy 8 | from typing import Dict 9 | from datetime import datetime 10 | from http import HTTPStatus 11 | from abc import ABC, abstractmethod 12 | from concurrent.futures import ThreadPoolExecutor, Future, wait 13 | from botocore.exceptions import ClientError 14 | from aws_lambda_powertools import Logger, Tracer, Metrics 15 | from aws_lambda_powertools.metrics import MetricUnit 16 | from personalization_error import ConfigError, PersonalizeError, JSONDecodeValidationError 17 | from auto_values import resolve_auto_values 18 | 19 | logger = Logger(child=True) 20 | tracer = Tracer() 21 | metrics = Metrics() 22 | 23 | PERSONALIZE_EVENT_TRACKER = 'personalize-event-tracker' 24 | KINESIS_STREAM = 'kinesis-stream' 25 | KINESIS_FIREHOSE = 'kinesis-firehose' 26 | 27 | class EventTarget(ABC): 28 | def __init__(self): 29 | pass 30 | 31 | @abstractmethod 32 | def put_events(self, config, api_event): 33 | pass 34 | 35 | def apply_auto_context(self, namespace_config: Dict, event_body: Dict, headers: Dict[str,str]): 36 | auto_context = resolve_auto_values(namespace_config.get('autoContext'), headers) 37 | if auto_context: 38 | for event in event_body.get('eventList'): 39 | if event.get('properties'): 40 | properties = json.loads(event.get('properties')) 41 | else: 42 | properties = {} 43 | 44 | for field, resolved in auto_context.items(): 45 | if not field in properties: 46 | if resolved.get('type') == 'string': 47 | properties[field] = '|'.join(resolved['values']) 48 | else: 49 | properties[field] = str(resolved['values'][0]) 50 | 51 | event['properties'] = json.dumps(properties) 52 | 53 | class PersonalizeEventTracker(EventTarget): 54 | _personalize_events = boto3.client('personalize-events') 55 | 56 | def __init__(self, trackingId: str): 57 | self.trackingId = trackingId 58 | 59 | @tracer.capture_method 60 | def put_events(self, namespace: str, namespace_config: Dict, api_event: Dict, event_body: Dict): 61 | if event_body.get('experimentConversions'): 62 | # The "experimentConversion" key is a custom extension supported only by this solution. 63 | # We need to remove this key before calling PutEvents for Personalize. Otherwise the API 64 | # call will fail for parameter validation. Make a copy of the event before removing the 65 | # key so that other event targets will still process the complete original event. 66 | event_body = copy(event_body) 67 | del event_body['experimentConversions'] 68 | 69 | event_body['trackingId'] = self.trackingId 70 | 71 | self.apply_auto_context(namespace_config, event_body, api_event.headers) 72 | 73 | logger.debug('Calling put_events on Personalize event tracker %s', self.trackingId) 74 | 75 | try: 76 | response = PersonalizeEventTracker._personalize_events.put_events(**event_body) 77 | logger.debug(response) 78 | except ClientError as e: 79 | if e.response['Error']['Code'] == 'ThrottlingException': 80 | metrics.add_dimension(name="TrackingId", value=self.trackingId) 81 | metrics.add_metric(name="PersonalizeEventTrackerThrottle", unit=MetricUnit.Count, value=1) 82 | raise PersonalizeError.from_client_error(e) 83 | 84 | class KinesisStream(EventTarget): 85 | _kinesis = boto3.client('kinesis') 86 | 87 | def __init__(self, stream_name: str): 88 | self.stream_name = stream_name 89 | 90 | @tracer.capture_method 91 | def put_events(self, namespace: str, namespace_config: Dict, api_event: Dict, event_body: Dict): 92 | self.apply_auto_context(namespace_config, event_body, api_event.headers) 93 | 94 | data = { 95 | 'namespace': namespace, 96 | 'path': api_event.path, 97 | 'headers': api_event.headers, 98 | 'queryStringParameters': api_event.query_string_parameters, 99 | 'body': event_body 100 | } 101 | 102 | logger.debug('Calling put_record on stream %s', self.stream_name) 103 | response = KinesisStream._kinesis.put_record( 104 | StreamName = self.stream_name, 105 | Data = json.dumps(data), 106 | PartitionKey = event_body['sessionId'] 107 | ) 108 | 109 | logger.debug(response) 110 | 111 | class KinesisFirehose(EventTarget): 112 | _firehose = boto3.client('firehose') 113 | 114 | def __init__(self, stream_name: str): 115 | self.stream_name = stream_name 116 | 117 | @tracer.capture_method 118 | def put_events(self, namespace: str, namespace_config: Dict, api_event: Dict, event_body: Dict): 119 | self.apply_auto_context(namespace_config, event_body, api_event.headers) 120 | 121 | data = { 122 | 'namespace': namespace, 123 | 'path': api_event.path, 124 | 'headers': api_event.headers, 125 | 'queryStringParameters': api_event.query_string_parameters, 126 | 'body': event_body 127 | } 128 | 129 | logger.debug('Calling put_record on Firehose %s', self.stream_name) 130 | response = KinesisFirehose._firehose.put_record( 131 | DeliveryStreamName = self.stream_name, 132 | Record = { 133 | 'Data': json.dumps(data) 134 | } 135 | ) 136 | 137 | logger.debug(response) 138 | 139 | @tracer.capture_method 140 | def process_targets(namespace: str, namespace_config: Dict, api_event: Dict): 141 | config_targets = namespace_config.get('eventTargets') 142 | if not config_targets: 143 | raise ConfigError(HTTPStatus.NOT_FOUND, 'NamespaceEventTargetsNotFound', 'No event targets are defined for this namespace path') 144 | 145 | try: 146 | event_body = api_event.json_body 147 | except json.decoder.JSONDecodeError as e: 148 | raise JSONDecodeValidationError.from_json_decoder_error('InvalidJSONRequestPayload', e) 149 | 150 | # Set sentAt if omitted from any of the events. 151 | if event_body.get('eventList'): 152 | for event in event_body['eventList']: 153 | if not 'sentAt' in event: 154 | event['sentAt'] = int(datetime.now().timestamp()) 155 | 156 | targets: EventTarget = [] 157 | 158 | for config_target in config_targets: 159 | type = config_target.get('type') 160 | 161 | if type == PERSONALIZE_EVENT_TRACKER: 162 | if event_body.get('eventList'): 163 | targets.append(PersonalizeEventTracker(config_target['trackingId'])) 164 | else: 165 | logger.warning('API event does not have any events ("eventList" missing or empty); skipping Personalize event tracker') 166 | elif type == KINESIS_STREAM: 167 | targets.append(KinesisStream(stream_name = config_target['streamName'])) 168 | elif type == KINESIS_FIREHOSE: 169 | targets.append(KinesisFirehose(stream_name = config_target['streamName'])) 170 | else: 171 | raise ConfigError(f'Event target type {type} is unsupported') 172 | 173 | if len(targets) == 1: 174 | logger.debug('Just one event target %s; executing synchronously', config_targets[0]) 175 | targets[0].put_events(namespace, namespace_config, api_event, event_body) 176 | else: 177 | logger.debug('%s event targets; executing concurrently', len(targets)) 178 | with ThreadPoolExecutor() as executor: 179 | futures: Future = [] 180 | for target in targets: 181 | futures.append(executor.submit(target.put_events, namespace, namespace_config, api_event, event_body)) 182 | 183 | logger.debug('Waiting for event targets to finish processing') 184 | wait(futures) 185 | logger.debug('All event targets completed processing') 186 | 187 | # Propagate any exceptions 188 | for future in futures: 189 | future.result() -------------------------------------------------------------------------------- /src/personalization_api_function/evidently.py: -------------------------------------------------------------------------------- 1 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # SPDX-License-Identifier: MIT-0 3 | 4 | import boto3 5 | import json 6 | 7 | from typing import Dict, List, Optional, Tuple, Union 8 | from datetime import datetime 9 | from http import HTTPStatus 10 | from aws_lambda_powertools import Logger, Tracer 11 | from personalization_config import PersonalizationConfig 12 | from personalization_error import ConfigError, ValidationError, JSONDecodeValidationError 13 | from background_tasks import BackgroundTasks 14 | 15 | tracer = Tracer() 16 | logger = Logger(child=True) 17 | 18 | evidently = boto3.client('evidently') 19 | 20 | EXPOSURE_VALUE = 0.0000001 21 | CONVERSION_VALUE = 1.0000001 22 | 23 | def _set_simple_json_path(path: str, value: Union[str, float], variation: Dict): 24 | """ Sets a value in a target dictionary based on a simple dot-notation JSON path """ 25 | path_elements = path.split('.') 26 | 27 | context = variation 28 | for element in path_elements[:-1]: 29 | if not context.get(element): 30 | context[element] = {} 31 | context = context[element] 32 | else: 33 | context = context.get(element) 34 | 35 | context[path_elements[-1]] = value 36 | 37 | def _create_event_data(metric_config: Dict[str,str], entity_id: str, value: float) -> Dict: 38 | event = {} 39 | _set_simple_json_path(metric_config['entityIdKey'], entity_id, event) 40 | _set_simple_json_path(metric_config['valueKey'], value, event) 41 | return event 42 | 43 | def create_exposure_event(metric_config: Dict, entity_id: str) -> Dict: 44 | return { 45 | 'type': 'aws.evidently.custom', 46 | 'data': json.dumps(_create_event_data(metric_config, entity_id, EXPOSURE_VALUE)), 47 | 'timestamp': datetime.now() 48 | } 49 | 50 | def create_conversion_event(metric_config: Dict, entity_id: str, value: float = CONVERSION_VALUE) -> Dict: 51 | return { 52 | 'type': 'aws.evidently.custom', 53 | 'data': json.dumps(_create_event_data(metric_config, entity_id, value)), 54 | 'timestamp': datetime.now() 55 | } 56 | 57 | @tracer.capture_method 58 | def record_evidently_events(project: str, events: List[Dict]): 59 | response = evidently.put_project_events( 60 | project = project, 61 | events = events 62 | ) 63 | 64 | logger.debug(response) 65 | 66 | @tracer.capture_method 67 | def evidently_evaluate_feature(feature: str, experiment_config: Dict, variations: Dict, user_id: str, background: BackgroundTasks) -> Tuple[Dict,Optional[Dict]]: 68 | try: 69 | response = evidently.evaluate_feature( 70 | entityId = user_id, 71 | project = experiment_config['project'], 72 | feature = feature 73 | ) 74 | 75 | variation = None 76 | value = None 77 | 78 | if response['value'].get('stringValue'): 79 | variation_id = response['value']['stringValue'] 80 | variation = variations.get(variation_id) 81 | if not variation and variation_id.isdigit(): 82 | try: 83 | variation_id, variation = list(variations.items())[int(variation_id)] 84 | except IndexError: 85 | raise ConfigError(HTTPStatus.INTERNAL_SERVER_ERROR, 'NoMatchedTarget', f'Evaluated feature variation value ({variation_id}) from Evidently is out of index range for configured variations') 86 | 87 | if not variation: 88 | raise ConfigError(HTTPStatus.INTERNAL_SERVER_ERROR, 'NoMatchedTarget', f'Evaluated feature variation value ("{variation_id}") from Evidently does not match a configured variation') 89 | 90 | elif response['value'].get('longValue'): 91 | idx = response['value']['longValue'] 92 | try: 93 | variation_id, variation = list(variations.items())[int(idx)] 94 | except IndexError: 95 | raise ConfigError(HTTPStatus.INTERNAL_SERVER_ERROR, 'NoMatchedTarget', f'Evaluated feature variation value ({value}) from Evidently is out of index range for configured variations') 96 | else: 97 | raise ConfigError(HTTPStatus.INTERNAL_SERVER_ERROR, 'UnsupportedEvaluationType', f'Evaluated feature variation value type from Evidently is not supported') 98 | 99 | logger.info('Evidently feature "%s" mapped to variation "%s" via Evidently variation "%s"', feature, variation_id, response.get('variation')) 100 | 101 | experiment = None 102 | 103 | if response['reason'] == 'EXPERIMENT_RULE_MATCH': 104 | experiment = { 105 | 'type': 'evidently', 106 | 'feature': feature, 107 | 'details': json.loads(response['details']) 108 | } 109 | 110 | if experiment_config.get('metrics'): 111 | events = [] 112 | for metric_name, metric_config in experiment_config['metrics'].items(): 113 | if metric_config.get('trackExposures', True): 114 | logger.info('Recording variation exposure for Evidently variation "%s" of experiment "%s" for metric "%s"', response.get('variation'), response.get('details'), metric_name) 115 | events.append(create_exposure_event(metric_config, user_id)) 116 | else: 117 | logger.info('Variation exposures for Evidently variation "%s" of experiment "%s" for metric "%s" are DISABLED; skipping', response.get('variation'), response.get('details'), metric_name) 118 | 119 | background.submit(record_evidently_events, experiment_config['project'], events) 120 | else: 121 | logger.warning('Evidently conversion metric details not defined in recommender configuration; unable to record exposure event for experiment "%s"', response.get('details')) 122 | 123 | return variation, experiment 124 | 125 | except evidently.exceptions.ResourceNotFoundException: 126 | logger.warning('Evidently project ("%s") and/or feature ("%s") do not exist; defaulting to first configured variation', experiment['project'], feature) 127 | return next(iter(variations.items()))[1], None 128 | 129 | @tracer.capture_method 130 | def process_conversions(namespace: str, namespace_config: Dict, api_event: Dict, config: PersonalizationConfig): 131 | try: 132 | event_body = api_event.json_body 133 | except json.decoder.JSONDecodeError as e: 134 | raise JSONDecodeValidationError.from_json_decoder_error('InvalidJSONRequestPayload', e) 135 | 136 | conversions = event_body.get('experimentConversions') 137 | if not conversions: 138 | logger.debug('API event does not include any experiment conversions') 139 | return 140 | 141 | if not isinstance(conversions, list): 142 | raise ValidationError('InvalidExperimentConversions', 'Must be a list') 143 | 144 | user_id = event_body.get('userId') 145 | if not user_id: 146 | raise ValidationError('UserIdRequired', 'userId is a required field in payload object') 147 | 148 | evidently_events = {} 149 | 150 | # Make a first pass over the conversions to validate them. 151 | for idx, conversion in enumerate(conversions): 152 | recommender = conversion.get('recommender') 153 | if not recommender: 154 | raise ValidationError('InvalidExperimentConversions', f'Experiment conversion at index {idx} is missing recommender') 155 | 156 | recommender_config = config.get_recommender_config(namespace, recommender) 157 | if not recommender_config: 158 | raise ValidationError('InvalidRecommender', f'Experiment conversion at index {idx} is referencing a recommender that does not exist') 159 | 160 | experiments = recommender_config.get('experiments') 161 | if not experiments: 162 | raise ConfigError(HTTPStatus.INTERNAL_SERVER_ERROR, 'ExperimentsNotFound', f'"experiments" not defined for recommender ("{recommender}")') 163 | 164 | feature = conversion.get('feature') 165 | if feature: 166 | experiment = experiments.get(feature) 167 | if not experiment: 168 | raise ValidationError('InvalidExperimentFeature', f'Experiment for feature {feature} for conversion at index {idx} is referencing a feature that is not in the configuration') 169 | elif len(experiments) == 1: 170 | experiment = next(iter(experiments.items()))[1] 171 | else: 172 | raise ValidationError('InvalidExperimentFeature', f'Experiment has multiple features configured but the feature name was not specified at conversion index {idx}') 173 | 174 | if experiment.get('method') == 'evidently': 175 | if not experiment.get('project'): 176 | raise ConfigError(HTTPStatus.INTERNAL_SERVER_ERROR, 'InvalidEvidentlyProject', 'Evidently project is missing from the configuration') 177 | 178 | metric = conversion.get('metric') 179 | if metric: 180 | metric_config = experiment.get('metrics').get(metric) 181 | if not metric_config: 182 | raise ValidationError('InvalidExperimentMetric', f'Experiment for feature {feature} for conversion at index {idx} is referencing a metric name {metric} that is not in the configuration') 183 | elif len(experiment.get('metrics')) == 1: 184 | metric_config = next(iter(experiment.get('metrics').items()))[1] 185 | else: 186 | raise ValidationError('InvalidExperimentMetric', f'Experiment for feature {feature} for conversion at index {idx} is does not specify a metric name') 187 | 188 | event = create_conversion_event(metric_config, user_id, conversion.get('value', CONVERSION_VALUE)) 189 | evidently_events.setdefault(experiment['project'], []).append(event) 190 | else: 191 | raise ConfigError(HTTPStatus.INTERNAL_SERVER_ERROR, 'UnsupportedEvaluationMethod', 'Variation evaluation method is not configured/supported') 192 | 193 | # Next send the events for each project to Evidently 194 | for item in evidently_events.items(): 195 | project = item[0] 196 | project_events = item[1] 197 | 198 | record_evidently_events(project, project_events) 199 | -------------------------------------------------------------------------------- /src/personalization_api_function/lambda_resolver.py: -------------------------------------------------------------------------------- 1 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # SPDX-License-Identifier: MIT-0 3 | 4 | """ AWS Lambda resolver that invokes a Lambda function to retrieve recommendations """ 5 | 6 | import boto3 7 | import json 8 | import codecs 9 | 10 | from typing import Dict, List, Union 11 | from http import HTTPStatus 12 | from aws_lambda_powertools import Logger, Tracer 13 | from personalization_error import LambdaError 14 | from personalization_constants import ACTION_RECOMMEND_ITEMS, ACTION_RELATED_ITEMS, ACTION_RERANK_ITEMS 15 | 16 | tracer = Tracer() 17 | logger = Logger(child=True) 18 | 19 | PAYLOAD_VERSION = '1.0' 20 | 21 | class LambdaResolver(): 22 | def __init__( 23 | self, 24 | lambda_client = boto3.client('lambda') 25 | ): 26 | self.lambda_client = lambda_client 27 | 28 | def _invoke_function(self, arn: str, context: Dict, payload: Dict) -> Dict: 29 | if context: 30 | if isinstance(context, str): 31 | context = json.loads(context) 32 | payload['context'] = context 33 | 34 | response = self.lambda_client.invoke( 35 | FunctionName = arn, 36 | InvocationType = 'RequestResponse', 37 | LogType = 'Tail', #'None'|'Tail', 38 | Payload = codecs.encode(json.dumps(payload)) 39 | ) 40 | 41 | logger.debug(response) 42 | 43 | status = response.get('StatusCode', 0) 44 | if status != HTTPStatus.OK: 45 | raise LambdaError(status, 'FunctionInvokeError', response.get('FunctionError')) 46 | 47 | return json.load(response.get('Payload')) 48 | 49 | @tracer.capture_method 50 | def get_recommend_items(self, recommender_path: str, recommender_config: Dict, variation_config: Dict, user_id: str, num_results: int = 25, context: Union[str,Dict] = None) -> Dict: 51 | arn = variation_config.get('arn') 52 | if not arn: 53 | raise LambdaError(HTTPStatus.NOT_FOUND, 'FunctionArnNotConfigured', 'Function ARN has not been configured for this namespace and recommender name') 54 | 55 | logger.debug('Invoking function %s for recommend-items recommendation type', arn) 56 | 57 | payload = { 58 | 'version': PAYLOAD_VERSION, 59 | 'action': ACTION_RECOMMEND_ITEMS, 60 | 'recommender': { 61 | 'path': recommender_path, 62 | 'config': recommender_config 63 | }, 64 | 'variation': variation_config, 65 | 'userId': user_id, 66 | 'numResults': num_results 67 | } 68 | 69 | return self._invoke_function(arn, context, payload) 70 | 71 | @tracer.capture_method 72 | def get_related_items(self, recommender_path: str, recommender_config: Dict, variation_config: Dict, item_id: str, num_results: int = 25, user_id: str = None, context: Union[str,Dict] = None) -> Dict: 73 | arn = variation_config.get('arn') 74 | if not arn: 75 | raise LambdaError(HTTPStatus.NOT_FOUND, 'FunctionArnNotConfigured', 'Function ARN has not been configured for this namespace and recommender name') 76 | 77 | logger.debug('Invoking function %s for related-items recommendation type', arn) 78 | 79 | payload = { 80 | 'version': PAYLOAD_VERSION, 81 | 'action': ACTION_RELATED_ITEMS, 82 | 'recommender': { 83 | 'path': recommender_path, 84 | 'config': recommender_config 85 | }, 86 | 'variation': variation_config, 87 | 'itemId': item_id, 88 | 'userId': (user_id if user_id else ''), 89 | 'numResults': num_results 90 | } 91 | 92 | return self._invoke_function(arn, context, payload) 93 | 94 | @tracer.capture_method 95 | def rerank_items(self, recommender_path: str, recommender_config: Dict, variation_config: Dict, user_id: str, input_list: List[str], context: Union[str,Dict] = None) -> Dict: 96 | arn = variation_config.get('arn') 97 | if not arn: 98 | raise LambdaError(HTTPStatus.NOT_FOUND, 'FunctionArnNotConfigured', 'Function ARN has not been configured for this namespace and recommender name') 99 | 100 | logger.debug('Invoking function %s for rerank-items recommendation type', arn) 101 | 102 | payload = { 103 | 'version': PAYLOAD_VERSION, 104 | 'action': ACTION_RERANK_ITEMS, 105 | 'recommender': { 106 | 'path': recommender_path, 107 | 'config': recommender_config 108 | }, 109 | 'variation': variation_config, 110 | 'userId': user_id, 111 | 'itemList': input_list 112 | } 113 | 114 | return self._invoke_function(arn, context, payload) 115 | -------------------------------------------------------------------------------- /src/personalization_api_function/personalization_error.py: -------------------------------------------------------------------------------- 1 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # SPDX-License-Identifier: MIT-0 3 | 4 | """ 5 | Error abstractions 6 | """ 7 | 8 | import json 9 | from http import HTTPStatus 10 | from botocore.exceptions import ClientError 11 | 12 | class PersonalizationError(Exception): 13 | def __init__( 14 | self, 15 | type: str, 16 | status_code: int, 17 | error_code: str, 18 | error_message: str, 19 | sdk_status_code: int = None 20 | ): 21 | super().__init__(error_message) 22 | self.type = type 23 | self.status_code = status_code 24 | self.error_code = error_code 25 | self.error_message = error_message 26 | self.sdk_status_code = sdk_status_code 27 | 28 | @classmethod 29 | def from_client_error(cls, e: ClientError): 30 | return cls(HTTPStatus.INTERNAL_SERVER_ERROR, e.response['Error']['Code'], e.response['Error']['Message'], e.response['ResponseMetadata']['HTTPStatusCode']) 31 | 32 | class ValidationError(PersonalizationError): 33 | def __init__( 34 | self, 35 | error_code: str, 36 | error_message: str, 37 | ): 38 | super().__init__('Validation', HTTPStatus.BAD_REQUEST, error_code, error_message) 39 | 40 | class JSONDecodeValidationError(ValidationError): 41 | def __init__( 42 | self, 43 | error_code: str, 44 | error_message: str 45 | ): 46 | super().__init__(error_code, error_message) 47 | 48 | @classmethod 49 | def from_json_decoder_error(cls, error_code: str, e: json.decoder.JSONDecodeError): 50 | return cls(error_code, f"{e.msg}: line {e.lineno} column {e.colno} (char {e.pos})") 51 | 52 | class ConfigError(PersonalizationError): 53 | def __init__( 54 | self, 55 | status_code: int, 56 | error_code: str, 57 | error_message: str, 58 | sdk_status_code: int = None 59 | ): 60 | super().__init__('Configuration', status_code, error_code, error_message, sdk_status_code) 61 | 62 | class PersonalizeError(PersonalizationError): 63 | def __init__( 64 | self, 65 | status_code: int, 66 | error_code: str, 67 | error_message: str, 68 | sdk_status_code: int = None 69 | ): 70 | super().__init__('Personalize', status_code, error_code, error_message, sdk_status_code) 71 | 72 | @classmethod 73 | def from_client_error(cls, e: ClientError): 74 | error_code = e.response['Error']['Code'] 75 | if error_code == 'ThrottlingException': 76 | return cls(HTTPStatus.TOO_MANY_REQUESTS, error_code, e.response['Error']['Message'], e.response['ResponseMetadata']['HTTPStatusCode']) 77 | 78 | return cls(HTTPStatus.INTERNAL_SERVER_ERROR, error_code, e.response['Error']['Message'], e.response['ResponseMetadata']['HTTPStatusCode']) 79 | 80 | class DynamoDbError(PersonalizationError): 81 | def __init__( 82 | self, 83 | status_code: int, 84 | error_code: str, 85 | error_message: str, 86 | sdk_status_code: int = None 87 | ): 88 | super().__init__('DynamoDB', status_code, error_code, error_message, sdk_status_code) 89 | 90 | class EvidentlyError(PersonalizationError): 91 | def __init__( 92 | self, 93 | status_code: int, 94 | error_code: str, 95 | error_message: str, 96 | sdk_status_code: int = None 97 | ): 98 | super().__init__('Evidently', status_code, error_code, error_message, sdk_status_code) 99 | 100 | class LambdaError(PersonalizationError): 101 | def __init__( 102 | self, 103 | status_code: int, 104 | error_code: str, 105 | error_message: str, 106 | sdk_status_code: int = None 107 | ): 108 | super().__init__('Function', status_code, error_code, error_message, sdk_status_code) 109 | 110 | class SageMakerError(PersonalizationError): 111 | def __init__( 112 | self, 113 | status_code: int, 114 | error_code: str, 115 | error_message: str, 116 | sdk_status_code: int = None 117 | ): 118 | super().__init__('SageMaker', status_code, error_code, error_message, sdk_status_code) 119 | -------------------------------------------------------------------------------- /src/personalization_api_function/personalize_resolver.py: -------------------------------------------------------------------------------- 1 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # SPDX-License-Identifier: MIT-0 3 | 4 | """ Amazon Personalize resolver that calls Personalize campaigns or recommenders """ 5 | 6 | import json 7 | import boto3 8 | 9 | from typing import Dict, List, Union 10 | from http import HTTPStatus 11 | from botocore.exceptions import ClientError 12 | from aws_lambda_powertools import Logger, Tracer, Metrics 13 | from aws_lambda_powertools.metrics import MetricUnit 14 | from personalization_error import PersonalizeError 15 | 16 | tracer = Tracer() 17 | logger = Logger(child=True) 18 | metrics = Metrics() 19 | 20 | class PersonalizeResolver(): 21 | def __init__( 22 | self, 23 | personalize = boto3.client('personalize-runtime') 24 | ): 25 | self.personalize_runtime = personalize 26 | 27 | @tracer.capture_method 28 | def get_recommend_items( 29 | self, 30 | variation_config: Dict, 31 | arn: str, 32 | user_id: str, 33 | num_results: int = 25, 34 | filter_arn: str = None, 35 | filter_values: Union[str,Dict] = None, 36 | context: Union[str,Dict] = None, 37 | include_metadata: bool = True 38 | ) -> Dict: 39 | 40 | if not arn: 41 | raise PersonalizeError(HTTPStatus.NOT_FOUND, 'RecommenderArnNotConfigured', 'Personalize recommender/campaign ARN has not been configured for this namespace and recommender name') 42 | 43 | params = { 44 | 'userId': user_id, 45 | 'numResults': num_results 46 | } 47 | 48 | is_recommender = arn.split(':')[5].startswith('recommender/') 49 | if is_recommender: 50 | params['recommenderArn'] = arn 51 | else: 52 | params['campaignArn'] = arn 53 | 54 | if filter_arn: 55 | params['filterArn'] = filter_arn 56 | if filter_values: 57 | if isinstance(filter_values, str): 58 | filter_values = json.loads(filter_values) 59 | params['filterValues'] = filter_values 60 | 61 | if context: 62 | if isinstance(context, str): 63 | context = json.loads(context) 64 | params['context'] = context 65 | 66 | metadata_config = variation_config.get('inferenceItemMetadata') 67 | if include_metadata and metadata_config and metadata_config.get('type') == 'personalize': 68 | item_columns = metadata_config.get('itemColumns') 69 | params['metadataColumns'] = { 70 | 'ITEMS': item_columns 71 | } 72 | 73 | logger.debug('Calling personalize.get_recommendations() with arguments: %s', params) 74 | 75 | try: 76 | response = self.personalize_runtime.get_recommendations(**params) 77 | logger.debug(response) 78 | del response['ResponseMetadata'] 79 | except ClientError as e: 80 | if e.response['Error']['Code'] == 'ThrottlingException': 81 | metrics.add_dimension(name="Arn", value=arn) 82 | metrics.add_metric(name="PersonalizeInferenceThrottledRequests", unit=MetricUnit.Count, value=1) 83 | raise PersonalizeError.from_client_error(e) 84 | 85 | return response 86 | 87 | @tracer.capture_method 88 | def get_related_items( 89 | self, 90 | variation_config: Dict, 91 | arn: str, 92 | item_id: str, 93 | num_results: int = 25, 94 | filter_arn: str = None, 95 | filter_values: Union[str,Dict] = None, 96 | user_id: str = None, 97 | context: Union[str,Dict] = None, 98 | include_metadata: bool = True 99 | ) -> Dict: 100 | 101 | if not arn: 102 | raise PersonalizeError(HTTPStatus.NOT_FOUND, 'RecommenderArnNotConfigured', 'Personalize recommender/campaign ARN has not been configured for this namespace and recommender name') 103 | 104 | params = { 105 | 'itemId': item_id, 106 | 'numResults': num_results 107 | } 108 | 109 | is_recommender = arn.split(':')[5].startswith('recommender/') 110 | if is_recommender: 111 | params['recommenderArn'] = arn 112 | else: 113 | params['campaignArn'] = arn 114 | 115 | if user_id: 116 | params['userId'] = user_id 117 | 118 | if filter_arn: 119 | params['filterArn'] = filter_arn 120 | if filter_values: 121 | if isinstance(filter_values, str): 122 | filter_values = json.loads(filter_values) 123 | params['filterValues'] = filter_values 124 | 125 | if context: 126 | if isinstance(context, str): 127 | context = json.loads(context) 128 | params['context'] = context 129 | 130 | metadata_config = variation_config.get('inferenceItemMetadata') 131 | if include_metadata and metadata_config and metadata_config.get('type') == 'personalize': 132 | item_columns = metadata_config.get('itemColumns') 133 | params['metadataColumns'] = { 134 | 'ITEMS': item_columns 135 | } 136 | 137 | logger.debug('Calling personalize.get_recommendations() with arguments: %s', params) 138 | 139 | try: 140 | response = self.personalize_runtime.get_recommendations(**params) 141 | logger.debug(response) 142 | del response['ResponseMetadata'] 143 | except ClientError as e: 144 | if e.response['Error']['Code'] == 'ThrottlingException': 145 | metrics.add_dimension(name="Arn", value=arn) 146 | metrics.add_metric(name="PersonalizeInferenceThrottledRequests", unit=MetricUnit.Count, value=1) 147 | 148 | raise PersonalizeError.from_client_error(e) 149 | 150 | return response 151 | 152 | @tracer.capture_method 153 | def rerank_items( 154 | self, 155 | variation_config: Dict, 156 | arn: str, 157 | user_id: str, 158 | input_list: List[str], 159 | filter_arn: str = None, 160 | filter_values: Union[str,Dict] = None, 161 | context: Union[str,Dict] = None, 162 | include_metadata: bool = True 163 | ) -> Dict: 164 | 165 | if not arn: 166 | raise PersonalizeError(HTTPStatus.NOT_FOUND, 'RecommenderArnNotConfigured', 'Personalize recommender/campaign ARN has not been configured for this namespace and recommender name') 167 | 168 | params = { 169 | 'userId': user_id, 170 | 'inputList': input_list 171 | } 172 | 173 | is_recommender = arn.split(':')[5].startswith('recommender/') 174 | if is_recommender: 175 | params['recommenderArn'] = arn 176 | else: 177 | params['campaignArn'] = arn 178 | 179 | if filter_arn: 180 | params['filterArn'] = filter_arn 181 | if filter_values: 182 | if isinstance(filter_values, str): 183 | filter_values = json.loads(filter_values) 184 | params['filterValues'] = filter_values 185 | 186 | if context: 187 | if isinstance(context, str): 188 | context = json.loads(context) 189 | params['context'] = context 190 | 191 | metadata_config = variation_config.get('inferenceItemMetadata') 192 | if include_metadata and metadata_config and metadata_config.get('type') == 'personalize': 193 | item_columns = metadata_config.get('itemColumns') 194 | params['metadataColumns'] = { 195 | 'ITEMS': item_columns 196 | } 197 | 198 | logger.debug('Calling personalize.get_personalized_ranking() with arguments: %s', params) 199 | 200 | try: 201 | response = self.personalize_runtime.get_personalized_ranking(**params) 202 | logger.debug(response) 203 | del response['ResponseMetadata'] 204 | except ClientError as e: 205 | if e.response['Error']['Code'] == 'ThrottlingException': 206 | metrics.add_dimension(name="Arn", value=arn) 207 | metrics.add_metric(name="PersonalizeInferenceThrottledRequests", unit=MetricUnit.Count, value=1) 208 | raise PersonalizeError.from_client_error(e) 209 | 210 | return response 211 | -------------------------------------------------------------------------------- /src/personalization_api_function/requirements.txt: -------------------------------------------------------------------------------- 1 | # Note: AWS Lambda Power Tools is required but is satisfied by a Lambda layer at runtime. 2 | # Require a recent version of boto3 to pickup latest API changes for Personalize 3 | boto3==1.34.78 4 | pytz -------------------------------------------------------------------------------- /src/personalization_api_function/response_decorator.py: -------------------------------------------------------------------------------- 1 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # SPDX-License-Identifier: MIT-0 3 | 4 | """ 5 | Personalization response decorators 6 | """ 7 | 8 | import boto3 9 | import botocore 10 | import os 11 | import time 12 | import math 13 | import dbm 14 | import json 15 | import gzip 16 | import shutil 17 | 18 | from typing import Any, Dict, List 19 | from http import HTTPStatus 20 | from abc import ABC, abstractmethod 21 | from concurrent.futures import ThreadPoolExecutor, Future, as_completed 22 | from botocore.exceptions import ClientError 23 | from aws_lambda_powertools import Logger, Tracer, Metrics 24 | from aws_lambda_powertools.metrics import MetricUnit 25 | from personalization_error import ConfigError, DynamoDbError 26 | from personalization_constants import LOCAL_DB_FILENAME, LOCAL_DB_GZIP_FILENAME 27 | from background_tasks import BackgroundTasks 28 | 29 | tracer = Tracer() 30 | logger = Logger(child=True) 31 | metrics = Metrics() 32 | 33 | # Should these be in the config? 34 | table_name_prefix = os.environ.get('ItemsTableNamePrefix', 'PersonalizationApiItemMetadata_') 35 | primary_key_name = os.environ.get('ItemsTablePrimaryKeyFieldName', 'id') 36 | 37 | PREPARE_CHECK_FREQUENCY = 5 # 5 seconds 38 | DEFAULT_LOCALDB_DOWNLOAD_FREQ = 300 # 5 minutes 39 | class ResponseDecorator(ABC): 40 | _decorators: Dict[str, Any] = {} 41 | _last_prepare_check = 0 42 | _last_localdb_download_attempt = {} 43 | 44 | @abstractmethod 45 | def decorate(self, response: Dict) -> Dict: 46 | pass 47 | 48 | def close(self): 49 | pass 50 | 51 | @staticmethod 52 | def prepare_datastores(config: Dict, background: BackgroundTasks): 53 | start = time.time() 54 | 55 | if start - ResponseDecorator._last_prepare_check > PREPARE_CHECK_FREQUENCY: 56 | bucket = os.environ['StagingBucket'] 57 | 58 | prepared_count = 0 59 | 60 | for namespace, namespace_config in config['namespaces'].items(): 61 | metadata_config = namespace_config.get('inferenceItemMetadata') 62 | if not metadata_config: 63 | continue 64 | 65 | type = metadata_config.get('type') 66 | 67 | if type == 'localdb': 68 | sync_interval = metadata_config.get('syncInterval', DEFAULT_LOCALDB_DOWNLOAD_FREQ) 69 | 70 | if start - ResponseDecorator._last_localdb_download_attempt.get(namespace, 0) > sync_interval: 71 | ResponseDecorator._last_localdb_download_attempt[namespace] = time.time() 72 | background.submit(ResponseDecorator._download_localdb, namespace = namespace, bucket = bucket) 73 | prepared_count += 1 74 | else: 75 | logger.debug('Localdb inference metadata sync check for namespace %s not due yet', namespace) 76 | 77 | elif type == 'dynamodb': 78 | ResponseDecorator._decorators[namespace] = DynamoDbResponseDecorator(table_name_prefix + namespace, primary_key_name) 79 | prepared_count += 1 80 | 81 | elif type == 'personalize': 82 | logger.debug('Personalize inference metadata does not require preparation') 83 | 84 | ResponseDecorator._last_prepare_check = prepare_done = time.time() 85 | 86 | if prepared_count > 0: 87 | logger.info('Prepared %s datastores in %0.2fms', prepared_count, prepare_done - start) 88 | 89 | else: 90 | logger.debug('Item metadata datastores not due for prepare check') 91 | 92 | @staticmethod 93 | def get_instance(namespace: str, config: Dict) -> Any: 94 | """ Creates and returns response decorator based on a namespace configuration """ 95 | namespace_config = config.get_namespace_config(namespace) 96 | if not namespace_config: 97 | return None 98 | 99 | metadata_config = namespace_config.get('inferenceItemMetadata') 100 | if not metadata_config: 101 | return None 102 | 103 | decorator = ResponseDecorator._decorators.get(namespace) 104 | if not decorator: 105 | type = metadata_config.get('type') 106 | if type == 'localdb': 107 | decorator = LocalDbResponseDecorator(namespace) 108 | elif type == 'dynamodb': 109 | decorator = DynamoDbResponseDecorator(table_name_prefix + namespace, primary_key_name) 110 | elif type == 'personalize': 111 | decorator = PersonalizeResponseDecorator(namespace) 112 | else: 113 | raise ConfigError(HTTPStatus.INTERNAL_SERVER_ERROR, 'UnsupportedInferenceItemMetadataType', 'Inference item metadata type is not supported') 114 | 115 | ResponseDecorator._decorators[namespace] = decorator 116 | 117 | return decorator 118 | 119 | @staticmethod 120 | def _download_localdb(namespace: str, bucket: str, s3: Any = None): 121 | if not s3: 122 | s3 = boto3.client('s3') 123 | 124 | local_dir = f'/tmp/{namespace}' 125 | if not os.path.isdir(local_dir): 126 | os.makedirs(local_dir) 127 | 128 | local_file = f'{local_dir}/{LOCAL_DB_FILENAME}' 129 | 130 | key = f'localdbs/{namespace}/{LOCAL_DB_GZIP_FILENAME}' 131 | 132 | logger.info('Downloading s3://%s/%s and uncompressing to %s', bucket, key, local_file) 133 | try: 134 | response = s3.get_object(Bucket = bucket, Key = key) 135 | stream = gzip.GzipFile(None, 'rb', fileobj = response['Body']) 136 | with open(local_file, 'wb') as out: 137 | shutil.copyfileobj(stream, out) 138 | 139 | old_decorator = ResponseDecorator._decorators.get(namespace) 140 | ResponseDecorator._decorators[namespace] = LocalDbResponseDecorator(namespace) 141 | if old_decorator: 142 | old_decorator.close() 143 | 144 | except ClientError as e: 145 | if e.response['Error']['Code'] == 'AccessDenied': 146 | logger.error('Staged localdb file s3://%s/%s either does not exist or access has been revoked', bucket, key) 147 | else: 148 | raise e 149 | 150 | class LocalDbResponseDecorator(ResponseDecorator): 151 | def __init__(self, namespace: str): 152 | self.namespace = namespace 153 | self.local_file = f'/tmp/{self.namespace}/{LOCAL_DB_FILENAME}' 154 | if os.path.isfile(self.local_file): 155 | self.dbm_file = dbm.open(self.local_file, 'r') 156 | else: 157 | self.dbm_file = None 158 | 159 | def __del__(self): 160 | self.close() 161 | 162 | def close(self): 163 | try: 164 | if self.dbm_file: 165 | self.dbm_file.close() 166 | except Exception: 167 | pass 168 | self.dbm = None 169 | 170 | @tracer.capture_method 171 | def decorate(self, response: Dict): 172 | if not self.dbm_file and os.path.isfile(self.local_file): 173 | self.dbm_file = dbm.open(self.local_file, 'r') 174 | 175 | if self.dbm_file: 176 | # Create lookup dictionary so results from DDB can be efficiently merged into response. 177 | lookup: Dict[str, List[int]] = {} 178 | items_key_name = 'itemList' if 'itemList' in response else 'personalizedRanking' 179 | if not items_key_name in response: 180 | raise ValueError(f'Response is missing "{items_key_name}" property') 181 | 182 | for idx,item in enumerate(response[items_key_name]): 183 | lookup.setdefault(item['itemId'], []).append(idx) 184 | 185 | unique_items = list(lookup.keys()) 186 | 187 | def get_item(id): 188 | s = self.dbm_file.get(id) 189 | return json.loads(s) if s else s 190 | 191 | for id in unique_items: 192 | item = get_item(id) 193 | if item: 194 | for idx in lookup[id]: 195 | response[items_key_name][idx]['metadata'] = item 196 | else: 197 | logger.error('Local DB file %s does not exist on local disk. Has item metadata been uploaded and staged in S3?', self.local_file) 198 | 199 | class DynamoDbResponseDecorator(ResponseDecorator): 200 | MAX_BATCH_SIZE = 50 201 | __dynamodb = boto3.resource('dynamodb') 202 | 203 | def __init__(self, table_name: str, primary_key_name: str): 204 | self.table_name = table_name 205 | self.primary_key_name = primary_key_name 206 | 207 | @tracer.capture_method 208 | def decorate(self, response: Dict): 209 | try: 210 | self._decorate(response) 211 | except DynamoDbResponseDecorator.__dynamodb.meta.client.exceptions.LimitExceedException as e: 212 | metrics.add_metric(name="DynamoDBLimitExceed", unit=MetricUnit.Count, value=1) 213 | raise DynamoDbError( 214 | HTTPStatus.TOO_MANY_REQUESTS, 215 | e.response['Error']['Code'], 216 | e.response['Error']['Message'], 217 | e.response['ResponseMetadata']['HTTPStatusCode'] 218 | ) 219 | except botocore.exceptions.ClientError as e: 220 | raise DynamoDbError( 221 | HTTPStatus.INTERNAL_SERVER_ERROR, 222 | e.response['Error']['Code'], 223 | e.response['Error']['Message'], 224 | e.response['ResponseMetadata']['HTTPStatusCode'] 225 | ) 226 | 227 | def _decorate(self, response: Dict): 228 | items_key_name = 'itemList' if 'itemList' in response else 'personalizedRanking' 229 | if not items_key_name in response: 230 | raise ValueError(f'Response is missing "{items_key_name}" property') 231 | 232 | # Create lookup dictionary so results from DDB can be efficiently merged into response. 233 | lookup = {} 234 | for idx,item in enumerate(response[items_key_name]): 235 | lookup.setdefault(item['itemId'], []).append(idx) 236 | 237 | unique_items = list(lookup.keys()) 238 | 239 | if len(unique_items) > self.MAX_BATCH_SIZE: 240 | chunk_size = int(math.ceil(len(unique_items) / math.ceil(len(unique_items)/self.MAX_BATCH_SIZE))) 241 | 242 | item_chunks = [unique_items[i:i + chunk_size] for i in range(0, len(unique_items), chunk_size)] 243 | 244 | logger.debug('Launching %d background threads to lookup metadata for %d unique items in chunks of max %d', 245 | len(item_chunks), len(unique_items), chunk_size) 246 | 247 | with ThreadPoolExecutor() as executor: 248 | futures: Future = [] 249 | for item_ids in item_chunks: 250 | batch_keys = { 251 | self.table_name: { 252 | 'Keys': [{self.primary_key_name: item_id} for item_id in item_ids] 253 | } 254 | } 255 | 256 | futures.append( 257 | executor.submit(self._batch_get, None, batch_keys) 258 | ) 259 | 260 | for future in as_completed(futures): 261 | retrieved = future.result() 262 | # Decorate each item with a "metadata" field containing info from DDB. 263 | for ddb_item in retrieved[self.table_name]: 264 | for idx in lookup[ddb_item[self.primary_key_name]]: 265 | response[items_key_name][idx]['metadata'] = ddb_item['attributes'] 266 | else: 267 | batch_keys = { 268 | self.table_name: { 269 | 'Keys': [{self.primary_key_name: item_id} for item_id in unique_items] 270 | } 271 | } 272 | 273 | retrieved = self._batch_get(DynamoDbResponseDecorator.__dynamodb, batch_keys) 274 | # Decorate each item with a "metadata" field containing info from DDB. 275 | for ddb_item in retrieved[self.table_name]: 276 | for idx in lookup[ddb_item[self.primary_key_name]]: 277 | response[items_key_name][idx]['metadata'] = ddb_item['attributes'] 278 | 279 | def _batch_get(self, dynamodb, batch_keys: Dict) -> Dict: 280 | """ 281 | Gets a batch of items from Amazon DynamoDB. Batches can contain keys from 282 | more than one table. 283 | 284 | When Amazon DynamoDB cannot process all items in a batch, a set of unprocessed 285 | keys is returned. This function uses an exponential backoff algorithm to retry 286 | getting the unprocessed keys until all are retrieved or the specified 287 | number of tries is reached. 288 | 289 | :param dynamodb: DynamoDB resource or None and one will be created (such as in thread) 290 | :param batch_keys: The set of keys to retrieve. A batch can contain at most 100 291 | keys. Otherwise, Amazon DynamoDB returns an error. 292 | :return: The dictionary of retrieved items grouped under their respective 293 | table names. 294 | """ 295 | if not dynamodb: 296 | dynamodb = boto3.resource('dynamodb') 297 | 298 | tries = 0 299 | max_tries = 3 300 | sleep_millis = 250 # Start with 250ms of sleep, then exponentially increase. 301 | retrieved = {key: [] for key in batch_keys} 302 | while tries < max_tries: 303 | response = dynamodb.batch_get_item(RequestItems=batch_keys) 304 | # Collect any retrieved items and retry unprocessed keys. 305 | for key in response.get('Responses', []): 306 | retrieved[key] += response['Responses'][key] 307 | 308 | unprocessed = response['UnprocessedKeys'] 309 | 310 | if len(unprocessed) > 0: 311 | batch_keys = unprocessed 312 | unprocessed_count = sum([len(batch_key['Keys']) for batch_key in batch_keys.values()]) 313 | logger.warn('%s unprocessed keys returned. Sleeping for %sms, then will retry', unprocessed_count, sleep_millis) 314 | 315 | tries += 1 316 | if tries < max_tries: 317 | logger.info('Sleeping for %sms', sleep_millis) 318 | time.sleep(sleep_millis / 1000.0) 319 | sleep_millis = min(sleep_millis * 2, 1500) 320 | else: 321 | break 322 | 323 | return retrieved 324 | 325 | class PersonalizeResponseDecorator(ResponseDecorator): 326 | def __init__(self, namespace: str): 327 | self.namespace = namespace 328 | 329 | @tracer.capture_method 330 | def decorate(self, response: Dict): 331 | # Nothing to do since Personalize already returns "metadata" for each item 332 | pass -------------------------------------------------------------------------------- /src/personalization_api_function/response_post_process.py: -------------------------------------------------------------------------------- 1 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # SPDX-License-Identifier: MIT-0 3 | 4 | """ AWS Lambda response post processor """ 5 | 6 | import boto3 7 | import json 8 | import codecs 9 | 10 | from typing import Dict 11 | from http import HTTPStatus 12 | from aws_lambda_powertools import Logger, Tracer 13 | from personalization_error import LambdaError 14 | from personalization_constants import ACTION_RECOMMEND_ITEMS, ACTION_RELATED_ITEMS, ACTION_RERANK_ITEMS 15 | 16 | tracer = Tracer() 17 | logger = Logger(child=True) 18 | 19 | PAYLOAD_VERSION = '1.0' 20 | 21 | class PostProcessor(): 22 | def __init__( 23 | self, 24 | lambda_client = boto3.client('lambda') 25 | ): 26 | self.lambda_client = lambda_client 27 | 28 | def _invoke_function(self, arn: str, payload: Dict) -> Dict: 29 | response = self.lambda_client.invoke( 30 | FunctionName = arn, 31 | InvocationType = 'RequestResponse', 32 | LogType = 'Tail', #'None'|'Tail', 33 | Payload = codecs.encode(json.dumps(payload)) 34 | ) 35 | 36 | logger.debug(response) 37 | 38 | status = response.get('StatusCode', 0) 39 | if status != HTTPStatus.OK: 40 | raise LambdaError(status, 'FunctionInvokeError', response.get('FunctionError')) 41 | 42 | return json.load(response.get('Payload')) 43 | 44 | @tracer.capture_method 45 | def process_recommend_items(self, recommender_path: str, recommender_config: Dict, variation_config: Dict, user_id: str, response: Dict) -> Dict: 46 | post_process_config = recommender_config.get('responsePostProcessor') 47 | arn = post_process_config.get('arn') 48 | if not arn: 49 | raise LambdaError(HTTPStatus.NOT_FOUND, 'FunctionArnNotConfigured', 'Post process function ARN has not been configured for this namespace and recommender name') 50 | 51 | logger.debug('Invoking post-proces function %s for recommend-items recommendation type', arn) 52 | 53 | payload = { 54 | 'version': PAYLOAD_VERSION, 55 | 'action': ACTION_RECOMMEND_ITEMS, 56 | 'recommender': { 57 | 'path': recommender_path, 58 | 'config': recommender_config 59 | }, 60 | 'variation': variation_config, 61 | 'userId': user_id, 62 | 'response': response 63 | } 64 | 65 | return self._invoke_function(arn, payload) 66 | 67 | @tracer.capture_method 68 | def process_related_items(self, recommender_path: str, recommender_config: Dict, variation_config: Dict, item_id: str, response: Dict) -> Dict: 69 | post_process_config = recommender_config.get('responsePostProcessor') 70 | arn = post_process_config.get('arn') 71 | if not arn: 72 | raise LambdaError(HTTPStatus.NOT_FOUND, 'FunctionArnNotConfigured', 'Post process function ARN has not been configured for this namespace and recommender name') 73 | 74 | logger.debug('Invoking function %s for related-items recommendation type', arn) 75 | 76 | payload = { 77 | 'version': PAYLOAD_VERSION, 78 | 'action': ACTION_RELATED_ITEMS, 79 | 'recommender': { 80 | 'path': recommender_path, 81 | 'config': recommender_config 82 | }, 83 | 'variation': variation_config, 84 | 'itemId': item_id, 85 | 'response': response 86 | } 87 | 88 | return self._invoke_function(arn, payload) 89 | 90 | @tracer.capture_method 91 | def process_rerank_items(self, recommender_path: str, recommender_config: Dict, variation_config: Dict, user_id: str, response: Dict) -> Dict: 92 | post_process_config = recommender_config.get('responsePostProcessor') 93 | arn = post_process_config.get('arn') 94 | if not arn: 95 | raise LambdaError(HTTPStatus.NOT_FOUND, 'FunctionArnNotConfigured', 'Post process function ARN has not been configured for this namespace and recommender name') 96 | 97 | logger.debug('Invoking function %s for rerank-items recommendation type', arn) 98 | 99 | payload = { 100 | 'version': PAYLOAD_VERSION, 101 | 'action': ACTION_RERANK_ITEMS, 102 | 'recommender': { 103 | 'path': recommender_path, 104 | 'config': recommender_config 105 | }, 106 | 'variation': variation_config, 107 | 'userId': user_id, 108 | 'response': response 109 | } 110 | 111 | return self._invoke_function(arn, payload) 112 | -------------------------------------------------------------------------------- /src/personalization_api_function/sagemaker_resolver.py: -------------------------------------------------------------------------------- 1 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # SPDX-License-Identifier: MIT-0 3 | 4 | """ Amazon SageMaker resolver that invokes an inference endpoint for a SageMaker model """ 5 | 6 | import boto3 7 | import json 8 | import codecs 9 | 10 | from typing import Dict, List, Union 11 | from http import HTTPStatus 12 | from aws_lambda_powertools import Logger, Tracer 13 | from personalization_error import SageMakerError 14 | from personalization_constants import ACTION_RECOMMEND_ITEMS, ACTION_RELATED_ITEMS, ACTION_RERANK_ITEMS 15 | 16 | tracer = Tracer() 17 | logger = Logger(child=True) 18 | 19 | PAYLOAD_VERSION = '1.0' 20 | 21 | class SageMakerResolver(): 22 | def __init__( 23 | self, 24 | sagemaker = boto3.client('sagemaker-runtime') 25 | ): 26 | self.sagemaker = sagemaker 27 | 28 | def _invoke_endpoint(self, endpoint_name: str, context: Dict, payload: Dict) -> Dict: 29 | if context: 30 | if isinstance(context, str): 31 | context = json.loads(context) 32 | payload['context'] = context 33 | 34 | response = self.sagemaker.invoke_endpoint( 35 | EndpointName = endpoint_name, 36 | ContentType = 'application/json', 37 | Accept = 'application/json', 38 | Body = codecs.encode(json.dumps(payload)) 39 | ) 40 | 41 | logger.debug(response) 42 | 43 | return json.load(response.get('Body')) 44 | 45 | @tracer.capture_method 46 | def get_recommend_items(self, recommender_path: str, recommender_config: Dict, variation_config: Dict, user_id: str, num_results: int = 25, context: Union[str,Dict] = None) -> Dict: 47 | endpoint_name = variation_config.get('endpointName') 48 | if not endpoint_name: 49 | raise SageMakerError(HTTPStatus.NOT_FOUND, 'EndpointNameNotConfigured', 'Endpoint name has not been configured for this namespace and recommender name') 50 | 51 | logger.debug('Invoking SageMaker endpoint %s for recommend-items recommendation type', endpoint_name) 52 | 53 | payload = { 54 | 'version': PAYLOAD_VERSION, 55 | 'action': ACTION_RECOMMEND_ITEMS, 56 | 'recommender': { 57 | 'path': recommender_path, 58 | 'config': recommender_config 59 | }, 60 | 'variation': variation_config, 61 | 'userId': user_id, 62 | 'numResults': num_results 63 | } 64 | 65 | return self._invoke_endpoint(endpoint_name, context, payload) 66 | 67 | @tracer.capture_method 68 | def get_related_items(self, recommender_path: str, recommender_config: Dict, variation_config: Dict, item_id: str, num_results: int = 25, user_id: str = None, context: Union[str,Dict] = None) -> Dict: 69 | endpoint_name = variation_config.get('endpointName') 70 | if not endpoint_name: 71 | raise SageMakerError(HTTPStatus.NOT_FOUND, 'EndpointNameNotConfigured', 'Endpoint name has not been configured for this namespace and recommender name') 72 | 73 | logger.debug('Invoking SageMaker endpoint %s for related-items recommendation type', endpoint_name) 74 | 75 | payload = { 76 | 'version': PAYLOAD_VERSION, 77 | 'action': ACTION_RELATED_ITEMS, 78 | 'recommender': { 79 | 'path': recommender_path, 80 | 'config': recommender_config 81 | }, 82 | 'variation': variation_config, 83 | 'itemId': item_id, 84 | 'userId': (user_id if user_id else ''), 85 | 'numResults': num_results 86 | } 87 | 88 | return self._invoke_endpoint(endpoint_name, context, payload) 89 | 90 | @tracer.capture_method 91 | def rerank_items(self, recommender_path: str, recommender_config: Dict, variation_config: Dict, user_id: str, input_list: List[str], context: Union[str,Dict] = None) -> Dict: 92 | endpoint_name = variation_config.get('endpointName') 93 | if not endpoint_name: 94 | raise SageMakerError(HTTPStatus.NOT_FOUND, 'EndpointNameNotConfigured', 'Endpoint name has not been configured for this namespace and recommender name') 95 | 96 | logger.debug('Invoking SageMaker endpoint %s for rerank-items recommendation type', endpoint_name) 97 | 98 | payload = { 99 | 'version': PAYLOAD_VERSION, 100 | 'action': ACTION_RERANK_ITEMS, 101 | 'recommender': { 102 | 'path': recommender_path, 103 | 'config': recommender_config 104 | }, 105 | 'variation': variation_config, 106 | 'userId': user_id, 107 | 'itemList': input_list 108 | } 109 | 110 | return self._invoke_endpoint(endpoint_name, context, payload) 111 | -------------------------------------------------------------------------------- /src/personalization_api_function/util.py: -------------------------------------------------------------------------------- 1 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # SPDX-License-Identifier: MIT-0 3 | 4 | import json 5 | import decimal 6 | 7 | class CompatEncoder(json.JSONEncoder): 8 | """ Compatibility encoder that supports Decimal type 9 | Usage: 10 | json.dumps(data, cls=CompatEncoder) 11 | """ 12 | def default(self, obj): 13 | if isinstance(obj, decimal.Decimal): 14 | if obj % 1 > 0: 15 | return float(obj) 16 | else: 17 | return int(obj) 18 | else: 19 | return super(CompatEncoder, self).default(obj) 20 | -------------------------------------------------------------------------------- /src/statemachine/README.md: -------------------------------------------------------------------------------- 1 | # Configuration synchronization state machine 2 | 3 | The [sync_resources.asl.json](./sync_resources.asl.json) file is an AWS Step Function state machine definition that synchronizes cache settings found in your [configuration](../../docs/configuration.md) to CloudFront and/or API Gateway as well as provision tables in DynamoDB to hold item metadata (if configured to do so). -------------------------------------------------------------------------------- /src/statemachine/sync_resources.asl.json: -------------------------------------------------------------------------------- 1 | { 2 | "Comment": "Personalization API configuration to dependent resource synchronization.", 3 | "StartAt": "SyncTasks", 4 | "States": { 5 | "SyncTasks": { 6 | "Type": "Parallel", 7 | "End": true, 8 | "Branches": [ 9 | { 10 | "StartAt": "SyncCacheSettings", 11 | "States": { 12 | "SyncCacheSettings": { 13 | "Type": "Task", 14 | "Resource": "${SyncCacheSettingsFunctionArn}", 15 | "InputPath": "$.detail.content", 16 | "End": true 17 | } 18 | } 19 | }, 20 | { 21 | "StartAt": "SyncDynamoDbTables", 22 | "States": { 23 | "SyncDynamoDbTables": { 24 | "Type": "Task", 25 | "Resource": "${SyncDyanamoDbTableFunctionArn}", 26 | "InputPath": "$.detail.content", 27 | "Retry": [{ 28 | "ErrorEquals": ["ResourcePending"], 29 | "IntervalSeconds": 5, 30 | "BackoffRate": 1.5, 31 | "MaxAttempts": 10 32 | }], 33 | "End": true 34 | } 35 | } 36 | } 37 | ] 38 | } 39 | } 40 | } -------------------------------------------------------------------------------- /src/swagger-ui/favicon-16x16.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/personalization-apis/2593e0f0a8f22a25440be1a2e6c52fcefd2ab65f/src/swagger-ui/favicon-16x16.png -------------------------------------------------------------------------------- /src/swagger-ui/favicon-32x32.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/personalization-apis/2593e0f0a8f22a25440be1a2e6c52fcefd2ab65f/src/swagger-ui/favicon-32x32.png -------------------------------------------------------------------------------- /src/swagger-ui/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 10 | Personalization APIs Demo - SwaggerUI 11 | 12 | 13 | 14 | 15 | 16 |
17 | 18 | 26 | 27 | -------------------------------------------------------------------------------- /src/sync_cache_settings_function/README.md: -------------------------------------------------------------------------------- 1 | # Synchronize cache settings function 2 | 3 | This AWS Lambda function is called via the [sync_resources.asl.json](../statemachine/sync_resources.asl.json) AWS Step Function state machine to synchronize cache settings in your [configuration](../../docs/configuration.md) to CloudFront and/or API Gateway as needed. The state machine is executed each time the configuration is deployed in AWS AppConfig. -------------------------------------------------------------------------------- /src/sync_cache_settings_function/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/personalization-apis/2593e0f0a8f22a25440be1a2e6c52fcefd2ab65f/src/sync_cache_settings_function/__init__.py -------------------------------------------------------------------------------- /src/sync_cache_settings_function/main.py: -------------------------------------------------------------------------------- 1 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # SPDX-License-Identifier: MIT-0 3 | 4 | import boto3 5 | import json 6 | import os 7 | from typing import Dict, List, Tuple 8 | from jsonpath_ng import parse 9 | from aws_lambda_powertools import Logger, Tracer 10 | 11 | tracer = Tracer() 12 | logger = Logger() 13 | 14 | apigw = boto3.client('apigateway') 15 | cloudfront = boto3.client('cloudfront') 16 | 17 | REST_API_ID = os.environ['RestApiId'] 18 | REST_API_STAGE = os.environ['RestApiStage'] 19 | CLOUD_FRONT_CACHE_POLICY_ID = os.environ['CloudFrontCachePolicyId'] 20 | CLOUD_FRONT_ORIGIN_REQUEST_POLICY_ID = os.environ['CloudFrontOriginRequestPolicyId'] 21 | 22 | APIGW_ROOT_PATHS = { 23 | 'recommend-items': '/~1recommend-items~1{namespace}~1{recommender}~1{userId}/GET', 24 | 'related-items': '/~1related-items~1{namespace}~1{recommender}~1{itemId}/GET', 25 | 'rerank-items': '/~1rerank-items~1{namespace}~1{recommender}~1{userId}~1{itemIds}/GET' 26 | } 27 | 28 | @tracer.capture_method 29 | def update_apigw_rest_stage_caching(personalization_config: Dict, cache_headers: List[str]): 30 | # Get current stage settings to determine cache cluster settings. 31 | response = apigw.get_stage( 32 | restApiId=REST_API_ID, 33 | stageName=REST_API_STAGE 34 | ) 35 | 36 | cache_cluster_current_enabled = response['cacheClusterEnabled'] 37 | cache_cluster_size = response['cacheClusterSize'] 38 | cache_cluster_status = response['cacheClusterStatus'] 39 | 40 | logger.info('Current APIGW cache cluster settings: enabled = %s, size = %s, status = %s', 41 | cache_cluster_current_enabled, cache_cluster_size, cache_cluster_status) 42 | 43 | ttls = { 44 | 'recommend-items': None, 45 | 'related-items': None, 46 | 'rerank-items': None 47 | } 48 | 49 | # Seed TTLs with the minimal value across root cache control 50 | jsonpath_expr = parse('$.cacheControl..maxAge') 51 | max_ages = [match.value for match in jsonpath_expr.find(personalization_config)] 52 | logger.info('Max ages for root cacheControl: %s', max_ages) 53 | if max_ages: 54 | for item in ttls.items(): 55 | ttls[item[0]] = min(max_ages) 56 | 57 | logger.debug('TTLs after applying root cacheControl: %s', ttls) 58 | 59 | for type in ttls.keys(): 60 | jsonpath_expr = parse(f'$..recommenders.{type}') 61 | recommenders = [match.value for match in jsonpath_expr.find(personalization_config)] 62 | 63 | for recommender in recommenders: 64 | jsonpath_expr = parse('$..cacheControl') 65 | cache_controls = [match.value for match in jsonpath_expr.find(recommender)] 66 | logger.debug('cacheControls for recommender %s/%s: %s', type, recommender.keys(), cache_controls) 67 | 68 | for cache_control in cache_controls: 69 | jsonpath_expr = parse('$..maxAge') 70 | max_ages = [match.value for match in jsonpath_expr.find(cache_control)] 71 | if max_ages: 72 | for item in ttls.items(): 73 | min_age = min(max_ages) 74 | ttls[type] = min_age if not ttls[type] else min(min_age, ttls[type]) 75 | 76 | patch = [] 77 | 78 | cache_cluster_enabled = False 79 | 80 | for item in ttls.items(): 81 | type = item[0] 82 | ttl = item[1] 83 | 84 | if ttl and ttl > 0: 85 | cache_cluster_enabled = True 86 | patch.extend([{ 87 | 'op': 'replace', 88 | 'path': f'{APIGW_ROOT_PATHS[type]}/caching/enabled', 89 | 'value': 'true' 90 | },{ 91 | 'op': 'replace', 92 | 'path': f'{APIGW_ROOT_PATHS[type]}/caching/ttlInSeconds', 93 | 'value': f'{ttl}' 94 | }]) 95 | else: 96 | patch.append({ 97 | 'op': 'replace', 98 | 'path': f'{APIGW_ROOT_PATHS[type]}/caching/enabled', 99 | 'value': 'false' 100 | }) 101 | 102 | patch.append({ 103 | 'op': 'replace', 104 | 'path': '/cacheClusterEnabled', 105 | 'value': ('true' if cache_cluster_enabled else 'false') 106 | }) 107 | 108 | if not cache_cluster_enabled and cache_cluster_current_enabled: 109 | logger.info('Stage caching is NOT configured for any methods but cache cluster is currently enabled for stage; disabling cache cluster') 110 | 111 | if cache_cluster_enabled and not cache_cluster_size: 112 | logger.info('Stage caching is configured for one or more methods but cache cluster is currently not enabled for stage; enabling cache cluster') 113 | patch.append({ 114 | 'op': 'replace', 115 | 'path': '/cacheClusterSize', 116 | 'value': '1.6' 117 | }) 118 | 119 | logger.debug('APIGW patchOperations: %s', patch) 120 | 121 | response = apigw.update_stage( 122 | restApiId=REST_API_ID, 123 | stageName=REST_API_STAGE, 124 | patchOperations=patch 125 | ) 126 | 127 | logger.debug(json.dumps(response, indent=2, default=str)) 128 | 129 | response = apigw.get_resources( 130 | restApiId=REST_API_ID, 131 | limit = 500 132 | ) 133 | 134 | paths_to_match = [ 135 | '/recommend-items/{namespace}/{recommender}/{userId}', 136 | '/related-items/{namespace}/{recommender}/{itemId}', 137 | '/rerank-items/{namespace}/{recommender}/{userId}/{itemIds}' 138 | ] 139 | resource_ids = [] 140 | for item in response['items']: 141 | methods = item.get('resourceMethods') 142 | if not methods or not methods.get('GET'): 143 | continue 144 | 145 | if item['path'] in paths_to_match: 146 | resource_ids.append(item['id']) 147 | 148 | for resource_id in resource_ids: 149 | response = apigw.get_integration( 150 | restApiId=REST_API_ID, 151 | resourceId=resource_id, 152 | httpMethod='GET' 153 | ) 154 | 155 | cache_keys = [] 156 | patch = [] 157 | 158 | for cache_key in response['cacheKeyParameters']: 159 | if not cache_key.startswith('method.request.header.'): 160 | cache_keys.append(cache_key) 161 | 162 | for cache_header in cache_headers: 163 | cache_keys.append(f'method.request.header.{cache_header}') 164 | 165 | apigw.update_integration( 166 | restApiId=REST_API_ID, 167 | resourceId=resource_id, 168 | httpMethod='GET', 169 | patchOperations=[ 170 | { 171 | 'op': 'add'|'remove'|'replace'|'move'|'copy'|'test', 172 | 'path': 'string', 173 | 'value': 'string', 174 | 'from': 'string' 175 | }, 176 | ] 177 | ) 178 | 179 | @tracer.capture_method 180 | def determine_required_cloudfront_headers(personalization_config: Dict) -> Tuple[List, List]: 181 | jsonpath_expr = parse('$..autoContext..rules[*].header') 182 | auto_context_headers = [match.value for match in jsonpath_expr.find(personalization_config)] 183 | origin_request_headers = set(i.lower() for i in auto_context_headers) 184 | cache_headers = set(origin_request_headers) 185 | 186 | jsonpath_expr = parse('$..autoContext..rules[*].type') 187 | auto_context_types = [match.value for match in jsonpath_expr.find(personalization_config)] 188 | for type in auto_context_types: 189 | if type == 'season-of-year': 190 | # Latitude needed for North/South hemisphere. However, due to high variability in 191 | # the latitude header value, do NOT include in cache headers to keep cache hit rate reasonable. 192 | origin_request_headers.add('CloudFront-Viewer-Latitude'.lower()) 193 | 194 | if type in [ 'hour-of-day', 'day-of-week', 'season-of-year' ]: 195 | origin_request_headers.add('CloudFront-Viewer-Time-Zone'.lower()) 196 | cache_headers.add('CloudFront-Viewer-Time-Zone'.lower()) 197 | 198 | # For CORS 199 | origin_request_headers.add('origin') 200 | origin_request_headers.add('access-control-request-headers') 201 | origin_request_headers.add('access-control-request-method') 202 | 203 | logger.info('Origin request headers: %s', origin_request_headers) 204 | logger.info('Cache headers: %s', cache_headers) 205 | 206 | return list(origin_request_headers), list(cache_headers) 207 | 208 | @tracer.capture_method 209 | def update_cloudfront_origin_request_policy(origin_request_headers: List[str]): 210 | response = cloudfront.get_origin_request_policy(Id = CLOUD_FRONT_ORIGIN_REQUEST_POLICY_ID) 211 | 212 | logger.debug(json.dumps(response, indent = 2, default=str)) 213 | 214 | policy_config = response['OriginRequestPolicy']['OriginRequestPolicyConfig'] 215 | 216 | update_required = False 217 | 218 | if len(origin_request_headers) > 0: 219 | update_required = (policy_config['HeadersConfig']['HeaderBehavior'] != 'whitelist' or 220 | not policy_config['HeadersConfig'].get('Headers') or 221 | not policy_config['HeadersConfig']['Headers'].get('Items') or 222 | sorted(policy_config['HeadersConfig']['Headers']['Items'], key=str.lower) != sorted(origin_request_headers, key=str.lower)) 223 | 224 | policy_config['HeadersConfig']['HeaderBehavior'] = 'whitelist' 225 | policy_config['HeadersConfig']['Headers'] = { 226 | 'Quantity': len(origin_request_headers), 227 | 'Items': origin_request_headers 228 | } 229 | else: 230 | update_required = policy_config['HeadersConfig']['HeaderBehavior'] != 'none' 231 | policy_config['HeadersConfig']['HeaderBehavior'] = 'none' 232 | policy_config['HeadersConfig'].pop('Headers', None) 233 | 234 | if update_required: 235 | logger.info('Updating CloudFront origin request policy %s', CLOUD_FRONT_ORIGIN_REQUEST_POLICY_ID) 236 | 237 | cloudfront.update_origin_request_policy( 238 | Id = CLOUD_FRONT_ORIGIN_REQUEST_POLICY_ID, 239 | IfMatch = response['ETag'], 240 | OriginRequestPolicyConfig = policy_config 241 | ) 242 | else: 243 | logger.info('Update of CloudFront origin request policy not required') 244 | 245 | @tracer.capture_method 246 | def update_cloudfront_cache_policy(personalization_config: Dict, cache_headers: List[str]): 247 | jsonpath_expr = parse('$..cacheControl.*') 248 | cache_controls = [match.value for match in jsonpath_expr.find(personalization_config)] 249 | logger.debug('cacheControl configs: %s', cache_controls) 250 | 251 | min_age = max_age = None 252 | 253 | # https://docs.aws.amazon.com/AmazonCloudFront/latest/DeveloperGuide/Expiration.html#ExpirationDownloadDist 254 | for cache_control in cache_controls: 255 | if isinstance(cache_control, dict): 256 | age = cache_control['maxAge'] 257 | min_age = age if not min_age else min(min_age, age) 258 | max_age = age if not max_age else max(max_age, age) 259 | 260 | logger.info('maxAge minimum = %s; maxAge maximum = %s', min_age, max_age) 261 | 262 | response = cloudfront.get_cache_policy(Id = CLOUD_FRONT_CACHE_POLICY_ID) 263 | 264 | logger.debug(json.dumps(response, indent = 2, default=str)) 265 | 266 | policy_config = response['CachePolicy']['CachePolicyConfig'] 267 | 268 | update_required = False 269 | 270 | if len(cache_headers) > 0: 271 | update_required = (policy_config['ParametersInCacheKeyAndForwardedToOrigin']['HeadersConfig']['HeaderBehavior'] != 'whitelist' or 272 | not policy_config['ParametersInCacheKeyAndForwardedToOrigin']['HeadersConfig'].get('Headers') or 273 | not policy_config['ParametersInCacheKeyAndForwardedToOrigin']['HeadersConfig']['Headers'].get('Items') or 274 | sorted(policy_config['ParametersInCacheKeyAndForwardedToOrigin']['HeadersConfig']['Headers']['Items'], key=str.lower) != sorted(cache_headers, key=str.lower)) 275 | 276 | policy_config['ParametersInCacheKeyAndForwardedToOrigin']['HeadersConfig']['HeaderBehavior'] = 'whitelist' 277 | policy_config['ParametersInCacheKeyAndForwardedToOrigin']['HeadersConfig']['Headers'] = { 278 | 'Quantity': len(cache_headers), 279 | 'Items': cache_headers 280 | } 281 | else: 282 | update_required = policy_config['ParametersInCacheKeyAndForwardedToOrigin']['HeadersConfig']['HeaderBehavior'] != 'none' 283 | policy_config['ParametersInCacheKeyAndForwardedToOrigin']['HeadersConfig']['HeaderBehavior'] = 'none' 284 | policy_config['ParametersInCacheKeyAndForwardedToOrigin']['HeadersConfig'].pop('Headers', None) 285 | 286 | update_required = update_required or policy_config['MinTTL'] != min_age or policy_config['MaxTTL'] != max_age 287 | 288 | policy_config['MinTTL'] = min_age 289 | policy_config['DefaultTTL'] = min_age 290 | policy_config['MaxTTL'] = max_age 291 | 292 | if update_required: 293 | logger.info('Updating CloudFront cache policy %s', CLOUD_FRONT_CACHE_POLICY_ID) 294 | 295 | cloudfront.update_cache_policy( 296 | Id = CLOUD_FRONT_CACHE_POLICY_ID, 297 | IfMatch = response['ETag'], 298 | CachePolicyConfig = policy_config 299 | ) 300 | else: 301 | logger.info('Update of CloudFront cache policy is not required') 302 | 303 | @logger.inject_lambda_context 304 | @tracer.capture_lambda_handler 305 | def lambda_handler(event, _): 306 | # The "event" is the configuration (passed as input to step function) 307 | logger.info(event) 308 | 309 | # Check the autoProvision flag on all cacheControl objects to make sure auto-provisioning is not disabled. 310 | jsonpath_expr = parse('$..cacheControl[*].autoProvision') 311 | auto_provision = [match.value for match in jsonpath_expr.find(event)] 312 | logger.debug('cacheControl.autoProvision values: %s', auto_provision) 313 | if len(auto_provision) == 0 or not False in auto_provision: 314 | origin_request_headers, cache_headers = determine_required_cloudfront_headers(event) 315 | 316 | if REST_API_ID and REST_API_STAGE: 317 | update_apigw_rest_stage_caching(event, cache_headers) 318 | pass 319 | else: 320 | # This can be the case when a dedicated CloudFront distribution is being used for caching. 321 | logger.info('REST API ID and/or stage name are not specified; skipping REST API sync') 322 | 323 | if CLOUD_FRONT_CACHE_POLICY_ID and CLOUD_FRONT_ORIGIN_REQUEST_POLICY_ID: 324 | update_cloudfront_origin_request_policy(origin_request_headers) 325 | update_cloudfront_cache_policy(event, cache_headers) 326 | else: 327 | # This can be the case when an API Gateway Edge endpoint type is being used (API Gateway manages its own CloudFront distribution) 328 | logger.info('CloudFront cache policy ID and/or origin request policy ID are not specified; skipping') 329 | else: 330 | logger.info('Auto provisioning of caching related resources is disabled for at least one cacheControl configuration; skipping CloudFront/API Gateway sync') 331 | -------------------------------------------------------------------------------- /src/sync_cache_settings_function/requirements.txt: -------------------------------------------------------------------------------- 1 | # Note: AWS Lambda Power Tools is required but is satisfied by a Lambda layer at runtime. 2 | jsonpath-ng -------------------------------------------------------------------------------- /src/sync_dynamodb_tables_function/README.md: -------------------------------------------------------------------------------- 1 | # Synchronize DynamoDB table resources function 2 | 3 | This AWS Lambda function is called via the [sync_resources.asl.json](../statemachine/sync_resources.asl.json) AWS Step Function state machine to synchronize DynamoDB table resources based on the item metadata settings in your [configuration](../../docs/configuration.md). For example, it will automatically create and update DynamoDB table settings to match your configuration. This function does not load data into DynamoDB tables, though. That is done by the [load_item_metadata_function](../load_item_metadata_function/). The state machine is executed each time the configuration is deployed in AWS AppConfig. -------------------------------------------------------------------------------- /src/sync_dynamodb_tables_function/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/personalization-apis/2593e0f0a8f22a25440be1a2e6c52fcefd2ab65f/src/sync_dynamodb_tables_function/__init__.py -------------------------------------------------------------------------------- /src/sync_dynamodb_tables_function/main.py: -------------------------------------------------------------------------------- 1 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # SPDX-License-Identifier: MIT-0 3 | 4 | import boto3 5 | import os 6 | 7 | from typing import Dict, List 8 | from aws_lambda_powertools import Logger, Tracer 9 | 10 | tracer = Tracer() 11 | logger = Logger() 12 | 13 | dynamodb = boto3.resource('dynamodb') 14 | 15 | primary_key_name = os.environ.get('ItemsTablePrimaryKeyFieldName', 'id') 16 | table_name_prefix = os.environ.get('ItemsTableNamePrefix', 'PersonalizationApiItemMetadata_') 17 | 18 | class ResourcePending(Exception): 19 | pass 20 | 21 | @tracer.capture_method 22 | def sync_tables(config: Dict) -> List[str]: 23 | table_names = [] 24 | namespaces = config.get('namespaces') 25 | if namespaces: 26 | for item in namespaces.items(): 27 | table_name = sync_table(item[0], item[1]) 28 | if table_name: 29 | table_names.append(table_name) 30 | else: 31 | for item in config.items(): 32 | table_name = sync_table(item[0], item[1]) 33 | if table_name: 34 | table_names.append(table_name) 35 | 36 | return table_names 37 | 38 | @tracer.capture_method 39 | def sync_table(path: str, ns_config: Dict) -> str: 40 | if not isinstance(ns_config, dict): 41 | return '' 42 | 43 | metadata_config = ns_config.get('inferenceItemMetadata') 44 | if not metadata_config: 45 | return '' 46 | 47 | if metadata_config.get('type') != 'dynamodb': 48 | logger.info('Inference metadata type is not dynamodb so skipping resource sync/provisioning') 49 | return '' 50 | 51 | table_name = table_name_prefix + path 52 | table_status = None 53 | 54 | if metadata_config.get('autoProvision') is not None and metadata_config.get('autoProvision') == False: 55 | logger.info('Auto provisioning inference item metadata resources for namespace "%s" is disabled; skipping') 56 | return table_name 57 | 58 | billing_mode = metadata_config.get('billingMode', 'PAY_PER_REQUEST') 59 | if not billing_mode in ['PAY_PER_REQUEST','PROVISIONED']: 60 | raise Exception(f'"inferenceItemMetadata.billingMode" ({billing_mode}) for namespace ("{path}") is invalid; must be PAY_PER_REQUEST or PROVISIONED') 61 | 62 | provisioned_throughput = metadata_config.get('provisionedThroughput') 63 | if billing_mode == 'PROVISIONED' and not provisioned_throughput: 64 | raise Exception(f'"inferenceItemMetadata.provisionedThroughput" is required for billingMode PROVISIONED but is not specfied for namespace ("{path}") in application configuration') 65 | 66 | try: 67 | response = dynamodb.meta.client.describe_table(TableName = table_name) 68 | 69 | table_status = response['Table']['TableStatus'] 70 | 71 | if response['Table']['BillingModeSummary']['BillingMode'] != billing_mode: 72 | logger.info('Updating BillingMode for table %s from %s to %s', table_name, response['Table']['BillingModeSummary']['BillingMode'], billing_mode) 73 | if billing_mode == 'PAY_PER_REQUEST': 74 | dynamodb.meta.client.update_table( 75 | TableName = table_name, 76 | BillingMode = billing_mode 77 | ) 78 | else: 79 | dynamodb.meta.client.update_table( 80 | TableName = table_name, 81 | BillingMode = billing_mode, 82 | ProvisionedThroughput = { 83 | 'ReadCapacityUnits': provisioned_throughput['readCapacityUnits'], 84 | 'WriteCapacityUnits': provisioned_throughput['writeCapacityUnits'] 85 | } 86 | ) 87 | elif (billing_mode == 'PROVISIONED' and 88 | (response['Table']['ProvisionedThroughput']['ReadCapacityUnits'] != provisioned_throughput['readCapacityUnits'] or 89 | response['Table']['ProvisionedThroughput']['WriteCapacityUnits'] != provisioned_throughput['writeCapacityUnits'])): 90 | logger.info('Provisioned read and/or write capacity units to not match configuration for table %s; updating table', table_name) 91 | dynamodb.meta.client.update_table( 92 | TableName = table_name, 93 | ProvisionedThroughput = { 94 | 'ReadCapacityUnits': provisioned_throughput['readCapacityUnits'], 95 | 'WriteCapacityUnits': provisioned_throughput['writeCapacityUnits'] 96 | } 97 | ) 98 | else: 99 | logger.info('Table %s already exists and billing mode (%s) and capacity units match configuration', table_name, billing_mode) 100 | 101 | except dynamodb.meta.client.exceptions.ResourceInUseException: 102 | raise ResourcePending(f'Table {table_name} is in-use and cannot be updated; status is {table_status}') 103 | 104 | except dynamodb.meta.client.exceptions.ResourceNotFoundException: 105 | logger.info('Table %s does not exist; creating table', table_name) 106 | 107 | create_params = { 108 | 'TableName': table_name, 109 | 'AttributeDefinitions': [ 110 | { 111 | 'AttributeName': primary_key_name, 112 | 'AttributeType': 'S', 113 | } 114 | ], 115 | 'KeySchema': [ 116 | { 117 | 'AttributeName': primary_key_name, 118 | 'KeyType': 'HASH', 119 | } 120 | ], 121 | 'BillingMode': billing_mode, 122 | 'Tags': [{ 123 | 'Key': 'CreatedBy', 124 | 'Value': 'Personalization-APIs-Solution' 125 | }] 126 | } 127 | 128 | logger.debug(create_params) 129 | 130 | if billing_mode == 'PROVISIONED': 131 | create_params['ProvisionedThroughput'] = { 132 | 'ReadCapacityUnits': provisioned_throughput['readCapacityUnits'], 133 | 'WriteCapacityUnits': provisioned_throughput['writeCapacityUnits'] 134 | } 135 | 136 | response = dynamodb.meta.client.create_table(**create_params) 137 | table_status = response['TableDescription']['TableStatus'] 138 | 139 | if table_status in [ 'CREATING', 'UPDATING' ]: 140 | logger.info('Table %s is being created/updated', table_name) 141 | raise ResourcePending(f'Table {table_name} is still being created') 142 | 143 | return table_name 144 | 145 | @logger.inject_lambda_context 146 | @tracer.capture_lambda_handler 147 | def lambda_handler(event, _): 148 | logger.info(event) 149 | table_names = sync_tables(event) 150 | return table_names -------------------------------------------------------------------------------- /src/sync_dynamodb_tables_function/requirements.txt: -------------------------------------------------------------------------------- 1 | # Note: AWS Lambda Power Tools is required but is satisfied by a Lambda layer at runtime. -------------------------------------------------------------------------------- /stage.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Staging script for copying deployment resources to an S3 bucket. 4 | # 5 | # Example usage: 6 | # ./stage.sh INPUT_TEMPLATE S3_BUCKET [OPTIONAL_S3_PATH/] 7 | # 8 | # The INPUT_TEMPLATE is the SAM template to use as input into the build process and 9 | # S3_BUCKET/OPTIONAL_S3_PATH is where all resources and templates will be uploaded. 10 | # If you don't specify the OPTIONAL_S3_PATH, it will be uploaded to the root of the bucket. 11 | # 12 | 13 | set -e 14 | 15 | INPUT_TEMPLATE=$1 16 | BUCKET=$2 17 | # Path with trailing / (if specified) 18 | S3PATH=$3 19 | 20 | echo "==============================================" 21 | echo "Executing the script with following arguments:" 22 | echo "==============================================" 23 | echo "INPUT_TEMPLATE = ${INPUT_TEMPLATE}" 24 | echo "BUCKET = ${BUCKET}" 25 | echo "S3PATH = ${S3PATH}" 26 | echo "==============================================" 27 | ######################################################################################################################################## 28 | 29 | if [ -z "$INPUT_TEMPLATE" ] || [ -z "$BUCKET" ]; then 30 | echo "Usage: $0 INPUT_TEMPLATE BUCKET [S3PATH]" 31 | echo " where INPUT_TEMPLATE is the input template filename, BUCKET is the S3 bucket to upload resources to, and S3PATH is optional path but if specified must have a trailing '/'" 32 | exit 1 33 | fi 34 | 35 | if [ ! -f "$INPUT_TEMPLATE" ]; then 36 | echo "Input template file $INPUT_TEMPLATE does not exist." 37 | fi 38 | 39 | BUCKET_LOCATION="$(aws s3api get-bucket-location --bucket ${BUCKET}|grep ":"|cut -d\" -f4)" 40 | if [ -z "$BUCKET_LOCATION" ]; then 41 | BUCKET_DOMAIN="s3.amazonaws.com" 42 | BUCKET_LOCATION="us-east-1" 43 | else 44 | BUCKET_DOMAIN="s3-${BUCKET_LOCATION}.amazonaws.com" 45 | fi 46 | 47 | if [ $(uname) == "Darwin" ]; then 48 | USE_CONTAINER="--use-container" 49 | else 50 | USE_CONTAINER="" 51 | fi 52 | 53 | echo "Building project artifacts" 54 | sam build --template-file $INPUT_TEMPLATE $USE_CONTAINER 55 | 56 | cd .aws-sam/build 57 | 58 | echo "Packaging and uploading project artifacts to s3://${BUCKET}/${S3PATH}" 59 | if [ "$S3PATH" == "" ]; then 60 | sam package --force-upload --output-template-file packaged.yaml --s3-bucket ${BUCKET} 61 | else 62 | sam package --force-upload --output-template-file packaged.yaml --s3-bucket ${BUCKET} --s3-prefix ${S3PATH%/} 63 | fi 64 | 65 | echo "Copying packaged CloudFormation template to s3://${BUCKET}/${S3PATH}template.yaml" 66 | aws s3 cp packaged.yaml s3://${BUCKET}/${S3PATH}template.yaml 67 | 68 | cd - 69 | 70 | echo "For CloudFormation: https://${BUCKET_DOMAIN}/${BUCKET}/${S3PATH}template.yaml" 71 | echo "Launch CloudFormation stack: https://console.aws.amazon.com/cloudformation/home?region=${BUCKET_LOCATION}#/stacks/create/review?templateURL=https://${BUCKET_DOMAIN}/${BUCKET}/${S3PATH}template.yaml&stackName=personalization-apis" -------------------------------------------------------------------------------- /template-edge.yaml: -------------------------------------------------------------------------------- 1 | AWSTemplateFormatVersion: "2010-09-09" 2 | Transform: AWS::Serverless-2016-10-31 3 | Description: > 4 | (P13N-RT-APIS-EDGE) - Lambda@Edge functions and related resources for the Personalization APIs solution. Since all Lambda@Edge 5 | functions must be deployed into the us-east-1 region, these resources are packaged separately here so that the primary Personalization 6 | APIs resources can be deployed in a separate region. This stack is only necessary if OAuth2 at the edge is being used. 7 | 8 | Parameters: 9 | CognitoRegion: 10 | Type: String 11 | Description: AWS region name where your Cognito user pool is deployed. 12 | AllowedPattern: ^[a-z]{2}-((?:gov|iso|isob)-)?[a-z]+-\d{1}?$ 13 | Default: "us-east-1" 14 | 15 | CognitoUserPoolId: 16 | Type: String 17 | Description: Amazon Cognito user pool ID for the user pool used to generated JWT tokens for your application. 18 | AllowedPattern: ^[\w-]+$ 19 | 20 | Globals: 21 | Function: 22 | Tags: 23 | CreatedBy: Personalization-APIs-Solution 24 | 25 | Resources: 26 | EdgeAuthFunction: 27 | Type: AWS::Serverless::Function 28 | DeletionPolicy: Retain 29 | UpdateReplacePolicy: Retain 30 | Properties: 31 | Description: Edge function that verifies JWT tokens in CloudFront to maximize caching and reduce latency 32 | Runtime: nodejs18.x 33 | Timeout: 3 34 | CodeUri: src/edge_auth_function 35 | Handler: index.handler 36 | MemorySize: 128 # Restricted by L@E 37 | Role: !GetAtt EdgeAuthExecutionRole.Arn 38 | AutoPublishAlias: Templated 39 | 40 | EdgeAuthExecutionRole: 41 | Type: AWS::IAM::Role 42 | Properties: 43 | AssumeRolePolicyDocument: 44 | Version: "2012-10-17" 45 | Statement: 46 | - Effect: Allow 47 | Principal: 48 | Service: 49 | - edgelambda.amazonaws.com 50 | - lambda.amazonaws.com 51 | Action: 52 | - sts:AssumeRole 53 | Path: "/" 54 | ManagedPolicyArns: 55 | - arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole 56 | 57 | UpdateEdgeFunction: 58 | Type: AWS::Serverless::Function 59 | Properties: 60 | CodeUri: src/edge_update_function 61 | Handler: main.lambda_handler 62 | Runtime: python3.11 63 | Policies: 64 | - Statement: 65 | - Effect: Allow 66 | Action: 67 | - lambda:GetFunction 68 | - lambda:UpdateFunctionCode 69 | Resource: 70 | - !GetAtt EdgeAuthFunction.Arn 71 | - !Sub "${EdgeAuthFunction.Arn}:Templated" 72 | 73 | UpdateEdgeFunctionCustom: 74 | Type: Custom::UpdateConfigCustom 75 | Properties: 76 | ServiceToken: !GetAtt UpdateEdgeFunction.Arn 77 | UserPoolId: !Ref CognitoUserPoolId 78 | CognitoRegion: !Ref CognitoRegion 79 | EdgeFunctionArn: !GetAtt EdgeAuthFunction.Arn 80 | 81 | ########################################################################## 82 | # Outputs # 83 | ########################################################################## 84 | Outputs: 85 | EdgeAuthFunction: 86 | Description: "Edge authentication function ARN" 87 | Value: !GetAtt EdgeAuthFunction.Arn 88 | 89 | EdgeAuthFunctionAlias: 90 | Description: Alias for templated version of the Function 91 | Value: !Ref EdgeAuthFunction.Alias 92 | 93 | UpdateEdgeFunction: 94 | Description: "Edge function update function ARN" 95 | Value: !GetAtt UpdateEdgeFunction.Arn 96 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/personalization-apis/2593e0f0a8f22a25440be1a2e6c52fcefd2ab65f/tests/__init__.py -------------------------------------------------------------------------------- /tests/requirements.txt: -------------------------------------------------------------------------------- 1 | jsonlines 2 | jsonpath-ng 3 | jsonschema==3.2.0 4 | pytest 5 | pytest-cov 6 | pytz -------------------------------------------------------------------------------- /tests/unit/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/personalization-apis/2593e0f0a8f22a25440be1a2e6c52fcefd2ab65f/tests/unit/__init__.py -------------------------------------------------------------------------------- /tests/unit/test_config.py: -------------------------------------------------------------------------------- 1 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # SPDX-License-Identifier: MIT-0 3 | 4 | from pytest import fixture 5 | from unittest.mock import MagicMock 6 | 7 | from layer.personalization_config import AppConfigPersonalizationConfig 8 | 9 | @fixture 10 | def config_no_version(): 11 | return { 12 | "namespaces": {} 13 | } 14 | 15 | @fixture 16 | def config_with_version(): 17 | return { 18 | "version": "1", 19 | "namespaces": {} 20 | } 21 | 22 | @fixture 23 | def config_cache_inheritance(): 24 | return { 25 | "cacheControl": { 26 | "userSpecified": { 27 | "maxAge": 10, 28 | "directives": "private" 29 | }, 30 | }, 31 | "namespaces": { 32 | "ns-inherit-root": { 33 | "recommenders": { 34 | "recommend-items": { 35 | "recommended-for-you": { 36 | } 37 | } 38 | } 39 | }, 40 | "ns-inherit-1": { 41 | "cacheControl": { 42 | "userSpecified": { 43 | "maxAge": 20, 44 | "directives": "private" 45 | }, 46 | }, 47 | "recommenders": { 48 | "recommend-items": { 49 | "recommended-for-you": { 50 | "cacheControl": { 51 | "userSpecified": { 52 | "maxAge": 30, 53 | "directives": "private" 54 | } 55 | } 56 | } 57 | }, 58 | "related-items": { 59 | "similar": { 60 | "cacheControl": { 61 | "userSpecified": { 62 | "maxAge": 40, 63 | "directives": "private" 64 | } 65 | } 66 | } 67 | } 68 | } 69 | }, 70 | "ns-inherit-2": { 71 | "recommenders": { 72 | "recommend-items": { 73 | "recommended-for-you": { 74 | "cacheControl": { 75 | "userSpecified": { 76 | "maxAge": 60, 77 | "directives": "private" 78 | } 79 | } 80 | } 81 | }, 82 | "related-items": { 83 | "similar": { 84 | } 85 | } 86 | } 87 | } 88 | } 89 | } 90 | 91 | @fixture 92 | def config_no_recommenders(): 93 | return { 94 | "namespaces": { 95 | "ns-1": { 96 | "eventTargets": [ 97 | { 98 | "type": "personalize-event-tracker", 99 | "trackingId": "[TRACKING_ID]" 100 | } 101 | ] 102 | } 103 | } 104 | } 105 | 106 | @fixture 107 | def obj(): 108 | obj = AppConfigPersonalizationConfig.__new__(AppConfigPersonalizationConfig) 109 | return obj 110 | 111 | def test_no_version(obj, config_no_version): 112 | obj.get_config = MagicMock(return_value = config_no_version) 113 | assert obj.get_version() is None 114 | 115 | def test_with_version(obj, config_with_version): 116 | obj.get_config = MagicMock(return_value = config_with_version) 117 | assert obj.get_version() == '1' 118 | 119 | def test_cache_inheritance(obj, config_cache_inheritance): 120 | obj.get_config = MagicMock(return_value = config_cache_inheritance) 121 | 122 | ns_config = obj.get_namespace_config('ns-inherit-root') 123 | assert ns_config['cacheControl']['userSpecified']['maxAge'] == 10 124 | 125 | rec_config = obj.get_recommender_config('ns-inherit-root', 'recommended-for-you', 'recommend-items') 126 | assert rec_config['cacheControl']['userSpecified']['maxAge'] == 10 127 | 128 | ns_config = obj.get_namespace_config('ns-inherit-1') 129 | assert ns_config['cacheControl']['userSpecified']['maxAge'] == 20 130 | 131 | rec_config = obj.get_recommender_config('ns-inherit-1', 'recommended-for-you', 'recommend-items') 132 | assert rec_config['cacheControl']['userSpecified']['maxAge'] == 30 133 | rec_config = obj.get_recommender_config('ns-inherit-1', 'similar', 'related-items') 134 | assert rec_config['cacheControl']['userSpecified']['maxAge'] == 40 135 | 136 | ns_config = obj.get_namespace_config('ns-inherit-2') 137 | assert ns_config['cacheControl']['userSpecified']['maxAge'] == 10 138 | 139 | rec_config = obj.get_recommender_config('ns-inherit-2', 'recommended-for-you', 'recommend-items') 140 | assert rec_config['cacheControl']['userSpecified']['maxAge'] == 60 141 | rec_config = obj.get_recommender_config('ns-inherit-2', 'similar', 'related-items') 142 | assert rec_config['cacheControl']['userSpecified']['maxAge'] == 10 143 | 144 | def test_no_recommenders(obj, config_no_recommenders): 145 | obj.get_config = MagicMock(return_value = config_no_recommenders) 146 | ns_config = obj.get_namespace_config('ns-1') 147 | assert not 'recommenders' in ns_config 148 | assert len(ns_config['eventTargets']) == 1 -------------------------------------------------------------------------------- /tests/unit/test_openapi.py: -------------------------------------------------------------------------------- 1 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # SPDX-License-Identifier: MIT-0 3 | 4 | import json 5 | from pytest import fixture 6 | from unittest.mock import MagicMock 7 | 8 | from config_validator_function.openapi import OpenApiGenerator 9 | 10 | @fixture 11 | def sample_config(): 12 | with open('samples/config_simple.json') as file: 13 | return json.loads(file.read()) 14 | 15 | def test_simple_noauth(sample_config): 16 | generator = OpenApiGenerator() 17 | generator._get_openapi_template_filename = MagicMock(return_value = 'src/config_validator_function/openapi_template.json') 18 | openapi = generator.generate( 19 | apis_config = sample_config, 20 | apigw_host = 'https://apigw-host.com', 21 | cloudfront_host = 'https://cloudfront-host.com', 22 | auth_scheme = 'NONE') 23 | 24 | assert len(openapi['servers']) == 2 25 | assert not 'security' in openapi 26 | 27 | def test_simple_apikey(sample_config): 28 | generator = OpenApiGenerator() 29 | generator._get_openapi_template_filename = MagicMock(return_value = 'src/config_validator_function/openapi_template.json') 30 | openapi = generator.generate( 31 | apis_config = sample_config, 32 | apigw_host = 'https://apigw-host.com', 33 | cloudfront_host = 'https://cloudfront-host.com', 34 | auth_scheme = 'ApiKey') 35 | 36 | assert len(openapi['servers']) == 2 37 | assert len(openapi['security']) == 1 38 | --------------------------------------------------------------------------------