├── .gitignore ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── LICENSE ├── README.md ├── cloudfront ├── app.py └── requirements.txt ├── input ├── app.py └── requirements.txt ├── mediatailor ├── app.py └── requirements.txt ├── slot_detection ├── app.py ├── requirements.txt ├── score.py ├── segment.py └── silence.py ├── template.yaml ├── video_transcoding_check ├── app.py └── requirements.txt ├── video_transcoding_start ├── app.py └── requirements.txt └── vmap_generation ├── ads.json ├── app.py ├── requirements.txt ├── vast_xml ├── ad.py ├── companionad.py ├── creative.py ├── icon.py ├── trackingevent.py └── vast.py └── vmap_xml ├── adbreak.py ├── events.py └── vmap.py /.gitignore: -------------------------------------------------------------------------------- 1 | # Created by https://www.gitignore.io/api/osx,linux,python,windows,jetbrains,visualstudiocode 2 | # Edit at https://www.gitignore.io/?templates=osx,linux,python,windows,jetbrains,visualstudiocode 3 | 4 | ### JetBrains ### 5 | # Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio and WebStorm 6 | # Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 7 | 8 | samconfig.toml 9 | 10 | # User-specific stuff 11 | .idea/**/workspace.xml 12 | .idea/**/tasks.xml 13 | .idea/**/usage.statistics.xml 14 | .idea/**/dictionaries 15 | .idea/**/shelf 16 | 17 | # Generated files 18 | .idea/**/contentModel.xml 19 | 20 | # Sensitive or high-churn files 21 | .idea/**/dataSources/ 22 | .idea/**/dataSources.ids 23 | .idea/**/dataSources.local.xml 24 | .idea/**/sqlDataSources.xml 25 | .idea/**/dynamic.xml 26 | .idea/**/uiDesigner.xml 27 | .idea/**/dbnavigator.xml 28 | 29 | # Gradle 30 | .idea/**/gradle.xml 31 | .idea/**/libraries 32 | 33 | # Gradle and Maven with auto-import 34 | # When using Gradle or Maven with auto-import, you should exclude module files, 35 | # since they will be recreated, and may cause churn. Uncomment if using 36 | # auto-import. 37 | # .idea/modules.xml 38 | # .idea/*.iml 39 | # .idea/modules 40 | # *.iml 41 | # *.ipr 42 | 43 | # CMake 44 | cmake-build-*/ 45 | 46 | # Mongo Explorer plugin 47 | .idea/**/mongoSettings.xml 48 | 49 | # File-based project format 50 | *.iws 51 | 52 | # IntelliJ 53 | out/ 54 | 55 | # mpeltonen/sbt-idea plugin 56 | .idea_modules/ 57 | 58 | # JIRA plugin 59 | atlassian-ide-plugin.xml 60 | 61 | # Cursive Clojure plugin 62 | .idea/replstate.xml 63 | 64 | # Crashlytics plugin (for Android Studio and IntelliJ) 65 | com_crashlytics_export_strings.xml 66 | crashlytics.properties 67 | crashlytics-build.properties 68 | fabric.properties 69 | 70 | # Editor-based Rest Client 71 | .idea/httpRequests 72 | 73 | # Android studio 3.1+ serialized cache file 74 | .idea/caches/build_file_checksums.ser 75 | 76 | ### JetBrains Patch ### 77 | # Comment Reason: https://github.com/joeblau/gitignore.io/issues/186#issuecomment-215987721 78 | 79 | # *.iml 80 | # modules.xml 81 | # .idea/misc.xml 82 | # *.ipr 83 | 84 | # Sonarlint plugin 85 | .idea/**/sonarlint/ 86 | 87 | # SonarQube Plugin 88 | .idea/**/sonarIssues.xml 89 | 90 | # Markdown Navigator plugin 91 | .idea/**/markdown-navigator.xml 92 | .idea/**/markdown-navigator/ 93 | 94 | ### Linux ### 95 | *~ 96 | 97 | # temporary files which can be created if a process still has a handle open of a deleted file 98 | .fuse_hidden* 99 | 100 | # KDE directory preferences 101 | .directory 102 | 103 | # Linux trash folder which might appear on any partition or disk 104 | .Trash-* 105 | 106 | # .nfs files are created when an open file is removed but is still being accessed 107 | .nfs* 108 | 109 | ### OSX ### 110 | # General 111 | .DS_Store 112 | .AppleDouble 113 | .LSOverride 114 | 115 | # Icon must end with two \r 116 | Icon 117 | 118 | # Thumbnails 119 | ._* 120 | 121 | # Files that might appear in the root of a volume 122 | .DocumentRevisions-V100 123 | .fseventsd 124 | .Spotlight-V100 125 | .TemporaryItems 126 | .Trashes 127 | .VolumeIcon.icns 128 | .com.apple.timemachine.donotpresent 129 | 130 | # Directories potentially created on remote AFP share 131 | .AppleDB 132 | .AppleDesktop 133 | Network Trash Folder 134 | Temporary Items 135 | .apdisk 136 | 137 | ### Python ### 138 | # Byte-compiled / optimized / DLL files 139 | __pycache__/ 140 | *.py[cod] 141 | *$py.class 142 | 143 | # C extensions 144 | *.so 145 | 146 | # Distribution / packaging 147 | .Python 148 | build/ 149 | develop-eggs/ 150 | dist/ 151 | downloads/ 152 | eggs/ 153 | .eggs/ 154 | lib/ 155 | lib64/ 156 | parts/ 157 | sdist/ 158 | var/ 159 | wheels/ 160 | pip-wheel-metadata/ 161 | share/python-wheels/ 162 | *.egg-info/ 163 | .installed.cfg 164 | *.egg 165 | MANIFEST 166 | 167 | # PyInstaller 168 | # Usually these files are written by a python script from a template 169 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 170 | *.manifest 171 | *.spec 172 | 173 | # Installer logs 174 | pip-log.txt 175 | pip-delete-this-directory.txt 176 | 177 | # Unit test / coverage reports 178 | htmlcov/ 179 | .tox/ 180 | .nox/ 181 | .coverage 182 | .coverage.* 183 | .cache 184 | nosetests.xml 185 | coverage.xml 186 | *.cover 187 | .hypothesis/ 188 | .pytest_cache/ 189 | 190 | # Translations 191 | *.mo 192 | *.pot 193 | 194 | # Scrapy stuff: 195 | .scrapy 196 | 197 | # Sphinx documentation 198 | docs/_build/ 199 | 200 | # PyBuilder 201 | target/ 202 | 203 | # pyenv 204 | .python-version 205 | 206 | # pipenv 207 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 208 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 209 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 210 | # install all needed dependencies. 211 | #Pipfile.lock 212 | 213 | # celery beat schedule file 214 | celerybeat-schedule 215 | 216 | # SageMath parsed files 217 | *.sage.py 218 | 219 | # Spyder project settings 220 | .spyderproject 221 | .spyproject 222 | 223 | # Rope project settings 224 | .ropeproject 225 | 226 | # Mr Developer 227 | .mr.developer.cfg 228 | .project 229 | .pydevproject 230 | 231 | # mkdocs documentation 232 | /site 233 | 234 | # mypy 235 | .mypy_cache/ 236 | .dmypy.json 237 | dmypy.json 238 | 239 | # Pyre type checker 240 | .pyre/ 241 | 242 | ### VisualStudioCode ### 243 | .vscode/* 244 | !.vscode/settings.json 245 | !.vscode/tasks.json 246 | !.vscode/launch.json 247 | !.vscode/extensions.json 248 | 249 | ### VisualStudioCode Patch ### 250 | # Ignore all local history of files 251 | .history 252 | 253 | ### Windows ### 254 | # Windows thumbnail cache files 255 | Thumbs.db 256 | Thumbs.db:encryptable 257 | ehthumbs.db 258 | ehthumbs_vista.db 259 | 260 | # Dump file 261 | *.stackdump 262 | 263 | # Folder config file 264 | [Dd]esktop.ini 265 | 266 | # Recycle Bin used on file shares 267 | $RECYCLE.BIN/ 268 | 269 | # Windows Installer files 270 | *.cab 271 | *.msi 272 | *.msix 273 | *.msm 274 | *.msp 275 | 276 | # Windows shortcuts 277 | *.lnk 278 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | ## Code of Conduct 2 | This project has adopted the [Amazon Open Source Code of Conduct](https://aws.github.io/code-of-conduct). 3 | For more information see the [Code of Conduct FAQ](https://aws.github.io/code-of-conduct-faq) or contact 4 | opensource-codeofconduct@amazon.com with any additional questions or comments. 5 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing Guidelines 2 | 3 | Thank you for your interest in contributing to our project. Whether it's a bug report, new feature, correction, or additional 4 | documentation, we greatly value feedback and contributions from our community. 5 | 6 | Please read through this document before submitting any issues or pull requests to ensure we have all the necessary 7 | information to effectively respond to your bug report or contribution. 8 | 9 | 10 | ## Reporting Bugs/Feature Requests 11 | 12 | We welcome you to use the GitHub issue tracker to report bugs or suggest features. 13 | 14 | When filing an issue, please check existing open, or recently closed, issues to make sure somebody else hasn't already 15 | reported the issue. Please try to include as much information as you can. Details like these are incredibly useful: 16 | 17 | * A reproducible test case or series of steps 18 | * The version of our code being used 19 | * Any modifications you've made relevant to the bug 20 | * Anything unusual about your environment or deployment 21 | 22 | 23 | ## Contributing via Pull Requests 24 | Contributions via pull requests are much appreciated. Before sending us a pull request, please ensure that: 25 | 26 | 1. You are working against the latest source on the *master* branch. 27 | 2. You check existing open, and recently merged, pull requests to make sure someone else hasn't addressed the problem already. 28 | 3. You open an issue to discuss any significant work - we would hate for your time to be wasted. 29 | 30 | To send us a pull request, please: 31 | 32 | 1. Fork the repository. 33 | 2. Modify the source; please focus on the specific change you are contributing. If you also reformat all the code, it will be hard for us to focus on your change. 34 | 3. Ensure local tests pass. 35 | 4. Commit to your fork using clear commit messages. 36 | 5. Send us a pull request, answering any default questions in the pull request interface. 37 | 6. Pay attention to any automated CI failures reported in the pull request, and stay involved in the conversation. 38 | 39 | GitHub provides additional document on [forking a repository](https://help.github.com/articles/fork-a-repo/) and 40 | [creating a pull request](https://help.github.com/articles/creating-a-pull-request/). 41 | 42 | 43 | ## Finding contributions to work on 44 | Looking at the existing issues is a great way to find something to contribute on. As our projects, by default, use the default GitHub issue labels (enhancement/bug/duplicate/help wanted/invalid/question/wontfix), looking at any 'help wanted' issues is a great place to start. 45 | 46 | 47 | ## Code of Conduct 48 | This project has adopted the [Amazon Open Source Code of Conduct](https://aws.github.io/code-of-conduct). 49 | For more information see the [Code of Conduct FAQ](https://aws.github.io/code-of-conduct-faq) or contact 50 | opensource-codeofconduct@amazon.com with any additional questions or comments. 51 | 52 | 53 | ## Security issue notifications 54 | If you discover a potential security issue in this project we ask that you notify AWS/Amazon Security via our [vulnerability reporting page](http://aws.amazon.com/security/vulnerability-reporting/). Please do **not** create a public github issue. 55 | 56 | 57 | ## Licensing 58 | 59 | See the [LICENSE](LICENSE) file for our project's licensing. We will ask you to confirm the licensing of your contribution. 60 | 61 | We may ask you to sign a [Contributor License Agreement (CLA)](http://en.wikipedia.org/wiki/Contributor_License_Agreement) for larger changes. 62 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy of 4 | this software and associated documentation files (the "Software"), to deal in 5 | the Software without restriction, including without limitation the rights to 6 | use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of 7 | the Software, and to permit persons to whom the Software is furnished to do so. 8 | 9 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 10 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS 11 | FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 12 | COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER 13 | IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 14 | CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 15 | 16 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Smart Ad Breaks 2 | 3 | ### Required parameters 4 | 5 | - **MediaInsightsEnginePython39Layer:** ARN of the MIE Lambda layer (MediaInsightsEnginePython39Layer output of the MIE main CloudFormation stack) 6 | - **WorkflowCustomResourceArn:** ARN of the MIE custom resource (WorkflowCustomResourceArn output of the MIE main CloudFormation stack) 7 | - **WorkflowEndpoint:** Workflow endpoint. (APIHandlerName output of MIE Workflow API CloudFormation nested stack) 8 | - **DataplaneEndpoint:** Dataplane endpoint (APIHandlerName output of the MIE Dataplane CloudFormation nested stack) 9 | - **DataplaneBucket:** Bucket for the dataplane (DataplaneBucket output of the MIE main CloudFormation stack) 10 | 11 | ## Setup 12 | 13 | ``` 14 | sam build --parameter-overrides 'ParameterKey=MediaInsightsEnginePython39Layer,ParameterValue=[ARN obtained from MIE stack]' 15 | 16 | sam deploy --guided 17 | ``` 18 | 19 | ## Content Security Legal Disclaimer 20 | The sample code; software libraries; command line tools; proofs of concept; templates; or other related technology (including any of the foregoing that are provided by our personnel) is provided to you as AWS Content under the AWS Customer Agreement, or the relevant written agreement between you and AWS (whichever applies). You should not use this AWS Content in your production accounts, or on production or other critical data. You are responsible for testing, securing, and optimizing the AWS Content, such as sample code, as appropriate for production grade use based on your specific quality control practices and standards. Deploying AWS Content may incur AWS charges for creating or using AWS chargeable resources, such as running Amazon EC2 instances or using Amazon S3 storage. 21 | 22 | ## Operational Metrics Collection 23 | This solution collects anonymous operational metrics to help AWS improve the quality and features of the solution. Data collection is subject to the AWS Privacy Policy (https://aws.amazon.com/privacy/). To opt out of this feature, simply remove the tag(s) starting with “uksb-” or “SO” from the description(s) in any CloudFormation templates or CDK TemplateOptions. 24 | -------------------------------------------------------------------------------- /cloudfront/app.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # SPDX-License-Identifier: Apache-2.0 3 | 4 | import json 5 | import boto3 6 | from botocore.exceptions import ClientError 7 | import cfnresponse 8 | 9 | def lambda_handler(event, context): 10 | print('event: {}'.format(event)) 11 | s3 = boto3.client('s3') 12 | bucket = event['ResourceProperties']['BucketName'] 13 | oai = event['ResourceProperties']['OriginAccessIdentity'] 14 | response = {} 15 | status = cfnresponse.SUCCESS 16 | if event['RequestType'] == 'Create' or event['RequestType'] == 'Update': 17 | try: 18 | policy = json.loads(s3.get_bucket_policy(Bucket=bucket)['Policy']) 19 | statement = { 20 | "Sid": oai, 21 | "Effect": "Allow", 22 | "Principal": { 23 | "AWS": "arn:aws:iam::cloudfront:user/CloudFront Origin Access Identity " + oai 24 | }, 25 | "Action": "s3:GetObject", 26 | "Resource": "arn:aws:s3:::" + bucket + "/*" 27 | } 28 | policy['Statement'].append(statement) 29 | s3.put_bucket_policy(Bucket=bucket, Policy=json.dumps(policy)) 30 | response = {} 31 | except ClientError as error: 32 | print('Exception: %s' % error) 33 | status = cfnresponse.FAILED 34 | response = {'Exception': str(error)} 35 | elif event['RequestType'] == 'Delete': 36 | try: 37 | policy = json.loads(s3.get_bucket_policy(Bucket=bucket)['Policy']) 38 | for statement in reversed(policy['Statement']): 39 | if 'Sid' in statement: 40 | if statement['Sid'] == oai: 41 | policy['Statement'].remove(statement) 42 | s3.put_bucket_policy(Bucket=bucket, Policy=json.dumps(policy)) 43 | response = {} 44 | except ClientError as error: 45 | print('Exception: %s' % error) 46 | status = cfnresponse.FAILED 47 | response = {'Exception': str(error)} 48 | print('response: {}'.format(response)) 49 | cfnresponse.send(event, context, status, response) -------------------------------------------------------------------------------- /cloudfront/requirements.txt: -------------------------------------------------------------------------------- 1 | cfnresponse>=1.0.2 -------------------------------------------------------------------------------- /input/app.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # SPDX-License-Identifier: Apache-2.0 3 | 4 | import os 5 | import json 6 | import time 7 | import uuid 8 | import boto3 9 | 10 | workflow_name = os.environ["WORKFLOW_NAME"] 11 | workflow_function = os.environ["WorkflowEndpoint"] 12 | dataplane_bucket = os.environ["DATAPLANE_BUCKET"] 13 | 14 | lambda_client = boto3.client('lambda') 15 | s3_client = boto3.client('s3') 16 | 17 | def lambda_handler(event, context): 18 | print(event) 19 | bucket_name = event['Records'][0]['s3']['bucket']['name'] 20 | object_key = event['Records'][0]['s3']['object']['key'] 21 | 22 | # Copy object to Dataplane bucket 23 | s3_client.copy_object( 24 | Bucket=dataplane_bucket, 25 | CopySource={ 26 | 'Bucket': bucket_name, 27 | 'Key': object_key 28 | }, 29 | Key=object_key, 30 | ) 31 | 32 | # Workflow input body 33 | body = { 34 | "Name": workflow_name, 35 | "Input":{ 36 | "Media":{ 37 | "Video":{ 38 | "S3Bucket": dataplane_bucket, 39 | "S3Key": object_key 40 | } 41 | } 42 | } 43 | } 44 | 45 | # Lambda request (with Chalice/API Gateway attributes) 46 | request = { 47 | "resource": "/workflow/execution", 48 | "path": "/workflow/execution", 49 | "httpMethod": "POST", 50 | "headers": { 51 | 'Content-Type': 'application/json' 52 | }, 53 | "multiValueHeaders": {}, 54 | "queryStringParameters": {}, 55 | "multiValueQueryStringParameters": {}, 56 | "pathParameters": {}, 57 | "stageVariables": {}, 58 | "requestContext": { 59 | 'resourcePath': "/workflow/execution", 60 | 'requestTime': time.time(), 61 | 'httpMethod': 'POST', 62 | 'requestId': 'lambda_' + str(uuid.uuid4()).split('-')[-1], 63 | }, 64 | "body": json.dumps(body), 65 | "isBase64Encoded": False 66 | } 67 | 68 | # Invoke Workflow lambda function 69 | response = lambda_client.invoke( 70 | FunctionName=workflow_function, 71 | InvocationType='RequestResponse', 72 | LogType='None', 73 | Payload=bytes(json.dumps(request), encoding='utf-8') 74 | ) 75 | print(response) 76 | print(json.loads(response['Payload'].read())) 77 | -------------------------------------------------------------------------------- /input/requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/aws-smart-ad-breaks/df9658e4954cfda5f80bedb7857d6fd58667ee41/input/requirements.txt -------------------------------------------------------------------------------- /mediatailor/app.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # SPDX-License-Identifier: Apache-2.0 3 | 4 | from urllib.parse import urlparse 5 | import boto3 6 | from botocore.exceptions import ClientError 7 | import cfnresponse 8 | 9 | def lambda_handler(event, context): 10 | print('event: {}'.format(event)) 11 | mediatailor = boto3.client('mediatailor') 12 | config_name = event['ResourceProperties']['ConfigurationName'] 13 | content_url = event['ResourceProperties']['VideoContentSource'] 14 | ads_url = event['ResourceProperties']['AdDecisionServer'] 15 | slate_ad = '' 16 | if 'SlateAd' in event['ResourceProperties']: 17 | slate_ad = event['ResourceProperties']['SlateAd'] 18 | cdn_content_prefix = '' 19 | if 'CDNContentSegmentPrefix' in event['ResourceProperties']: 20 | cdn_content_prefix = event['ResourceProperties']['CDNContentSegmentPrefix'] 21 | cdn_ad_prefix = '' 22 | if 'CDNAdSegmentPrefix' in event['ResourceProperties']: 23 | cdn_ad_prefix = event['ResourceProperties']['CDNAdSegmentPrefix'] 24 | response = {} 25 | status = cfnresponse.SUCCESS 26 | if event['RequestType'] == 'Create' or event['RequestType'] == 'Update': 27 | try: 28 | if cdn_content_prefix != '' or cdn_ad_prefix != '': 29 | res = mediatailor.put_playback_configuration( 30 | Name=config_name, 31 | VideoContentSourceUrl=content_url, 32 | AdDecisionServerUrl=ads_url, 33 | SlateAdUrl=slate_ad, # ok even if this is empty 34 | CdnConfiguration={ 35 | 'AdSegmentUrlPrefix': cdn_ad_prefix, 36 | 'ContentSegmentUrlPrefix': cdn_content_prefix 37 | } 38 | ) 39 | else: 40 | res = mediatailor.put_playback_configuration( 41 | Name=config_name, 42 | VideoContentSourceUrl=content_url, 43 | AdDecisionServerUrl=ads_url, 44 | SlateAdUrl=slate_ad, # ok even if this is empty 45 | ) 46 | print('res: {}'.format(res)) 47 | hls_prefix = urlparse(res['HlsConfiguration']['ManifestEndpointPrefix']) 48 | hls_domain = hls_prefix.netloc 49 | hls_path = hls_prefix.path 50 | response = { 51 | 'SessionInitializationPrefix': res['SessionInitializationEndpointPrefix'], 52 | 'HLSPlaybackDomain': hls_domain, 53 | 'HLSPlaybackPath': hls_path, 54 | 'HLSPlaybackPrefix': res['HlsConfiguration']['ManifestEndpointPrefix'], 55 | 'DashPlaybackPrefix': res['DashConfiguration']['ManifestEndpointPrefix'] 56 | } 57 | except ClientError as error: 58 | print('Exception: %s' % error) 59 | status = cfnresponse.FAILED 60 | response = {'Exception': str(error)} 61 | elif event['RequestType'] == 'Delete': 62 | try: 63 | response = mediatailor.delete_playback_configuration( 64 | Name=config_name 65 | ) 66 | except ClientError as error: 67 | print('Exception: %s' % error) 68 | status = cfnresponse.FAILED 69 | response = {'Exception': str(error)} 70 | print('response: {}'.format(response)) 71 | cfnresponse.send(event, context, status, response) -------------------------------------------------------------------------------- /mediatailor/requirements.txt: -------------------------------------------------------------------------------- 1 | cfnresponse>=1.0.2 -------------------------------------------------------------------------------- /slot_detection/app.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # SPDX-License-Identifier: Apache-2.0 3 | 4 | import os 5 | 6 | from MediaInsightsEngineLambdaHelper import MediaInsightsOperationHelper 7 | from MediaInsightsEngineLambdaHelper import MasExecutionError 8 | from MediaInsightsEngineLambdaHelper import DataPlane 9 | 10 | from silence import detect_silences 11 | from segment import detect_technical_cues, detect_shots 12 | from score import calculate_scores 13 | 14 | REKOGNITION_OPERATORS = { 15 | "celebrityRecognition", 16 | "faceDetection", 17 | "labelDetection", 18 | "contentModeration", 19 | "technicalCueDetection", 20 | "shotDetection" 21 | } 22 | 23 | dataplane = DataPlane() 24 | 25 | def lambda_handler(event, context): 26 | print("We got the following event:\n", event) 27 | operator_object = MediaInsightsOperationHelper(event) 28 | # Get media metadata from input event 29 | try: 30 | workflow_id = operator_object.workflow_execution_id 31 | asset_id = operator_object.asset_id 32 | loudness_bucket = operator_object.input["Media"]["Loudness"]["S3Bucket"] 33 | loudness_key = operator_object.input["Media"]["Loudness"]["S3Key"] 34 | except Exception as exception: 35 | operator_object.update_workflow_status("Error") 36 | operator_object.add_workflow_metadata( 37 | SlotDetectionError="Missing a required metadata key {e}".format(e=exception)) 38 | raise MasExecutionError(operator_object.return_output_object()) 39 | # Get asset metadata from dataplane 40 | try: 41 | asset_metadata = __get_asset_metadata(asset_id) 42 | except Exception as exception: 43 | operator_object.update_workflow_status("Error") 44 | operator_object.add_workflow_metadata( 45 | SlotDetectionError="Unable to retrieve metadata for asset {}: {}".format(asset_id, exception)) 46 | raise MasExecutionError(operator_object.return_output_object()) 47 | try: 48 | # Get detected reasons' timestamps from media and asset metadata 49 | silences = detect_silences(loudness_bucket, loudness_key) 50 | black_frames, end_credits = detect_technical_cues(asset_metadata) 51 | shots = detect_shots(asset_metadata) 52 | reasons_timestamps = { 53 | "Silence": silences, 54 | "BlackFrame": black_frames, 55 | "ShotChange": shots, 56 | "EndCredits": end_credits 57 | } 58 | media_info = asset_metadata["shotDetection"]["VideoMetadata"][0] 59 | # Create slots from reasons' timestamps 60 | print("reasons_timestamps: {}".format(reasons_timestamps)) 61 | slots = [] 62 | for reason in reasons_timestamps: 63 | for timestamp in reasons_timestamps[reason]: 64 | slots.append({ 65 | "Timestamp": float(timestamp), 66 | "Score": 1.0, 67 | "Reasons": [reason] 68 | }) 69 | print("slots: {}".format(slots)) 70 | # Consolidate slots and calculate scores 71 | slots = calculate_scores(slots, media_info, asset_metadata) 72 | print("scored_slots: {}".format(slots)) 73 | except Exception as exception: 74 | operator_object.update_workflow_status("Error") 75 | operator_object.add_workflow_metadata(SlotDetectionError=str(exception)) 76 | raise MasExecutionError(operator_object.return_output_object()) 77 | 78 | operator_object.add_workflow_metadata( 79 | AssetId=asset_id, 80 | WorkflowExecutionId=workflow_id) 81 | operator_object.update_workflow_status("Complete") 82 | 83 | metadata_upload = dataplane.store_asset_metadata( 84 | asset_id=asset_id, 85 | operator_name=operator_object.name, 86 | workflow_id=workflow_id, 87 | results={"slots": slots} 88 | ) 89 | print("metadata_upload: {}".format(metadata_upload)) 90 | if metadata_upload["Status"] == "Success": 91 | print("Uploaded metadata for asset: {asset}".format(asset=asset_id)) 92 | elif metadata_upload["Status"] == "Failed": 93 | operator_object.update_workflow_status("Error") 94 | operator_object.add_workflow_metadata( 95 | SlotDetectionError="Unable to upload metadata for asset {}: {}".format(asset_id, metadata_upload)) 96 | raise MasExecutionError(operator_object.return_output_object()) 97 | else: 98 | operator_object.update_workflow_status("Error") 99 | operator_object.add_workflow_metadata( 100 | SlotDetectionError="Unable to upload metadata for asset {}: {}".format(asset_id, metadata_upload)) 101 | raise MasExecutionError(operator_object.return_output_object()) 102 | 103 | return operator_object.return_output_object() 104 | 105 | def __get_asset_metadata(asset_id): 106 | asset_metadata = {operator: {} for operator in REKOGNITION_OPERATORS} 107 | params = {"asset_id": asset_id} 108 | while True: 109 | response = dataplane.retrieve_asset_metadata(**params) 110 | if "operator" in response and response["operator"] in REKOGNITION_OPERATORS: 111 | __update_and_merge_lists(asset_metadata[response["operator"]], response["results"]) 112 | if "cursor" not in response: 113 | break 114 | params["cursor"] = response["cursor"] 115 | return asset_metadata 116 | 117 | def __update_and_merge_lists(dict1, dict2): 118 | for key in dict2: 119 | if key in dict1: 120 | if type(dict1[key]) is list and type(dict2[key]) is list: 121 | dict1[key].extend(dict2[key]) 122 | elif type(dict1[key]) is dict and type(dict1[key]) is dict: 123 | __update_and_merge_lists(dict1[key], dict2[key]) 124 | else: 125 | dict1[key] = dict2[key] 126 | else: 127 | dict1[key] = dict2[key] -------------------------------------------------------------------------------- /slot_detection/requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/aws-smart-ad-breaks/df9658e4954cfda5f80bedb7857d6fd58667ee41/slot_detection/requirements.txt -------------------------------------------------------------------------------- /slot_detection/score.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # SPDX-License-Identifier: Apache-2.0 3 | 4 | import os 5 | import math 6 | 7 | SCORE_ADJUSTMENTS = { 8 | "Silence": 0.7, 9 | "BlackFrame": 0.8, 10 | "ShotChange": 0.7, 11 | "EndCredits": 1.0 12 | } 13 | 14 | context_interval = int(os.environ["CONTEXT_INTERVAL_IN_SECONDS"]) 15 | min_confidence = int(os.environ["CONTEXT_MIN_CONFIDENCE"]) 16 | 17 | def calculate_scores(slots, media_info, asset_metadata): 18 | # Sorting by slot time 19 | slots.sort(key=lambda slot: slot["Timestamp"]) 20 | 21 | min_slot_interval = 0.50 22 | block_ratio = 0.25 23 | 24 | consolidated = [] 25 | prev_slot = {} 26 | 27 | # Getting video duration 28 | total_duration = float(media_info["DurationMillis"]) / 1000.0 29 | 30 | for slot in slots: 31 | # Adjusting base slot score 32 | slot["Score"] = slot["Score"] * SCORE_ADJUSTMENTS[slot["Reasons"][0]] 33 | 34 | # Score adjustment: distance from beginning 35 | if slot["Timestamp"] / total_duration < block_ratio: 36 | slot["Score"] = slot["Score"] * math.pow( 37 | slot["Timestamp"] / total_duration, 0.30) 38 | 39 | if "Timestamp" in prev_slot: 40 | dist_from_prev = slot["Timestamp"] - prev_slot["Timestamp"] 41 | # Consolidating with previous slot if distance < min_slot_interval 42 | if dist_from_prev < min_slot_interval: 43 | print("Consolidating slots: {}\n{}".format(prev_slot, slot)) 44 | 45 | prev_slot["Timestamp"] = slot["Timestamp"] 46 | if slot["Reasons"][0] not in prev_slot["Reasons"]: 47 | prev_slot["Reasons"].append(slot["Reasons"][0]) 48 | prev_slot["Score"] = __disjunction(prev_slot["Score"], slot["Score"]) 49 | continue 50 | # Score adjustment: distance between slots 51 | elif dist_from_prev / total_duration < block_ratio: 52 | if slot["Score"] < prev_slot["Score"]: 53 | slot["Score"] = slot["Score"] * math.pow( 54 | dist_from_prev / total_duration, 0.05) 55 | else: 56 | prev_slot["Score"] = prev_slot["Score"] * math.pow( 57 | dist_from_prev / total_duration, 0.05) 58 | 59 | # Score adjustment: labels before and after 60 | slot["Context"] = __get_context_metadata(slot["Timestamp"], asset_metadata) 61 | pre_labels = set(label["Name"] for label in slot["Context"]["Labels"]["Before"]) 62 | post_labels = set(label["Name"] for label in slot["Context"]["Labels"]["After"]) 63 | if pre_labels or post_labels: 64 | distance = 1.0 - (len(pre_labels.intersection(post_labels)) / len(pre_labels.union(post_labels))) 65 | slot["Score"] = __disjunction(slot["Score"], math.pow(distance, 4.0)) 66 | 67 | consolidated.append(slot) 68 | prev_slot = slot 69 | 70 | return consolidated 71 | 72 | def __disjunction(x, y): 73 | return 1.0 - ((1.0 - x) * (1.0 - y)) 74 | 75 | def __get_context_metadata(slot_timestamp, asset_metadata): 76 | rek_operator_keys = { 77 | "celebrityRecognition": ["Celebrities", "Celebrity"], 78 | "faceDetection": ["Faces", "Face"], 79 | "labelDetection": ["Labels", "Label"], 80 | "contentModeration": ["ModerationLabels", "ModerationLabel"] 81 | } 82 | context = {} 83 | for operator in rek_operator_keys: 84 | list_key = rek_operator_keys[operator][0] 85 | before = {} 86 | after = {} 87 | for result in asset_metadata[operator][list_key]: 88 | result_timestamp = result["Timestamp"] / 1000.0 89 | item_key = rek_operator_keys[operator][1] 90 | if slot_timestamp - context_interval <= result_timestamp <= slot_timestamp: 91 | for label in __labels_from_result(result, item_key): 92 | if label["Name"] in before: 93 | if before[label["Name"]]["Confidence"] < label["Confidence"]: 94 | before[label["Name"]] = label 95 | else: 96 | before[label["Name"]] = label 97 | elif slot_timestamp <= result_timestamp <= slot_timestamp + context_interval: 98 | for label in __labels_from_result(result, item_key): 99 | if label["Name"] in after: 100 | if after[label["Name"]]["Confidence"] < label["Confidence"]: 101 | after[label["Name"]] = label 102 | else: 103 | after[label["Name"]] = label 104 | elif result_timestamp > slot_timestamp + context_interval: 105 | break 106 | context[list_key] = { 107 | "Before": list(before.values()), 108 | "After": list(after.values()) 109 | } 110 | return context 111 | 112 | def __labels_from_result(result, item_key): 113 | labels = [] 114 | if "Emotions" in result[item_key]: 115 | for emotion in result[item_key]["Emotions"]: 116 | if float(emotion["Confidence"]) >= min_confidence: 117 | labels.append({"Name": emotion["Type"], "Confidence": emotion["Confidence"]}) 118 | else: 119 | if float(result[item_key]["Confidence"]) >= min_confidence: 120 | labels.append({"Name": result[item_key]["Name"], "Confidence": result[item_key]["Confidence"]}) 121 | return labels -------------------------------------------------------------------------------- /slot_detection/segment.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # SPDX-License-Identifier: Apache-2.0 3 | 4 | def detect_technical_cues(asset_metadata): 5 | black_frames = [] 6 | end_credits = [] 7 | for segment in asset_metadata["technicalCueDetection"]["Segments"]: 8 | if segment["TechnicalCueSegment"]["Type"] == "BlackFrames": 9 | black_frames.append(float(segment["StartTimestampMillis"]) / 1000.0) 10 | elif segment["TechnicalCueSegment"]["Type"] == "EndCredits": 11 | end_credits.append(float(segment["StartTimestampMillis"]) / 1000.0) 12 | return (black_frames, end_credits) 13 | 14 | def detect_shots(asset_metadata): 15 | shots = [] 16 | for segment in asset_metadata["shotDetection"]["Segments"]: 17 | if segment["Type"] == "SHOT": 18 | shots.append(float(segment["StartTimestampMillis"]) / 1000.0) 19 | return shots 20 | -------------------------------------------------------------------------------- /slot_detection/silence.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # SPDX-License-Identifier: Apache-2.0 3 | 4 | import os 5 | import tempfile 6 | import boto3 7 | 8 | s3 = boto3.client("s3") 9 | 10 | def detect_silences(loudness_bucket, loudness_key): 11 | start_threshold = float(os.environ['START_THRESHOLD_IN_SECONDS']) 12 | # silent audio has loudness lower than threshold 13 | silent_threshold = float(os.environ['SILENT_THRESHOLD']) 14 | silences = [] 15 | with tempfile.TemporaryFile() as tmp: 16 | s3.download_fileobj(loudness_bucket, loudness_key, tmp) 17 | tmp.seek(0) 18 | is_silent = False 19 | for line in tmp.readlines()[1:]: 20 | line = line.decode('utf-8') 21 | short_term_loudness = float(line.split(',')[3]) 22 | if short_term_loudness < silent_threshold: 23 | if not is_silent: 24 | second = float(line.split(',')[0]) 25 | if second > start_threshold: 26 | silences.append(second) 27 | is_silent = True 28 | else: 29 | is_silent = False 30 | return silences -------------------------------------------------------------------------------- /template.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # SPDX-License-Identifier: Apache-2.0 3 | 4 | AWSTemplateFormatVersion: 2010-09-09 5 | Transform: AWS::Serverless-2016-10-31 6 | 7 | Description: (uksb-1tsflhnas/v1/backend) This template deploys the aws-smart-ad-breaks 8 | 9 | Metadata: 10 | ID: uksb-1tsflhnas 11 | Version: 1 12 | Stack: backend 13 | 14 | Parameters: 15 | MediaInsightsEnginePython39Layer: 16 | Type: String 17 | Description: "ARN of the MIE Lambda layer (MediaInsightsEnginePython39Layer output of the main MIE CloudFormation stack)" 18 | WorkflowCustomResourceArn: 19 | Type: String 20 | Description: "ARN of the MIE custom resource (WorkflowCustomResourceArn output of the main MIE CloudFormation stack)" 21 | WorkflowEndpoint: 22 | Type: "String" 23 | Description: "Workflow endpoint (APIHandlerName output of the MIE Workflow CloudFormation nested stack)" 24 | DataplaneEndpoint: 25 | Type: "String" 26 | Description: "Dataplane endpoint (APIHandlerName output of the MIE Dataplane CloudFormation nested stack)" 27 | DataplaneBucket: 28 | Type: "String" 29 | Description: "Bucket for the dataplane (DataplaneBucket output of the main MIE CloudFormation stack)" 30 | 31 | Globals: 32 | Function: 33 | Runtime: python3.9 34 | Handler: app.lambda_handler 35 | Timeout: 300 36 | MemorySize: 1024 37 | Layers: 38 | - !Ref MediaInsightsEnginePython39Layer 39 | Environment: 40 | Variables: 41 | DATAPLANE_BUCKET: !Ref DataplaneBucket 42 | DataplaneEndpoint: !Ref DataplaneEndpoint 43 | 44 | Resources: 45 | ############# 46 | # IAM Roles # 47 | ############# 48 | MediaConvertS3Role: 49 | Type: AWS::IAM::Role 50 | Properties: 51 | AssumeRolePolicyDocument: 52 | Version: 2012-10-17 53 | Statement: 54 | - Effect: Allow 55 | Principal: 56 | Service: 57 | - mediaconvert.amazonaws.com 58 | Action: 59 | - 'sts:AssumeRole' 60 | Policies: 61 | - PolicyName: MediaConvertS3RolePolicy 62 | PolicyDocument: 63 | Statement: 64 | - Effect: Allow 65 | Action: 66 | - s3:GetObject 67 | - s3:PutObject 68 | Resource: 69 | - !Sub "arn:aws:s3:::${DataplaneBucket}/*" 70 | LambdaMediaConvertRole: 71 | Type: AWS::IAM::Role 72 | Properties: 73 | AssumeRolePolicyDocument: 74 | Version: 2012-10-17 75 | Statement: 76 | - Effect: Allow 77 | Principal: 78 | Service: 79 | - lambda.amazonaws.com 80 | Action: 81 | - sts:AssumeRole 82 | ManagedPolicyArns: 83 | - arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole 84 | - arn:aws:iam::aws:policy/AmazonS3ReadOnlyAccess 85 | Policies: 86 | - PolicyName: LambdaMediaConvertRolePolicy 87 | PolicyDocument: 88 | Statement: 89 | - Effect: Allow 90 | Action: 91 | - mediaconvert:GetJob 92 | - mediaconvert:ListJobs 93 | - mediaconvert:DescribeEndpoints 94 | - mediaconvert:CreateJob 95 | Resource: 96 | - "*" 97 | - Effect: Allow 98 | Action: 99 | - iam:PassRole 100 | Resource: 101 | - !GetAtt MediaConvertS3Role.Arn 102 | - Effect: Allow 103 | Action: 104 | - s3:GetObject 105 | - s3:PutObject 106 | Resource: 107 | - !Sub "arn:aws:s3:::${DataplaneBucket}/*" 108 | - Effect: Allow 109 | Action: 110 | - lambda:InvokeFunction 111 | Resource: 112 | - !Sub "arn:aws:lambda:${AWS::Region}:${AWS::AccountId}:function:${DataplaneEndpoint}*" 113 | LambdaDataplaneRole: 114 | Type: AWS::IAM::Role 115 | Properties: 116 | AssumeRolePolicyDocument: 117 | Version: 2012-10-17 118 | Statement: 119 | - Effect: Allow 120 | Principal: 121 | Service: 122 | - lambda.amazonaws.com 123 | Action: 124 | - sts:AssumeRole 125 | ManagedPolicyArns: 126 | - arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole 127 | - arn:aws:iam::aws:policy/AmazonS3ReadOnlyAccess 128 | Policies: 129 | - PolicyName: LambdaDataplaneRolePolicy 130 | PolicyDocument: 131 | Statement: 132 | - Effect: Allow 133 | Action: 134 | - s3:GetObject 135 | - s3:PutObject 136 | Resource: 137 | - !Sub "arn:aws:s3:::${DataplaneBucket}/*" 138 | - Effect: Allow 139 | Action: 140 | - lambda:InvokeFunction 141 | Resource: 142 | - !Sub "arn:aws:lambda:${AWS::Region}:${AWS::AccountId}:function:${DataplaneEndpoint}*" 143 | LambdaWorkflowRole: 144 | Type: AWS::IAM::Role 145 | Properties: 146 | AssumeRolePolicyDocument: 147 | Version: 2012-10-17 148 | Statement: 149 | - Effect: Allow 150 | Principal: 151 | Service: 152 | - lambda.amazonaws.com 153 | Action: 154 | - sts:AssumeRole 155 | ManagedPolicyArns: 156 | - arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole 157 | - arn:aws:iam::aws:policy/AmazonS3ReadOnlyAccess 158 | Policies: 159 | - PolicyName: LambdaWorkflowRolePolicy 160 | PolicyDocument: 161 | Statement: 162 | - Effect: Allow 163 | Action: 164 | - s3:GetObject 165 | - s3:PutObject 166 | Resource: 167 | - !Sub "arn:aws:s3:::${DataplaneBucket}/*" 168 | - Effect: Allow 169 | Action: 170 | - lambda:InvokeFunction 171 | Resource: 172 | - !Sub "arn:aws:lambda:${AWS::Region}:${AWS::AccountId}:function:${WorkflowEndpoint}*" 173 | StateMachineExecutionRole: 174 | Type: AWS::IAM::Role 175 | Properties: 176 | AssumeRolePolicyDocument: 177 | Version: 2012-10-17 178 | Statement: 179 | - Effect: Allow 180 | Principal: 181 | Service: 182 | - states.amazonaws.com 183 | Action: 184 | - sts:AssumeRole 185 | Policies: 186 | - PolicyName: StateMachineExecutionRolePolicy 187 | PolicyDocument: 188 | Statement: 189 | - Effect: Allow 190 | Action: 191 | - lambda:InvokeFunction 192 | Resource: 193 | - "arn:aws:lambda:*:*:function:*" 194 | 195 | ################################### 196 | # Input bucket and Lambda funcion # 197 | ################################### 198 | InputBucket: 199 | Type: AWS::S3::Bucket 200 | Properties: 201 | BucketEncryption: 202 | ServerSideEncryptionConfiguration: 203 | - ServerSideEncryptionByDefault: 204 | SSEAlgorithm: 'AES256' 205 | VersioningConfiguration: 206 | Status: Enabled 207 | InputBucketPolicy: 208 | Type: AWS::S3::BucketPolicy 209 | Properties: 210 | Bucket: !Ref InputBucket 211 | PolicyDocument: 212 | Statement: 213 | - Effect: Deny 214 | Action: 215 | - s3:* 216 | Resource: 217 | - !Sub "arn:aws:s3:::${InputBucket}/*" 218 | - !Sub "arn:aws:s3:::${InputBucket}" 219 | Principal: "*" 220 | Condition: 221 | Bool: 222 | aws:SecureTransport: false 223 | InputFunction: 224 | Type: AWS::Serverless::Function 225 | Properties: 226 | CodeUri: input/ 227 | Role: !GetAtt LambdaWorkflowRole.Arn 228 | Environment: 229 | Variables: 230 | WORKFLOW_NAME: "SmartAdBreaksWorkflow" 231 | WorkflowEndpoint: !Ref WorkflowEndpoint 232 | Events: 233 | S3Bucket: 234 | Type: S3 235 | Properties: 236 | Bucket: !Ref InputBucket 237 | Events: s3:ObjectCreated:* 238 | 239 | ######################################## 240 | # Lambda functions of custom operators # 241 | ######################################## 242 | VideoTranscodingStartFunction: 243 | Type: AWS::Serverless::Function 244 | Properties: 245 | CodeUri: video_transcoding_start/ 246 | Role: !GetAtt LambdaMediaConvertRole.Arn 247 | Environment: 248 | Variables: 249 | MEDIA_CONVERT_ROLE_ARN: !GetAtt MediaConvertS3Role.Arn 250 | VideoTranscodingCheckFunction: 251 | Type: AWS::Serverless::Function 252 | Properties: 253 | CodeUri: video_transcoding_check/ 254 | Role: !GetAtt LambdaMediaConvertRole.Arn 255 | SlotDetectionFunction: 256 | Type: AWS::Serverless::Function 257 | Properties: 258 | CodeUri: slot_detection/ 259 | Role: !GetAtt LambdaDataplaneRole.Arn 260 | Environment: 261 | Variables: 262 | # Silence 263 | START_THRESHOLD_IN_SECONDS: 3 264 | SILENT_THRESHOLD: -50 265 | # Context 266 | CONTEXT_MIN_CONFIDENCE: 70 267 | CONTEXT_INTERVAL_IN_SECONDS: 2 268 | VmapGenerationFunction: 269 | Type: AWS::Serverless::Function 270 | Properties: 271 | CodeUri: vmap_generation/ 272 | Role: !GetAtt LambdaDataplaneRole.Arn 273 | Environment: 274 | Variables: 275 | TOP_SLOTS_QTY: 3 276 | 277 | ############# 278 | # Operators # 279 | ############# 280 | VideoTranscodingOperator: 281 | Type: Custom::CustomResource 282 | Properties: 283 | ServiceToken: !Ref WorkflowCustomResourceArn 284 | ResourceType: "Operation" 285 | Name: "videoTranscoding" 286 | Type: "Async" 287 | Configuration: 288 | { 289 | "MediaType": "Video", 290 | "Enabled": true 291 | } 292 | StartLambdaArn: !GetAtt VideoTranscodingStartFunction.Arn 293 | MonitorLambdaArn: !GetAtt VideoTranscodingCheckFunction.Arn 294 | StateMachineExecutionRoleArn: !GetAtt StateMachineExecutionRole.Arn 295 | SlotDetectionOperator: 296 | Type: Custom::CustomResource 297 | Properties: 298 | ServiceToken: !Ref WorkflowCustomResourceArn 299 | ResourceType: "Operation" 300 | Name: "slotDetection" 301 | Type: "Sync" 302 | Configuration: 303 | { 304 | "MediaType": "Video", 305 | "Enabled": true 306 | } 307 | StartLambdaArn: !GetAtt SlotDetectionFunction.Arn 308 | StateMachineExecutionRoleArn: !GetAtt StateMachineExecutionRole.Arn 309 | VmapGenerationOperator: 310 | Type: Custom::CustomResource 311 | Properties: 312 | ServiceToken: !Ref WorkflowCustomResourceArn 313 | ResourceType: "Operation" 314 | Name: "vmapGeneration" 315 | Type: "Sync" 316 | Configuration: 317 | { 318 | "MediaType": "Video", 319 | "Enabled": true 320 | } 321 | StartLambdaArn: !GetAtt VmapGenerationFunction.Arn 322 | StateMachineExecutionRoleArn: !GetAtt StateMachineExecutionRole.Arn 323 | 324 | ########## 325 | # Stages # 326 | ########## 327 | SmartAdBreaksStage1: 328 | DependsOn: 329 | - VideoTranscodingOperator 330 | Type: Custom::CustomResource 331 | Properties: 332 | ServiceToken: !Ref WorkflowCustomResourceArn 333 | ResourceType: "Stage" 334 | Name: "SmartAdBreaksStage1" 335 | Operations: 336 | - !GetAtt VideoTranscodingOperator.Name 337 | SmartAdBreaksStage2: 338 | Type: Custom::CustomResource 339 | Properties: 340 | ServiceToken: !Ref WorkflowCustomResourceArn 341 | ResourceType: "Stage" 342 | Name: "SmartAdBreaksStage2" 343 | Operations: 344 | - labelDetection 345 | - celebrityRecognition 346 | - faceDetection 347 | - contentModeration 348 | - shotDetection 349 | - technicalCueDetection 350 | SmartAdBreaksStage3: 351 | DependsOn: 352 | - SlotDetectionOperator 353 | Type: Custom::CustomResource 354 | Properties: 355 | ServiceToken: !Ref WorkflowCustomResourceArn 356 | ResourceType: "Stage" 357 | Name: "SmartAdBreaksStage3" 358 | Operations: 359 | - !GetAtt SlotDetectionOperator.Name 360 | SmartAdBreaksStage4: 361 | DependsOn: 362 | - VmapGenerationOperator 363 | Type: Custom::CustomResource 364 | Properties: 365 | ServiceToken: !Ref WorkflowCustomResourceArn 366 | ResourceType: "Stage" 367 | Name: "SmartAdBreaksStage4" 368 | Operations: 369 | - !GetAtt VmapGenerationOperator.Name 370 | 371 | ############ 372 | # Workflow # 373 | ############ 374 | SmartAdBreaksWorkflow: 375 | DependsOn: 376 | - SmartAdBreaksStage1 377 | - SmartAdBreaksStage2 378 | - SmartAdBreaksStage3 379 | - SmartAdBreaksStage4 380 | Type: Custom::CustomResource 381 | Properties: 382 | ServiceToken: !Ref WorkflowCustomResourceArn 383 | ResourceType: "Workflow" 384 | Name: "SmartAdBreaksWorkflow" 385 | Stages: !Sub 386 | - |- 387 | { 388 | "${stagename1}": 389 | { 390 | "Next": "${stagename2}" 391 | }, 392 | "${stagename2}": 393 | { 394 | "Next": "${stagename3}" 395 | }, 396 | "${stagename3}": 397 | { 398 | "Next": "${stagename4}" 399 | }, 400 | "${stagename4}": 401 | { 402 | "End": true 403 | } 404 | } 405 | - stagename1: !GetAtt SmartAdBreaksStage1.Name 406 | stagename2: !GetAtt SmartAdBreaksStage2.Name 407 | stagename3: !GetAtt SmartAdBreaksStage3.Name 408 | stagename4: !GetAtt SmartAdBreaksStage4.Name 409 | StartAt: !GetAtt SmartAdBreaksStage1.Name 410 | 411 | ############### 412 | # MediaTailor # 413 | ############### 414 | MediaTailorConfig: 415 | Type: Custom::MediaTailorConfig 416 | Properties: 417 | ServiceToken: !GetAtt MediaTailorConfigFunction.Arn 418 | ConfigurationName: !Sub "${AWS::StackName}-config" 419 | VideoContentSource: !Sub "https://${DataplaneBucket}.s3.amazonaws.com/private" 420 | AdDecisionServer: !Sub "https://${DataplaneBucket}.s3.amazonaws.com/private/assets/[player_params.asset_id]/vmap/ad_breaks.vmap" 421 | MediaTailorConfigUpdateWithCDN: 422 | Type: Custom::MediaTailorConfig 423 | Properties: 424 | ServiceToken: !GetAtt MediaTailorConfigFunction.Arn 425 | ConfigurationName: !Sub "${AWS::StackName}-config" 426 | VideoContentSource: !Sub "https://${CloudFrontDistribution.DomainName}" 427 | AdDecisionServer: !Sub "https://${CloudFrontDistribution.DomainName}/assets/[player_params.asset_id]/vmap/ad_breaks.vmap" 428 | CDNContentSegmentPrefix: !Sub "https://${CloudFrontDistribution.DomainName}" 429 | CDNAdSegmentPrefix: !Sub "https://${CloudFrontDistribution.DomainName}" 430 | MediaTailorConfigFunctionRole: 431 | Type: AWS::IAM::Role 432 | Properties: 433 | ManagedPolicyArns: 434 | - arn:aws:iam::aws:policy/CloudWatchLogsFullAccess 435 | - arn:aws:iam::aws:policy/AWSLambda_FullAccess 436 | Policies: 437 | - PolicyName: MediaTailorAccess 438 | PolicyDocument: 439 | Version: 2012-10-17 440 | Statement: 441 | - Effect: Allow 442 | Action: 443 | - mediatailor:PutPlaybackConfiguration 444 | - mediatailor:DeletePlaybackConfiguration 445 | Resource: "*" 446 | AssumeRolePolicyDocument: 447 | Version: 2012-10-17 448 | Statement: 449 | - Effect: Allow 450 | Principal: 451 | Service: 452 | - lambda.amazonaws.com 453 | Action: sts:AssumeRole 454 | MediaTailorConfigFunction: 455 | Type: AWS::Serverless::Function 456 | Properties: 457 | CodeUri: mediatailor/ 458 | Role: !GetAtt MediaTailorConfigFunctionRole.Arn 459 | 460 | ############## 461 | # CloudFront # 462 | ############## 463 | CloudFrontOAI: 464 | Type: AWS::CloudFront::CloudFrontOriginAccessIdentity 465 | Properties: 466 | CloudFrontOriginAccessIdentityConfig: 467 | Comment: !Sub "access-identity-${DataplaneBucket}.s3.amazonaws.com" 468 | CloudFrontDistribution: 469 | Type: AWS::CloudFront::Distribution 470 | Properties: 471 | DistributionConfig: 472 | Enabled: true 473 | DefaultRootObject: '' 474 | ViewerCertificate: 475 | CloudFrontDefaultCertificate: true 476 | Origins: 477 | - Id: DataplaneBucket 478 | DomainName: !Sub "${DataplaneBucket}.s3.amazonaws.com" 479 | OriginPath: '/private' 480 | S3OriginConfig: 481 | OriginAccessIdentity: !Sub "origin-access-identity/cloudfront/${CloudFrontOAI}" 482 | - Id: MediaTailorConfig 483 | DomainName: !GetAtt MediaTailorConfig.HLSPlaybackDomain 484 | OriginPath: '' 485 | CustomOriginConfig: 486 | OriginSSLProtocols: 487 | - TLSv1.2 488 | OriginProtocolPolicy: https-only 489 | - Id: MediaTailorAdSegments 490 | DomainName: !Sub "segments.mediatailor.${AWS::Region}.amazonaws.com" 491 | OriginPath: '' 492 | CustomOriginConfig: 493 | OriginSSLProtocols: 494 | - TLSv1.2 495 | OriginProtocolPolicy: https-only 496 | DefaultCacheBehavior: 497 | TargetOriginId: MediaTailorAdSegments 498 | ViewerProtocolPolicy: allow-all 499 | ForwardedValues: 500 | Cookies: 501 | Forward: none 502 | QueryString: true 503 | CacheBehaviors: 504 | - TargetOriginId: MediaTailorConfig 505 | ViewerProtocolPolicy: allow-all 506 | CachePolicyId: 4135ea2d-6df8-44a3-9df3-4b5a84be39ad # Managed-CachingDisabled 507 | OriginRequestPolicyId: 775133bc-15f2-49f9-abea-afb2e0bf67d2 # Managed-Elemental-MediaTailor-PersonalizedManifests 508 | ForwardedValues: 509 | Cookies: 510 | Forward: none 511 | QueryString: true 512 | PathPattern: "/v1/*" 513 | - TargetOriginId: DataplaneBucket 514 | ViewerProtocolPolicy: allow-all 515 | ForwardedValues: 516 | Cookies: 517 | Forward: none 518 | QueryString: true 519 | PathPattern: "/assets/*" 520 | CloudFrontBucketPermission: 521 | Type: Custom::CloudFrontBucketPermission 522 | Properties: 523 | ServiceToken: !GetAtt CloudFrontBucketPermissionFunction.Arn 524 | BucketName: !Ref DataplaneBucket 525 | OriginAccessIdentity: !Ref CloudFrontOAI 526 | CloudFrontBucketPermissionFunctionRole: 527 | Type: AWS::IAM::Role 528 | Properties: 529 | ManagedPolicyArns: 530 | - arn:aws:iam::aws:policy/CloudWatchLogsFullAccess 531 | - arn:aws:iam::aws:policy/AWSLambda_FullAccess 532 | - arn:aws:iam::aws:policy/AmazonS3FullAccess 533 | AssumeRolePolicyDocument: 534 | Version: 2012-10-17 535 | Statement: 536 | - Effect: Allow 537 | Principal: 538 | Service: 539 | - lambda.amazonaws.com 540 | Action: sts:AssumeRole 541 | CloudFrontBucketPermissionFunction: 542 | Type: AWS::Serverless::Function 543 | Properties: 544 | CodeUri: cloudfront/ 545 | Role: !GetAtt CloudFrontBucketPermissionFunctionRole.Arn 546 | 547 | Outputs: 548 | InputBucket: 549 | Value: !Ref InputBucket 550 | CloudFrontDomainName: 551 | Value: !Sub "https://${CloudFrontDistribution.DomainName}" 552 | CloudFrontHLSPlaybackPrefix: 553 | Value: !Sub "https://${CloudFrontDistribution.DomainName}${MediaTailorConfigUpdateWithCDN.HLSPlaybackPath}" 554 | -------------------------------------------------------------------------------- /video_transcoding_check/app.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # SPDX-License-Identifier: Apache-2.0 3 | 4 | ############################################################################### 5 | # PURPOSE: 6 | # Adds HLS, proxy encode and audio outputs from MediaConvert to 7 | # workflow metadata so downstream operators can use them as inputs. 8 | ############################################################################### 9 | 10 | import os 11 | import boto3 12 | 13 | from MediaInsightsEngineLambdaHelper import MediaInsightsOperationHelper 14 | from MediaInsightsEngineLambdaHelper import MasExecutionError 15 | 16 | region = os.environ["AWS_REGION"] 17 | mediaconvert = boto3.client("mediaconvert", region_name=region) 18 | 19 | def lambda_handler(event, context): 20 | print("We got the following event:\n", event) 21 | operator_object = MediaInsightsOperationHelper(event) 22 | 23 | # Get MediaConvert job id 24 | try: 25 | workflow_id = operator_object.workflow_execution_id 26 | job_id = operator_object.metadata["VideoTranscodingJobId"] 27 | input_file = operator_object.metadata["VideoTranscodingInputFile"] 28 | except KeyError as e: 29 | operator_object.update_workflow_status("Error") 30 | operator_object.add_workflow_metadata(VideoTranscodingError="Missing a required metadata key {e}".format(e=e)) 31 | raise MasExecutionError(operator_object.return_output_object()) 32 | # Get asset id 33 | try: 34 | asset_id = operator_object.asset_id 35 | except KeyError as e: 36 | print("No asset_id in this workflow") 37 | asset_id = '' 38 | 39 | # Get mediaconvert endpoint from cache if available 40 | if ("MEDIACONVERT_ENDPOINT" in os.environ): 41 | mediaconvert_endpoint = os.environ["MEDIACONVERT_ENDPOINT"] 42 | customer_mediaconvert = boto3.client("mediaconvert", region_name=region, endpoint_url=mediaconvert_endpoint) 43 | else: 44 | try: 45 | response = mediaconvert.describe_endpoints() 46 | except Exception as e: 47 | print("Exception:\n", e) 48 | operator_object.update_workflow_status("Error") 49 | operator_object.add_workflow_metadata(VideoTranscodingError=str(e)) 50 | raise MasExecutionError(operator_object.return_output_object()) 51 | else: 52 | mediaconvert_endpoint = response["Endpoints"][0]["Url"] 53 | # Cache the mediaconvert endpoint in order to avoid getting throttled on 54 | # the DescribeEndpoints API. 55 | os.environ["MEDIACONVERT_ENDPOINT"] = mediaconvert_endpoint 56 | customer_mediaconvert = boto3.client("mediaconvert", region_name=region, endpoint_url=mediaconvert_endpoint) 57 | 58 | # Get MediaConvert job results 59 | try: 60 | response = customer_mediaconvert.get_job(Id=job_id) 61 | except Exception as e: 62 | print("Exception:\n", e) 63 | operator_object.update_workflow_status("Error") 64 | operator_object.add_workflow_metadata(VideoTranscodingError=e, VideoTranscodingJobId=job_id) 65 | raise MasExecutionError(operator_object.return_output_object()) 66 | else: 67 | if response["Job"]["Status"] == 'IN_PROGRESS' or response["Job"]["Status"] == 'PROGRESSING': 68 | operator_object.update_workflow_status("Executing") 69 | operator_object.add_workflow_metadata(VideoTranscodingJobId=job_id, 70 | VideoTranscodingInputFile=input_file, 71 | AssetId=asset_id, 72 | WorkflowExecutionId=workflow_id) 73 | return operator_object.return_output_object() 74 | elif response["Job"]["Status"] == 'COMPLETE': 75 | # Get HLS object 76 | hls_output_uri = response["Job"]["Settings"]["OutputGroups"][0]["OutputGroupSettings"]["HlsGroupSettings"]["Destination"] 77 | hls_bucket = hls_output_uri.split("/")[2] 78 | hls_path = "/".join(hls_output_uri.split("/")[3:]) 79 | hls_key = hls_path + ".m3u8" 80 | operator_object.add_media_object("HLS", hls_bucket, hls_key) 81 | # Get proxy object 82 | proxy_output_uri = response["Job"]["Settings"]["OutputGroups"][1]["OutputGroupSettings"]["FileGroupSettings"]["Destination"] 83 | proxy_extension = response["Job"]["Settings"]["OutputGroups"][1]["Outputs"][0]["Extension"] 84 | proxy_modifier = response["Job"]["Settings"]["OutputGroups"][1]["Outputs"][0]["NameModifier"] 85 | proxy_bucket = proxy_output_uri.split("/")[2] 86 | proxy_path = "/".join(proxy_output_uri.split("/")[3:]) 87 | proxy_key = proxy_path + proxy_modifier + "." + proxy_extension 88 | operator_object.add_media_object("ProxyEncode", proxy_bucket, proxy_key) 89 | # Get audio object 90 | audio_output_uri = response["Job"]["Settings"]["OutputGroups"][2]["OutputGroupSettings"]["FileGroupSettings"]["Destination"] 91 | audio_extension = response["Job"]["Settings"]["OutputGroups"][2]["Outputs"][0]["Extension"] 92 | audio_modifier = response["Job"]["Settings"]["OutputGroups"][2]["Outputs"][0]["NameModifier"] 93 | audio_bucket = audio_output_uri.split("/")[2] 94 | audio_path = "/".join(audio_output_uri.split("/")[3:]) 95 | audio_key = audio_path + audio_modifier + "." + audio_extension 96 | loudness_key = audio_path + audio_modifier + "_loudness.csv" 97 | operator_object.add_media_object("Audio", audio_bucket, audio_key) 98 | operator_object.add_media_object("Loudness", audio_bucket, loudness_key) 99 | # Set workflow status complete 100 | operator_object.add_workflow_metadata(VideoTranscodingJobId=job_id) 101 | operator_object.update_workflow_status("Complete") 102 | return operator_object.return_output_object() 103 | else: 104 | operator_object.update_workflow_status("Error") 105 | operator_object.add_workflow_metadata( 106 | VideoTranscodingError="Unhandled exception, unable to get status from mediaconvert: {response}".format(response=response), 107 | VideoTranscodingJobId=job_id) 108 | raise MasExecutionError(operator_object.return_output_object()) -------------------------------------------------------------------------------- /video_transcoding_check/requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/aws-smart-ad-breaks/df9658e4954cfda5f80bedb7857d6fd58667ee41/video_transcoding_check/requirements.txt -------------------------------------------------------------------------------- /video_transcoding_start/app.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # SPDX-License-Identifier: Apache-2.0 3 | 4 | ############################################################################### 5 | # PURPOSE: 6 | # This operator uses MediaConvert to do four things: 7 | # 1) transcode video into an HLS playlist to be used with MediaTailor 8 | # 2) transcode video into an MP4 format supported by Rekognition 9 | # 3) extract audio track from video (with loudness analysis) 10 | # The transcode MP4 video is a "proxy encode" and is used by Rekognition 11 | # operators instead of the original video uploaded by a user. 12 | # 13 | # OUTPUT: 14 | # Audio will be saved to the following path: 15 | # "s3://" + $DATAPLANE_BUCKET + "/private/assets/"" + asset_id + "/workflows/" + workflow_id + "/" 16 | # 17 | ############################################################################### 18 | 19 | import os 20 | import boto3 21 | 22 | from MediaInsightsEngineLambdaHelper import MediaInsightsOperationHelper 23 | from MediaInsightsEngineLambdaHelper import MasExecutionError 24 | 25 | region = os.environ['AWS_REGION'] 26 | mediaconvert_role = os.environ['MEDIA_CONVERT_ROLE_ARN'] 27 | 28 | mediaconvert = boto3.client("mediaconvert", region_name=region) 29 | 30 | def lambda_handler(event, context): 31 | print("We got the following event:\n", event) 32 | operator_object = MediaInsightsOperationHelper(event) 33 | 34 | try: 35 | workflow_id = str(operator_object.workflow_execution_id) 36 | asset_id = operator_object.asset_id 37 | bucket = operator_object.input["Media"]["Video"]["S3Bucket"] 38 | key = operator_object.input["Media"]["Video"]["S3Key"] 39 | except KeyError as e: 40 | operator_object.update_workflow_status("Error") 41 | operator_object.add_workflow_metadata(VideoTranscodingError="Missing a required metadata key {e}".format(e=e)) 42 | raise MasExecutionError(operator_object.return_output_object()) 43 | 44 | file_input = "s3://" + bucket + "/" + key 45 | hls_destination = "s3://" + bucket + "/private/assets/" + asset_id + "/hls/playlist" 46 | proxy_destination = "s3://" + bucket + "/private/assets/" + asset_id + "/proxy/" + asset_id 47 | audio_destination = "s3://" + bucket + "/private/assets/" + asset_id + "/audio/" + asset_id 48 | 49 | # Get mediaconvert endpoint from cache if available 50 | if ("MEDIACONVERT_ENDPOINT" in os.environ): 51 | mediaconvert_endpoint = os.environ["MEDIACONVERT_ENDPOINT"] 52 | customer_mediaconvert = boto3.client("mediaconvert", region_name=region, endpoint_url=mediaconvert_endpoint) 53 | else: 54 | try: 55 | response = mediaconvert.describe_endpoints() 56 | except Exception as e: 57 | print("Exception:\n", e) 58 | operator_object.update_workflow_status("Error") 59 | operator_object.add_workflow_metadata(VideoTranscodingError=str(e)) 60 | raise MasExecutionError(operator_object.return_output_object()) 61 | else: 62 | mediaconvert_endpoint = response["Endpoints"][0]["Url"] 63 | # Cache the mediaconvert endpoint in order to avoid getting throttled on 64 | # the DescribeEndpoints API. 65 | os.environ["MEDIACONVERT_ENDPOINT"] = mediaconvert_endpoint 66 | customer_mediaconvert = boto3.client("mediaconvert", region_name=region, endpoint_url=mediaconvert_endpoint) 67 | 68 | try: 69 | response = customer_mediaconvert.create_job( 70 | Role=mediaconvert_role, 71 | Settings={ 72 | "OutputGroups": [ 73 | { 74 | "Name": "Apple HLS", 75 | "Outputs": [ 76 | { 77 | "Preset": "System-Avc_16x9_1080p_29_97fps_8500kbps", 78 | "NameModifier": "_hls" 79 | } 80 | ], 81 | "OutputGroupSettings": { 82 | "Type": "HLS_GROUP_SETTINGS", 83 | "HlsGroupSettings": { 84 | "ManifestDurationFormat": "INTEGER", 85 | "SegmentLength": 1, 86 | "TimedMetadataId3Period": 10, 87 | "CaptionLanguageSetting": "OMIT", 88 | "TimedMetadataId3Frame": "PRIV", 89 | "CodecSpecification": "RFC_4281", 90 | "OutputSelection": "MANIFESTS_AND_SEGMENTS", 91 | "ProgramDateTimePeriod": 600, 92 | "MinSegmentLength": 0, 93 | "MinFinalSegmentLength": 0, 94 | "DirectoryStructure": "SINGLE_DIRECTORY", 95 | "ProgramDateTime": "EXCLUDE", 96 | "SegmentControl": "SEGMENTED_FILES", 97 | "ManifestCompression": "NONE", 98 | "ClientCache": "ENABLED", 99 | "StreamInfResolution": "INCLUDE", 100 | "Destination": hls_destination 101 | } 102 | } 103 | }, 104 | { 105 | "CustomName": "Proxy", 106 | "Name": "File Group", 107 | "Outputs": [ 108 | { 109 | "VideoDescription": { 110 | "ScalingBehavior": "DEFAULT", 111 | "TimecodeInsertion": "DISABLED", 112 | "AntiAlias": "ENABLED", 113 | "Sharpness": 50, 114 | "CodecSettings": { 115 | "Codec": "H_264", 116 | "H264Settings": { 117 | "InterlaceMode": "PROGRESSIVE", 118 | "NumberReferenceFrames": 3, 119 | "Syntax": "DEFAULT", 120 | "Softness": 0, 121 | "GopClosedCadence": 1, 122 | "GopSize": 90, 123 | "Slices": 1, 124 | "GopBReference": "DISABLED", 125 | "SlowPal": "DISABLED", 126 | "SpatialAdaptiveQuantization": "ENABLED", 127 | "TemporalAdaptiveQuantization": "ENABLED", 128 | "FlickerAdaptiveQuantization": "DISABLED", 129 | "EntropyEncoding": "CABAC", 130 | "Bitrate": 5000000, 131 | "FramerateControl": "SPECIFIED", 132 | "RateControlMode": "CBR", 133 | "CodecProfile": "MAIN", 134 | "Telecine": "NONE", 135 | "MinIInterval": 0, 136 | "AdaptiveQuantization": "HIGH", 137 | "CodecLevel": "AUTO", 138 | "FieldEncoding": "PAFF", 139 | "SceneChangeDetect": "ENABLED", 140 | "QualityTuningLevel": "SINGLE_PASS", 141 | "FramerateConversionAlgorithm": "DUPLICATE_DROP", 142 | "UnregisteredSeiTimecode": "DISABLED", 143 | "GopSizeUnits": "FRAMES", 144 | "ParControl": "SPECIFIED", 145 | "NumberBFramesBetweenReferenceFrames": 2, 146 | "RepeatPps": "DISABLED", 147 | "FramerateNumerator": 30, 148 | "FramerateDenominator": 1, 149 | "ParNumerator": 1, 150 | "ParDenominator": 1 151 | } 152 | }, 153 | "AfdSignaling": "NONE", 154 | "DropFrameTimecode": "ENABLED", 155 | "RespondToAfd": "NONE", 156 | "ColorMetadata": "INSERT" 157 | }, 158 | "AudioDescriptions": [ 159 | { 160 | "AudioTypeControl": "FOLLOW_INPUT", 161 | "CodecSettings": { 162 | "Codec": "AAC", 163 | "AacSettings": { 164 | "AudioDescriptionBroadcasterMix": "NORMAL", 165 | "RateControlMode": "CBR", 166 | "CodecProfile": "LC", 167 | "CodingMode": "CODING_MODE_2_0", 168 | "RawFormat": "NONE", 169 | "SampleRate": 48000, 170 | "Specification": "MPEG4", 171 | "Bitrate": 64000 172 | } 173 | }, 174 | "LanguageCodeControl": "FOLLOW_INPUT", 175 | "AudioSourceName": "Audio Selector 1" 176 | } 177 | ], 178 | "ContainerSettings": { 179 | "Container": "MP4", 180 | "Mp4Settings": { 181 | "CslgAtom": "INCLUDE", 182 | "FreeSpaceBox": "EXCLUDE", 183 | "MoovPlacement": "PROGRESSIVE_DOWNLOAD" 184 | } 185 | }, 186 | "Extension": "mp4", 187 | "NameModifier": "_proxy" 188 | } 189 | ], 190 | "OutputGroupSettings": { 191 | "Type": "FILE_GROUP_SETTINGS", 192 | "FileGroupSettings": { 193 | "Destination": proxy_destination 194 | } 195 | } 196 | }, 197 | { 198 | "CustomName": "Audio", 199 | "Name": "File Group", 200 | "Outputs": [ 201 | { 202 | "ContainerSettings": { 203 | "Container": "MP4", 204 | "Mp4Settings": { 205 | "CslgAtom": "INCLUDE", 206 | "CttsVersion": 0, 207 | "FreeSpaceBox": "EXCLUDE", 208 | "MoovPlacement": "PROGRESSIVE_DOWNLOAD" 209 | } 210 | }, 211 | "AudioDescriptions": [ 212 | { 213 | "AudioTypeControl": "FOLLOW_INPUT", 214 | "AudioSourceName": "Audio Selector 1", 215 | "AudioNormalizationSettings": { 216 | "Algorithm": "ITU_BS_1770_2", 217 | "AlgorithmControl": "MEASURE_ONLY", 218 | "LoudnessLogging": "LOG", 219 | "PeakCalculation": "NONE" 220 | }, 221 | "CodecSettings": { 222 | "Codec": "AAC", 223 | "AacSettings": { 224 | "AudioDescriptionBroadcasterMix": "NORMAL", 225 | "Bitrate": 96000, 226 | "RateControlMode": "CBR", 227 | "CodecProfile": "LC", 228 | "CodingMode": "CODING_MODE_2_0", 229 | "RawFormat": "NONE", 230 | "SampleRate": 48000, 231 | "Specification": "MPEG4" 232 | } 233 | }, 234 | "LanguageCodeControl": "FOLLOW_INPUT" 235 | } 236 | ], 237 | "Extension": "mp4", 238 | "NameModifier": "_audio" 239 | } 240 | ], 241 | "OutputGroupSettings": { 242 | "Type": "FILE_GROUP_SETTINGS", 243 | "FileGroupSettings": { 244 | "Destination": audio_destination 245 | } 246 | } 247 | } 248 | ], 249 | "Inputs": [{ 250 | "AudioSelectors": { 251 | "Audio Selector 1": { 252 | "Offset": 0, 253 | "DefaultSelection": "DEFAULT", 254 | "ProgramSelection": 1 255 | } 256 | }, 257 | "VideoSelector": { 258 | "ColorSpace": "FOLLOW", 259 | "Rotate": "DEGREE_0", 260 | "AlphaBehavior": "DISCARD" 261 | }, 262 | "FilterEnable": "AUTO", 263 | "PsiControl": "USE_PSI", 264 | "FilterStrength": 0, 265 | "DeblockFilter": "DISABLED", 266 | "DenoiseFilter": "DISABLED", 267 | "TimecodeSource": "EMBEDDED", 268 | "FileInput": file_input 269 | }] 270 | } 271 | ) 272 | # TODO: Add support for boto client error handling 273 | except Exception as e: 274 | print("Exception:\n", e) 275 | operator_object.update_workflow_status("Error") 276 | operator_object.add_workflow_metadata(VideoTranscodingError=str(e)) 277 | raise MasExecutionError(operator_object.return_output_object()) 278 | else: 279 | job_id = response['Job']['Id'] 280 | operator_object.update_workflow_status("Executing") 281 | operator_object.add_workflow_metadata(VideoTranscodingJobId=job_id, VideoTranscodingInputFile=file_input, AssetId=asset_id, WorkflowExecutionId=workflow_id) 282 | return operator_object.return_output_object() -------------------------------------------------------------------------------- /video_transcoding_start/requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/aws-smart-ad-breaks/df9658e4954cfda5f80bedb7857d6fd58667ee41/video_transcoding_start/requirements.txt -------------------------------------------------------------------------------- /vmap_generation/ads.json: -------------------------------------------------------------------------------- 1 | { 2 | "ads": [ 3 | { 4 | "url": "https://d2qohgpffhaffh.cloudfront.net/Ads/AD-caribbean-15.mp4", 5 | "labels": ["Coast", "Beach", "Shoreline", "Sea", "Ocean", "Tropical", "Outdoors", "Boat", "Sailboat", "Palm Tree", "Tree", "Summer"] 6 | }, 7 | { 8 | "url": "https://d2qohgpffhaffh.cloudfront.net/Ads/AD-music-15.mp4", 9 | "labels": ["Musician", "Musical Instrument", "Music Band", "Guitar", "Electric Guitar", "Microphone", "Bass Guitar", "Drum", "Concert", "Stage"] 10 | }, 11 | { 12 | "url": "https://d2qohgpffhaffh.cloudfront.net/Ads/AD-carracing-15.mp4", 13 | "labels": ["Car", "Vehicle", "Tarmac", "Formula One", "Asphalt", "Kart", "Automobile"] 14 | }, 15 | { 16 | "url": "https://d2qohgpffhaffh.cloudfront.net/Ads/AD-perfume-15.mp4", 17 | "labels": ["Perfume", "Fashion", "Cosmetics", "Hair"] 18 | }, 19 | { 20 | "url": "https://d2qohgpffhaffh.cloudfront.net/Ads/AD-polarbear-15.mp4", 21 | "labels": ["Polar Bear", "Bear", "Ice", "Iceberg", "Snow", "Wildlife", "Animal"] 22 | }, 23 | { 24 | "url": "https://d2qohgpffhaffh.cloudfront.net/Ads/AD-robots-15.mp4", 25 | "labels": ["Robot", "Joystick", "Computer", "Electronics", "Hardware", "Video Gaming"] 26 | }, 27 | { 28 | "url": "https://d2qohgpffhaffh.cloudfront.net/Ads/AD-skiing-15.mp4", 29 | "labels": ["Skiing", "Snow", "Sport", "Sports", "Mountain", "Winter", "Travel", "Outdoors", "Nature"] 30 | }, 31 | { 32 | "url": "https://d2qohgpffhaffh.cloudfront.net/Ads/AD-sports-15.mp4", 33 | "labels": ["Sport", "Sports", "Basketball", "Football", "Baseball", "Hockey", "Soccer", "Tennis", "Golf"] 34 | } 35 | ] 36 | } -------------------------------------------------------------------------------- /vmap_generation/app.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # SPDX-License-Identifier: Apache-2.0 3 | 4 | import os 5 | import json 6 | import datetime 7 | import random 8 | import boto3 9 | 10 | from MediaInsightsEngineLambdaHelper import MediaInsightsOperationHelper 11 | from MediaInsightsEngineLambdaHelper import MasExecutionError 12 | from MediaInsightsEngineLambdaHelper import DataPlane 13 | 14 | from vmap_xml.vmap import VMAP 15 | from vast_xml.vast import VAST 16 | 17 | ADS_FILE = 'ads.json' 18 | 19 | top_slots_qty = int(os.environ['TOP_SLOTS_QTY']) 20 | 21 | s3 = boto3.client('s3') 22 | dataplane = DataPlane() 23 | 24 | # Reading ads from JSON file 25 | with open(ADS_FILE) as json_file: 26 | ads = json.load(json_file)['ads'] 27 | 28 | def lambda_handler(event, context): 29 | print("We got the following event:\n", event) 30 | operator_object = MediaInsightsOperationHelper(event) 31 | # Get media metadata from input event 32 | try: 33 | asset_id = operator_object.asset_id 34 | bucket = operator_object.input["Media"]["Video"]["S3Bucket"] 35 | except Exception as exception: 36 | operator_object.update_workflow_status("Error") 37 | operator_object.add_workflow_metadata( 38 | VmapGenerationError="Missing a required metadata key {e}".format(e=exception)) 39 | raise MasExecutionError(operator_object.return_output_object()) 40 | # Get slots metadata from dataplane 41 | try: 42 | slots = {} 43 | params = {"asset_id": asset_id, "operator_name": "slotDetection"} 44 | while True: 45 | resp = dataplane.retrieve_asset_metadata(**params) 46 | if "operator" in resp and resp["operator"] == "slotDetection": 47 | __update_and_merge_lists(slots, resp["results"]) 48 | if "cursor" not in resp: 49 | break 50 | params["cursor"] = resp["cursor"] 51 | print("slots: {}".format(slots)) 52 | except Exception as exception: 53 | operator_object.update_workflow_status("Error") 54 | operator_object.add_workflow_metadata( 55 | VmapGenerationError="Unable to retrieve metadata for asset {}: {}".format(asset_id, exception)) 56 | raise MasExecutionError(operator_object.return_output_object()) 57 | try: 58 | # Select slots with highest scores 59 | slots["slots"].sort(key=lambda slot: slot["Score"]) 60 | top_slots = slots["slots"][-top_slots_qty:] 61 | # Generate VMAP and add object 62 | key = 'private/assets/{}/vmap/ad_breaks.vmap'.format(asset_id) 63 | __write_vmap(top_slots, bucket, key) 64 | operator_object.add_media_object("VMAP", bucket, key) 65 | # Set workflow status complete 66 | operator_object.update_workflow_status("Complete") 67 | return operator_object.return_output_object() 68 | except Exception as exception: 69 | print("Exception:\n", exception) 70 | operator_object.update_workflow_status("Error") 71 | operator_object.add_workflow_metadata(VmapGenerationError=exception) 72 | raise MasExecutionError(operator_object.return_output_object()) 73 | 74 | def __update_and_merge_lists(dict1, dict2): 75 | for key in dict2: 76 | if key in dict1: 77 | if type(dict1[key]) is list and type(dict2[key]) is list: 78 | dict1[key].extend(dict2[key]) 79 | elif type(dict1[key]) is dict and type(dict1[key]) is dict: 80 | __update_and_merge_lists(dict1[key], dict2[key]) 81 | else: 82 | dict1[key] = dict2[key] 83 | else: 84 | dict1[key] = dict2[key] 85 | 86 | def __write_vmap(slots, bucket, key): 87 | vmap = VMAP() 88 | i = 1 89 | for slot in slots: 90 | # Merging labels from before and after the slot into a single list 91 | before_labels = [label['Name'] for label in slot['Context']['Labels']['Before']] 92 | after_labels = [label['Name'] for label in slot['Context']['Labels']['After']] 93 | labels = before_labels + list(set(after_labels) - set(before_labels)) 94 | # Adding ad break to VMAP file 95 | ad_break = vmap.attachAdBreak({ 96 | 'timeOffset': __format_timedelta(datetime.timedelta(seconds=float(slot['Timestamp']))), 97 | 'breakType': 'linear', 98 | 'breakId': 'midroll-{}'.format(i) 99 | }) 100 | # Adding VAST ad source 101 | vast = VAST() 102 | ad_break.attachAdSource( 103 | 'midroll-{}-ad-1'.format(i), 104 | 'false', 105 | 'true', 106 | 'VASTAdData', 107 | vast) 108 | ad = vast.attachAd({ 109 | 'id': str(i), 110 | 'structure': 'inline', 111 | 'AdSystem': {'name': '2.0'}, 112 | 'AdTitle': 'midroll-{}-ad-1'.format(i) 113 | }) 114 | ad.attachImpression({}) 115 | creative = ad.attachCreative('Linear', { 116 | 'Duration' : '00:00:15' 117 | }) 118 | # Setting media file URL referencing the ad server, passing labels as parameters 119 | creative.attachMediaFile(__select_ad(labels), { 120 | 'id': 'midroll-{}-ad-1'.format(i), 121 | 'type': 'video/mp4', 122 | 'delivery': 'progressive', 123 | 'width': '1920', 124 | 'height': '1080' 125 | }) 126 | i += 1 127 | # Converting VMAP content to XML 128 | vmap_content = vmap.xml() 129 | print(vmap_content) 130 | # Putting VMAP file into dataplane bucket 131 | s3.put_object( 132 | Body=bytes(vmap_content), 133 | Bucket=bucket, 134 | Key=key 135 | ) 136 | 137 | def __format_timedelta(delta): 138 | hours, rem = divmod(delta.seconds, 3600) 139 | minutes, seconds = divmod(rem, 60) 140 | milliseconds = int(delta.microseconds / 1000) 141 | return '{:02d}:{:02d}:{:02d}.{:03d}'.format(hours, minutes, seconds, milliseconds) 142 | 143 | def __select_ad(labels): 144 | print('labels: {}'.format(labels)) 145 | # Searching ads to find the one with most similar labels 146 | top_similarity = -1.0 147 | top_ad = None 148 | slot_labels = set(labels) 149 | random.shuffle(ads) # Shuffle to return a random ad in case none has similarity 150 | for ad in ads: 151 | print('ad: {}'.format(ad)) 152 | ad_labels = set(ad['labels']) 153 | similarity = len(slot_labels.intersection(ad_labels)) / len(slot_labels.union(ad_labels)) 154 | if similarity > top_similarity: 155 | top_similarity = similarity 156 | top_ad = ad 157 | print('top_ad: {}'.format(top_ad)) 158 | print('top_similarity: {}'.format(top_similarity)) 159 | # Return URL to selected ad video file 160 | return top_ad['url'] 161 | -------------------------------------------------------------------------------- /vmap_generation/requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/aws-smart-ad-breaks/df9658e4954cfda5f80bedb7857d6fd58667ee41/vmap_generation/requirements.txt -------------------------------------------------------------------------------- /vmap_generation/vast_xml/ad.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright 2015 Timu Eren 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | 18 | from vast_xml.creative import Creative 19 | 20 | REQUIRED_INLINE = ['AdSystem', 'AdTitle'] 21 | REQUIRED_WRAPPER = ['AdSystem', 'VASTAdTagURI'] 22 | 23 | 24 | def validateSettings(settings, requireds): 25 | keys = settings.keys() 26 | for required in requireds: 27 | if required not in keys: 28 | raise Exception("Missing required settings: {required}".format(required=required)) 29 | 30 | 31 | def validateInLineSettings(settings): 32 | validateSettings(settings, REQUIRED_INLINE) 33 | 34 | 35 | def validateWrapperSettings(settings): 36 | validateSettings(settings, REQUIRED_WRAPPER) 37 | 38 | 39 | class Ad(object): 40 | def __init__(self, settings={}): 41 | self.errors = [] 42 | self.surveys = [] 43 | self.impressions = [] 44 | self.creatives = [] 45 | 46 | if settings["structure"].lower() == 'wrapper': 47 | validateWrapperSettings(settings) 48 | self.VASTAdTagURI = settings["VASTAdTagURI"] 49 | else: 50 | validateInLineSettings(settings) 51 | 52 | self.id = settings["id"] 53 | self.sequence = settings.get("sequence", None) 54 | self.structure = settings["structure"] 55 | self.AdSystem = settings["AdSystem"] 56 | self.AdTitle = settings["AdTitle"] 57 | 58 | # optional elements 59 | self.Error = settings.get("Error", None) 60 | self.Description = settings.get("Description", None) 61 | self.Advertiser = settings.get("Advertiser", None) 62 | 63 | self.Pricing = settings.get("Pricing", None) 64 | self.Extensions = settings.get("Extensions", None) 65 | 66 | def attachSurvey(self, settings): 67 | survey = {"url": settings.url} 68 | if "type" in settings: 69 | survey["type"] = settings["type"] 70 | self.surveys.append(survey) 71 | 72 | def attachImpression(self, settings): 73 | self.impressions.append(settings) 74 | return self 75 | 76 | def attachCreative(self, _type, options): 77 | creative = Creative(_type, options) 78 | self.creatives.append(creative) 79 | return creative -------------------------------------------------------------------------------- /vmap_generation/vast_xml/companionad.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright 2015 Timu Eren 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License") 7 | # you may not use self file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | 18 | from vast_xml.trackingevent import TrackingEvent 19 | 20 | 21 | class CompanionAd(object): 22 | def __init__(self, resource, settings={}): 23 | self.resource = resource 24 | self.type = settings.get("type", None) 25 | self.url = settings.get("url", None) 26 | self.AdParameters = settings.get("AdParameters", None) 27 | self.AltText = settings.get("AltText", None) 28 | self.CompanionClickThrough = settings.get("CompanionClickThrough", None) 29 | self.CompanionClickTracking = settings.get("CompanionClickTracking", None) 30 | self.width = settings.get("width", None) 31 | self.height = settings.get("height", None) 32 | self.trackingEvents = [] 33 | 34 | def attachTrackingEvent(self, type, url): 35 | self.trackingEvents.append(TrackingEvent(type, url)) -------------------------------------------------------------------------------- /vmap_generation/vast_xml/creative.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright 2015 Timu Eren 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use self file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | 18 | from vast_xml.icon import Icon 19 | from vast_xml.trackingevent import TrackingEvent 20 | 21 | VALID_VIDEO_CLICKS = ['ClickThrough', 'ClickTracking', 'CustomClick'] 22 | 23 | 24 | class Creative(object): 25 | def __init__(self, _type, settings=None): 26 | settings = {} if settings is None else settings 27 | self.type = _type 28 | self.mediaFiles = [] 29 | self.trackingEvents = [] 30 | self.videoClicks = [] 31 | self.clickThroughs = [] 32 | self.clicks = [] 33 | self.resources = [] 34 | self.icons = [] 35 | self.AdParameters = settings.get("AdParameters", None) 36 | self._adParameters = None 37 | self.attributes = {} 38 | self.duration = settings.get("Duration", None) 39 | self.skipoffset = settings.get("skipoffset", None) 40 | self.nonLinearClickThrough = None 41 | self.nonLinearClickTracking = None 42 | 43 | if _type == "Linear" and self.duration is None: 44 | raise Exception('A Duration is required for all creatives. Consider defaulting to "00:00:00"') 45 | 46 | if "id" in settings: 47 | self.attributes["id"] = settings["id"] 48 | 49 | if "width" in settings: 50 | self.attributes["width"] = settings["width"] 51 | if "height" in settings: 52 | self.attributes["height"] = settings["height"] 53 | if "expandedWidth" in settings: 54 | self.attributes["expandedWidth"] = settings["expandedWidth"] 55 | if "expandedHeight" in settings: 56 | self.attributes["expandedHeight"] = settings["expandedHeight"] 57 | if "scalable" in settings: 58 | self.attributes["scalable"] = settings["scalable"] 59 | if "maintainAspectRatio" in settings: 60 | self.attributes["maintainAspectRatio"] = settings["maintainAspectRatio"] 61 | if "minSuggestedDuration" in settings: 62 | self.attributes["minSuggestedDuration"] = settings["minSuggestedDuration"] 63 | if "apiFramework" in settings: 64 | self.attributes["apiFramework"] = settings["apiFramework"] 65 | 66 | def attachMediaFile(self, url, settings={}): 67 | media_file = {"attributes": {}} 68 | media_file["url"] = url 69 | media_file["attributes"]["type"] = settings.get("type", 'video/mp4') 70 | media_file["attributes"]["width"] = settings.get("width", '640') 71 | media_file["attributes"]["height"] = settings.get("height", '360') 72 | media_file["attributes"]["delivery"] = settings.get("delivery", 'progressive') 73 | if "id" not in settings: 74 | raise Exception('an `id` is required for all media files') 75 | 76 | media_file["attributes"]["id"] = settings["id"] 77 | if "bitrate" in settings: 78 | media_file["attributes"]["bitrate"] = settings["bitrate"] 79 | if "minBitrate" in settings: 80 | media_file["attributes"]["minBitrate"] = settings["minBitrate"] 81 | if "maxBitrate" in settings: 82 | media_file["attributes"]["maxBitrate"] = settings["maxBitrate"] 83 | if "scalable" in settings: 84 | media_file["attributes"]["scalable"] = settings["scalable"] 85 | if "codec" in settings: 86 | media_file["attributes"]["codec"] = settings["codec"] 87 | if "apiFramework" in settings: 88 | media_file["attributes"]["apiFramework"] = settings["apiFramework"] 89 | if "maintainAspectRatio" in settings: 90 | media_file["attributes"]["maintainAspectRatio"] = settings["maintainAspectRatio"] 91 | 92 | self.mediaFiles.append(media_file) 93 | return self 94 | 95 | def attachTrackingEvent(self, _type, url, offset=None): 96 | self.trackingEvents.append(TrackingEvent(_type, url, offset)) 97 | return self 98 | 99 | def attachVideoClick(self, _type, url, _id=''): 100 | if _type not in VALID_VIDEO_CLICKS: 101 | raise Exception('The supplied VideoClick `type` is not a valid VAST VideoClick type.') 102 | self.videoClicks.append({"type": _type, "url": url, "id": _id}) 103 | return self 104 | 105 | def attachClickThrough(self, url): 106 | self.clickThroughs.append(url) 107 | return self 108 | 109 | def attachClick(self, uri, _type=None): 110 | if isinstance(uri, basestring): 111 | _type = 'NonLinearClickThrough' 112 | self.clicks = [{"type": _type, "uri": uri}] 113 | return self 114 | 115 | def attachResource(self, _type, uri, creative_type=None): 116 | resource = {"type": _type, "uri": uri} 117 | if _type == 'HTMLResource': 118 | resource["html"] = uri 119 | if creative_type is not None: 120 | resource["creativeType"] = creative_type 121 | self.resources.append(resource) 122 | return self 123 | 124 | def attachIcon(self, settings): 125 | icon = Icon(settings) 126 | self.icons.append(icon) 127 | return icon 128 | 129 | def adParameters(self, data, xml_encoded): 130 | self._adParameters = {"data": data, "xmlEncoded": xml_encoded} 131 | return self 132 | 133 | def attachNonLinearClickThrough(self, url): 134 | self.nonLinearClickThrough = url 135 | 136 | def attachNonLinearClickTracking(self, url): 137 | self.nonLinearClickTracking = url -------------------------------------------------------------------------------- /vmap_generation/vast_xml/icon.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright 2015 Timu Eren 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | 18 | REQURED_ATTRIBUTES = ["program", "width", "height", "xPosition", "yPosition"] 19 | 20 | 21 | class Icon(object): 22 | def __init__(self, settings=dict()): 23 | keys = settings.keys() 24 | for required in keys: 25 | if required not in keys: 26 | raise Exception("Missing required attribute '{attr}'".format(attr=required)) 27 | 28 | self.attributes = {} 29 | self.attributes.update(settings) 30 | self.resource = None 31 | self.clickThrough = None 32 | self.click = None 33 | self.view = None 34 | 35 | def setResource(self, _type, uri, creativeType=None): 36 | if _type not in ('StaticResource', "IFrameResource", "HTMLResource"): 37 | raise Exception("Invalid resource type") 38 | 39 | resource = {"type": _type, "uri": uri} 40 | if _type == 'HTMLResource': 41 | resource["html"] = uri 42 | if creativeType: 43 | resource["creativeType"] = creativeType 44 | self.resource = resource 45 | 46 | def setClickThrough(self, uri): 47 | self.clickThrough = uri 48 | return self 49 | 50 | def setClickTracking(self, uri): 51 | self.click = uri 52 | return self 53 | 54 | def setViewTracking(self, uri): 55 | self.view = uri 56 | return self -------------------------------------------------------------------------------- /vmap_generation/vast_xml/trackingevent.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright 2015 Timu Eren 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | 18 | VALID_TRACKING_EVENT_TYPES = [ 19 | 'creativeView', 20 | 'start', 21 | 'firstQuartile', 22 | 'midpoint', 23 | 'thirdQuartile', 24 | 'complete', 25 | 'mute', 26 | 'unmute', 27 | 'pause', 28 | 'rewind', 29 | 'resume', 30 | 'fullscreen', 31 | 'exitFullscreen', 32 | 'skip', 33 | 'progress', 34 | 'expand', 35 | 'collapse', 36 | 'acceptInvitationLinear', 37 | 'closeLinear' 38 | ] 39 | 40 | 41 | class TrackingEvent(object): 42 | def __init__(self, event, url, offset=None): 43 | self.offset = None 44 | self.event = None 45 | self.url = None 46 | 47 | if event not in VALID_TRACKING_EVENT_TYPES: 48 | raise Exception("""The supplied Tracking `event` {event} is not a valid Tracking event. 49 | Valid tracking events: {events}""".format( 50 | event=event, 51 | events=",".join(VALID_TRACKING_EVENT_TYPES) 52 | )) 53 | 54 | if event == "progress": 55 | if offset is None: 56 | raise Exception("Offset must be present for `progress` TrackingEvent.") 57 | self.offset = offset 58 | self.event = event 59 | self.url = url -------------------------------------------------------------------------------- /vmap_generation/vast_xml/vast.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright 2015 Timu Eren 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | 18 | from vast_xml.ad import Ad 19 | from xml.dom.minidom import Document 20 | 21 | 22 | class VAST(object): 23 | def __init__(self, settings={}): 24 | self.ads = [] 25 | self.version = settings.get("version", "3.0") 26 | self.VASTErrorURI = settings.get("VASTErrorURI", None) 27 | 28 | def attachAd(self, settings): 29 | ad = Ad(settings) 30 | self.ads.append(ad) 31 | return ad 32 | 33 | def xml(self): 34 | doc = Document() 35 | doc.appendChild(self.toElement(doc)) 36 | return doc.toxml('utf-8') 37 | 38 | def toElement(self, doc): 39 | vastElement = doc.createElement("VAST") 40 | vastElement.setAttribute("version", self.version) 41 | if len(self.ads) == 0 and self.VASTErrorURI: 42 | errorElement = doc.createElement("Error") 43 | errorElement.appendChild(doc.createCDATASection(self.VASTErrorURI)) 44 | vastElement.appendChild(errorElement) 45 | return vastElement 46 | for ad in self.ads: 47 | adOptions = {"id": ad.id} 48 | if ad.sequence: 49 | adOptions["sequence"] = str(ad.sequence) 50 | 51 | adElement = doc.createElement("Ad") 52 | vastElement.appendChild(adElement) 53 | 54 | inLineElement = doc.createElement("InLine") 55 | adSystemElement = doc.createElement("AdSystem") 56 | adSystemElement.appendChild(doc.createTextNode(ad.AdSystem["name"])) 57 | inLineElement.appendChild(adSystemElement) 58 | 59 | adTitleElement = doc.createElement("AdTitle") 60 | adTitleElement.appendChild(doc.createTextNode(ad.AdTitle)) 61 | inLineElement.appendChild(adTitleElement) 62 | 63 | descriptionElement = doc.createElement("Description") 64 | descriptionElement.appendChild(doc.createCDATASection(ad.Description or "")) 65 | inLineElement.appendChild(descriptionElement) 66 | 67 | for survey in ad.surveys: 68 | surveyElement = doc.createElement("Survey") 69 | if survey.type: 70 | surveyElement.setAttribute("type", survey.type) 71 | surveyElement.appendChild(doc.createCDATASection(survey.url)) 72 | inLineElement.appendChild(surveyElement) 73 | 74 | if ad.Error: 75 | errorElement = doc.createElement("Error") 76 | errorElement.appendChild(doc.createCDATASection(ad.Error)) 77 | inLineElement.appendChild(errorElement) 78 | 79 | for impression in ad.impressions: 80 | impressionElement = doc.createElement("Impression") 81 | if "url" in impression: 82 | impressionElement.appendChild(doc.createCDATASection(impression["url"])) 83 | inLineElement.appendChild(impressionElement) 84 | 85 | creativesElement = doc.createElement("Creatives") 86 | linearCreatives = [c for c in ad.creatives if c.type == "Linear"] 87 | nonLinearCreatives = [c for c in ad.creatives if c.type == "NonLinear"] 88 | companionAdCreatives = [c for c in ad.creatives if c.type == "CompanionAd"] 89 | for creative in linearCreatives: 90 | creativeElement = doc.createElement("Creative") 91 | linearElement = doc.createElement("Linear") 92 | if creative.skipoffset: 93 | linearElement.setAttribute("skipoffset", creative.skipoffset) 94 | if len(creative.icons) > 0: 95 | iconsElement = doc.createElement("Icons") 96 | linearElement.appendChild(iconsElement) 97 | for icon in creative.icons: 98 | iconElement = doc.createElement("Icon") 99 | iconsElement.appendChild(iconElement) 100 | for key, value in icon.attributes.items(): 101 | iconElement.setAttribute(key, value) 102 | resourceElement = doc.createElement(icon.resource["type"]) 103 | iconElement.appendChild(resourceElement) 104 | resourceElement.appendChild(doc.createCDATASection(icon.resource["uri"])) 105 | if "creativeType" in icon.resource: 106 | resourceElement.setAttribute("creativeType", icon.resource["creativeType"]) 107 | if icon.click or icon.clickThrough: 108 | iconClicksElement = doc.createElement("IconClicks") 109 | iconElement.appendChild(iconClicksElement) 110 | if icon.clickThrough: 111 | iconClickThroughElement = doc.createElement("IconClickThrough") 112 | iconClicksElement.appendChild(iconClickThroughElement) 113 | iconClickThroughElement.appendChild(doc.createCDATASection(icon.clickThrough)) 114 | if icon.click: 115 | iconClickTrackingElement = doc.createElement("IconClickTracking") 116 | iconClicksElement.appendChild(iconClickTrackingElement) 117 | iconClickTrackingElement.appendChild(doc.createCDATASection(icon.click)) 118 | if icon.view: 119 | response.IconViewTracking(self.cdata(icon.view)) 120 | durationElement = doc.createElement("Duration") 121 | durationElement.appendChild(doc.createTextNode(creative.duration)) 122 | linearElement.appendChild(durationElement) 123 | trackingEventsElement = doc.createElementNS("http://www.iab.net/videosuite/vmap", "vmap:TrackingEvents") 124 | linearElement.appendChild(trackingEventsElement) 125 | for event in creative.trackingEvents: 126 | trackingElement = doc.createElementNS("http://www.iab.net/videosuite/vmap", "vmap:Tracking") 127 | trackingElement.setAttribute("event", event.event) 128 | if event.offset: 129 | trackingElement.setAttribute("offset", event.offset) 130 | trackingElement.appendChild(doc.createCDATASection(event.url)) 131 | trackingEventsElement.appendChild(trackingElement) 132 | if creative.AdParameters: 133 | adParametersElement = doc.createElement("AdParameters") 134 | adParametersElement.setAttribute("xmlEncoded", creative.AdParameters["xmlEncoded"]) 135 | adParametersElement.appendChild(doc.createTextNode(creative.AdParameters)) 136 | linearElement.appendChild(adParametersElement) 137 | for click in creative.videoClicks: 138 | clickElement = doc.createElement(click["type"]) 139 | clickElement.setAttribute("id", click.get("id", "")) 140 | clickElement.appendChild(doc.createCDATASection(click["url"])) 141 | nonLinearElement.appendChild(clickElement) 142 | mediaFilesElement = doc.createElement("MediaFiles") 143 | for media in creative.mediaFiles: 144 | mediaFileElement = doc.createElement("MediaFile") 145 | mediaFileElement.appendChild(doc.createCDATASection(media["url"])) 146 | for key, value in media["attributes"].items(): 147 | mediaFileElement.setAttribute(key, value) 148 | mediaFilesElement.appendChild(mediaFileElement) 149 | linearElement.appendChild(mediaFilesElement) 150 | creativeElement.appendChild(linearElement) 151 | creativesElement.appendChild(creativeElement) 152 | 153 | for creative in nonLinearCreatives: 154 | creativeElement = doc.createElement("Creative") 155 | nonLinearAdsElement = doc.createElement("NonLinearAds") 156 | nonLinearElement = doc.createElement("NonLinear") 157 | for key, value in creative.attributes.items(): 158 | nonLinearElement.setAttribute(key, value) 159 | for resource in creative.resources: 160 | resourceElement = doc.createElement(resource["type"]) 161 | if "creativeType" in resource: 162 | resourceElement.setAttribute("creativeType", resource["creativeType"]) 163 | resourceElement.appendChild(doc.createCDATASection(resource["uri"])) 164 | nonLinearElement.appendChild(resourceElement) 165 | for click in creative.clicks: 166 | clickElement = doc.createElement(click["type"]) 167 | clickElement.appendChild(doc.createCDATASection(click["uri"])) 168 | nonLinearElement.appendChild(clickElement) 169 | if creative.AdParameters: 170 | adParametersElement = doc.createElement("AdParameters") 171 | adParametersElement.setAttribute("xmlEncoded", creative.AdParameters["xmlEncoded"]) 172 | adParametersElement.appendChild(doc.createTextNode(creative.AdParameters["data"])) 173 | nonLinearElement.appendChild(adParametersElement) 174 | if creative.nonLinearClickThrough: 175 | clickThroughElement = doc.createElement("NonLinearClickThrough") 176 | clickThroughElement.appendChild(doc.createCDATASection(creative.nonLinearClickThrough)) 177 | nonLinearElement.appendChild(clickThroughElement) 178 | if creative.nonLinearClickTracking: 179 | clickTrackingElement = doc.createElement("NonLinearClickTracking") 180 | clickTrackingElement.appendChild(doc.createCDATASection(creative.nonLinearClickTracking)) 181 | nonLinearElement.appendChild(clickTrackingElement) 182 | nonLinearAdsElement.appendChild(nonLinearElement) 183 | creativeElement.appendChild(nonLinearAdsElement) 184 | creativesElement.appendChild(creativeElement) 185 | 186 | if len(companionAdCreatives) > 0: 187 | companionAdsElement = doc.createElement("CompanionAds") 188 | for creative in companionAdCreatives: 189 | companionElement = doc.createElement("Companion") 190 | for key, value in creative.attributes.items(): 191 | companionElement.setAttribute(key, value) 192 | for resource in creative.resources: 193 | resourceElement = doc.createElement(resource["type"]) 194 | if "creativeType" in resource: 195 | resourceElement.setAttribute("creativeType", resource["creativeType"]) 196 | resourceElement.appendChild(doc.createCDATASection(resource["uri"])) 197 | companionElement.appendChild(resourceElement) 198 | if "adParameters" in resource: 199 | adParametersElement = doc.createElement("AdParameters") 200 | adParametersElement.setAttribute("xmlEncoded", resource["adParameters"]["xmlEncoded"]) 201 | adParametersElement.appendChild(doc.createTextNode(resource["adParameters"]["data"])) 202 | companionElement.appendChild(adParametersElement) 203 | for click in creative.clickThroughs: 204 | clickThroughElement = doc.createElement("CompanionClickThrough") 205 | clickThroughElement.appendChild(doc.createCDATASection(click)) 206 | companionElement.appendChild(clickThroughElement) 207 | if creative.nonLinearClickTracking: 208 | clickTrackingElement = doc.createElement("CompanionClickTracking") 209 | clickTrackingElement.appendChild(doc.createCDATASection(creative.nonLinearClickTracking)) 210 | companionElement.appendChild(clickTrackingElement) 211 | companionAdsElement.appendChild(companionElement) 212 | creativesElement.appendChild(companionAdsElement) 213 | 214 | inLineElement.appendChild(creativesElement) 215 | adElement.appendChild(inLineElement) 216 | return vastElement 217 | -------------------------------------------------------------------------------- /vmap_generation/vmap_xml/adbreak.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright 2015 Timu Eren 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | 18 | from vmap_xml.events import TrackingEvent 19 | 20 | REQURED_ATTRIBUTES = ("timeOffset", "breakType") 21 | BREAK_TYPES = ("linear", "nonlinear", "display") 22 | 23 | class AdBreak(object): 24 | def __init__(self, settings={}): 25 | self.attributes = {} 26 | self.trackingEvents = [] 27 | self.adSource = None 28 | for _type in REQURED_ATTRIBUTES: 29 | if _type not in settings.keys(): 30 | raise Exception("Missing attribute '{attr}' on AdBreak".format(attr=_type)) 31 | 32 | if settings.get("breakType") not in BREAK_TYPES: 33 | raise Exception("Invalid breakType value, break type can be only one of them {types}" 34 | .format(types=",".join(BREAK_TYPES))) 35 | 36 | self.attributes["timeOffset"] = settings.get("timeOffset") 37 | self.attributes["breakType"] = settings.get("breakType") 38 | 39 | if settings.get("breakId", None): 40 | self.attributes["breakId"] = settings.get("breakId") 41 | if settings.get("repeatAfter", None): 42 | self.attributes["repeatAfter"] = settings.get("repeatAfter", None) 43 | 44 | def attachAdSource(self, _id, allow_mutiple_ads, follow_redirects, _type, source, attributes={}): 45 | if _type != 'VASTAdData' and "templateType" not in attributes: 46 | raise Exception("templateType required by {type}".format(type=_type)) 47 | source = dict(_id=_id, 48 | allow_mutiple_ads=allow_mutiple_ads, 49 | follow_redirects=follow_redirects, 50 | attributes=attributes, 51 | source=source, 52 | type=_type 53 | ) 54 | self.adSource = source 55 | 56 | def attachEvent(self, event, url): 57 | self.trackingEvents.append(TrackingEvent(event, url)) 58 | -------------------------------------------------------------------------------- /vmap_generation/vmap_xml/events.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright 2015 Timu Eren 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | 18 | VALID_TRACKING_EVENT_TYPES = [ 19 | 'breakStart', 20 | 'breakEnd', 21 | 'error' 22 | ] 23 | 24 | class TrackingEvent(object): 25 | def __init__(self, event, url): 26 | if event not in VALID_TRACKING_EVENT_TYPES: 27 | raise Exception("""The supplied Tracking `event` {event} is not a valid Tracking event. 28 | Valid tracking events: {events}""".format( 29 | event=event, 30 | events=",".join(VALID_TRACKING_EVENT_TYPES) 31 | )) 32 | 33 | self.event = event 34 | self.url = url 35 | -------------------------------------------------------------------------------- /vmap_generation/vmap_xml/vmap.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright 2015 Timu Eren 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | 18 | from vmap_xml.adbreak import AdBreak 19 | from xml.dom.minidom import Document 20 | 21 | 22 | class VMAP(object): 23 | def __init__(self, settings={}, version="1.0"): 24 | self.adBreaks = [] 25 | self.version = version 26 | 27 | def attachAdBreak(self, settings={}): 28 | adBreak = AdBreak(settings) 29 | self.adBreaks.append(adBreak) 30 | return adBreak 31 | 32 | def xml(self): 33 | doc = Document() 34 | doc.appendChild(self.toElement(doc)) 35 | return doc.toxml('utf-8') 36 | 37 | def toElement(self, doc): 38 | vmapElement = doc.createElementNS('http://www.iab.net/videosuite/vmap', 'vmap:VMAP') 39 | vmapElement.setAttribute("version", self.version) 40 | vmapElement.setAttribute("xmlns:vmap", "http://www.iab.net/videosuite/vmap") 41 | for adBreak in self.adBreaks: 42 | _type = adBreak.adSource["type"] 43 | adBreakElement = doc.createElementNS("http://www.iab.net/videosuite/vmap", "vmap:AdBreak") 44 | for key, value in adBreak.attributes.items(): 45 | adBreakElement.setAttribute(key, value) 46 | vmapElement.appendChild(adBreakElement) 47 | 48 | adSourceElement = doc.createElementNS("http://www.iab.net/videosuite/vmap", "vmap:AdSource") 49 | adSourceElement.setAttribute("id", adBreak.adSource["_id"]) 50 | adSourceElement.setAttribute("allowMultipleAds", adBreak.adSource["allow_mutiple_ads"]) 51 | adSourceElement.setAttribute("followRedirects", adBreak.adSource["follow_redirects"]) 52 | adBreakElement.appendChild(adSourceElement) 53 | 54 | adTypedElement = doc.createElementNS("http://www.iab.net/videosuite/vmap", "vmap:{type}".format(type=_type)) 55 | for key, value in adBreak.adSource["attributes"].items(): 56 | adTypedElement.setAttribute(key, value) 57 | adSourceElement.appendChild(adTypedElement) 58 | 59 | if _type == "VASTAdData": 60 | vastElement = adBreak.adSource["source"].toElement(doc) 61 | adTypedElement.appendChild(vastElement) 62 | elif _type == 'AdTagURI': 63 | adTypedElement.appendChild(doc.createCDATASection(adBreak.adSource["source"])) 64 | 65 | if len(adBreak.trackingEvents) > 0: 66 | trackingEventsElement = doc.createElementNS("http://www.iab.net/videosuite/vmap", "vmap:TrackingEvents") 67 | adBreakElement.appendChild(trackingEventsElement) 68 | for event in adBreak.trackingEvents: 69 | trackingElement = doc.createElementNS("http://www.iab.net/videosuite/vmap", "vmap:Tracking") 70 | trackingElement.setAttribute("event", event.event) 71 | trackingElement.appendChild(doc.createCDATASection(event.url)) 72 | trackingEventsElement.appendChild(trackingElement) 73 | 74 | return vmapElement 75 | --------------------------------------------------------------------------------