├── .github ├── .git-hooks │ └── detect-api-keys.py ├── CODEOWNERS ├── ISSUE_TEMPLATE │ ├── bug_report_template.md │ └── feature_request_template.md ├── pull_request_template.md ├── stale.yaml └── workflows │ ├── build.yml │ └── codeql-analysis.yml ├── .gitignore ├── .pre-commit-config.yaml ├── CONTRIBUTING.md ├── LICENSE ├── LICENSE-3rdparty.csv ├── NOTICE ├── README.md ├── awslambdanorpc.go ├── awslambdawithrpc.go ├── ddlambda.go ├── ddlambda_example_test.go ├── ddlambda_test.go ├── go.mod ├── go.sum ├── internal ├── extension │ ├── extension.go │ └── extension_test.go ├── logger │ └── log.go ├── metrics │ ├── api.go │ ├── api_test.go │ ├── batcher.go │ ├── batcher_test.go │ ├── constants.go │ ├── context.go │ ├── context_test.go │ ├── kms_decrypter.go │ ├── kms_decrypter_test.go │ ├── listener.go │ ├── listener_test.go │ ├── model.go │ ├── processor.go │ ├── processor_test.go │ └── time.go ├── testdata │ ├── apig-event-no-headers.json │ ├── apig-event-with-headers.json │ ├── invalid.json │ ├── non-proxy-no-headers.json │ ├── non-proxy-with-headers.json │ ├── non-proxy-with-missing-sampling-priority.json │ └── non-proxy-with-mixed-case-headers.json ├── trace │ ├── constants.go │ ├── context.go │ ├── context_test.go │ ├── listener.go │ └── listener_test.go ├── version │ └── version.go └── wrapper │ ├── wrap_handler.go │ └── wrap_handler_test.go ├── scripts └── release.sh └── tests └── integration_tests ├── .gitignore ├── README.md ├── error ├── go.mod ├── go.sum └── main.go ├── hello ├── go.mod ├── go.sum └── main.go ├── input_events └── api-gateway-get.json ├── package.json ├── parse-json.js ├── run_integration_tests.sh ├── serverless.yml └── snapshots ├── logs ├── error.log └── hello.log └── return_values ├── error_api-gateway-get.json └── hello_api-gateway-get.json /.github/.git-hooks/detect-api-keys.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | from __future__ import print_function 3 | 4 | import argparse 5 | import re 6 | import sys 7 | 8 | 9 | def detect_aws_access_key(line): 10 | match = re.search(r"(? 2 | 3 | ### What does this PR do? 4 | 5 | 6 | 7 | ### Motivation 8 | 9 | 10 | 11 | ### Testing Guidelines 12 | 13 | 14 | 15 | ### Additional Notes 16 | 17 | 18 | 19 | ### Types of changes 20 | 21 | - [ ] Bug fix 22 | - [ ] New feature 23 | - [ ] Breaking change 24 | - [ ] Misc (docs, refactoring, dependency upgrade, etc.) 25 | 26 | ### Checklist 27 | 28 | - [ ] This PR's description is comprehensive 29 | - [ ] This PR contains breaking changes that are documented in the description 30 | - [ ] This PR introduces new APIs or parameters that are documented and unlikely to change in the foreseeable future 31 | - [ ] This PR impacts documentation, and it has been updated (or a ticket has been logged) 32 | - [ ] This PR's changes are covered by the automated tests 33 | - [ ] This PR collects user input/sensitive content into Datadog 34 | -------------------------------------------------------------------------------- /.github/stale.yaml: -------------------------------------------------------------------------------- 1 | # Number of days of inactivity before an issue becomes stale 2 | daysUntilStale: 60 3 | # Number of days of inactivity before a stale issue is closed 4 | daysUntilClose: 7 5 | # Issues with these labels will never be considered stale 6 | exemptLabels: 7 | - pinned 8 | # Label to use when marking an issue as stale 9 | staleLabel: wontfix 10 | # Comment to post when marking an issue as stale. Set to `false` to disable 11 | markComment: > 12 | This issue has been automatically marked as stale and it will be closed 13 | if no further activity occurs. Thank you for your contributions! You can 14 | also find us in the \#serverless channel from the 15 | [Datadog community Slack](https://chat.datadoghq.com/). 16 | # Comment to post when closing a stale issue. Set to `false` to disable 17 | closeComment: false 18 | 19 | -------------------------------------------------------------------------------- /.github/workflows/build.yml: -------------------------------------------------------------------------------- 1 | name: build 2 | 3 | on: [push, pull_request] 4 | 5 | jobs: 6 | lint: 7 | runs-on: ubuntu-latest 8 | 9 | steps: 10 | - name: Checkout 11 | uses: actions/checkout@v3 12 | 13 | - name: Set up Go 14 | uses: actions/setup-go@v3 15 | with: 16 | go-version: '1.21' 17 | 18 | - name: golangci-lint 19 | uses: golangci/golangci-lint-action@v3 20 | 21 | unit-test: 22 | runs-on: ubuntu-latest 23 | 24 | steps: 25 | - name: Checkout 26 | uses: actions/checkout@v3 27 | 28 | - name: Set up Go 29 | uses: actions/setup-go@v3 30 | with: 31 | go-version: '1.21' 32 | 33 | - name: Run tests 34 | run: go test -race -coverprofile=coverage.txt -covermode=atomic ./... 35 | 36 | - name: Upload code coverage report 37 | run: bash <(curl -s https://codecov.io/bash) 38 | 39 | integration-test: 40 | runs-on: ubuntu-latest 41 | 42 | steps: 43 | - name: Checkout 44 | uses: actions/checkout@v3 45 | 46 | - name: Set up Node 22 47 | uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 48 | with: 49 | node-version: 22 50 | 51 | - name: Set up Go 52 | uses: actions/setup-go@v3 53 | with: 54 | go-version: '1.21' 55 | 56 | - name: Cache Node modules 57 | id: cache-node-modules 58 | uses: actions/cache@v3 59 | with: 60 | path: "**/node_modules" 61 | key: ${{ runner.os }}-modules-${{ hashFiles('**/yarn.lock') }} 62 | 63 | - name: Install Serverless Framework 64 | run: sudo yarn global add serverless@^3.38.0 --prefix /usr/local 65 | 66 | - name: Install dependencies 67 | if: steps.cache-node-modules.outputs.cache-hit != 'true' 68 | working-directory: tests/integration_tests/ 69 | run: yarn install 70 | 71 | - name: Run tests 72 | env: 73 | BUILD_LAYERS: true 74 | DD_API_KEY: ${{ secrets.DD_API_KEY }} 75 | AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} 76 | AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} 77 | working-directory: tests/integration_tests/ 78 | run: ./run_integration_tests.sh 79 | -------------------------------------------------------------------------------- /.github/workflows/codeql-analysis.yml: -------------------------------------------------------------------------------- 1 | # For most projects, this workflow file will not need changing; you simply need 2 | # to commit it to your repository. 3 | # 4 | # You may wish to alter this file to override the set of languages analyzed, 5 | # or to provide custom queries or build logic. 6 | # 7 | # ******** NOTE ******** 8 | # We have attempted to detect the languages in your repository. Please check 9 | # the `language` matrix defined below to confirm you have the correct set of 10 | # supported CodeQL languages. 11 | # 12 | name: "CodeQL" 13 | 14 | on: 15 | push: 16 | branches: [ main, master ] 17 | pull_request: 18 | # The branches below must be a subset of the branches above 19 | branches: [ main ] 20 | schedule: 21 | - cron: '33 8 * * 1' 22 | 23 | jobs: 24 | analyze: 25 | name: Analyze 26 | runs-on: ubuntu-latest 27 | permissions: 28 | actions: read 29 | contents: read 30 | security-events: write 31 | 32 | strategy: 33 | fail-fast: false 34 | matrix: 35 | language: [ 'go' ] 36 | # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ] 37 | # Learn more about CodeQL language support at https://git.io/codeql-language-support 38 | 39 | steps: 40 | - name: Checkout repository 41 | uses: actions/checkout@v3 42 | 43 | # Initializes the CodeQL tools for scanning. 44 | - name: Initialize CodeQL 45 | uses: github/codeql-action/init@v2 46 | with: 47 | languages: ${{ matrix.language }} 48 | # If you wish to specify custom queries, you can do so here or in a config file. 49 | # By default, queries listed here will override any specified in a config file. 50 | # Prefix the list here with "+" to use these queries and those in the config file. 51 | # queries: ./path/to/local/query, your-org/your-repo/queries@main 52 | 53 | # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). 54 | # If this step fails, then you should remove it and run the build manually (see below) 55 | - name: Autobuild 56 | uses: github/codeql-action/autobuild@v2 57 | 58 | # ℹ️ Command-line programs to run using the OS shell. 59 | # 📚 https://git.io/JvXDl 60 | 61 | # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines 62 | # and modify them (or add more) to build your code if your project 63 | # uses a compiled language 64 | 65 | #- run: | 66 | # make bootstrap 67 | # make release 68 | 69 | - name: Perform CodeQL Analysis 70 | uses: github/codeql-action/analyze@v2 71 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # go 2 | bin/ 3 | 4 | # profiling 5 | *.test 6 | *.out 7 | 8 | # generic 9 | .DS_Store 10 | *.cov 11 | *.lock 12 | *.swp 13 | .idea 14 | 15 | /contrib/google.golang.org/grpc.v12/vendor/ 16 | .vscode 17 | coverage.txt 18 | 19 | **/.serverless 20 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | repos: 2 | - repo: git@github.com:pre-commit/pre-commit-hooks 3 | rev: v2.1.0 4 | hooks: 5 | - id: check-merge-conflict 6 | files: \.go$ 7 | - repo: local 8 | hooks: 9 | - id: detect-api-keys 10 | name: detect-api-keys 11 | description: Checks for AWS or Datadog API keys 12 | entry: ".github/.git-hooks/detect-api-keys.py" 13 | language: python 14 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing 2 | 3 | We love pull requests. For new features, consider opening an issue to discuss the idea first. When you're ready to open a pull requset, here's a quick guide. 4 | 5 | 1. Fork, clone and branch off: 6 | ```bash 7 | git clone git@github.com:/datadog-lambda-go.git 8 | git checkout -b 9 | ``` 10 | 1. Make your changes, update tests and ensure the tests pass: 11 | ```bash 12 | go test ./... 13 | ``` 14 | 1. Build and test your own serverless application with your modified version of `datadog-lambda-go`. 15 | 1. Push to your fork and [submit a pull request][pr]. 16 | 17 | [pr]: https://github.com/your-username/datadog-lambda-go/compare/DataDog:main...main 18 | 19 | At this point you're waiting on us. We may suggest some changes or improvements or alternatives. 20 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "{}" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright 2021 Datadog, Inc. 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | 203 | SPDX-License-Identifier: Apache-2.0 -------------------------------------------------------------------------------- /LICENSE-3rdparty.csv: -------------------------------------------------------------------------------- 1 | Component,Origin,License,Copyright 2 | aws-lambda-go,github.com/aws/aws-lambda-go,Apache-2.0,"Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. Lambda functions are made available under a modified MIT license. See LICENSE-LAMBDACODE for details. The remainder of the project is made available under the terms of the Apache License, version 2.0. See LICENSE for details." 3 | aws-sdk-go,github.com/aws/aws-sdk-go,Apache-2.0,"Copyright 2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. Copyright 2014-2015 Stripe, Inc." 4 | aws-xray-sdk-go,github.com/aws/aws-xray-sdk-go,Apache-2.0,"Copyright 2017-2017 Amazon.com, Inc. or its affiliates. All Rights Reserved." 5 | backoff,github.com/cenkalti/backoff,MIT,"Copyright (c) 2014 Cenk Altı" 6 | seelog,github.com/cihub/seelog,BSD-3-Clause,"Copyright (c) 2012, Cloud Instruments Co., Ltd. . All rights reserved." 7 | go-spew,github.com/davecgh/go-spew,ISC,"Copyright (c) 2012-2016 Dave Collins " 8 | go-jmespath,github.com/jmespath/go-jmespath,Apache-2.0,"Copyright 2015 James Saryerwinnie" 9 | go-difflib,github.com/pmezard/go-difflib,BSD-3-Clause,"Copyright (c) 2013, Patrick Mezard. All rights reserved." 10 | testify,github.com/stretchr/testify,MIT,"Copyright (c) 2012-2018 Mat Ryer and Tyler Bunnell" 11 | gobreaker,github.com/sony/gobreaker,MIT,"Copyright 2015 Sony Corporation" 12 | -------------------------------------------------------------------------------- /NOTICE: -------------------------------------------------------------------------------- 1 | Datadog datadog-lambda-go 2 | Copyright 2021 Datadog, Inc. 3 | 4 | This product includes software developed at Datadog (https://www.datadoghq.com/). -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # datadog-lambda-go 2 | 3 | ![build](https://github.com/DataDog/datadog-lambda-go/workflows/build/badge.svg) 4 | [![Code Coverage](https://img.shields.io/codecov/c/github/DataDog/datadog-lambda-go)](https://codecov.io/gh/DataDog/datadog-lambda-go) 5 | [![Slack](https://chat.datadoghq.com/badge.svg?bg=632CA6)](https://chat.datadoghq.com/) 6 | [![Godoc](https://img.shields.io/badge/godoc-reference-blue.svg)](https://godoc.org/github.com/DataDog/datadog-lambda-go) 7 | [![License](https://img.shields.io/badge/license-Apache--2.0-blue)](https://github.com/DataDog/datadog-lambda-go/blob/main/LICENSE) 8 | 9 | Datadog Lambda Library for Go enables enhanced Lambda metrics, distributed tracing, and custom metric submission from AWS Lambda functions. 10 | 11 | ## Installation 12 | 13 | Follow the [installation instructions](https://docs.datadoghq.com/serverless/installation/go/), and view your function's enhanced metrics, traces and logs in Datadog. 14 | 15 | ## Configurations 16 | 17 | See the [advanced configuration options](https://docs.datadoghq.com/serverless/configuration) to tag your telemetry, capture request/response payloads, filter or scrub sensitive information from logs or traces, and more. 18 | 19 | ## Opening Issues 20 | 21 | If you encounter a bug with this package, we want to hear about it. Before opening a new issue, search the existing issues to avoid duplicates. 22 | 23 | When opening an issue, include the datadog-lambda-go version, `go version`, and stack trace if available. In addition, include the steps to reproduce when appropriate. 24 | 25 | You can also open an issue for a feature request. 26 | 27 | ## Contributing 28 | 29 | If you find an issue with this package and have a fix, please feel free to open a pull request following the [procedures](https://github.com/DataDog/datadog-lambda-go/blob/main/CONTRIBUTING.md). 30 | 31 | ## Community 32 | 33 | For product feedback and questions, join the `#serverless` channel in the [Datadog community on Slack](https://chat.datadoghq.com/). 34 | 35 | ## License 36 | 37 | Unless explicitly stated otherwise all files in this repository are licensed under the Apache License Version 2.0. 38 | 39 | This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2021 Datadog, Inc. 40 | -------------------------------------------------------------------------------- /awslambdanorpc.go: -------------------------------------------------------------------------------- 1 | //go:build lambda.norpc 2 | // +build lambda.norpc 3 | 4 | /* 5 | * Unless explicitly stated otherwise all files in this repository are licensed 6 | * under the Apache License Version 2.0. 7 | * 8 | * This product includes software developed at Datadog (https://www.datadoghq.com/). 9 | * Copyright 2021 Datadog, Inc. 10 | */ 11 | 12 | package ddlambda 13 | 14 | const awsLambdaRpcSupport = false 15 | -------------------------------------------------------------------------------- /awslambdawithrpc.go: -------------------------------------------------------------------------------- 1 | //go:build !lambda.norpc 2 | // +build !lambda.norpc 3 | 4 | /* 5 | * Unless explicitly stated otherwise all files in this repository are licensed 6 | * under the Apache License Version 2.0. 7 | * 8 | * This product includes software developed at Datadog (https://www.datadoghq.com/). 9 | * Copyright 2021 Datadog, Inc. 10 | */ 11 | 12 | package ddlambda 13 | 14 | const awsLambdaRpcSupport = true 15 | -------------------------------------------------------------------------------- /ddlambda_example_test.go: -------------------------------------------------------------------------------- 1 | package ddlambda_test 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "gopkg.in/DataDog/dd-trace-go.v1/ddtrace/tracer" 7 | "strings" 8 | "testing" 9 | 10 | "github.com/aws/aws-lambda-go/events" 11 | 12 | ddlambda "github.com/DataDog/datadog-lambda-go" 13 | ) 14 | 15 | var exampleSQSExtractor = func(ctx context.Context, ev json.RawMessage) map[string]string { 16 | eh := events.SQSEvent{} 17 | 18 | headers := map[string]string{} 19 | 20 | if err := json.Unmarshal(ev, &eh); err != nil { 21 | return headers 22 | } 23 | 24 | // Using SQS as a trigger with a batchSize=1 so its important we check for this as a single SQS message 25 | // will drive the execution of the handler. 26 | if len(eh.Records) != 1 { 27 | return headers 28 | } 29 | 30 | record := eh.Records[0] 31 | 32 | lowercaseHeaders := map[string]string{} 33 | for k, v := range record.MessageAttributes { 34 | if v.StringValue != nil { 35 | lowercaseHeaders[strings.ToLower(k)] = *v.StringValue 36 | } 37 | } 38 | 39 | return lowercaseHeaders 40 | } 41 | 42 | func TestCustomExtractorExample(t *testing.T) { 43 | handler := func(ctx context.Context, event events.SQSEvent) error { 44 | // Use the parent span retrieved from the SQS Message Attributes. 45 | span, _ := tracer.SpanFromContext(ctx) 46 | span.SetTag("key", "value") 47 | return nil 48 | } 49 | 50 | cfg := &ddlambda.Config{ 51 | TraceContextExtractor: exampleSQSExtractor, 52 | } 53 | ddlambda.WrapFunction(handler, cfg) 54 | } 55 | -------------------------------------------------------------------------------- /ddlambda_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Unless explicitly stated otherwise all files in this repository are licensed 3 | * under the Apache License Version 2.0. 4 | * 5 | * This product includes software developed at Datadog (https://www.datadoghq.com/). 6 | * Copyright 2021 Datadog, Inc. 7 | */ 8 | package ddlambda 9 | 10 | import ( 11 | "context" 12 | "fmt" 13 | "net/http" 14 | "net/http/httptest" 15 | "os" 16 | "testing" 17 | 18 | "github.com/stretchr/testify/assert" 19 | ) 20 | 21 | func TestInvokeDryRun(t *testing.T) { 22 | t.Setenv(UniversalInstrumentation, "false") 23 | t.Setenv(DatadogTraceEnabledEnvVar, "false") 24 | 25 | called := false 26 | _, err := InvokeDryRun(func(ctx context.Context) { 27 | called = true 28 | globalCtx := GetContext() 29 | assert.Equal(t, globalCtx, ctx) 30 | }, nil) 31 | assert.NoError(t, err) 32 | assert.True(t, called) 33 | } 34 | 35 | func TestMetricsSilentFailWithoutWrapper(t *testing.T) { 36 | Metric("my-metric", 100, "my:tag") 37 | } 38 | 39 | func TestMetricsSubmitWithWrapper(t *testing.T) { 40 | called := false 41 | server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 42 | called = true 43 | w.WriteHeader(http.StatusCreated) 44 | })) 45 | defer server.Close() 46 | 47 | _, err := InvokeDryRun(func(ctx context.Context) { 48 | Metric("my-metric", 100, "my:tag") 49 | }, &Config{ 50 | APIKey: "abc-123", 51 | Site: server.URL, 52 | }) 53 | assert.NoError(t, err) 54 | assert.True(t, called) 55 | } 56 | 57 | func TestToMetricConfigLocalTest(t *testing.T) { 58 | testcases := []struct { 59 | envs map[string]string 60 | cval bool 61 | }{ 62 | { 63 | envs: map[string]string{"DD_LOCAL_TEST": "True"}, 64 | cval: true, 65 | }, 66 | { 67 | envs: map[string]string{"DD_LOCAL_TEST": "true"}, 68 | cval: true, 69 | }, 70 | { 71 | envs: map[string]string{"DD_LOCAL_TEST": "1"}, 72 | cval: true, 73 | }, 74 | { 75 | envs: map[string]string{"DD_LOCAL_TEST": "False"}, 76 | cval: false, 77 | }, 78 | { 79 | envs: map[string]string{"DD_LOCAL_TEST": "false"}, 80 | cval: false, 81 | }, 82 | { 83 | envs: map[string]string{"DD_LOCAL_TEST": "0"}, 84 | cval: false, 85 | }, 86 | { 87 | envs: map[string]string{"DD_LOCAL_TEST": ""}, 88 | cval: false, 89 | }, 90 | { 91 | envs: map[string]string{}, 92 | cval: false, 93 | }, 94 | } 95 | 96 | cfg := Config{} 97 | for _, tc := range testcases { 98 | t.Run(fmt.Sprintf("%#v", tc.envs), func(t *testing.T) { 99 | for k, v := range tc.envs { 100 | os.Setenv(k, v) 101 | } 102 | mc := cfg.toMetricsConfig(true) 103 | assert.Equal(t, tc.cval, mc.LocalTest) 104 | }) 105 | } 106 | } 107 | 108 | func TestCalculateFipsMode(t *testing.T) { 109 | // Save original environment to restore later 110 | originalRegion := os.Getenv("AWS_REGION") 111 | originalFipsMode := os.Getenv(FIPSModeEnvVar) 112 | defer func() { 113 | os.Setenv("AWS_REGION", originalRegion) 114 | os.Setenv(FIPSModeEnvVar, originalFipsMode) 115 | }() 116 | 117 | testCases := []struct { 118 | name string 119 | configFIPSMode *bool 120 | region string 121 | fipsModeEnv string 122 | expected bool 123 | }{ 124 | { 125 | name: "Config explicit true", 126 | configFIPSMode: boolPtr(true), 127 | region: "us-east-1", 128 | fipsModeEnv: "", 129 | expected: true, 130 | }, 131 | { 132 | name: "Config explicit false", 133 | configFIPSMode: boolPtr(false), 134 | region: "us-gov-west-1", 135 | fipsModeEnv: "", 136 | expected: false, 137 | }, 138 | { 139 | name: "GovCloud default true", 140 | configFIPSMode: nil, 141 | region: "us-gov-east-1", 142 | fipsModeEnv: "", 143 | expected: true, 144 | }, 145 | { 146 | name: "Non-GovCloud default false", 147 | configFIPSMode: nil, 148 | region: "us-east-1", 149 | fipsModeEnv: "", 150 | expected: false, 151 | }, 152 | { 153 | name: "Env var override to true", 154 | configFIPSMode: nil, 155 | region: "us-east-1", 156 | fipsModeEnv: "true", 157 | expected: true, 158 | }, 159 | { 160 | name: "Env var override to false", 161 | configFIPSMode: nil, 162 | region: "us-gov-west-1", 163 | fipsModeEnv: "false", 164 | expected: false, 165 | }, 166 | { 167 | name: "Invalid env var in GovCloud", 168 | configFIPSMode: nil, 169 | region: "us-gov-west-1", 170 | fipsModeEnv: "invalid", 171 | expected: true, 172 | }, 173 | { 174 | name: "Invalid env var in non-GovCloud", 175 | configFIPSMode: nil, 176 | region: "us-east-1", 177 | fipsModeEnv: "invalid", 178 | expected: false, 179 | }, 180 | { 181 | name: "Config takes precedence over env and region", 182 | configFIPSMode: boolPtr(true), 183 | region: "us-east-1", 184 | fipsModeEnv: "false", 185 | expected: true, 186 | }, 187 | } 188 | 189 | for _, tc := range testCases { 190 | t.Run(tc.name, func(t *testing.T) { 191 | os.Setenv("AWS_REGION", tc.region) 192 | os.Setenv(FIPSModeEnvVar, tc.fipsModeEnv) 193 | 194 | cfg := &Config{FIPSMode: tc.configFIPSMode} 195 | result := cfg.calculateFipsMode() 196 | 197 | assert.Equal(t, tc.expected, result, "calculateFipsMode returned incorrect value") 198 | }) 199 | } 200 | } 201 | 202 | // Helper function to create bool pointers 203 | func boolPtr(b bool) *bool { 204 | return &b 205 | } 206 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/DataDog/datadog-lambda-go 2 | 3 | go 1.23.0 4 | 5 | require ( 6 | github.com/DataDog/datadog-go/v5 v5.5.0 7 | github.com/aws/aws-lambda-go v1.46.0 8 | github.com/aws/aws-sdk-go-v2/config v1.26.6 9 | github.com/aws/aws-sdk-go-v2/service/kms v1.27.9 10 | github.com/aws/aws-xray-sdk-go/v2 v2.0.0 11 | github.com/cenkalti/backoff/v4 v4.2.1 12 | github.com/sony/gobreaker v0.5.0 13 | github.com/stretchr/testify v1.9.0 14 | go.opentelemetry.io/otel v1.27.0 15 | gopkg.in/DataDog/dd-trace-go.v1 v1.72.1 16 | ) 17 | 18 | require go.uber.org/atomic v1.11.0 // indirect 19 | 20 | require ( 21 | github.com/DataDog/appsec-internal-go v1.9.0 // indirect 22 | github.com/DataDog/datadog-agent/pkg/obfuscate v0.58.0 // indirect 23 | github.com/DataDog/datadog-agent/pkg/proto v0.58.0 // indirect 24 | github.com/DataDog/datadog-agent/pkg/remoteconfig/state v0.58.0 // indirect 25 | github.com/DataDog/datadog-agent/pkg/trace v0.58.0 // indirect 26 | github.com/DataDog/datadog-agent/pkg/util/log v0.58.0 // indirect 27 | github.com/DataDog/datadog-agent/pkg/util/scrubber v0.58.0 // indirect 28 | github.com/DataDog/go-libddwaf/v3 v3.5.1 // indirect 29 | github.com/DataDog/go-runtime-metrics-internal v0.0.4-0.20241206090539-a14610dc22b6 // indirect 30 | github.com/DataDog/go-sqllexer v0.0.14 // indirect 31 | github.com/DataDog/go-tuf v1.1.0-0.5.2 // indirect 32 | github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/attributes v0.20.0 // indirect 33 | github.com/DataDog/sketches-go v1.4.5 // indirect 34 | github.com/Microsoft/go-winio v0.6.1 // indirect 35 | github.com/andybalholm/brotli v1.1.0 // indirect 36 | github.com/aws/aws-sdk-go-v2 v1.24.1 // indirect 37 | github.com/aws/aws-sdk-go-v2/credentials v1.16.16 // indirect 38 | github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.11 // indirect 39 | github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.10 // indirect 40 | github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.10 // indirect 41 | github.com/aws/aws-sdk-go-v2/internal/ini v1.7.3 // indirect 42 | github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.4 // indirect 43 | github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.10 // indirect 44 | github.com/aws/aws-sdk-go-v2/service/sso v1.18.7 // indirect 45 | github.com/aws/aws-sdk-go-v2/service/ssooidc v1.21.7 // indirect 46 | github.com/aws/aws-sdk-go-v2/service/sts v1.26.7 // indirect 47 | github.com/aws/smithy-go v1.19.0 // indirect 48 | github.com/cespare/xxhash/v2 v2.3.0 // indirect 49 | github.com/cihub/seelog v0.0.0-20170130134532-f561c5e57575 // indirect 50 | github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect 51 | github.com/dustin/go-humanize v1.0.1 // indirect 52 | github.com/eapache/queue/v2 v2.0.0-20230407133247-75960ed334e4 // indirect 53 | github.com/ebitengine/purego v0.6.0-alpha.5 // indirect 54 | github.com/go-logr/logr v1.4.1 // indirect 55 | github.com/go-logr/stdr v1.2.2 // indirect 56 | github.com/go-ole/go-ole v1.2.6 // indirect 57 | github.com/gogo/protobuf v1.3.2 // indirect 58 | github.com/golang/protobuf v1.5.4 // indirect 59 | github.com/google/uuid v1.6.0 // indirect 60 | github.com/hashicorp/go-secure-stdlib/parseutil v0.1.7 // indirect 61 | github.com/hashicorp/go-secure-stdlib/strutil v0.1.2 // indirect 62 | github.com/hashicorp/go-sockaddr v1.0.2 // indirect 63 | github.com/json-iterator/go v1.1.12 // indirect 64 | github.com/klauspost/compress v1.17.6 // indirect 65 | github.com/lufia/plan9stats v0.0.0-20220913051719-115f729f3c8c // indirect 66 | github.com/mitchellh/mapstructure v1.5.1-0.20231216201459-8508981c8b6c // indirect 67 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect 68 | github.com/modern-go/reflect2 v1.0.2 // indirect 69 | github.com/outcaste-io/ristretto v0.2.3 // indirect 70 | github.com/philhofer/fwd v1.1.3-0.20240612014219-fbbf4953d986 // indirect 71 | github.com/pkg/errors v0.9.1 // indirect 72 | github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect 73 | github.com/power-devops/perfstat v0.0.0-20220216144756-c35f1ee13d7c // indirect 74 | github.com/ryanuber/go-glob v1.0.0 // indirect 75 | github.com/secure-systems-lab/go-securesystemslib v0.8.0 // indirect 76 | github.com/shirou/gopsutil/v3 v3.24.4 // indirect 77 | github.com/shoenig/go-m1cpu v0.1.6 // indirect 78 | github.com/tinylib/msgp v1.2.1 // indirect 79 | github.com/tklauser/go-sysconf v0.3.12 // indirect 80 | github.com/tklauser/numcpus v0.6.1 // indirect 81 | github.com/valyala/bytebufferpool v1.0.0 // indirect 82 | github.com/valyala/fasthttp v1.52.0 // indirect 83 | github.com/yusufpapurcu/wmi v1.2.4 // indirect 84 | go.opentelemetry.io/collector/component v0.104.0 // indirect 85 | go.opentelemetry.io/collector/config/configtelemetry v0.104.0 // indirect 86 | go.opentelemetry.io/collector/pdata v1.11.0 // indirect 87 | go.opentelemetry.io/collector/pdata/pprofile v0.104.0 // indirect 88 | go.opentelemetry.io/collector/semconv v0.104.0 // indirect 89 | go.opentelemetry.io/otel/metric v1.27.0 // indirect 90 | go.opentelemetry.io/otel/trace v1.27.0 // indirect 91 | go.uber.org/multierr v1.11.0 // indirect 92 | go.uber.org/zap v1.27.0 // indirect 93 | golang.org/x/mod v0.20.0 // indirect 94 | golang.org/x/net v0.33.0 // indirect 95 | golang.org/x/sync v0.10.0 // indirect 96 | golang.org/x/sys v0.28.0 // indirect 97 | golang.org/x/text v0.21.0 // indirect 98 | golang.org/x/time v0.6.0 // indirect 99 | golang.org/x/tools v0.24.0 // indirect 100 | golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect 101 | google.golang.org/genproto/googleapis/rpc v0.0.0-20240520151616-dc85e6b867a5 // indirect 102 | google.golang.org/grpc v1.64.1 // indirect 103 | google.golang.org/protobuf v1.34.2 // indirect 104 | gopkg.in/ini.v1 v1.67.0 // indirect 105 | gopkg.in/yaml.v2 v2.4.0 // indirect 106 | gopkg.in/yaml.v3 v3.0.1 // indirect 107 | ) 108 | -------------------------------------------------------------------------------- /internal/extension/extension.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Unless explicitly stated otherwise all files in this repository are licensed 3 | * under the Apache License Version 2.0. 4 | * 5 | * This product includes software developed at Datadog (https://www.datadoghq.com/). 6 | * Copyright 2021 Datadog, Inc. 7 | */ 8 | 9 | package extension 10 | 11 | import ( 12 | "bytes" 13 | "context" 14 | "encoding/base64" 15 | "encoding/json" 16 | "fmt" 17 | "net/http" 18 | "os" 19 | "reflect" 20 | "runtime" 21 | "strconv" 22 | "strings" 23 | "time" 24 | 25 | "github.com/DataDog/datadog-lambda-go/internal/logger" 26 | 27 | "gopkg.in/DataDog/dd-trace-go.v1/ddtrace" 28 | ) 29 | 30 | type ddTraceContext string 31 | 32 | const ( 33 | DdTraceId ddTraceContext = "x-datadog-trace-id" 34 | DdParentId ddTraceContext = "x-datadog-parent-id" 35 | DdSpanId ddTraceContext = "x-datadog-span-id" 36 | DdSamplingPriority ddTraceContext = "x-datadog-sampling-priority" 37 | DdInvocationError ddTraceContext = "x-datadog-invocation-error" 38 | DdInvocationErrorMsg ddTraceContext = "x-datadog-invocation-error-msg" 39 | DdInvocationErrorType ddTraceContext = "x-datadog-invocation-error-type" 40 | DdInvocationErrorStack ddTraceContext = "x-datadog-invocation-error-stack" 41 | 42 | DdSeverlessSpan ddTraceContext = "dd-tracer-serverless-span" 43 | DdLambdaResponse ddTraceContext = "dd-response" 44 | ) 45 | 46 | const ( 47 | // We don't want call to the Serverless Agent to block indefinitely for any reasons, 48 | // so here's a configuration of the timeout when calling the Serverless Agent. We also 49 | // want to let it having some time for its cold start so we should not set this too low. 50 | timeout = 3000 * time.Millisecond 51 | 52 | helloUrl = "http://localhost:8124/lambda/hello" 53 | flushUrl = "http://localhost:8124/lambda/flush" 54 | startInvocationUrl = "http://localhost:8124/lambda/start-invocation" 55 | endInvocationUrl = "http://localhost:8124/lambda/end-invocation" 56 | 57 | extensionPath = "/opt/extensions/datadog-agent" 58 | ) 59 | 60 | type ExtensionManager struct { 61 | helloRoute string 62 | flushRoute string 63 | extensionPath string 64 | startInvocationUrl string 65 | endInvocationUrl string 66 | httpClient HTTPClient 67 | isExtensionRunning bool 68 | isUniversalInstrumentation bool 69 | } 70 | 71 | type HTTPClient interface { 72 | Do(req *http.Request) (*http.Response, error) 73 | } 74 | 75 | func BuildExtensionManager(isUniversalInstrumentation bool) *ExtensionManager { 76 | em := &ExtensionManager{ 77 | helloRoute: helloUrl, 78 | flushRoute: flushUrl, 79 | startInvocationUrl: startInvocationUrl, 80 | endInvocationUrl: endInvocationUrl, 81 | extensionPath: extensionPath, 82 | httpClient: &http.Client{Timeout: timeout}, 83 | isUniversalInstrumentation: isUniversalInstrumentation, 84 | } 85 | em.checkAgentRunning() 86 | return em 87 | } 88 | 89 | func (em *ExtensionManager) checkAgentRunning() { 90 | if _, err := os.Stat(em.extensionPath); err != nil { 91 | logger.Debug("Will use the API") 92 | em.isExtensionRunning = false 93 | } else { 94 | logger.Debug("Will use the Serverless Agent") 95 | em.isExtensionRunning = true 96 | 97 | // Tell the extension not to create an execution span if universal instrumentation is disabled 98 | if !em.isUniversalInstrumentation { 99 | req, _ := http.NewRequest(http.MethodGet, em.helloRoute, nil) 100 | if response, err := em.httpClient.Do(req); err == nil && response.StatusCode == 200 { 101 | logger.Debug("Hit the extension /hello route") 102 | } else { 103 | logger.Debug("Will use the API since the Serverless Agent was detected but the hello route was unreachable") 104 | em.isExtensionRunning = false 105 | } 106 | } 107 | } 108 | } 109 | 110 | func (em *ExtensionManager) SendStartInvocationRequest(ctx context.Context, eventPayload json.RawMessage) context.Context { 111 | body := bytes.NewBuffer(eventPayload) 112 | req, _ := http.NewRequest(http.MethodPost, em.startInvocationUrl, body) 113 | 114 | if response, err := em.httpClient.Do(req); err == nil && response.StatusCode == 200 { 115 | // Propagate dd-trace context from the extension response if found in the response headers 116 | traceId := response.Header.Get(string(DdTraceId)) 117 | if traceId != "" { 118 | ctx = context.WithValue(ctx, DdTraceId, traceId) 119 | } 120 | parentId := traceId 121 | if pid := response.Header.Get(string(DdParentId)); pid != "" { 122 | parentId = pid 123 | } 124 | if parentId != "" { 125 | ctx = context.WithValue(ctx, DdParentId, parentId) 126 | } 127 | samplingPriority := response.Header.Get(string(DdSamplingPriority)) 128 | if samplingPriority != "" { 129 | ctx = context.WithValue(ctx, DdSamplingPriority, samplingPriority) 130 | } 131 | } 132 | return ctx 133 | } 134 | 135 | func (em *ExtensionManager) SendEndInvocationRequest(ctx context.Context, functionExecutionSpan ddtrace.Span, cfg ddtrace.FinishConfig) { 136 | // Handle Lambda response 137 | lambdaResponse := ctx.Value(DdLambdaResponse) 138 | content, responseErr := json.Marshal(lambdaResponse) 139 | if responseErr != nil { 140 | content = []byte("{}") 141 | } 142 | body := bytes.NewBuffer(content) 143 | req, _ := http.NewRequest(http.MethodPost, em.endInvocationUrl, body) 144 | 145 | // Mark the invocation as an error if any 146 | if cfg.Error != nil { 147 | req.Header.Set(string(DdInvocationError), "true") 148 | req.Header.Set(string(DdInvocationErrorMsg), cfg.Error.Error()) 149 | req.Header.Set(string(DdInvocationErrorType), reflect.TypeOf(cfg.Error).String()) 150 | req.Header.Set(string(DdInvocationErrorStack), takeStacktrace(cfg)) 151 | } 152 | 153 | // Extract the DD trace context and pass them to the extension via request headers 154 | traceId, ok := ctx.Value(DdTraceId).(string) 155 | if ok { 156 | req.Header.Set(string(DdTraceId), traceId) 157 | if parentId, ok := ctx.Value(DdParentId).(string); ok { 158 | req.Header.Set(string(DdParentId), parentId) 159 | } 160 | if spanId, ok := ctx.Value(DdSpanId).(string); ok { 161 | req.Header.Set(string(DdSpanId), spanId) 162 | } 163 | if samplingPriority, ok := ctx.Value(DdSamplingPriority).(string); ok { 164 | req.Header.Set(string(DdSamplingPriority), samplingPriority) 165 | } 166 | } else { 167 | spanContext := functionExecutionSpan.Context() 168 | req.Header.Set(string(DdTraceId), fmt.Sprint(spanContext.TraceID())) 169 | req.Header.Set(string(DdSpanId), fmt.Sprint(spanContext.SpanID())) 170 | 171 | // Try to get sampling priority 172 | // Check if the context implements SamplingPriority method 173 | if pc, ok := spanContext.(interface{ SamplingPriority() (int, bool) }); ok && pc != nil { 174 | if priority, ok := pc.SamplingPriority(); ok { 175 | req.Header.Set(string(DdSamplingPriority), fmt.Sprint(priority)) 176 | } 177 | } 178 | } 179 | 180 | resp, err := em.httpClient.Do(req) 181 | if err != nil || resp.StatusCode != 200 { 182 | logger.Error(fmt.Errorf("could not send end invocation payload to the extension: %v", err)) 183 | } 184 | } 185 | 186 | // defaultStackLength specifies the default maximum size of a stack trace. 187 | const defaultStackLength = 32 188 | 189 | // takeStacktrace takes a stack trace of maximum n entries, skipping the first skip entries. 190 | // If n is 0, up to 20 entries are retrieved. 191 | func takeStacktrace(opts ddtrace.FinishConfig) string { 192 | if opts.StackFrames == 0 { 193 | opts.StackFrames = defaultStackLength 194 | } 195 | var builder strings.Builder 196 | pcs := make([]uintptr, opts.StackFrames) 197 | 198 | // +3 to exclude runtime.Callers, takeStacktrace and SendEndInvocationRequest 199 | numFrames := runtime.Callers(3+int(opts.SkipStackFrames), pcs) 200 | if numFrames == 0 { 201 | return "" 202 | } 203 | frames := runtime.CallersFrames(pcs[:numFrames]) 204 | for i := 0; ; i++ { 205 | frame, more := frames.Next() 206 | if i != 0 { 207 | builder.WriteByte('\n') 208 | } 209 | builder.WriteString(frame.Function) 210 | builder.WriteByte('\n') 211 | builder.WriteByte('\t') 212 | builder.WriteString(frame.File) 213 | builder.WriteByte(':') 214 | builder.WriteString(strconv.Itoa(frame.Line)) 215 | if !more { 216 | break 217 | } 218 | } 219 | 220 | return base64.StdEncoding.EncodeToString([]byte(builder.String())) 221 | } 222 | 223 | func (em *ExtensionManager) IsExtensionRunning() bool { 224 | return em.isExtensionRunning 225 | } 226 | 227 | func (em *ExtensionManager) Flush() error { 228 | req, _ := http.NewRequest(http.MethodGet, em.flushRoute, nil) 229 | if response, err := em.httpClient.Do(req); err != nil { 230 | err := fmt.Errorf("was not able to reach the Agent to flush: %s", err) 231 | logger.Error(err) 232 | return err 233 | } else if response.StatusCode != 200 { 234 | err := fmt.Errorf("the Agent didn't returned HTTP 200: %s", response.Status) 235 | logger.Error(err) 236 | return err 237 | } 238 | return nil 239 | } 240 | -------------------------------------------------------------------------------- /internal/extension/extension_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Unless explicitly stated otherwise all files in this repository are licensed 3 | * under the Apache License Version 2.0. 4 | * 5 | * This product includes software developed at Datadog (https://www.datadoghq.com/). 6 | * Copyright 2021 Datadog, Inc. 7 | */ 8 | 9 | package extension 10 | 11 | import ( 12 | "bytes" 13 | "context" 14 | "encoding/base64" 15 | "fmt" 16 | "net/http" 17 | "os" 18 | "testing" 19 | 20 | "github.com/DataDog/datadog-lambda-go/internal/logger" 21 | "github.com/stretchr/testify/assert" 22 | "gopkg.in/DataDog/dd-trace-go.v1/ddtrace" 23 | "gopkg.in/DataDog/dd-trace-go.v1/ddtrace/tracer" 24 | ) 25 | 26 | type ClientErrorMock struct { 27 | } 28 | 29 | type ClientSuccessMock struct { 30 | } 31 | 32 | type ClientSuccess202Mock struct { 33 | } 34 | 35 | type ClientSuccessStartInvoke struct { 36 | headers http.Header 37 | } 38 | 39 | type ClientSuccessEndInvoke struct { 40 | } 41 | 42 | const ( 43 | mockTraceId = "1" 44 | mockParentId = "2" 45 | mockSamplingPriority = "3" 46 | ) 47 | 48 | func (c *ClientErrorMock) Do(req *http.Request) (*http.Response, error) { 49 | return nil, fmt.Errorf("KO") 50 | } 51 | 52 | func (c *ClientSuccessMock) Do(req *http.Request) (*http.Response, error) { 53 | return &http.Response{StatusCode: 200}, nil 54 | } 55 | 56 | func (c *ClientSuccess202Mock) Do(req *http.Request) (*http.Response, error) { 57 | return &http.Response{StatusCode: 202, Status: "KO"}, nil 58 | } 59 | 60 | func (c *ClientSuccessStartInvoke) Do(req *http.Request) (*http.Response, error) { 61 | return &http.Response{StatusCode: 200, Status: "KO", Header: c.headers}, nil 62 | } 63 | 64 | func (c *ClientSuccessEndInvoke) Do(req *http.Request) (*http.Response, error) { 65 | return &http.Response{StatusCode: 200, Status: "KO"}, nil 66 | } 67 | 68 | func captureLog(f func()) string { 69 | var buf bytes.Buffer 70 | logger.SetOutput(&buf) 71 | f() 72 | logger.SetOutput(os.Stdout) 73 | return buf.String() 74 | } 75 | 76 | func TestBuildExtensionManager(t *testing.T) { 77 | em := BuildExtensionManager(false) 78 | assert.Equal(t, "http://localhost:8124/lambda/hello", em.helloRoute) 79 | assert.Equal(t, "http://localhost:8124/lambda/flush", em.flushRoute) 80 | assert.Equal(t, "http://localhost:8124/lambda/start-invocation", em.startInvocationUrl) 81 | assert.Equal(t, "http://localhost:8124/lambda/end-invocation", em.endInvocationUrl) 82 | assert.Equal(t, "/opt/extensions/datadog-agent", em.extensionPath) 83 | assert.Equal(t, false, em.isUniversalInstrumentation) 84 | assert.NotNil(t, em.httpClient) 85 | } 86 | 87 | func TestIsAgentRunningFalse(t *testing.T) { 88 | em := &ExtensionManager{ 89 | httpClient: &ClientErrorMock{}, 90 | } 91 | assert.False(t, em.IsExtensionRunning()) 92 | } 93 | 94 | func TestIsAgentRunningFalseSinceTheAgentIsNotHere(t *testing.T) { 95 | em := &ExtensionManager{ 96 | extensionPath: "/impossible/path/test", 97 | } 98 | em.checkAgentRunning() 99 | assert.False(t, em.IsExtensionRunning()) 100 | } 101 | 102 | func TestIsAgentRunningTrue(t *testing.T) { 103 | existingPath, err := os.Getwd() 104 | assert.Nil(t, err) 105 | 106 | em := &ExtensionManager{ 107 | httpClient: &ClientSuccessMock{}, 108 | extensionPath: existingPath, 109 | } 110 | em.checkAgentRunning() 111 | assert.True(t, em.IsExtensionRunning()) 112 | } 113 | 114 | func TestFlushErrorNot200(t *testing.T) { 115 | em := &ExtensionManager{ 116 | httpClient: &ClientSuccess202Mock{}, 117 | } 118 | err := em.Flush() 119 | assert.Equal(t, "the Agent didn't returned HTTP 200: KO", err.Error()) 120 | } 121 | 122 | func TestFlushError(t *testing.T) { 123 | em := &ExtensionManager{ 124 | httpClient: &ClientErrorMock{}, 125 | } 126 | err := em.Flush() 127 | assert.Equal(t, "was not able to reach the Agent to flush: KO", err.Error()) 128 | } 129 | 130 | func TestFlushSuccess(t *testing.T) { 131 | em := &ExtensionManager{ 132 | httpClient: &ClientSuccessMock{}, 133 | } 134 | err := em.Flush() 135 | assert.Nil(t, err) 136 | } 137 | 138 | func TestExtensionStartInvoke(t *testing.T) { 139 | em := &ExtensionManager{ 140 | startInvocationUrl: startInvocationUrl, 141 | httpClient: &ClientSuccessStartInvoke{}, 142 | } 143 | ctx := em.SendStartInvocationRequest(context.TODO(), []byte{}) 144 | traceId := ctx.Value(DdTraceId) 145 | parentId := ctx.Value(DdParentId) 146 | samplingPriority := ctx.Value(DdSamplingPriority) 147 | err := em.Flush() 148 | 149 | assert.Nil(t, err) 150 | assert.Nil(t, traceId) 151 | assert.Nil(t, parentId) 152 | assert.Nil(t, samplingPriority) 153 | } 154 | 155 | func TestExtensionStartInvokeWithTraceContext(t *testing.T) { 156 | headers := http.Header{} 157 | headers.Set(string(DdTraceId), mockTraceId) 158 | headers.Set(string(DdParentId), mockParentId) 159 | headers.Set(string(DdSamplingPriority), mockSamplingPriority) 160 | 161 | em := &ExtensionManager{ 162 | startInvocationUrl: startInvocationUrl, 163 | httpClient: &ClientSuccessStartInvoke{ 164 | headers: headers, 165 | }, 166 | } 167 | ctx := em.SendStartInvocationRequest(context.TODO(), []byte{}) 168 | traceId := ctx.Value(DdTraceId) 169 | parentId := ctx.Value(DdParentId) 170 | samplingPriority := ctx.Value(DdSamplingPriority) 171 | err := em.Flush() 172 | 173 | assert.Nil(t, err) 174 | assert.Equal(t, mockTraceId, traceId) 175 | assert.Equal(t, mockParentId, parentId) 176 | assert.Equal(t, mockSamplingPriority, samplingPriority) 177 | } 178 | 179 | func TestExtensionStartInvokeWithTraceContextNoParentID(t *testing.T) { 180 | headers := http.Header{} 181 | headers.Set(string(DdTraceId), mockTraceId) 182 | headers.Set(string(DdSamplingPriority), mockSamplingPriority) 183 | 184 | em := &ExtensionManager{ 185 | startInvocationUrl: startInvocationUrl, 186 | httpClient: &ClientSuccessStartInvoke{ 187 | headers: headers, 188 | }, 189 | } 190 | ctx := em.SendStartInvocationRequest(context.TODO(), []byte{}) 191 | traceId := ctx.Value(DdTraceId) 192 | parentId := ctx.Value(DdParentId) 193 | samplingPriority := ctx.Value(DdSamplingPriority) 194 | err := em.Flush() 195 | 196 | assert.Nil(t, err) 197 | assert.Equal(t, mockTraceId, traceId) 198 | assert.Equal(t, mockTraceId, parentId) 199 | assert.Equal(t, mockSamplingPriority, samplingPriority) 200 | } 201 | 202 | func TestExtensionEndInvocation(t *testing.T) { 203 | em := &ExtensionManager{ 204 | endInvocationUrl: endInvocationUrl, 205 | httpClient: &ClientSuccessEndInvoke{}, 206 | } 207 | span := tracer.StartSpan("aws.lambda") 208 | logOutput := captureLog(func() { em.SendEndInvocationRequest(context.TODO(), span, ddtrace.FinishConfig{}) }) 209 | span.Finish() 210 | 211 | assert.Equal(t, "", logOutput) 212 | } 213 | 214 | func TestExtensionEndInvocationError(t *testing.T) { 215 | em := &ExtensionManager{ 216 | endInvocationUrl: endInvocationUrl, 217 | httpClient: &ClientErrorMock{}, 218 | } 219 | span := tracer.StartSpan("aws.lambda") 220 | logOutput := captureLog(func() { em.SendEndInvocationRequest(context.TODO(), span, ddtrace.FinishConfig{}) }) 221 | span.Finish() 222 | 223 | assert.Contains(t, logOutput, "could not send end invocation payload to the extension") 224 | } 225 | 226 | type mockSpanContext struct { 227 | ddtrace.SpanContext 228 | } 229 | 230 | func (m mockSpanContext) TraceID() uint64 { return 123 } 231 | func (m mockSpanContext) SpanID() uint64 { return 456 } 232 | func (m mockSpanContext) SamplingPriority() (int, bool) { return -1, true } 233 | 234 | type mockSpan struct{ ddtrace.Span } 235 | 236 | func (m mockSpan) Context() ddtrace.SpanContext { return mockSpanContext{} } 237 | 238 | func TestExtensionEndInvocationSamplingPriority(t *testing.T) { 239 | headers := http.Header{} 240 | em := &ExtensionManager{httpClient: capturingClient{hdr: headers}} 241 | span := &mockSpan{} 242 | 243 | // When priority in context, use that value 244 | ctx := context.WithValue(context.Background(), DdTraceId, "123") 245 | ctx = context.WithValue(ctx, DdSamplingPriority, "2") 246 | em.SendEndInvocationRequest(ctx, span, ddtrace.FinishConfig{}) 247 | assert.Equal(t, "2", headers.Get("X-Datadog-Sampling-Priority")) 248 | 249 | // When no context, get priority from span 250 | em.SendEndInvocationRequest(context.Background(), span, ddtrace.FinishConfig{}) 251 | assert.Equal(t, "-1", headers.Get("X-Datadog-Sampling-Priority")) 252 | } 253 | 254 | type capturingClient struct { 255 | hdr http.Header 256 | } 257 | 258 | func (c capturingClient) Do(req *http.Request) (*http.Response, error) { 259 | for k, v := range req.Header { 260 | c.hdr[k] = v 261 | } 262 | return &http.Response{StatusCode: 200}, nil 263 | } 264 | 265 | func TestExtensionEndInvocationErrorHeaders(t *testing.T) { 266 | hdr := http.Header{} 267 | em := &ExtensionManager{httpClient: capturingClient{hdr: hdr}} 268 | span := tracer.StartSpan("aws.lambda") 269 | cfg := ddtrace.FinishConfig{Error: fmt.Errorf("ooooops")} 270 | 271 | em.SendEndInvocationRequest(context.TODO(), span, cfg) 272 | 273 | assert.Equal(t, hdr.Get("X-Datadog-Invocation-Error"), "true") 274 | assert.Equal(t, hdr.Get("X-Datadog-Invocation-Error-Msg"), "ooooops") 275 | assert.Equal(t, hdr.Get("X-Datadog-Invocation-Error-Type"), "*errors.errorString") 276 | 277 | data, err := base64.StdEncoding.DecodeString(hdr.Get("X-Datadog-Invocation-Error-Stack")) 278 | assert.Nil(t, err) 279 | assert.Contains(t, string(data), "github.com/DataDog/datadog-lambda-go") 280 | assert.Contains(t, string(data), "TestExtensionEndInvocationErrorHeaders") 281 | } 282 | 283 | func TestExtensionEndInvocationErrorHeadersNilError(t *testing.T) { 284 | hdr := http.Header{} 285 | em := &ExtensionManager{httpClient: capturingClient{hdr: hdr}} 286 | span := tracer.StartSpan("aws.lambda") 287 | cfg := ddtrace.FinishConfig{Error: nil} 288 | 289 | em.SendEndInvocationRequest(context.TODO(), span, cfg) 290 | 291 | assert.Equal(t, hdr.Get("X-Datadog-Invocation-Error"), "") 292 | assert.Equal(t, hdr.Get("X-Datadog-Invocation-Error-Msg"), "") 293 | assert.Equal(t, hdr.Get("X-Datadog-Invocation-Error-Type"), "") 294 | assert.Equal(t, hdr.Get("X-Datadog-Invocation-Error-Stack"), "") 295 | } 296 | -------------------------------------------------------------------------------- /internal/logger/log.go: -------------------------------------------------------------------------------- 1 | package logger 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "io" 7 | "log" 8 | "os" 9 | ) 10 | 11 | // LogLevel represents the level of logging that should be performed 12 | type LogLevel int 13 | 14 | const ( 15 | // LevelDebug logs all information 16 | LevelDebug LogLevel = iota 17 | // LevelWarn only logs warnings and errors 18 | LevelWarn LogLevel = iota 19 | ) 20 | 21 | var ( 22 | logLevel = LevelWarn 23 | output io.Writer = os.Stdout 24 | ) 25 | 26 | // SetLogLevel set the level of logging for the ddlambda 27 | func SetLogLevel(ll LogLevel) { 28 | logLevel = ll 29 | } 30 | 31 | // SetOutput changes the writer for the logger 32 | func SetOutput(w io.Writer) { 33 | log.SetOutput(w) 34 | output = w 35 | } 36 | 37 | // Error logs a structured error message to stdout 38 | func Error(err error) { 39 | finalMessage := logStructure{ 40 | Status: "error", 41 | Message: fmt.Sprintf("datadog: %s", err.Error()), 42 | } 43 | result, _ := json.Marshal(finalMessage) 44 | 45 | log.Println(string(result)) 46 | } 47 | 48 | // Debug logs a structured log message to stdout 49 | func Debug(message string) { 50 | if logLevel > LevelDebug { 51 | return 52 | } 53 | finalMessage := logStructure{ 54 | Status: "debug", 55 | Message: fmt.Sprintf("datadog: %s", message), 56 | } 57 | 58 | result, _ := json.Marshal(finalMessage) 59 | 60 | log.Println(string(result)) 61 | } 62 | 63 | // Warn logs a structured log message to stdout 64 | func Warn(message string) { 65 | if logLevel > LevelWarn { 66 | return 67 | } 68 | finalMessage := logStructure{ 69 | Status: "warning", 70 | Message: fmt.Sprintf("datadog: %s", message), 71 | } 72 | 73 | result, _ := json.Marshal(finalMessage) 74 | 75 | log.Println(string(result)) 76 | } 77 | 78 | // Raw prints a raw message to the logs. 79 | func Raw(message string) { 80 | fmt.Fprintln(output, message) 81 | } 82 | 83 | type logStructure struct { 84 | Status string `json:"status"` 85 | Message string `json:"message"` 86 | } 87 | -------------------------------------------------------------------------------- /internal/metrics/api.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Unless explicitly stated otherwise all files in this repository are licensed 3 | * under the Apache License Version 2.0. 4 | * 5 | * This product includes software developed at Datadog (https://www.datadoghq.com/). 6 | * Copyright 2021 Datadog, Inc. 7 | */ 8 | 9 | package metrics 10 | 11 | import ( 12 | "bytes" 13 | "context" 14 | "encoding/json" 15 | "fmt" 16 | "io" 17 | "net/http" 18 | "time" 19 | 20 | "github.com/DataDog/datadog-lambda-go/internal/logger" 21 | ) 22 | 23 | type ( 24 | // Client sends metrics to Datadog 25 | Client interface { 26 | SendMetrics(metrics []APIMetric) error 27 | } 28 | 29 | // APIClient send metrics to Datadog, via the Datadog API 30 | APIClient struct { 31 | apiKey string 32 | apiKeyDecryptChan <-chan string 33 | baseAPIURL string 34 | httpClient *http.Client 35 | context context.Context 36 | } 37 | 38 | // APIClientOptions contains instantiation options from creating an APIClient. 39 | APIClientOptions struct { 40 | baseAPIURL string 41 | apiKey string 42 | kmsAPIKey string 43 | decrypter Decrypter 44 | httpClientTimeout time.Duration 45 | } 46 | 47 | postMetricsModel struct { 48 | Series []APIMetric `json:"series"` 49 | } 50 | ) 51 | 52 | // MakeAPIClient creates a new API client with the given api and app keys 53 | func MakeAPIClient(ctx context.Context, options APIClientOptions) *APIClient { 54 | httpClient := &http.Client{ 55 | Timeout: options.httpClientTimeout, 56 | } 57 | client := &APIClient{ 58 | apiKey: options.apiKey, 59 | baseAPIURL: options.baseAPIURL, 60 | httpClient: httpClient, 61 | context: ctx, 62 | } 63 | if len(options.apiKey) == 0 && len(options.kmsAPIKey) != 0 { 64 | client.apiKeyDecryptChan = client.decryptAPIKey(options.decrypter, options.kmsAPIKey) 65 | } 66 | 67 | return client 68 | } 69 | 70 | // SendMetrics posts a batch metrics payload to the Datadog API 71 | func (cl *APIClient) SendMetrics(metrics []APIMetric) error { 72 | 73 | // If the api key was provided as a kms key, wait for it to finish decrypting 74 | if cl.apiKeyDecryptChan != nil { 75 | cl.apiKey = <-cl.apiKeyDecryptChan 76 | cl.apiKeyDecryptChan = nil 77 | } 78 | 79 | content, err := marshalAPIMetricsModel(metrics) 80 | if err != nil { 81 | return fmt.Errorf("Couldn't marshal metrics model: %v", err) 82 | } 83 | body := bytes.NewBuffer(content) 84 | 85 | // For the moment we only support distribution metrics. 86 | // Other metric types use the "series" endpoint, which takes an identical payload. 87 | req, err := http.NewRequest("POST", cl.makeRoute("distribution_points"), body) 88 | if err != nil { 89 | return fmt.Errorf("Couldn't create send metrics request:%v", err) 90 | } 91 | req = req.WithContext(cl.context) 92 | 93 | defer req.Body.Close() 94 | 95 | logger.Debug(fmt.Sprintf("Sending payload with body %s", content)) 96 | 97 | cl.addAPICredentials(req) 98 | 99 | resp, err := cl.httpClient.Do(req) 100 | 101 | if err != nil { 102 | return fmt.Errorf("Failed to send metrics to API") 103 | } 104 | defer resp.Body.Close() 105 | 106 | if resp.StatusCode < 200 || resp.StatusCode > 299 { 107 | if resp.StatusCode == 403 { 108 | logger.Debug(fmt.Sprintf("authorization failed with api key of length %d characters", len(cl.apiKey))) 109 | } 110 | bodyBytes, err := io.ReadAll(resp.Body) 111 | body := "" 112 | if err == nil { 113 | body = string(bodyBytes) 114 | } 115 | return fmt.Errorf("Failed to send metrics to API. Status Code %d, Body %s", resp.StatusCode, body) 116 | } 117 | 118 | return err 119 | } 120 | 121 | func (cl *APIClient) decryptAPIKey(decrypter Decrypter, kmsAPIKey string) <-chan string { 122 | 123 | ch := make(chan string) 124 | 125 | go func() { 126 | result, err := decrypter.Decrypt(kmsAPIKey) 127 | if err != nil { 128 | logger.Error(fmt.Errorf("Couldn't decrypt api kms key %s", err)) 129 | } 130 | ch <- result 131 | close(ch) 132 | }() 133 | return ch 134 | } 135 | 136 | func (cl *APIClient) addAPICredentials(req *http.Request) { 137 | query := req.URL.Query() 138 | query.Add(apiKeyParam, cl.apiKey) 139 | req.URL.RawQuery = query.Encode() 140 | } 141 | 142 | func (cl *APIClient) makeRoute(route string) string { 143 | url := fmt.Sprintf("%s/%s", cl.baseAPIURL, route) 144 | logger.Debug(fmt.Sprintf("posting to url %s", url)) 145 | return url 146 | } 147 | 148 | func marshalAPIMetricsModel(metrics []APIMetric) ([]byte, error) { 149 | pm := postMetricsModel{} 150 | pm.Series = metrics 151 | return json.Marshal(pm) 152 | } 153 | -------------------------------------------------------------------------------- /internal/metrics/api_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Unless explicitly stated otherwise all files in this repository are licensed 3 | * under the Apache License Version 2.0. 4 | * 5 | * This product includes software developed at Datadog (https://www.datadoghq.com/). 6 | * Copyright 2021 Datadog, Inc. 7 | */ 8 | 9 | package metrics 10 | 11 | import ( 12 | "context" 13 | "io" 14 | "net/http" 15 | "net/http/httptest" 16 | "testing" 17 | 18 | "github.com/stretchr/testify/assert" 19 | ) 20 | 21 | const ( 22 | mockAPIKey = "12345" 23 | mockEncryptedAPIKey = "mockEncrypted" 24 | mockDecryptedAPIKey = "mockDecrypted" 25 | ) 26 | 27 | type ( 28 | mockDecrypter struct { 29 | returnValue string 30 | returnError error 31 | } 32 | ) 33 | 34 | func (md *mockDecrypter) Decrypt(cipherText string) (string, error) { 35 | return md.returnValue, md.returnError 36 | } 37 | 38 | func TestAddAPICredentials(t *testing.T) { 39 | cl := MakeAPIClient(context.Background(), APIClientOptions{baseAPIURL: "", apiKey: mockAPIKey}) 40 | req, _ := http.NewRequest("GET", "http://some-api.com/endpoint", nil) 41 | cl.addAPICredentials(req) 42 | assert.Equal(t, "http://some-api.com/endpoint?api_key=12345", req.URL.String()) 43 | } 44 | 45 | func TestSendMetricsSuccess(t *testing.T) { 46 | called := false 47 | server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 48 | called = true 49 | w.WriteHeader(http.StatusCreated) 50 | body, _ := io.ReadAll(r.Body) 51 | s := string(body) 52 | 53 | assert.Equal(t, "/distribution_points?api_key=12345", r.URL.String()) 54 | assert.Equal(t, "{\"series\":[{\"metric\":\"metric-1\",\"tags\":[\"a\",\"b\",\"c\"],\"type\":\"distribution\",\"points\":[[1,[2]],[3,[4]],[5,[6]]]}]}", s) 55 | 56 | })) 57 | defer server.Close() 58 | 59 | am := []APIMetric{ 60 | { 61 | Name: "metric-1", 62 | Host: nil, 63 | Tags: []string{"a", "b", "c"}, 64 | MetricType: DistributionType, 65 | Points: []interface{}{ 66 | []interface{}{float64(1), []interface{}{float64(2)}}, 67 | []interface{}{float64(3), []interface{}{float64(4)}}, 68 | []interface{}{float64(5), []interface{}{float64(6)}}, 69 | }, 70 | }, 71 | } 72 | 73 | cl := MakeAPIClient(context.Background(), APIClientOptions{baseAPIURL: server.URL, apiKey: mockAPIKey}) 74 | err := cl.SendMetrics(am) 75 | 76 | assert.NoError(t, err) 77 | assert.True(t, called) 78 | } 79 | 80 | func TestSendMetricsBadRequest(t *testing.T) { 81 | called := false 82 | server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 83 | called = true 84 | w.WriteHeader(http.StatusForbidden) 85 | body, _ := io.ReadAll(r.Body) 86 | s := string(body) 87 | 88 | assert.Equal(t, "/distribution_points?api_key=12345", r.URL.String()) 89 | assert.Equal(t, "{\"series\":[{\"metric\":\"metric-1\",\"tags\":[\"a\",\"b\",\"c\"],\"type\":\"distribution\",\"points\":[[1,[2]],[3,[4]],[5,[6]]]}]}", s) 90 | 91 | })) 92 | defer server.Close() 93 | 94 | am := []APIMetric{ 95 | { 96 | Name: "metric-1", 97 | Host: nil, 98 | Tags: []string{"a", "b", "c"}, 99 | MetricType: DistributionType, 100 | Points: []interface{}{ 101 | []interface{}{float64(1), []interface{}{float64(2)}}, 102 | []interface{}{float64(3), []interface{}{float64(4)}}, 103 | []interface{}{float64(5), []interface{}{float64(6)}}, 104 | }, 105 | }, 106 | } 107 | 108 | cl := MakeAPIClient(context.Background(), APIClientOptions{baseAPIURL: server.URL, apiKey: mockAPIKey}) 109 | err := cl.SendMetrics(am) 110 | 111 | assert.Error(t, err) 112 | assert.True(t, called) 113 | } 114 | 115 | func TestSendMetricsCantReachServer(t *testing.T) { 116 | called := false 117 | server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 118 | called = true 119 | })) 120 | defer server.Close() 121 | 122 | am := []APIMetric{ 123 | { 124 | Name: "metric-1", 125 | Host: nil, 126 | Tags: []string{"a", "b", "c"}, 127 | MetricType: DistributionType, 128 | Points: []interface{}{ 129 | []interface{}{float64(1), []interface{}{float64(2)}}, 130 | []interface{}{float64(3), []interface{}{float64(4)}}, 131 | []interface{}{float64(5), []interface{}{float64(6)}}, 132 | }, 133 | }, 134 | } 135 | 136 | cl := MakeAPIClient(context.Background(), APIClientOptions{baseAPIURL: "httpa:///badly-formatted-url", apiKey: mockAPIKey}) 137 | err := cl.SendMetrics(am) 138 | 139 | assert.Error(t, err) 140 | assert.False(t, called) 141 | } 142 | 143 | func TestDecryptsUsingKMSKey(t *testing.T) { 144 | called := false 145 | server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 146 | called = true 147 | assert.Equal(t, "/distribution_points?api_key=mockDecrypted", r.URL.String()) 148 | })) 149 | defer server.Close() 150 | 151 | am := []APIMetric{ 152 | { 153 | Name: "metric-1", 154 | Host: nil, 155 | Tags: []string{"a", "b", "c"}, 156 | MetricType: DistributionType, 157 | Points: []interface{}{ 158 | []interface{}{float64(1), []interface{}{float64(2)}}, 159 | []interface{}{float64(3), []interface{}{float64(4)}}, 160 | []interface{}{float64(5), []interface{}{float64(6)}}, 161 | }, 162 | }, 163 | } 164 | md := mockDecrypter{} 165 | md.returnValue = mockDecryptedAPIKey 166 | 167 | cl := MakeAPIClient(context.Background(), APIClientOptions{baseAPIURL: server.URL, apiKey: "", kmsAPIKey: mockEncryptedAPIKey, decrypter: &md}) 168 | err := cl.SendMetrics(am) 169 | 170 | assert.NoError(t, err) 171 | assert.True(t, called) 172 | } 173 | -------------------------------------------------------------------------------- /internal/metrics/batcher.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Unless explicitly stated otherwise all files in this repository are licensed 3 | * under the Apache License Version 2.0. 4 | * 5 | * This product includes software developed at Datadog (https://www.datadoghq.com/). 6 | * Copyright 2021 Datadog, Inc. 7 | */ 8 | 9 | package metrics 10 | 11 | import ( 12 | "fmt" 13 | "sort" 14 | "strings" 15 | "time" 16 | ) 17 | 18 | type ( 19 | // Batcher aggregates metrics with common properties,(metric name, tags, type etc) 20 | Batcher struct { 21 | metrics map[string]Metric 22 | batchInterval time.Duration 23 | } 24 | // BatchKey identifies a batch of metrics 25 | BatchKey struct { 26 | metricType MetricType 27 | name string 28 | tags []string 29 | host *string 30 | } 31 | ) 32 | 33 | // MakeBatcher creates a new batcher object 34 | func MakeBatcher(batchInterval time.Duration) *Batcher { 35 | return &Batcher{ 36 | batchInterval: batchInterval, 37 | metrics: map[string]Metric{}, 38 | } 39 | } 40 | 41 | // AddMetric adds a point to a given metric 42 | func (b *Batcher) AddMetric(metric Metric) { 43 | sk := b.getStringKey(metric.ToBatchKey()) 44 | if existing, ok := b.metrics[sk]; ok { 45 | existing.Join(metric) 46 | } else { 47 | b.metrics[sk] = metric 48 | } 49 | } 50 | 51 | // ToAPIMetrics converts the current batch of metrics into API metrics 52 | func (b *Batcher) ToAPIMetrics() []APIMetric { 53 | 54 | ar := []APIMetric{} 55 | interval := b.batchInterval / time.Second 56 | 57 | for _, metric := range b.metrics { 58 | values := metric.ToAPIMetric(interval) 59 | ar = append(ar, values...) 60 | } 61 | return ar 62 | } 63 | 64 | func (b *Batcher) getStringKey(bk BatchKey) string { 65 | tagKey := getTagKey(bk.tags) 66 | 67 | if bk.host != nil { 68 | return fmt.Sprintf("(%s)-(%s)-(%s)-(%s)", bk.metricType, bk.name, tagKey, *bk.host) 69 | } 70 | return fmt.Sprintf("(%s)-(%s)-(%s)", bk.metricType, bk.name, tagKey) 71 | } 72 | 73 | func getTagKey(tags []string) string { 74 | sortedTags := make([]string, len(tags)) 75 | copy(sortedTags, tags) 76 | sort.Strings(sortedTags) 77 | return strings.Join(sortedTags, ":") 78 | } 79 | -------------------------------------------------------------------------------- /internal/metrics/batcher_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Unless explicitly stated otherwise all files in this repository are licensed 3 | * under the Apache License Version 2.0. 4 | * 5 | * This product includes software developed at Datadog (https://www.datadoghq.com/). 6 | * Copyright 2021 Datadog, Inc. 7 | */ 8 | 9 | package metrics 10 | 11 | import ( 12 | "testing" 13 | "time" 14 | 15 | "github.com/stretchr/testify/assert" 16 | ) 17 | 18 | func TestGetMetricDifferentTagOrder(t *testing.T) { 19 | 20 | tm := time.Now() 21 | batcher := MakeBatcher(10) 22 | dm1 := Distribution{ 23 | Name: "metric-1", 24 | Values: []MetricValue{{Timestamp: tm, Value: 1}, {Timestamp: tm, Value: 2}}, 25 | Tags: []string{"a", "b", "c"}, 26 | } 27 | dm2 := Distribution{ 28 | Name: "metric-1", 29 | Values: []MetricValue{{Timestamp: tm, Value: 3}, {Timestamp: tm, Value: 4}}, 30 | Tags: []string{"c", "b", "a"}, 31 | } 32 | 33 | batcher.AddMetric(&dm1) 34 | batcher.AddMetric(&dm2) 35 | 36 | assert.Equal(t, []MetricValue{{Timestamp: tm, Value: 1}, {Timestamp: tm, Value: 2}, {Timestamp: tm, Value: 3}, {Timestamp: tm, Value: 4}}, dm1.Values) 37 | } 38 | 39 | func TestGetMetricFailDifferentName(t *testing.T) { 40 | 41 | tm := time.Now() 42 | batcher := MakeBatcher(10) 43 | 44 | dm1 := Distribution{ 45 | Name: "metric-1", 46 | Values: []MetricValue{{Timestamp: tm, Value: 1}, {Timestamp: tm, Value: 2}}, 47 | Tags: []string{"a", "b", "c"}, 48 | } 49 | dm2 := Distribution{ 50 | Name: "metric-2", 51 | Values: []MetricValue{{Timestamp: tm, Value: 3}, {Timestamp: tm, Value: 4}}, 52 | Tags: []string{"c", "b", "a"}, 53 | } 54 | 55 | batcher.AddMetric(&dm1) 56 | batcher.AddMetric(&dm2) 57 | 58 | assert.Equal(t, []MetricValue{{Timestamp: tm, Value: 1}, {Timestamp: tm, Value: 2}}, dm1.Values) 59 | 60 | } 61 | 62 | func TestGetMetricFailDifferentHost(t *testing.T) { 63 | tm := time.Now() 64 | batcher := MakeBatcher(10) 65 | 66 | host1 := "my-host-1" 67 | host2 := "my-host-2" 68 | 69 | dm1 := Distribution{ 70 | Values: []MetricValue{{Timestamp: tm, Value: 1}, {Timestamp: tm, Value: 2}}, 71 | 72 | Tags: []string{"a", "b", "c"}, 73 | Host: &host1, 74 | } 75 | dm2 := Distribution{ 76 | Name: "metric-1", 77 | Values: []MetricValue{{Timestamp: tm, Value: 3}, {Timestamp: tm, Value: 4}}, 78 | Tags: []string{"a", "b", "c"}, 79 | Host: &host2, 80 | } 81 | 82 | batcher.AddMetric(&dm1) 83 | batcher.AddMetric(&dm2) 84 | 85 | assert.Equal(t, []MetricValue{{Timestamp: tm, Value: 1}, {Timestamp: tm, Value: 2}}, dm1.Values) 86 | } 87 | 88 | func TestGetMetricSameHost(t *testing.T) { 89 | 90 | tm := time.Now() 91 | batcher := MakeBatcher(10) 92 | 93 | host := "my-host" 94 | 95 | dm1 := Distribution{ 96 | Name: "metric-1", 97 | Values: []MetricValue{{Timestamp: tm, Value: 1}, {Timestamp: tm, Value: 2}}, 98 | Tags: []string{"a", "b", "c"}, 99 | Host: &host, 100 | } 101 | dm2 := Distribution{ 102 | Name: "metric-1", 103 | Values: []MetricValue{{Timestamp: tm, Value: 3}, {Timestamp: tm, Value: 4}}, 104 | Tags: []string{"a", "b", "c"}, 105 | Host: &host, 106 | } 107 | 108 | batcher.AddMetric(&dm1) 109 | batcher.AddMetric(&dm2) 110 | 111 | assert.Equal(t, []MetricValue{{Timestamp: tm, Value: 1}, {Timestamp: tm, Value: 2}, {Timestamp: tm, Value: 3}, {Timestamp: tm, Value: 4}}, dm1.Values) 112 | } 113 | 114 | func TestToAPIMetricsSameInterval(t *testing.T) { 115 | tm := time.Now() 116 | hostname := "host-1" 117 | 118 | batcher := MakeBatcher(10) 119 | dm := Distribution{ 120 | Name: "metric-1", 121 | Tags: []string{"a", "b", "c"}, 122 | Host: &hostname, 123 | Values: []MetricValue{}, 124 | } 125 | 126 | dm.AddPoint(tm, 1) 127 | dm.AddPoint(tm, 2) 128 | dm.AddPoint(tm, 3) 129 | 130 | batcher.AddMetric(&dm) 131 | 132 | floatTime := float64(tm.Unix()) 133 | result := batcher.ToAPIMetrics() 134 | expected := []APIMetric{ 135 | { 136 | Name: "metric-1", 137 | Host: &hostname, 138 | Tags: []string{"a", "b", "c"}, 139 | MetricType: DistributionType, 140 | Interval: nil, 141 | Points: []interface{}{ 142 | []interface{}{floatTime, []interface{}{float64(1)}}, 143 | []interface{}{floatTime, []interface{}{float64(2)}}, 144 | []interface{}{floatTime, []interface{}{float64(3)}}, 145 | }, 146 | }, 147 | } 148 | 149 | assert.Equal(t, expected, result) 150 | } 151 | -------------------------------------------------------------------------------- /internal/metrics/constants.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Unless explicitly stated otherwise all files in this repository are licensed 3 | * under the Apache License Version 2.0. 4 | * 5 | * This product includes software developed at Datadog (https://www.datadoghq.com/). 6 | * Copyright 2021 Datadog, Inc. 7 | */ 8 | 9 | package metrics 10 | 11 | import "time" 12 | 13 | const ( 14 | apiKeyParam = "api_key" 15 | defaultRetryInterval = time.Millisecond * 250 16 | defaultBatchInterval = time.Second * 15 17 | defaultHttpClientTimeout = time.Second * 5 18 | defaultCircuitBreakerInterval = time.Second * 30 19 | defaultCircuitBreakerTimeout = time.Second * 60 20 | defaultCircuitBreakerTotalFailures = 4 21 | ) 22 | 23 | // MetricType enumerates all the available metric types 24 | type MetricType string 25 | 26 | const ( 27 | 28 | // DistributionType represents a distribution metric 29 | DistributionType MetricType = "distribution" 30 | ) 31 | -------------------------------------------------------------------------------- /internal/metrics/context.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Unless explicitly stated otherwise all files in this repository are licensed 3 | * under the Apache License Version 2.0. 4 | * 5 | * This product includes software developed at Datadog (https://www.datadoghq.com/). 6 | * Copyright 2021 Datadog, Inc. 7 | */ 8 | 9 | package metrics 10 | 11 | import "context" 12 | 13 | type contextKeytype int 14 | 15 | var metricsListenerKey = new(contextKeytype) 16 | 17 | // GetListener retrieves the metrics listener from a context object. 18 | func GetListener(ctx context.Context) *Listener { 19 | result := ctx.Value(metricsListenerKey) 20 | if result == nil { 21 | return nil 22 | } 23 | return result.(*Listener) 24 | } 25 | 26 | // AddListener adds a metrics listener to a context object 27 | func AddListener(ctx context.Context, listener *Listener) context.Context { 28 | return context.WithValue(ctx, metricsListenerKey, listener) 29 | } 30 | -------------------------------------------------------------------------------- /internal/metrics/context_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Unless explicitly stated otherwise all files in this repository are licensed 3 | * under the Apache License Version 2.0. 4 | * 5 | * This product includes software developed at Datadog (https://www.datadoghq.com/). 6 | * Copyright 2021 Datadog, Inc. 7 | */ 8 | 9 | package metrics 10 | 11 | import ( 12 | "context" 13 | "testing" 14 | 15 | "github.com/DataDog/datadog-lambda-go/internal/extension" 16 | "github.com/stretchr/testify/assert" 17 | ) 18 | 19 | func TestGetProcessorEmptyContext(t *testing.T) { 20 | ctx := context.Background() 21 | result := GetListener(ctx) 22 | assert.Nil(t, result) 23 | } 24 | 25 | func TestGetProcessorSuccess(t *testing.T) { 26 | lst := MakeListener(Config{}, &extension.ExtensionManager{}) 27 | ctx := AddListener(context.Background(), &lst) 28 | result := GetListener(ctx) 29 | assert.NotNil(t, result) 30 | } 31 | -------------------------------------------------------------------------------- /internal/metrics/kms_decrypter.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Unless explicitly stated otherwise all files in this repository are licensed 3 | * under the Apache License Version 2.0. 4 | * 5 | * This product includes software developed at Datadog (https://www.datadoghq.com/). 6 | * Copyright 2021 Datadog, Inc. 7 | */ 8 | package metrics 9 | 10 | import ( 11 | "context" 12 | "encoding/base64" 13 | "fmt" 14 | "github.com/aws/aws-sdk-go-v2/aws" 15 | "os" 16 | 17 | "github.com/DataDog/datadog-lambda-go/internal/logger" 18 | "github.com/aws/aws-sdk-go-v2/config" 19 | "github.com/aws/aws-sdk-go-v2/service/kms" 20 | ) 21 | 22 | type ( 23 | // Decrypter attempts to decrypt a key 24 | Decrypter interface { 25 | Decrypt(cipherText string) (string, error) 26 | } 27 | 28 | kmsDecrypter struct { 29 | kmsClient *kms.Client 30 | } 31 | 32 | clientDecrypter interface { 33 | Decrypt(context.Context, *kms.DecryptInput, ...func(*kms.Options)) (*kms.DecryptOutput, error) 34 | } 35 | ) 36 | 37 | // functionNameEnvVar is the environment variable that stores the Lambda function name 38 | const functionNameEnvVar string = "AWS_LAMBDA_FUNCTION_NAME" 39 | 40 | // encryptionContextKey is the key added to the encryption context by the Lambda console UI 41 | const encryptionContextKey string = "LambdaFunctionName" 42 | 43 | // MakeKMSDecrypter creates a new decrypter which uses the AWS KMS service to decrypt variables 44 | func MakeKMSDecrypter(fipsMode bool) Decrypter { 45 | fipsEndpoint := aws.FIPSEndpointStateUnset 46 | if fipsMode { 47 | fipsEndpoint = aws.FIPSEndpointStateEnabled 48 | logger.Debug("Using FIPS endpoint for KMS decryption.") 49 | } 50 | 51 | cfg, err := config.LoadDefaultConfig(context.Background(), config.WithUseFIPSEndpoint(fipsEndpoint)) 52 | if err != nil { 53 | logger.Error(fmt.Errorf("could not create a new aws config: %v", err)) 54 | panic(err) 55 | } 56 | return &kmsDecrypter{ 57 | kmsClient: kms.NewFromConfig(cfg), 58 | } 59 | } 60 | 61 | func (kd *kmsDecrypter) Decrypt(ciphertext string) (string, error) { 62 | return decryptKMS(kd.kmsClient, ciphertext) 63 | } 64 | 65 | // decryptKMS decodes and deciphers the base64-encoded ciphertext given as a parameter using KMS. 66 | // For this to work properly, the Lambda function must have the appropriate IAM permissions. 67 | func decryptKMS(kmsClient clientDecrypter, ciphertext string) (string, error) { 68 | decodedBytes, err := base64.StdEncoding.DecodeString(ciphertext) 69 | if err != nil { 70 | return "", fmt.Errorf("failed to encode cipher text to base64: %v", err) 71 | } 72 | 73 | // When the API key is encrypted using the AWS console, the function name is added as an 74 | // encryption context. When the API key is encrypted using the AWS CLI, no encryption context 75 | // is added. We need to try decrypting the API key both with and without the encryption context. 76 | 77 | // Try without encryption context, in case API key was encrypted using the AWS CLI 78 | functionName := os.Getenv(functionNameEnvVar) 79 | params := &kms.DecryptInput{ 80 | CiphertextBlob: decodedBytes, 81 | } 82 | ctx := context.Background() 83 | response, err := kmsClient.Decrypt(ctx, params) 84 | 85 | if err != nil { 86 | logger.Debug("Failed to decrypt ciphertext without encryption context, retrying with encryption context") 87 | // Try with encryption context, in case API key was encrypted using the AWS Console 88 | params = &kms.DecryptInput{ 89 | CiphertextBlob: decodedBytes, 90 | EncryptionContext: map[string]string{ 91 | encryptionContextKey: functionName, 92 | }, 93 | } 94 | response, err = kmsClient.Decrypt(ctx, params) 95 | if err != nil { 96 | return "", fmt.Errorf("failed to decrypt ciphertext with kms: %v", err) 97 | } 98 | } 99 | 100 | plaintext := string(response.Plaintext) 101 | return plaintext, nil 102 | } 103 | -------------------------------------------------------------------------------- /internal/metrics/kms_decrypter_test.go: -------------------------------------------------------------------------------- 1 | // Unless explicitly stated otherwise all files in this repository are licensed 2 | // under the Apache License Version 2.0. 3 | // This product includes software developed at Datadog (https://www.datadoghq.com/). 4 | // Copyright 2016-present Datadog, Inc. 5 | 6 | package metrics 7 | 8 | import ( 9 | "bytes" 10 | "context" 11 | "errors" 12 | "os" 13 | "testing" 14 | 15 | "github.com/aws/aws-sdk-go-v2/service/kms" 16 | "github.com/stretchr/testify/assert" 17 | ) 18 | 19 | // mockEncryptedAPIKeyBase64 represents an API key encrypted with KMS and encoded as a base64 string 20 | const mockEncryptedAPIKeyBase64 = "MjIyMjIyMjIyMjIyMjIyMg==" 21 | 22 | // mockDecodedEncryptedAPIKey represents the encrypted API key after it has been decoded from base64 23 | const mockDecodedEncryptedAPIKey = "2222222222222222" 24 | 25 | // expectedDecryptedAPIKey represents the true value of the API key after decryption by KMS 26 | const expectedDecryptedAPIKey = "1111111111111111" 27 | 28 | // mockFunctionName represents the name of the current function 29 | var mockFunctionName = "my-Function" 30 | 31 | type mockKMSClientWithEncryptionContext struct{} 32 | 33 | func (mockKMSClientWithEncryptionContext) Decrypt(_ context.Context, params *kms.DecryptInput, _ ...func(*kms.Options)) (*kms.DecryptOutput, error) { 34 | encryptionContextPointer, exists := params.EncryptionContext[encryptionContextKey] 35 | if !exists { 36 | return nil, errors.New("InvalidCiphertextException") 37 | } 38 | if encryptionContextPointer != mockFunctionName { 39 | return nil, errors.New("InvalidCiphertextException") 40 | } 41 | if bytes.Equal(params.CiphertextBlob, []byte(mockDecodedEncryptedAPIKey)) { 42 | return &kms.DecryptOutput{ 43 | Plaintext: []byte(expectedDecryptedAPIKey), 44 | }, nil 45 | } 46 | return nil, errors.New("KMS error") 47 | } 48 | 49 | type mockKMSClientNoEncryptionContext struct{} 50 | 51 | func (mockKMSClientNoEncryptionContext) Decrypt(_ context.Context, params *kms.DecryptInput, _ ...func(*kms.Options)) (*kms.DecryptOutput, error) { 52 | if params.EncryptionContext[encryptionContextKey] != "" { 53 | return nil, errors.New("InvalidCiphertextException") 54 | } 55 | if bytes.Equal(params.CiphertextBlob, []byte(mockDecodedEncryptedAPIKey)) { 56 | return &kms.DecryptOutput{ 57 | Plaintext: []byte(expectedDecryptedAPIKey), 58 | }, nil 59 | } 60 | return nil, errors.New("KMS error") 61 | } 62 | 63 | func TestDecryptKMSWithEncryptionContext(t *testing.T) { 64 | os.Setenv(functionNameEnvVar, mockFunctionName) 65 | defer os.Setenv(functionNameEnvVar, "") 66 | 67 | client := mockKMSClientWithEncryptionContext{} 68 | result, _ := decryptKMS(client, mockEncryptedAPIKeyBase64) 69 | assert.Equal(t, expectedDecryptedAPIKey, result) 70 | } 71 | 72 | func TestDecryptKMSNoEncryptionContext(t *testing.T) { 73 | client := mockKMSClientNoEncryptionContext{} 74 | result, _ := decryptKMS(client, mockEncryptedAPIKeyBase64) 75 | assert.Equal(t, expectedDecryptedAPIKey, result) 76 | } 77 | -------------------------------------------------------------------------------- /internal/metrics/listener.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Unless explicitly stated otherwise all files in this repository are licensed 3 | * under the Apache License Version 2.0. 4 | * 5 | * This product includes software developed at Datadog (https://www.datadoghq.com/). 6 | * Copyright 2021 Datadog, Inc. 7 | */ 8 | 9 | package metrics 10 | 11 | import ( 12 | "context" 13 | "encoding/json" 14 | "fmt" 15 | "runtime" 16 | "strconv" 17 | "strings" 18 | "time" 19 | 20 | "github.com/aws/aws-lambda-go/lambdacontext" 21 | 22 | "github.com/DataDog/datadog-go/v5/statsd" 23 | "github.com/DataDog/datadog-lambda-go/internal/extension" 24 | "github.com/DataDog/datadog-lambda-go/internal/logger" 25 | "github.com/DataDog/datadog-lambda-go/internal/version" 26 | ) 27 | 28 | type ( 29 | // Listener implements wrapper.HandlerListener, injecting metrics into the context 30 | Listener struct { 31 | apiClient *APIClient 32 | statsdClient *statsd.Client 33 | config *Config 34 | processor Processor 35 | isAgentRunning bool 36 | extensionManager *extension.ExtensionManager 37 | } 38 | 39 | // Config gives options for how the listener should work 40 | Config struct { 41 | APIKey string 42 | KMSAPIKey string 43 | Site string 44 | ShouldRetryOnFailure bool 45 | ShouldUseLogForwarder bool 46 | BatchInterval time.Duration 47 | EnhancedMetrics bool 48 | HTTPClientTimeout time.Duration 49 | CircuitBreakerInterval time.Duration 50 | CircuitBreakerTimeout time.Duration 51 | CircuitBreakerTotalFailures uint32 52 | LocalTest bool 53 | FIPSMode bool 54 | } 55 | 56 | logMetric struct { 57 | MetricName string `json:"m"` 58 | Value float64 `json:"v"` 59 | Timestamp int64 `json:"e"` 60 | Tags []string `json:"t"` 61 | } 62 | ) 63 | 64 | // MakeListener initializes a new metrics lambda listener 65 | func MakeListener(config Config, extensionManager *extension.ExtensionManager) Listener { 66 | 67 | var apiClient *APIClient 68 | if !config.FIPSMode { 69 | apiClient = MakeAPIClient(context.Background(), APIClientOptions{ 70 | baseAPIURL: config.Site, 71 | apiKey: config.APIKey, 72 | decrypter: MakeKMSDecrypter(config.FIPSMode), 73 | kmsAPIKey: config.KMSAPIKey, 74 | httpClientTimeout: config.HTTPClientTimeout, 75 | }) 76 | } 77 | 78 | if config.HTTPClientTimeout <= 0 { 79 | config.HTTPClientTimeout = defaultHttpClientTimeout 80 | } 81 | if config.CircuitBreakerInterval <= 0 { 82 | config.CircuitBreakerInterval = defaultCircuitBreakerInterval 83 | } 84 | if config.CircuitBreakerTimeout <= 0 { 85 | config.CircuitBreakerTimeout = defaultCircuitBreakerTimeout 86 | } 87 | if config.CircuitBreakerTotalFailures <= 0 { 88 | config.CircuitBreakerTotalFailures = defaultCircuitBreakerTotalFailures 89 | } 90 | if config.BatchInterval <= 0 { 91 | config.BatchInterval = defaultBatchInterval 92 | } 93 | 94 | var statsdClient *statsd.Client 95 | // immediate call to the Agent, if not a 200, fallback to API 96 | // TODO(remy): we may want to use an environment var to force the use of the 97 | // Agent instead of using this "discovery" implementation. 98 | if extensionManager.IsExtensionRunning() { 99 | var err error 100 | if statsdClient, err = statsd.New("127.0.0.1:8125"); err != nil { 101 | statsdClient = nil // force nil if an error occurred during statsd client init 102 | } 103 | } 104 | 105 | return Listener{ 106 | apiClient: apiClient, 107 | config: &config, 108 | isAgentRunning: statsdClient != nil, 109 | statsdClient: statsdClient, 110 | processor: nil, 111 | extensionManager: extensionManager, 112 | } 113 | } 114 | 115 | // canSendMetrics reports whether l can send metrics. 116 | func (l *Listener) canSendMetrics() bool { 117 | return l.isAgentRunning || l.config.ShouldUseLogForwarder || !l.config.FIPSMode || (l.apiClient != nil && (l.apiClient.apiKey != "" || l.config.KMSAPIKey != "")) 118 | } 119 | 120 | // HandlerStarted adds metrics service to the context 121 | func (l *Listener) HandlerStarted(ctx context.Context, msg json.RawMessage) context.Context { 122 | if !l.canSendMetrics() { 123 | logger.Error(fmt.Errorf("datadog api key isn't set, won't be able to send metrics")) 124 | } 125 | 126 | ctx = AddListener(ctx, l) 127 | 128 | if !l.config.FIPSMode { 129 | ts := MakeTimeService() 130 | pr := MakeProcessor(ctx, l.apiClient, ts, l.config.BatchInterval, l.config.ShouldRetryOnFailure, l.config.CircuitBreakerInterval, l.config.CircuitBreakerTimeout, l.config.CircuitBreakerTotalFailures) 131 | l.processor = pr 132 | 133 | // Setting the context on the client will mean that future requests will be cancelled correctly 134 | // if the lambda times out. 135 | l.apiClient.context = ctx 136 | 137 | pr.StartProcessing() 138 | } 139 | 140 | l.submitEnhancedMetrics("invocations", ctx) 141 | 142 | return ctx 143 | } 144 | 145 | // HandlerFinished implemented as part of the wrapper.HandlerListener interface 146 | func (l *Listener) HandlerFinished(ctx context.Context, err error) { 147 | if l.isAgentRunning { 148 | // use the agent 149 | // flush the metrics from the DogStatsD client to the Agent 150 | if l.statsdClient != nil { 151 | if err := l.statsdClient.Flush(); err != nil { 152 | logger.Error(fmt.Errorf("can't flush the DogStatsD client: %s", err)) 153 | } 154 | } 155 | // send a message to the Agent to flush the metrics 156 | if l.config.LocalTest { 157 | if err := l.extensionManager.Flush(); err != nil { 158 | logger.Error(fmt.Errorf("error while flushing the metrics: %s", err)) 159 | } 160 | } 161 | } else { 162 | // use the api 163 | if l.processor != nil { 164 | if err != nil { 165 | l.submitEnhancedMetrics("errors", ctx) 166 | } 167 | l.processor.FinishProcessing() 168 | } 169 | } 170 | } 171 | 172 | // AddDistributionMetric sends a distribution metric 173 | func (l *Listener) AddDistributionMetric(metric string, value float64, timestamp time.Time, forceLogForwarder bool, tags ...string) { 174 | 175 | // We add our own runtime tag to the metric for version tracking 176 | tags = append(tags, getRuntimeTag()) 177 | 178 | if l.isAgentRunning { 179 | err := l.statsdClient.Distribution(metric, value, tags, 1) 180 | if err != nil { 181 | logger.Error(fmt.Errorf("could not send metric %s: %s", metric, err.Error())) 182 | } 183 | return 184 | } 185 | 186 | if l.config.ShouldUseLogForwarder || forceLogForwarder { 187 | logger.Debug("sending metric via log forwarder") 188 | unixTime := timestamp.Unix() 189 | lm := logMetric{ 190 | MetricName: metric, 191 | Value: value, 192 | Timestamp: unixTime, 193 | Tags: tags, 194 | } 195 | result, err := json.Marshal(lm) 196 | if err != nil { 197 | logger.Error(fmt.Errorf("failed to marshall metric for log forwarder with error %v", err)) 198 | return 199 | } 200 | payload := string(result) 201 | logger.Raw(payload) 202 | return 203 | } 204 | 205 | if l.config.FIPSMode { 206 | logger.Debug(fmt.Sprintf("skipping metric %s due to FIPS mode - direct API calls are disabled", metric)) 207 | return 208 | } 209 | 210 | m := Distribution{ 211 | Name: metric, 212 | Tags: tags, 213 | Values: []MetricValue{}, 214 | } 215 | m.AddPoint(timestamp, value) 216 | logger.Debug(fmt.Sprintf("adding metric \"%s\", with value %f", metric, value)) 217 | l.processor.AddMetric(&m) 218 | } 219 | 220 | func getRuntimeTag() string { 221 | v := runtime.Version() 222 | return fmt.Sprintf("dd_lambda_layer:datadog-%s", v) 223 | } 224 | 225 | func (l *Listener) submitEnhancedMetrics(metricName string, ctx context.Context) { 226 | if l.config.EnhancedMetrics { 227 | tags := getEnhancedMetricsTags(ctx) 228 | l.AddDistributionMetric(fmt.Sprintf("aws.lambda.enhanced.%s", metricName), 1, time.Now(), true, tags...) 229 | } 230 | } 231 | 232 | func getEnhancedMetricsTags(ctx context.Context) []string { 233 | isColdStart := ctx.Value("cold_start") 234 | 235 | if lc, ok := lambdacontext.FromContext(ctx); ok { 236 | // ex: arn:aws:lambda:us-east-1:123497558138:function:golang-layer:alias 237 | splitArn := strings.Split(lc.InvokedFunctionArn, ":") 238 | 239 | // malformed arn string 240 | if len(splitArn) < 5 { 241 | logger.Debug("malformed arn string in the LambdaContext") 242 | return []string{} 243 | } 244 | 245 | var alias string 246 | var executedVersion string 247 | 248 | functionName := fmt.Sprintf("functionname:%s", lambdacontext.FunctionName) 249 | region := fmt.Sprintf("region:%s", splitArn[3]) 250 | accountId := fmt.Sprintf("account_id:%s", splitArn[4]) 251 | memorySize := fmt.Sprintf("memorysize:%d", lambdacontext.MemoryLimitInMB) 252 | coldStart := fmt.Sprintf("cold_start:%t", isColdStart.(bool)) 253 | resource := fmt.Sprintf("resource:%s", lambdacontext.FunctionName) 254 | datadogLambda := fmt.Sprintf("datadog_lambda:v%s", version.DDLambdaVersion) 255 | 256 | tags := []string{functionName, region, accountId, memorySize, coldStart, datadogLambda} 257 | 258 | // Check if our slice contains an alias or version 259 | if len(splitArn) > 7 { 260 | alias = splitArn[7] 261 | 262 | // If we have an alias... 263 | switch alias != "" { 264 | // If the alias is $Latest, drop the $ for ddog tag conventio 265 | case strings.HasPrefix(alias, "$"): 266 | alias = strings.TrimPrefix(alias, "$") 267 | // If this is not a version number, we will have an alias and executed version 268 | case isNotNumeric(alias): 269 | executedVersion = fmt.Sprintf("executedversion:%s", lambdacontext.FunctionVersion) 270 | tags = append(tags, executedVersion) 271 | } 272 | 273 | resource = fmt.Sprintf("resource:%s:%s", lambdacontext.FunctionName, alias) 274 | } 275 | 276 | tags = append(tags, resource) 277 | 278 | return tags 279 | } 280 | 281 | logger.Debug("could not retrieve the LambdaContext from Context") 282 | return []string{} 283 | } 284 | 285 | func isNotNumeric(s string) bool { 286 | _, err := strconv.ParseInt(s, 0, 64) 287 | return err != nil 288 | } 289 | -------------------------------------------------------------------------------- /internal/metrics/listener_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Unless explicitly stated otherwise all files in this repository are licensed 3 | * under the Apache License Version 2.0. 4 | * 5 | * This product includes software developed at Datadog (https://www.datadoghq.com/). 6 | * Copyright 2021 Datadog, Inc. 7 | */ 8 | 9 | package metrics 10 | 11 | import ( 12 | "bytes" 13 | "context" 14 | "encoding/json" 15 | "errors" 16 | "fmt" 17 | "net" 18 | "net/http" 19 | "net/http/httptest" 20 | "os" 21 | "strings" 22 | "testing" 23 | "time" 24 | 25 | "github.com/DataDog/datadog-lambda-go/internal/extension" 26 | "github.com/DataDog/datadog-lambda-go/internal/logger" 27 | "github.com/DataDog/datadog-lambda-go/internal/version" 28 | "github.com/aws/aws-lambda-go/lambdacontext" 29 | 30 | "github.com/stretchr/testify/assert" 31 | ) 32 | 33 | func captureOutput(f func()) string { 34 | var buf bytes.Buffer 35 | logger.SetOutput(&buf) 36 | f() 37 | logger.SetOutput(os.Stderr) 38 | return buf.String() 39 | } 40 | 41 | func TestHandlerAddsItselfToContext(t *testing.T) { 42 | listener := MakeListener(Config{}, &extension.ExtensionManager{}) 43 | ctx := listener.HandlerStarted(context.Background(), json.RawMessage{}) 44 | pr := GetListener(ctx) 45 | assert.NotNil(t, pr) 46 | } 47 | 48 | func TestHandlerFinishesProcessing(t *testing.T) { 49 | listener := MakeListener(Config{}, &extension.ExtensionManager{}) 50 | ctx := listener.HandlerStarted(context.Background(), json.RawMessage{}) 51 | 52 | listener.HandlerFinished(ctx, nil) 53 | assert.False(t, listener.processor.IsProcessing()) 54 | } 55 | 56 | func TestAddDistributionMetricWithAPI(t *testing.T) { 57 | 58 | called := false 59 | server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 60 | assert.Equal(t, "/distribution_points?api_key=12345", r.URL.String()) 61 | called = true 62 | w.WriteHeader(http.StatusCreated) 63 | })) 64 | defer server.Close() 65 | 66 | listener := MakeListener(Config{APIKey: "12345", Site: server.URL}, &extension.ExtensionManager{}) 67 | ctx := listener.HandlerStarted(context.Background(), json.RawMessage{}) 68 | listener.AddDistributionMetric("the-metric", 2, time.Now(), false, "tag:a", "tag:b") 69 | listener.HandlerFinished(ctx, nil) 70 | assert.True(t, called) 71 | } 72 | 73 | func TestAddDistributionMetricWithLogForwarder(t *testing.T) { 74 | called := false 75 | server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 76 | called = true 77 | w.WriteHeader(http.StatusCreated) 78 | })) 79 | defer server.Close() 80 | 81 | listener := MakeListener(Config{APIKey: "12345", Site: server.URL, ShouldUseLogForwarder: true}, &extension.ExtensionManager{}) 82 | ctx := listener.HandlerStarted(context.Background(), json.RawMessage{}) 83 | listener.AddDistributionMetric("the-metric", 2, time.Now(), false, "tag:a", "tag:b") 84 | listener.HandlerFinished(ctx, nil) 85 | assert.False(t, called) 86 | } 87 | func TestAddDistributionMetricWithForceLogForwarder(t *testing.T) { 88 | called := false 89 | server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 90 | called = true 91 | w.WriteHeader(http.StatusCreated) 92 | })) 93 | defer server.Close() 94 | 95 | listener := MakeListener(Config{APIKey: "12345", Site: server.URL, ShouldUseLogForwarder: false}, &extension.ExtensionManager{}) 96 | ctx := listener.HandlerStarted(context.Background(), json.RawMessage{}) 97 | listener.AddDistributionMetric("the-metric", 2, time.Now(), true, "tag:a", "tag:b") 98 | listener.HandlerFinished(ctx, nil) 99 | assert.False(t, called) 100 | } 101 | 102 | func TestAddDistributionMetricWithFIPSMode(t *testing.T) { 103 | // Setup a test server to detect if any API calls are made 104 | apiCallAttempted := false 105 | server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 106 | apiCallAttempted = true 107 | w.WriteHeader(http.StatusCreated) 108 | })) 109 | defer server.Close() 110 | 111 | // Create a listener with FIPS mode enabled 112 | listener := MakeListener(Config{ 113 | APIKey: "12345", 114 | Site: server.URL, 115 | FIPSMode: true, 116 | }, &extension.ExtensionManager{}) 117 | 118 | // Verify the API client wasn't created 119 | assert.Nil(t, listener.apiClient, "API client should be nil when FIPS mode is enabled") 120 | 121 | // Initialize the listener 122 | ctx := listener.HandlerStarted(context.Background(), json.RawMessage{}) 123 | 124 | // Verify processor wasn't initialized 125 | assert.Nil(t, listener.processor, "Processor should be nil when FIPS mode is enabled") 126 | 127 | // Log calls to validate we're getting the expected log message 128 | var logOutput string 129 | logger.SetLogLevel(logger.LevelDebug) 130 | logOutput = captureOutput(func() { 131 | listener.AddDistributionMetric("fips-test-metric", 42, time.Now(), false, "tag:fips") 132 | }) 133 | 134 | // Check that we logged the skipping message 135 | assert.Contains(t, logOutput, "skipping metric fips-test-metric due to FIPS mode", "Expected log about skipping metric") 136 | assert.Contains(t, logOutput, "direct API calls are disabled", "Expected log about disabled API calls") 137 | 138 | // Finish the handler 139 | listener.HandlerFinished(ctx, nil) 140 | 141 | // Check that no API call was attempted 142 | assert.False(t, apiCallAttempted, "No API call should be attempted when FIPS mode is enabled") 143 | } 144 | 145 | func TestGetEnhancedMetricsTags(t *testing.T) { 146 | //nolint 147 | ctx := context.WithValue(context.Background(), "cold_start", false) 148 | 149 | lambdacontext.MemoryLimitInMB = 256 150 | lambdacontext.FunctionName = "go-lambda-test" 151 | lc := &lambdacontext.LambdaContext{ 152 | InvokedFunctionArn: "arn:aws:lambda:us-east-1:123497558138:function:go-lambda-test:$Latest", 153 | } 154 | tags := getEnhancedMetricsTags(lambdacontext.NewContext(ctx, lc)) 155 | 156 | assert.ElementsMatch(t, tags, []string{"functionname:go-lambda-test", "region:us-east-1", "memorysize:256", "cold_start:false", "account_id:123497558138", "resource:go-lambda-test:Latest", "datadog_lambda:v" + version.DDLambdaVersion}) 157 | } 158 | 159 | func TestGetEnhancedMetricsTagsWithAlias(t *testing.T) { 160 | //nolint 161 | ctx := context.WithValue(context.Background(), "cold_start", false) 162 | 163 | lambdacontext.MemoryLimitInMB = 256 164 | lambdacontext.FunctionName = "go-lambda-test" 165 | lambdacontext.FunctionVersion = "1" 166 | lc := &lambdacontext.LambdaContext{ 167 | InvokedFunctionArn: "arn:aws:lambda:us-east-1:123497558138:function:go-lambda-test:my-alias", 168 | } 169 | 170 | tags := getEnhancedMetricsTags((lambdacontext.NewContext(ctx, lc))) 171 | assert.ElementsMatch(t, tags, []string{"functionname:go-lambda-test", "region:us-east-1", "memorysize:256", "cold_start:false", "account_id:123497558138", "resource:go-lambda-test:my-alias", "executedversion:1", "datadog_lambda:v" + version.DDLambdaVersion}) 172 | } 173 | 174 | func TestGetEnhancedMetricsTagsNoLambdaContext(t *testing.T) { 175 | //nolint 176 | ctx := context.WithValue(context.Background(), "cold_start", true) 177 | tags := getEnhancedMetricsTags(ctx) 178 | 179 | assert.Empty(t, tags) 180 | } 181 | 182 | func TestSubmitEnhancedMetrics(t *testing.T) { 183 | called := false 184 | server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 185 | called = true 186 | w.WriteHeader(http.StatusCreated) 187 | })) 188 | defer server.Close() 189 | ml := MakeListener( 190 | Config{ 191 | APIKey: "abc-123", 192 | Site: server.URL, 193 | EnhancedMetrics: true, 194 | }, 195 | &extension.ExtensionManager{}, 196 | ) 197 | //nolint 198 | ctx := context.WithValue(context.Background(), "cold_start", false) 199 | 200 | output := captureOutput(func() { 201 | ctx = ml.HandlerStarted(ctx, json.RawMessage{}) 202 | ml.HandlerFinished(ctx, nil) 203 | }) 204 | 205 | assert.False(t, called) 206 | expected := "{\"m\":\"aws.lambda.enhanced.invocations\",\"v\":1," 207 | assert.True(t, strings.Contains(output, expected)) 208 | } 209 | 210 | func TestDoNotSubmitEnhancedMetrics(t *testing.T) { 211 | called := false 212 | server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 213 | called = true 214 | w.WriteHeader(http.StatusCreated) 215 | })) 216 | defer server.Close() 217 | 218 | ml := MakeListener( 219 | Config{ 220 | APIKey: "abc-123", 221 | Site: server.URL, 222 | EnhancedMetrics: false, 223 | }, 224 | &extension.ExtensionManager{}, 225 | ) 226 | //nolint 227 | ctx := context.WithValue(context.Background(), "cold_start", true) 228 | 229 | output := captureOutput(func() { 230 | ctx = ml.HandlerStarted(ctx, json.RawMessage{}) 231 | ml.HandlerFinished(ctx, nil) 232 | }) 233 | 234 | assert.False(t, called) 235 | expected := "{\"m\":\"aws.lambda.enhanced.invocations\",\"v\":1," 236 | assert.False(t, strings.Contains(output, expected)) 237 | } 238 | 239 | func TestSubmitEnhancedMetricsOnlyErrors(t *testing.T) { 240 | called := false 241 | server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 242 | called = true 243 | w.WriteHeader(http.StatusCreated) 244 | })) 245 | defer server.Close() 246 | 247 | ml := MakeListener( 248 | Config{ 249 | APIKey: "abc-123", 250 | Site: server.URL, 251 | EnhancedMetrics: false, 252 | }, 253 | &extension.ExtensionManager{}, 254 | ) 255 | //nolint 256 | ctx := context.WithValue(context.Background(), "cold_start", true) 257 | 258 | output := captureOutput(func() { 259 | ctx = ml.HandlerStarted(ctx, json.RawMessage{}) 260 | ml.config.EnhancedMetrics = true 261 | err := errors.New("something went wrong") 262 | ml.HandlerFinished(ctx, err) 263 | }) 264 | 265 | assert.False(t, called) 266 | expected := "{\"m\":\"aws.lambda.enhanced.errors\",\"v\":1," 267 | assert.True(t, strings.Contains(output, expected)) 268 | } 269 | 270 | func TestListenerHandlerFinishedFlushes(t *testing.T) { 271 | var called bool 272 | 273 | ts := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 274 | called = true 275 | })) 276 | ts.Listener.Close() 277 | ts.Listener, _ = net.Listen("tcp", "127.0.0.1:8124") 278 | 279 | ts.Start() 280 | defer ts.Close() 281 | 282 | listener := MakeListener(Config{}, extension.BuildExtensionManager(false)) 283 | listener.isAgentRunning = true 284 | for _, localTest := range []bool{true, false} { 285 | t.Run(fmt.Sprintf("%#v", localTest), func(t *testing.T) { 286 | called = false 287 | listener.config.LocalTest = localTest 288 | listener.HandlerFinished(context.TODO(), nil) 289 | assert.Equal(t, called, localTest) 290 | }) 291 | } 292 | } 293 | -------------------------------------------------------------------------------- /internal/metrics/model.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Unless explicitly stated otherwise all files in this repository are licensed 3 | * under the Apache License Version 2.0. 4 | * 5 | * This product includes software developed at Datadog (https://www.datadoghq.com/). 6 | * Copyright 2021 Datadog, Inc. 7 | */ 8 | 9 | package metrics 10 | 11 | import ( 12 | "time" 13 | ) 14 | 15 | type ( 16 | // Metric represents a metric that can have any kind of 17 | Metric interface { 18 | AddPoint(timestamp time.Time, value float64) 19 | ToAPIMetric(interval time.Duration) []APIMetric 20 | ToBatchKey() BatchKey 21 | Join(metric Metric) 22 | } 23 | 24 | // APIMetric is a metric that can be marshalled to send to the metrics API 25 | APIMetric struct { 26 | Name string `json:"metric"` 27 | Host *string `json:"host,omitempty"` 28 | Tags []string `json:"tags,omitempty"` 29 | MetricType MetricType `json:"type"` 30 | Interval *float64 `json:"interval,omitempty"` 31 | Points []interface{} `json:"points"` 32 | } 33 | 34 | // MetricValue represents a datapoint for a metric 35 | MetricValue struct { 36 | Value float64 37 | Timestamp time.Time 38 | } 39 | 40 | // Distribution is a type of metric that is aggregated over multiple hosts 41 | Distribution struct { 42 | Name string 43 | Tags []string 44 | Host *string 45 | Values []MetricValue 46 | } 47 | ) 48 | 49 | // AddPoint adds a point to the distribution metric 50 | func (d *Distribution) AddPoint(timestamp time.Time, value float64) { 51 | d.Values = append(d.Values, MetricValue{Timestamp: timestamp, Value: value}) 52 | } 53 | 54 | // ToBatchKey returns a key that can be used to batch the metric 55 | func (d *Distribution) ToBatchKey() BatchKey { 56 | return BatchKey{ 57 | name: d.Name, 58 | host: d.Host, 59 | tags: d.Tags, 60 | metricType: DistributionType, 61 | } 62 | } 63 | 64 | // Join creates a union between two metric sets 65 | func (d *Distribution) Join(metric Metric) { 66 | otherDist, ok := metric.(*Distribution) 67 | if !ok { 68 | return 69 | } 70 | for _, val := range otherDist.Values { 71 | d.AddPoint(val.Timestamp, val.Value) 72 | } 73 | 74 | } 75 | 76 | // ToAPIMetric converts a distribution into an API ready format. 77 | func (d *Distribution) ToAPIMetric(interval time.Duration) []APIMetric { 78 | points := make([]interface{}, len(d.Values)) 79 | 80 | for i, val := range d.Values { 81 | currentTime := float64(val.Timestamp.Unix()) 82 | 83 | points[i] = []interface{}{currentTime, []interface{}{val.Value}} 84 | } 85 | 86 | return []APIMetric{ 87 | APIMetric{ 88 | Name: d.Name, 89 | Host: d.Host, 90 | Tags: d.Tags, 91 | MetricType: DistributionType, 92 | Points: points, 93 | Interval: nil, 94 | }, 95 | } 96 | } 97 | -------------------------------------------------------------------------------- /internal/metrics/processor.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Unless explicitly stated otherwise all files in this repository are licensed 3 | * under the Apache License Version 2.0. 4 | * 5 | * This product includes software developed at Datadog (https://www.datadoghq.com/). 6 | * Copyright 2021 Datadog, Inc. 7 | */ 8 | 9 | package metrics 10 | 11 | import ( 12 | "context" 13 | "fmt" 14 | "sync" 15 | "time" 16 | 17 | "github.com/DataDog/datadog-lambda-go/internal/logger" 18 | "github.com/cenkalti/backoff/v4" 19 | "github.com/sony/gobreaker" 20 | ) 21 | 22 | type ( 23 | // Processor is used to batch metrics on a background thread, and send them on to a client periodically. 24 | Processor interface { 25 | // AddMetric sends a metric to the agent 26 | AddMetric(metric Metric) 27 | // StartProcessing begins processing metrics asynchronously 28 | StartProcessing() 29 | // FinishProcessing shuts down the agent, and tries to flush any remaining metrics 30 | FinishProcessing() 31 | // Whether the processor is still processing 32 | IsProcessing() bool 33 | } 34 | 35 | processor struct { 36 | context context.Context 37 | metricsChan chan Metric 38 | timeService TimeService 39 | waitGroup sync.WaitGroup 40 | batchInterval time.Duration 41 | client Client 42 | batcher *Batcher 43 | shouldRetryOnFail bool 44 | isProcessing bool 45 | breaker *gobreaker.CircuitBreaker 46 | } 47 | ) 48 | 49 | // MakeProcessor creates a new metrics context 50 | func MakeProcessor(ctx context.Context, client Client, timeService TimeService, batchInterval time.Duration, shouldRetryOnFail bool, circuitBreakerInterval time.Duration, circuitBreakerTimeout time.Duration, circuitBreakerTotalFailures uint32) Processor { 51 | batcher := MakeBatcher(batchInterval) 52 | 53 | breaker := MakeCircuitBreaker(circuitBreakerInterval, circuitBreakerTimeout, circuitBreakerTotalFailures) 54 | 55 | return &processor{ 56 | context: ctx, 57 | metricsChan: make(chan Metric, 2000), 58 | batchInterval: batchInterval, 59 | waitGroup: sync.WaitGroup{}, 60 | client: client, 61 | batcher: batcher, 62 | shouldRetryOnFail: shouldRetryOnFail, 63 | timeService: timeService, 64 | isProcessing: false, 65 | breaker: breaker, 66 | } 67 | } 68 | 69 | func MakeCircuitBreaker(circuitBreakerInterval time.Duration, circuitBreakerTimeout time.Duration, circuitBreakerTotalFailures uint32) *gobreaker.CircuitBreaker { 70 | readyToTrip := func(counts gobreaker.Counts) bool { 71 | return counts.TotalFailures > circuitBreakerTotalFailures 72 | } 73 | 74 | st := gobreaker.Settings{ 75 | Name: "post distribution_points", 76 | Interval: circuitBreakerInterval, 77 | Timeout: circuitBreakerTimeout, 78 | ReadyToTrip: readyToTrip, 79 | } 80 | return gobreaker.NewCircuitBreaker(st) 81 | } 82 | 83 | func (p *processor) AddMetric(metric Metric) { 84 | // We use a large buffer in the metrics channel, to make this operation non-blocking. 85 | // However, if the channel does fill up, this will become a blocking operation. 86 | p.metricsChan <- metric 87 | } 88 | 89 | func (p *processor) StartProcessing() { 90 | if !p.isProcessing { 91 | p.isProcessing = true 92 | p.waitGroup.Add(1) 93 | go p.processMetrics() 94 | } 95 | 96 | } 97 | 98 | func (p *processor) FinishProcessing() { 99 | if !p.isProcessing { 100 | p.StartProcessing() 101 | } 102 | // Closes the metrics channel, and waits for the last send to complete 103 | close(p.metricsChan) 104 | p.waitGroup.Wait() 105 | } 106 | 107 | func (p *processor) IsProcessing() bool { 108 | return p.isProcessing 109 | } 110 | 111 | func (p *processor) processMetrics() { 112 | 113 | ticker := p.timeService.NewTicker(p.batchInterval) 114 | 115 | doneChan := p.context.Done() 116 | shouldExit := false 117 | for !shouldExit { 118 | shouldSendBatch := false 119 | // Batches metrics until timeout is reached 120 | select { 121 | case <-doneChan: 122 | // This process is being cancelled by the context,(probably due to a lambda deadline), exit without flushing. 123 | shouldExit = true 124 | case m, ok := <-p.metricsChan: 125 | if !ok { 126 | // The channel has now been closed 127 | shouldSendBatch = true 128 | shouldExit = true 129 | } else { 130 | p.batcher.AddMetric(m) 131 | } 132 | case <-ticker.C: 133 | // We are ready to send a batch to our backend 134 | shouldSendBatch = true 135 | } 136 | // Since the go select statement picks randomly if multiple values are available, it's possible the done channel was 137 | // closed, but another channel was selected instead. We double check the done channel, to make sure this isn't he case. 138 | select { 139 | case <-doneChan: 140 | shouldExit = true 141 | shouldSendBatch = false 142 | default: 143 | // Non-blocking 144 | } 145 | 146 | if shouldSendBatch { 147 | _, err := p.breaker.Execute(func() (interface{}, error) { 148 | if shouldExit && p.shouldRetryOnFail { 149 | // If we are shutting down, and we just failed to send our last batch, do a retry 150 | bo := backoff.WithMaxRetries(backoff.NewConstantBackOff(defaultRetryInterval), 2) 151 | err := backoff.Retry(p.sendMetricsBatch, bo) 152 | if err != nil { 153 | return nil, fmt.Errorf("after retry: %v", err) 154 | } 155 | } else { 156 | err := p.sendMetricsBatch() 157 | if err != nil { 158 | return nil, fmt.Errorf("with no retry: %v", err) 159 | } 160 | } 161 | return nil, nil 162 | }) 163 | if err != nil { 164 | logger.Error(fmt.Errorf("failed to flush metrics to datadog API: %v", err)) 165 | } 166 | } 167 | } 168 | ticker.Stop() 169 | p.isProcessing = false 170 | p.waitGroup.Done() 171 | } 172 | 173 | func (p *processor) sendMetricsBatch() error { 174 | mts := p.batcher.ToAPIMetrics() 175 | if len(mts) > 0 { 176 | oldBatcher := p.batcher 177 | p.batcher = MakeBatcher(p.batchInterval) 178 | 179 | err := p.client.SendMetrics(mts) 180 | if err != nil { 181 | if p.shouldRetryOnFail { 182 | // If we want to retry on error, keep the metrics in the batcher until they are sent correctly. 183 | p.batcher = oldBatcher 184 | } 185 | return err 186 | } 187 | } 188 | return nil 189 | } 190 | -------------------------------------------------------------------------------- /internal/metrics/processor_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Unless explicitly stated otherwise all files in this repository are licensed 3 | * under the Apache License Version 2.0. 4 | * 5 | * This product includes software developed at Datadog (https://www.datadoghq.com/). 6 | * Copyright 2021 Datadog, Inc. 7 | */ 8 | 9 | package metrics 10 | 11 | import ( 12 | "context" 13 | "errors" 14 | "math" 15 | "testing" 16 | "time" 17 | 18 | "github.com/stretchr/testify/assert" 19 | ) 20 | 21 | type ( 22 | mockClient struct { 23 | batches chan []APIMetric 24 | sendMetricsCalledCount int 25 | err error 26 | } 27 | 28 | mockTimeService struct { 29 | now time.Time 30 | tickerChan chan time.Time 31 | } 32 | ) 33 | 34 | func makeMockClient() mockClient { 35 | return mockClient{ 36 | batches: make(chan []APIMetric, 10), 37 | err: nil, 38 | } 39 | } 40 | 41 | func makeMockTimeService() mockTimeService { 42 | return mockTimeService{ 43 | now: time.Now(), 44 | tickerChan: make(chan time.Time), 45 | } 46 | } 47 | 48 | func (mc *mockClient) SendMetrics(mts []APIMetric) error { 49 | mc.sendMetricsCalledCount++ 50 | mc.batches <- mts 51 | return mc.err 52 | } 53 | 54 | func (ts *mockTimeService) NewTicker(duration time.Duration) *time.Ticker { 55 | return &time.Ticker{ 56 | C: ts.tickerChan, 57 | } 58 | } 59 | 60 | func (ts *mockTimeService) Now() time.Time { 61 | return ts.now 62 | } 63 | 64 | func TestProcessorBatches(t *testing.T) { 65 | mc := makeMockClient() 66 | mts := makeMockTimeService() 67 | 68 | mts.now, _ = time.Parse(time.RFC3339, "2006-01-02T15:04:05Z") 69 | nowUnix := float64(mts.now.Unix()) 70 | 71 | processor := MakeProcessor(context.Background(), &mc, &mts, 1000, false, time.Hour*1000, time.Hour*1000, math.MaxUint32) 72 | 73 | d1 := Distribution{ 74 | Name: "metric-1", 75 | Tags: []string{"a", "b", "c"}, 76 | Values: []MetricValue{{Timestamp: mts.now, Value: 1}, {Timestamp: mts.now, Value: 2}, {Timestamp: mts.now, Value: 3}}, 77 | } 78 | d2 := Distribution{ 79 | Name: "metric-1", 80 | Tags: []string{"a", "b", "c"}, 81 | Values: []MetricValue{{Timestamp: mts.now, Value: 4}, {Timestamp: mts.now, Value: 5}, {Timestamp: mts.now, Value: 6}}, 82 | } 83 | 84 | processor.AddMetric(&d1) 85 | processor.AddMetric(&d2) 86 | 87 | processor.StartProcessing() 88 | processor.FinishProcessing() 89 | 90 | firstBatch := <-mc.batches 91 | 92 | assert.Equal(t, []APIMetric{{ 93 | Name: "metric-1", 94 | Tags: []string{"a", "b", "c"}, 95 | MetricType: DistributionType, 96 | Points: []interface{}{ 97 | []interface{}{nowUnix, []interface{}{float64(1)}}, 98 | []interface{}{nowUnix, []interface{}{float64(2)}}, 99 | []interface{}{nowUnix, []interface{}{float64(3)}}, 100 | []interface{}{nowUnix, []interface{}{float64(4)}}, 101 | []interface{}{nowUnix, []interface{}{float64(5)}}, 102 | []interface{}{nowUnix, []interface{}{float64(6)}}, 103 | }, 104 | }}, firstBatch) 105 | } 106 | 107 | func TestProcessorBatchesPerTick(t *testing.T) { 108 | mc := makeMockClient() 109 | mts := makeMockTimeService() 110 | 111 | firstTime, _ := time.Parse(time.RFC3339, "2006-01-02T15:04:05Z") 112 | firstTimeUnix := float64(firstTime.Unix()) 113 | secondTime, _ := time.Parse(time.RFC3339, "2007-01-02T15:04:05Z") 114 | secondTimeUnix := float64(secondTime.Unix()) 115 | mts.now = firstTime 116 | 117 | processor := MakeProcessor(context.Background(), &mc, &mts, 1000, false, time.Hour*1000, time.Hour*1000, math.MaxUint32) 118 | 119 | d1 := Distribution{ 120 | Name: "metric-1", 121 | Tags: []string{"a", "b", "c"}, 122 | Values: []MetricValue{{Timestamp: firstTime, Value: 1}, {Timestamp: firstTime, Value: 2}}, 123 | } 124 | d2 := Distribution{ 125 | Name: "metric-1", 126 | Tags: []string{"a", "b", "c"}, 127 | Values: []MetricValue{{Timestamp: firstTime, Value: 3}}, 128 | } 129 | d3 := Distribution{ 130 | Name: "metric-1", 131 | Tags: []string{"a", "b", "c"}, 132 | Values: []MetricValue{{Timestamp: secondTime, Value: 4}, {Timestamp: secondTime, Value: 5}}, 133 | } 134 | d4 := Distribution{ 135 | Name: "metric-1", 136 | Tags: []string{"a", "b", "c"}, 137 | Values: []MetricValue{{Timestamp: secondTime, Value: 6}}, 138 | } 139 | 140 | processor.StartProcessing() 141 | 142 | processor.AddMetric(&d1) 143 | processor.AddMetric(&d2) 144 | 145 | // This wait is necessary to make sure both metrics have been added to the batch 146 | <-time.Tick(time.Millisecond * 10) 147 | // Sending time to the ticker channel will flush the batch. 148 | mts.tickerChan <- firstTime 149 | firstBatch := <-mc.batches 150 | mts.now = secondTime 151 | 152 | processor.AddMetric(&d3) 153 | processor.AddMetric(&d4) 154 | 155 | processor.FinishProcessing() 156 | secondBatch := <-mc.batches 157 | batches := [][]APIMetric{firstBatch, secondBatch} 158 | 159 | assert.Equal(t, [][]APIMetric{ 160 | []APIMetric{ 161 | { 162 | Name: "metric-1", 163 | Tags: []string{"a", "b", "c"}, 164 | MetricType: DistributionType, 165 | Points: []interface{}{ 166 | []interface{}{firstTimeUnix, []interface{}{float64(1)}}, 167 | []interface{}{firstTimeUnix, []interface{}{float64(2)}}, 168 | []interface{}{firstTimeUnix, []interface{}{float64(3)}}, 169 | }, 170 | }}, 171 | []APIMetric{ 172 | { 173 | Name: "metric-1", 174 | Tags: []string{"a", "b", "c"}, 175 | MetricType: DistributionType, 176 | Points: []interface{}{ 177 | []interface{}{secondTimeUnix, []interface{}{float64(4)}}, 178 | []interface{}{secondTimeUnix, []interface{}{float64(5)}}, 179 | []interface{}{secondTimeUnix, []interface{}{float64(6)}}, 180 | }, 181 | }}, 182 | }, batches) 183 | } 184 | 185 | func TestProcessorPerformsRetry(t *testing.T) { 186 | mc := makeMockClient() 187 | mts := makeMockTimeService() 188 | 189 | mts.now, _ = time.Parse(time.RFC3339, "2006-01-02T15:04:05Z") 190 | 191 | shouldRetry := true 192 | processor := MakeProcessor(context.Background(), &mc, &mts, 1000, shouldRetry, time.Hour*1000, time.Hour*1000, math.MaxUint32) 193 | 194 | d1 := Distribution{ 195 | Name: "metric-1", 196 | Tags: []string{"a", "b", "c"}, 197 | Values: []MetricValue{{Timestamp: mts.now, Value: 1}, {Timestamp: mts.now, Value: 2}, {Timestamp: mts.now, Value: 3}}, 198 | } 199 | 200 | mc.err = errors.New("Some error") 201 | 202 | processor.AddMetric(&d1) 203 | 204 | processor.FinishProcessing() 205 | 206 | assert.Equal(t, 3, mc.sendMetricsCalledCount) 207 | } 208 | 209 | func TestProcessorCancelsWithContext(t *testing.T) { 210 | mc := makeMockClient() 211 | mts := makeMockTimeService() 212 | 213 | mts.now, _ = time.Parse(time.RFC3339, "2006-01-02T15:04:05Z") 214 | 215 | shouldRetry := true 216 | ctx, cancelFunc := context.WithCancel(context.Background()) 217 | processor := MakeProcessor(ctx, &mc, &mts, 1000, shouldRetry, time.Hour*1000, time.Hour*1000, math.MaxUint32) 218 | 219 | d1 := Distribution{ 220 | Name: "metric-1", 221 | Tags: []string{"a", "b", "c"}, 222 | Values: []MetricValue{{Timestamp: mts.now, Value: 1}, {Timestamp: mts.now, Value: 2}, {Timestamp: mts.now, Value: 3}}, 223 | } 224 | 225 | processor.AddMetric(&d1) 226 | // After calling cancelFunc, no metrics should be processed/sent 227 | cancelFunc() 228 | //<-time.Tick(time.Millisecond * 100) 229 | 230 | processor.FinishProcessing() 231 | 232 | assert.Equal(t, 0, mc.sendMetricsCalledCount) 233 | } 234 | 235 | func TestProcessorBatchesWithOpeningCircuitBreaker(t *testing.T) { 236 | mc := makeMockClient() 237 | mts := makeMockTimeService() 238 | 239 | mts.now, _ = time.Parse(time.RFC3339, "2006-01-02T15:04:05Z") 240 | 241 | // Will open the circuit breaker at number of total failures > 1 242 | circuitBreakerTotalFailures := uint32(1) 243 | processor := MakeProcessor(context.Background(), &mc, &mts, 1000, false, time.Hour*1000, time.Hour*1000, circuitBreakerTotalFailures) 244 | 245 | d1 := Distribution{ 246 | Name: "metric-1", 247 | Tags: []string{"a", "b", "c"}, 248 | Values: []MetricValue{{Timestamp: mts.now, Value: 1}, {Timestamp: mts.now, Value: 2}, {Timestamp: mts.now, Value: 3}}, 249 | } 250 | 251 | mc.err = errors.New("Some error") 252 | 253 | processor.AddMetric(&d1) 254 | 255 | processor.FinishProcessing() 256 | 257 | // It should have retried 3 times, but circuit breaker opened at the second time 258 | assert.Equal(t, 1, mc.sendMetricsCalledCount) 259 | } 260 | -------------------------------------------------------------------------------- /internal/metrics/time.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Unless explicitly stated otherwise all files in this repository are licensed 3 | * under the Apache License Version 2.0. 4 | * 5 | * This product includes software developed at Datadog (https://www.datadoghq.com/). 6 | * Copyright 2021 Datadog, Inc. 7 | */ 8 | 9 | package metrics 10 | 11 | import "time" 12 | 13 | type ( 14 | //TimeService wraps common time related operations 15 | TimeService interface { 16 | NewTicker(duration time.Duration) *time.Ticker 17 | Now() time.Time 18 | } 19 | 20 | timeService struct { 21 | } 22 | ) 23 | 24 | // MakeTimeService creates a new time service 25 | func MakeTimeService() TimeService { 26 | return &timeService{} 27 | } 28 | 29 | func (ts *timeService) NewTicker(duration time.Duration) *time.Ticker { 30 | return time.NewTicker(duration) 31 | } 32 | 33 | func (ts *timeService) Now() time.Time { 34 | return time.Now() 35 | } 36 | -------------------------------------------------------------------------------- /internal/testdata/apig-event-no-headers.json: -------------------------------------------------------------------------------- 1 | { 2 | "body": "{{.body}}", 3 | "resource": "{{.resource}}", 4 | "requestContext": { 5 | "resourceId": "123456", 6 | "apiId": "1234567890", 7 | "resourcePath": "{{.resource}}", 8 | "httpMethod": "{{.method}}", 9 | "requestId": "c6af9ac6-7b61-11e6-9a41-93e8deadbeef", 10 | "accountId": "123456789012", 11 | "identity": { 12 | "apiKey": null, 13 | "userArn": null, 14 | "cognitoAuthenticationType": null, 15 | "caller": null, 16 | "userAgent": "Custom User Agent String", 17 | "user": null, 18 | "cognitoIdentityPoolId": null, 19 | "cognitoIdentityId": null, 20 | "cognitoAuthenticationProvider": null, 21 | "sourceIp": "127.0.0.1", 22 | "accountId": null 23 | }, 24 | "stage": "prod" 25 | }, 26 | "queryStringParameters": { 27 | "foo": "bar" 28 | }, 29 | "headers": { 30 | "Via": "1.1 08f323deadbeefa7af34d5feb414ce27.cloudfront.net (CloudFront)", 31 | "Accept-Language": "en-US,en;q=0.8", 32 | "CloudFront-Is-Desktop-Viewer": "true", 33 | "CloudFront-Is-SmartTV-Viewer": "false", 34 | "CloudFront-Is-Mobile-Viewer": "false", 35 | "X-Forwarded-For": "127.0.0.1, 127.0.0.2", 36 | "CloudFront-Viewer-Country": "US", 37 | "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8", 38 | "Upgrade-Insecure-Requests": "1", 39 | "X-Forwarded-Port": "443", 40 | "Host": "1234567890.execute-api.us-east-1.amazonaws.com", 41 | "X-Forwarded-Proto": "https", 42 | "X-Amz-Cf-Id": "cDehVQoZnx43VYQb9j2-nvCh-9z396Uhbp027Y2JvkCPNLmGJHqlaA==", 43 | "CloudFront-Is-Tablet-Viewer": "false", 44 | "Cache-Control": "max-age=0", 45 | "User-Agent": "Custom User Agent String", 46 | "CloudFront-Forwarded-Proto": "https", 47 | "Accept-Encoding": "gzip, deflate, sdch" 48 | }, 49 | "pathParameters": { 50 | "proxy": "{{.path}}" 51 | }, 52 | "httpMethod": "{{.method}}", 53 | "stageVariables": { 54 | "baz": "qux" 55 | }, 56 | "path": "{{.path}}" 57 | } -------------------------------------------------------------------------------- /internal/testdata/apig-event-with-headers.json: -------------------------------------------------------------------------------- 1 | { 2 | "body": "{{.body}}", 3 | "resource": "{{.resource}}", 4 | "requestContext": { 5 | "resourceId": "123456", 6 | "apiId": "1234567890", 7 | "resourcePath": "{{.resource}}", 8 | "httpMethod": "{{.method}}", 9 | "requestId": "c6af9ac6-7b61-11e6-9a41-93e8deadbeef", 10 | "accountId": "123456789012", 11 | "identity": { 12 | "apiKey": null, 13 | "userArn": null, 14 | "cognitoAuthenticationType": null, 15 | "caller": null, 16 | "userAgent": "Custom User Agent String", 17 | "user": null, 18 | "cognitoIdentityPoolId": null, 19 | "cognitoIdentityId": null, 20 | "cognitoAuthenticationProvider": null, 21 | "sourceIp": "127.0.0.1", 22 | "accountId": null 23 | }, 24 | "stage": "prod" 25 | }, 26 | "queryStringParameters": { 27 | "foo": "bar" 28 | }, 29 | "headers": { 30 | "Via": "1.1 08f323deadbeefa7af34d5feb414ce27.cloudfront.net (CloudFront)", 31 | "Accept-Language": "en-US,en;q=0.8", 32 | "CloudFront-Is-Desktop-Viewer": "true", 33 | "CloudFront-Is-SmartTV-Viewer": "false", 34 | "CloudFront-Is-Mobile-Viewer": "false", 35 | "X-Forwarded-For": "127.0.0.1, 127.0.0.2", 36 | "CloudFront-Viewer-Country": "US", 37 | "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8", 38 | "Upgrade-Insecure-Requests": "1", 39 | "X-Forwarded-Port": "443", 40 | "Host": "1234567890.execute-api.us-east-1.amazonaws.com", 41 | "X-Forwarded-Proto": "https", 42 | "X-Amz-Cf-Id": "cDehVQoZnx43VYQb9j2-nvCh-9z396Uhbp027Y2JvkCPNLmGJHqlaA==", 43 | "CloudFront-Is-Tablet-Viewer": "false", 44 | "Cache-Control": "max-age=0", 45 | "User-Agent": "Custom User Agent String", 46 | "CloudFront-Forwarded-Proto": "https", 47 | "Accept-Encoding": "gzip, deflate, sdch", 48 | "x-datadog-trace-id": "1231452342", 49 | "x-datadog-parent-id": "45678910", 50 | "x-datadog-sampling-priority": "2" 51 | }, 52 | "pathParameters": { 53 | "proxy": "{{.path}}" 54 | }, 55 | "httpMethod": "{{.method}}", 56 | "stageVariables": { 57 | "baz": "qux" 58 | }, 59 | "path": "{{.path}}" 60 | } -------------------------------------------------------------------------------- /internal/testdata/invalid.json: -------------------------------------------------------------------------------- 1 | { 2 | "tag-left-open": [ 3 | " -------------------------------------------------------------------------------- /internal/testdata/non-proxy-no-headers.json: -------------------------------------------------------------------------------- 1 | { 2 | "my-custom-event": { 3 | "hello": 100 4 | }, 5 | "fake-id": "12345678910" 6 | } -------------------------------------------------------------------------------- /internal/testdata/non-proxy-with-headers.json: -------------------------------------------------------------------------------- 1 | { 2 | "my-custom-event": { 3 | "hello": 100 4 | }, 5 | "fake-id": "12345678910", 6 | "headers": { 7 | "x-datadog-trace-id": "1231452342", 8 | "x-datadog-parent-id": "45678910", 9 | "x-datadog-sampling-priority": "2" 10 | } 11 | } -------------------------------------------------------------------------------- /internal/testdata/non-proxy-with-missing-sampling-priority.json: -------------------------------------------------------------------------------- 1 | { 2 | "my-custom-event": { 3 | "hello": 100 4 | }, 5 | "fake-id": "12345678910", 6 | "headers": { 7 | "X-Datadog-Trace-Id": "1231452342", 8 | "X-Datadog-Parent-Id": "45678910" 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /internal/testdata/non-proxy-with-mixed-case-headers.json: -------------------------------------------------------------------------------- 1 | { 2 | "my-custom-event": { 3 | "hello": 100 4 | }, 5 | "fake-id": "12345678910", 6 | "headers": { 7 | "X-Datadog-Trace-Id": "1231452342", 8 | "X-Datadog-Parent-Id": "45678910", 9 | "X-Datadog-Sampling-Priority": "2" 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /internal/trace/constants.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Unless explicitly stated otherwise all files in this repository are licensed 3 | * under the Apache License Version 2.0. 4 | * 5 | * This product includes software developed at Datadog (https://www.datadoghq.com/). 6 | * Copyright 2021 Datadog, Inc. 7 | */ 8 | 9 | package trace 10 | 11 | const ( 12 | traceIDHeader = "x-datadog-trace-id" 13 | parentIDHeader = "x-datadog-parent-id" 14 | samplingPriorityHeader = "x-datadog-sampling-priority" 15 | ) 16 | 17 | const ( 18 | userReject = "-1" 19 | // autoReject = "0" 20 | // autoKeep = "1" 21 | userKeep = "2" 22 | ) 23 | 24 | const ( 25 | xraySubsegmentName = "datadog-metadata" 26 | xraySubsegmentKey = "trace" 27 | xraySubsegmentNamespace = "datadog" 28 | ) 29 | -------------------------------------------------------------------------------- /internal/trace/context.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Unless explicitly stated otherwise all files in this repository are licensed 3 | * under the Apache License Version 2.0. 4 | * 5 | * This product includes software developed at Datadog (https://www.datadoghq.com/). 6 | * Copyright 2021 Datadog, Inc. 7 | */ 8 | 9 | package trace 10 | 11 | import ( 12 | "context" 13 | "encoding/binary" 14 | "encoding/hex" 15 | "encoding/json" 16 | "fmt" 17 | "strconv" 18 | "strings" 19 | 20 | "github.com/DataDog/datadog-lambda-go/internal/extension" 21 | "github.com/DataDog/datadog-lambda-go/internal/logger" 22 | "github.com/aws/aws-xray-sdk-go/v2/header" 23 | "github.com/aws/aws-xray-sdk-go/v2/xray" 24 | "gopkg.in/DataDog/dd-trace-go.v1/ddtrace" 25 | "gopkg.in/DataDog/dd-trace-go.v1/ddtrace/tracer" 26 | ) 27 | 28 | type ( 29 | eventWithHeaders struct { 30 | Headers map[string]string `json:"headers"` 31 | } 32 | 33 | // TraceContext is map of headers containing a Datadog trace context. 34 | TraceContext map[string]string 35 | 36 | // ContextExtractor is a func type for extracting a root TraceContext. 37 | ContextExtractor func(ctx context.Context, ev json.RawMessage) map[string]string 38 | ) 39 | 40 | type contextKeytype int 41 | 42 | // traceContextKey is the key used to store a TraceContext in a TraceContext object 43 | var traceContextKey = new(contextKeytype) 44 | 45 | // DefaultTraceExtractor is the default trace extractor. Extracts root trace from API Gateway headers. 46 | var DefaultTraceExtractor = getHeadersFromEventHeaders 47 | 48 | // contextWithRootTraceContext uses the incoming event and context object payloads to determine 49 | // the root TraceContext and then adds that TraceContext to the context object. 50 | func contextWithRootTraceContext(ctx context.Context, ev json.RawMessage, mergeXrayTraces bool, extractor ContextExtractor) (context.Context, error) { 51 | datadogTraceContext, gotDatadogTraceContext := getTraceContext(ctx, extractor(ctx, ev)) 52 | 53 | xrayTraceContext, errGettingXrayContext := convertXrayTraceContextFromLambdaContext(ctx) 54 | if errGettingXrayContext != nil { 55 | logger.Error(fmt.Errorf("Couldn't convert X-Ray trace context: %v", errGettingXrayContext)) 56 | } 57 | 58 | if gotDatadogTraceContext && errGettingXrayContext == nil { 59 | err := createDummySubsegmentForXrayConverter(ctx, datadogTraceContext) 60 | if err != nil { 61 | logger.Error(fmt.Errorf("Couldn't create segment: %v", err)) 62 | } 63 | } 64 | 65 | if !mergeXrayTraces { 66 | logger.Debug("Merge X-Ray Traces is off, using trace context from Datadog only") 67 | return context.WithValue(ctx, traceContextKey, datadogTraceContext), nil 68 | } 69 | 70 | if !gotDatadogTraceContext { 71 | logger.Debug("Merge X-Ray Traces is on, but did not get incoming Datadog trace context; using X-Ray trace context instead") 72 | return context.WithValue(ctx, traceContextKey, xrayTraceContext), nil 73 | } 74 | 75 | logger.Debug("Using merged Datadog/X-Ray trace context") 76 | mergedTraceContext := TraceContext{} 77 | mergedTraceContext[traceIDHeader] = datadogTraceContext[traceIDHeader] 78 | mergedTraceContext[samplingPriorityHeader] = datadogTraceContext[samplingPriorityHeader] 79 | mergedTraceContext[parentIDHeader] = xrayTraceContext[parentIDHeader] 80 | return context.WithValue(ctx, traceContextKey, mergedTraceContext), nil 81 | } 82 | 83 | // ConvertCurrentXrayTraceContext returns the current X-Ray trace context converted to Datadog headers, taking into account 84 | // the current subsegment. It is designed for sending Datadog trace headers from functions instrumented with the X-Ray SDK. 85 | func ConvertCurrentXrayTraceContext(ctx context.Context) TraceContext { 86 | if xrayTraceContext, err := convertXrayTraceContextFromLambdaContext(ctx); err == nil { 87 | // If there is an active X-Ray segment, use it as the parent 88 | parentID := xrayTraceContext[parentIDHeader] 89 | segment := xray.GetSegment(ctx) 90 | if segment != nil { 91 | newParentID, err := convertXRayEntityIDToDatadogParentID(segment.ID) 92 | if err == nil { 93 | parentID = newParentID 94 | } 95 | } 96 | 97 | newTraceContext := map[string]string{} 98 | newTraceContext[traceIDHeader] = xrayTraceContext[traceIDHeader] 99 | newTraceContext[samplingPriorityHeader] = xrayTraceContext[samplingPriorityHeader] 100 | newTraceContext[parentIDHeader] = parentID 101 | 102 | return newTraceContext 103 | } 104 | return map[string]string{} 105 | } 106 | 107 | // createDummySubsegmentForXrayConverter creates a dummy X-Ray subsegment containing Datadog trace context metadata. 108 | // This metadata is used by the Datadog X-Ray converter to parent the X-Ray trace under the Datadog trace. 109 | // This subsegment will be dropped by the X-Ray converter and will not appear in Datadog. 110 | func createDummySubsegmentForXrayConverter(ctx context.Context, traceCtx TraceContext) error { 111 | _, segment := xray.BeginSubsegment(ctx, xraySubsegmentName) 112 | 113 | traceID := traceCtx[traceIDHeader] 114 | parentID := traceCtx[parentIDHeader] 115 | sampled := traceCtx[samplingPriorityHeader] 116 | metadata := map[string]string{ 117 | "trace-id": traceID, 118 | "parent-id": parentID, 119 | "sampling-priority": sampled, 120 | } 121 | 122 | err := segment.AddMetadataToNamespace(xraySubsegmentNamespace, xraySubsegmentKey, metadata) 123 | if err != nil { 124 | return fmt.Errorf("couldn't save trace context to XRay: %v", err) 125 | } 126 | segment.Close(nil) 127 | return nil 128 | } 129 | 130 | func getTraceContext(ctx context.Context, headers map[string]string) (TraceContext, bool) { 131 | tc := TraceContext{} 132 | 133 | traceID := headers[traceIDHeader] 134 | if traceID == "" { 135 | if val, ok := ctx.Value(extension.DdTraceId).(string); ok { 136 | traceID = val 137 | } 138 | } 139 | if traceID == "" { 140 | return tc, false 141 | } 142 | 143 | parentID := headers[parentIDHeader] 144 | if parentID == "" { 145 | if val, ok := ctx.Value(extension.DdParentId).(string); ok { 146 | parentID = val 147 | } 148 | } 149 | if parentID == "" { 150 | return tc, false 151 | } 152 | 153 | samplingPriority := headers[samplingPriorityHeader] 154 | if samplingPriority == "" { 155 | if val, ok := ctx.Value(extension.DdSamplingPriority).(string); ok { 156 | samplingPriority = val 157 | } 158 | } 159 | if samplingPriority == "" { 160 | samplingPriority = "1" //sampler-keep 161 | } 162 | 163 | tc[samplingPriorityHeader] = samplingPriority 164 | tc[traceIDHeader] = traceID 165 | tc[parentIDHeader] = parentID 166 | 167 | return tc, true 168 | } 169 | 170 | // getHeadersFromEventHeaders extracts the Datadog trace context from an incoming Lambda event payload 171 | // and creates a dummy X-Ray subsegment containing this information. 172 | // This is used as the DefaultTraceExtractor. 173 | func getHeadersFromEventHeaders(ctx context.Context, ev json.RawMessage) map[string]string { 174 | eh := eventWithHeaders{} 175 | 176 | headers := map[string]string{} 177 | 178 | err := json.Unmarshal(ev, &eh) 179 | if err != nil { 180 | return headers 181 | } 182 | 183 | lowercaseHeaders := map[string]string{} 184 | for k, v := range eh.Headers { 185 | lowercaseHeaders[strings.ToLower(k)] = v 186 | } 187 | 188 | return lowercaseHeaders 189 | } 190 | 191 | func convertXrayTraceContextFromLambdaContext(ctx context.Context) (TraceContext, error) { 192 | traceCtx := map[string]string{} 193 | 194 | header := getXrayTraceHeaderFromContext(ctx) 195 | if header == nil { 196 | return traceCtx, fmt.Errorf("Couldn't read X-Ray trace context from Lambda context object") 197 | } 198 | 199 | traceID, err := convertXRayTraceIDToDatadogTraceID(header.TraceID) 200 | if err != nil { 201 | return traceCtx, fmt.Errorf("Couldn't read trace id from X-Ray: %v", err) 202 | } 203 | parentID, err := convertXRayEntityIDToDatadogParentID(header.ParentID) 204 | if err != nil { 205 | return traceCtx, fmt.Errorf("Couldn't read parent id from X-Ray: %v", err) 206 | } 207 | samplingPriority := convertXRaySamplingDecision(header.SamplingDecision) 208 | 209 | traceCtx[traceIDHeader] = traceID 210 | traceCtx[parentIDHeader] = parentID 211 | traceCtx[samplingPriorityHeader] = samplingPriority 212 | return traceCtx, nil 213 | } 214 | 215 | // getXrayTraceHeaderFromContext is used to extract xray segment metadata from the lambda context object. 216 | // By default, the context object won't have any Segment, (xray.GetSegment(ctx) will return nil). However it 217 | // will have the "LambdaTraceHeader" object, which contains the traceID/parentID/sampling info. 218 | func getXrayTraceHeaderFromContext(ctx context.Context) *header.Header { 219 | var traceHeader string 220 | 221 | if traceHeaderValue := ctx.Value(xray.LambdaTraceHeaderKey); traceHeaderValue != nil { 222 | traceHeader = traceHeaderValue.(string) 223 | return header.FromString(traceHeader) 224 | } 225 | return nil 226 | } 227 | 228 | // Converts the last 63 bits of an X-Ray trace ID (hex) to a Datadog trace id (uint64). 229 | func convertXRayTraceIDToDatadogTraceID(traceID string) (string, error) { 230 | parts := strings.Split(traceID, "-") 231 | 232 | if len(parts) != 3 { 233 | return "0", fmt.Errorf("invalid x-ray trace id; expected 3 components in id") 234 | } 235 | if len(parts[2]) != 24 { 236 | return "0", fmt.Errorf("x-ray trace id should be 96 bits") 237 | } 238 | 239 | traceIDLength := len(parts[2]) - 16 240 | traceID = parts[2][traceIDLength : traceIDLength+16] // Per XRay Team: use the last 64 bits of the trace id 241 | apmTraceID, err := convertHexIDToUint64(traceID) 242 | if err != nil { 243 | return "0", fmt.Errorf("while converting xray trace id: %v", err) 244 | } 245 | apmTraceID = 0x7FFFFFFFFFFFFFFF & apmTraceID // The APM Trace ID is restricted to 63 bits, so make sure the 64th bit is always 0 246 | return strconv.FormatUint(apmTraceID, 10), nil 247 | } 248 | 249 | func convertHexIDToUint64(hexNumber string) (uint64, error) { 250 | ba, err := hex.DecodeString(hexNumber) 251 | if err != nil { 252 | return 0, fmt.Errorf("couldn't convert hex to uint64: %v", err) 253 | } 254 | return binary.BigEndian.Uint64(ba), nil // TODO: Verify that this is correct 255 | } 256 | 257 | // Converts an X-Ray entity ID (hex) to a Datadog parent id (uint64). 258 | func convertXRayEntityIDToDatadogParentID(entityID string) (string, error) { 259 | if len(entityID) < 16 { 260 | return "0", fmt.Errorf("couldn't convert to trace id, too short") 261 | } 262 | val, err := convertHexIDToUint64(entityID[len(entityID)-16:]) 263 | if err != nil { 264 | return "0", fmt.Errorf("couldn't convert entity id to trace id: %v", err) 265 | } 266 | return strconv.FormatUint(val, 10), nil 267 | } 268 | 269 | // Converts an X-Ray sampling decision into its Datadog counterpart. 270 | func convertXRaySamplingDecision(decision header.SamplingDecision) string { 271 | if decision == header.Sampled { 272 | return userKeep 273 | } 274 | return userReject 275 | } 276 | 277 | // ConvertTraceContextToSpanContext converts a TraceContext object to a SpanContext that can be used by dd-trace. 278 | func ConvertTraceContextToSpanContext(traceCtx TraceContext) (ddtrace.SpanContext, error) { 279 | spanCtx, err := propagator.Extract(tracer.TextMapCarrier(traceCtx)) 280 | 281 | if err != nil { 282 | logger.Debug("Could not convert TraceContext to a SpanContext (most likely TraceContext was empty)") 283 | return nil, err 284 | } 285 | 286 | return spanCtx, nil 287 | } 288 | 289 | // propagator is able to extract a SpanContext object from a TraceContext object 290 | var propagator = tracer.NewPropagator(&tracer.PropagatorConfig{ 291 | TraceHeader: traceIDHeader, 292 | ParentHeader: parentIDHeader, 293 | PriorityHeader: samplingPriorityHeader, 294 | }) 295 | -------------------------------------------------------------------------------- /internal/trace/context_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Unless explicitly stated otherwise all files in this repository are licensed 3 | * under the Apache License Version 2.0. 4 | * 5 | * This product includes software developed at Datadog (https://www.datadoghq.com/). 6 | * Copyright 2021 Datadog, Inc. 7 | */ 8 | 9 | package trace 10 | 11 | import ( 12 | "context" 13 | "encoding/json" 14 | "os" 15 | "testing" 16 | 17 | "github.com/DataDog/datadog-lambda-go/internal/extension" 18 | "github.com/aws/aws-xray-sdk-go/v2/header" 19 | "github.com/aws/aws-xray-sdk-go/v2/xray" 20 | "github.com/stretchr/testify/assert" 21 | ) 22 | 23 | const ( 24 | mockXRayEntityID = "0b11cc4230d3e09e" 25 | mockXRayTraceID = "1-5ce31dc2-2c779014b90ce44db5e03875" 26 | convertedXRayEntityID = "797643193680388254" 27 | convertedXRayTraceID = "4110911582297405557" 28 | ) 29 | 30 | func mockLambdaXRayTraceContext(ctx context.Context, traceID, parentID string, sampled bool) context.Context { 31 | decision := header.NotSampled 32 | if sampled { 33 | decision = header.Sampled 34 | } 35 | 36 | traceHeader := header.Header{ 37 | TraceID: traceID, 38 | ParentID: parentID, 39 | SamplingDecision: decision, 40 | AdditionalData: make(TraceContext), 41 | } 42 | headerString := traceHeader.String() 43 | //nolint 44 | return context.WithValue(ctx, xray.LambdaTraceHeaderKey, headerString) 45 | } 46 | 47 | func mockTraceContext(traceID, parentID, samplingPriority string) context.Context { 48 | ctx := context.Background() 49 | if traceID != "" { 50 | ctx = context.WithValue(ctx, extension.DdTraceId, traceID) 51 | } 52 | if parentID != "" { 53 | ctx = context.WithValue(ctx, extension.DdParentId, parentID) 54 | } 55 | if samplingPriority != "" { 56 | ctx = context.WithValue(ctx, extension.DdSamplingPriority, samplingPriority) 57 | } 58 | return ctx 59 | } 60 | 61 | func loadRawJSON(t *testing.T, filename string) *json.RawMessage { 62 | bytes, err := os.ReadFile(filename) 63 | if err != nil { 64 | assert.Fail(t, "Couldn't find JSON file") 65 | return nil 66 | } 67 | msg := json.RawMessage{} 68 | err = msg.UnmarshalJSON(bytes) 69 | assert.NoError(t, err) 70 | return &msg 71 | } 72 | func TestGetDatadogTraceContextForTraceMetadataNonProxyEvent(t *testing.T) { 73 | ctx := mockLambdaXRayTraceContext(context.Background(), mockXRayTraceID, mockXRayEntityID, true) 74 | ev := loadRawJSON(t, "../testdata/apig-event-with-headers.json") 75 | 76 | headers, ok := getTraceContext(ctx, getHeadersFromEventHeaders(ctx, *ev)) 77 | assert.True(t, ok) 78 | 79 | expected := TraceContext{ 80 | traceIDHeader: "1231452342", 81 | parentIDHeader: "45678910", 82 | samplingPriorityHeader: "2", 83 | } 84 | assert.Equal(t, expected, headers) 85 | } 86 | 87 | func TestGetDatadogTraceContextForTraceMetadataWithMixedCaseHeaders(t *testing.T) { 88 | ctx := mockLambdaXRayTraceContext(context.Background(), mockXRayTraceID, mockXRayEntityID, true) 89 | ev := loadRawJSON(t, "../testdata/non-proxy-with-mixed-case-headers.json") 90 | 91 | headers, ok := getTraceContext(ctx, getHeadersFromEventHeaders(ctx, *ev)) 92 | assert.True(t, ok) 93 | 94 | expected := TraceContext{ 95 | traceIDHeader: "1231452342", 96 | parentIDHeader: "45678910", 97 | samplingPriorityHeader: "2", 98 | } 99 | assert.Equal(t, expected, headers) 100 | } 101 | 102 | func TestGetDatadogTraceContextForTraceMetadataWithMissingSamplingPriority(t *testing.T) { 103 | ctx := mockLambdaXRayTraceContext(context.Background(), mockXRayTraceID, mockXRayEntityID, true) 104 | ev := loadRawJSON(t, "../testdata/non-proxy-with-missing-sampling-priority.json") 105 | 106 | headers, ok := getTraceContext(ctx, getHeadersFromEventHeaders(ctx, *ev)) 107 | assert.True(t, ok) 108 | 109 | expected := TraceContext{ 110 | traceIDHeader: "1231452342", 111 | parentIDHeader: "45678910", 112 | samplingPriorityHeader: "1", 113 | } 114 | assert.Equal(t, expected, headers) 115 | } 116 | 117 | func TestGetDatadogTraceContextForInvalidData(t *testing.T) { 118 | ctx := mockLambdaXRayTraceContext(context.Background(), mockXRayTraceID, mockXRayEntityID, true) 119 | ev := loadRawJSON(t, "../testdata/invalid.json") 120 | 121 | _, ok := getTraceContext(ctx, getHeadersFromEventHeaders(ctx, *ev)) 122 | assert.False(t, ok) 123 | } 124 | 125 | func TestGetDatadogTraceContextForMissingData(t *testing.T) { 126 | ctx := mockLambdaXRayTraceContext(context.Background(), mockXRayTraceID, mockXRayEntityID, true) 127 | ev := loadRawJSON(t, "../testdata/non-proxy-no-headers.json") 128 | 129 | _, ok := getTraceContext(ctx, getHeadersFromEventHeaders(ctx, *ev)) 130 | assert.False(t, ok) 131 | } 132 | 133 | func TestGetDatadogTraceContextFromContextObject(t *testing.T) { 134 | testcases := []struct { 135 | traceID string 136 | parentID string 137 | samplingPriority string 138 | expectTC TraceContext 139 | expectOk bool 140 | }{ 141 | { 142 | "trace", 143 | "parent", 144 | "sampling", 145 | TraceContext{ 146 | "x-datadog-trace-id": "trace", 147 | "x-datadog-parent-id": "parent", 148 | "x-datadog-sampling-priority": "sampling", 149 | }, 150 | true, 151 | }, 152 | { 153 | "", 154 | "parent", 155 | "sampling", 156 | TraceContext{}, 157 | false, 158 | }, 159 | { 160 | "trace", 161 | "", 162 | "sampling", 163 | TraceContext{}, 164 | false, 165 | }, 166 | { 167 | "trace", 168 | "parent", 169 | "", 170 | TraceContext{ 171 | "x-datadog-trace-id": "trace", 172 | "x-datadog-parent-id": "parent", 173 | "x-datadog-sampling-priority": "1", 174 | }, 175 | true, 176 | }, 177 | } 178 | 179 | ev := loadRawJSON(t, "../testdata/non-proxy-no-headers.json") 180 | for _, test := range testcases { 181 | t.Run(test.traceID+test.parentID+test.samplingPriority, func(t *testing.T) { 182 | ctx := mockTraceContext(test.traceID, test.parentID, test.samplingPriority) 183 | tc, ok := getTraceContext(ctx, getHeadersFromEventHeaders(ctx, *ev)) 184 | assert.Equal(t, test.expectTC, tc) 185 | assert.Equal(t, test.expectOk, ok) 186 | }) 187 | } 188 | } 189 | 190 | func TestConvertXRayTraceID(t *testing.T) { 191 | output, err := convertXRayTraceIDToDatadogTraceID(mockXRayTraceID) 192 | assert.NoError(t, err) 193 | assert.Equal(t, convertedXRayTraceID, output) 194 | } 195 | 196 | func TestConvertXRayTraceIDTooShort(t *testing.T) { 197 | output, err := convertXRayTraceIDToDatadogTraceID("1-5ce31dc2-5e03875") 198 | assert.Error(t, err) 199 | assert.Equal(t, "0", output) 200 | } 201 | 202 | func TestConvertXRayTraceIDInvalidFormat(t *testing.T) { 203 | output, err := convertXRayTraceIDToDatadogTraceID("1-2c779014b90ce44db5e03875") 204 | assert.Error(t, err) 205 | assert.Equal(t, "0", output) 206 | } 207 | func TestConvertXRayTraceIDIncorrectCharacters(t *testing.T) { 208 | output, err := convertXRayTraceIDToDatadogTraceID("1-5ce31dc2-c779014b90ce44db5e03875;") 209 | assert.Error(t, err) 210 | assert.Equal(t, "0", output) 211 | } 212 | 213 | func TestConvertXRayEntityID(t *testing.T) { 214 | output, err := convertXRayEntityIDToDatadogParentID(mockXRayEntityID) 215 | assert.NoError(t, err) 216 | assert.Equal(t, convertedXRayEntityID, output) 217 | } 218 | 219 | func TestConvertXRayEntityIDInvalidFormat(t *testing.T) { 220 | output, err := convertXRayEntityIDToDatadogParentID(";b11cc4230d3e09e") 221 | assert.Error(t, err) 222 | assert.Equal(t, "0", output) 223 | } 224 | 225 | func TestConvertXRayEntityIDTooShort(t *testing.T) { 226 | output, err := convertXRayEntityIDToDatadogParentID("c4230d3e09e") 227 | assert.Error(t, err) 228 | assert.Equal(t, "0", output) 229 | } 230 | 231 | func TestXrayTraceContextNoSegment(t *testing.T) { 232 | ctx := context.Background() 233 | 234 | _, err := convertXrayTraceContextFromLambdaContext(ctx) 235 | assert.Error(t, err) 236 | } 237 | func TestXrayTraceContextWithSegment(t *testing.T) { 238 | 239 | ctx := mockLambdaXRayTraceContext(context.Background(), mockXRayTraceID, mockXRayEntityID, true) 240 | 241 | headers, err := convertXrayTraceContextFromLambdaContext(ctx) 242 | assert.NoError(t, err) 243 | assert.Equal(t, "2", headers[samplingPriorityHeader]) 244 | assert.NotNil(t, headers[traceIDHeader]) 245 | assert.NotNil(t, headers[parentIDHeader]) 246 | } 247 | 248 | func TestContextWithRootTraceContextNoDatadogContext(t *testing.T) { 249 | ctx := mockLambdaXRayTraceContext(context.Background(), mockXRayTraceID, mockXRayEntityID, true) 250 | ev := loadRawJSON(t, "../testdata/apig-event-no-headers.json") 251 | 252 | newCTX, _ := contextWithRootTraceContext(ctx, *ev, false, DefaultTraceExtractor) 253 | traceContext, _ := newCTX.Value(traceContextKey).(TraceContext) 254 | 255 | expected := TraceContext{} 256 | assert.Equal(t, expected, traceContext) 257 | } 258 | 259 | func TestContextWithRootTraceContextWithDatadogContext(t *testing.T) { 260 | ctx := mockLambdaXRayTraceContext(context.Background(), mockXRayTraceID, mockXRayEntityID, true) 261 | ev := loadRawJSON(t, "../testdata/apig-event-with-headers.json") 262 | 263 | newCTX, _ := contextWithRootTraceContext(ctx, *ev, false, DefaultTraceExtractor) 264 | traceContext, _ := newCTX.Value(traceContextKey).(TraceContext) 265 | 266 | expected := TraceContext{ 267 | traceIDHeader: "1231452342", 268 | parentIDHeader: "45678910", 269 | samplingPriorityHeader: "2", 270 | } 271 | assert.Equal(t, expected, traceContext) 272 | } 273 | 274 | func TestContextWithRootTraceContextMergeXrayTracesNoDatadogContext(t *testing.T) { 275 | ctx := mockLambdaXRayTraceContext(context.Background(), mockXRayTraceID, mockXRayEntityID, true) 276 | ev := loadRawJSON(t, "../testdata/apig-event-no-headers.json") 277 | 278 | newCTX, _ := contextWithRootTraceContext(ctx, *ev, true, DefaultTraceExtractor) 279 | traceContext, _ := newCTX.Value(traceContextKey).(TraceContext) 280 | 281 | expected := TraceContext{ 282 | traceIDHeader: convertedXRayTraceID, 283 | parentIDHeader: convertedXRayEntityID, 284 | samplingPriorityHeader: "2", 285 | } 286 | assert.Equal(t, expected, traceContext) 287 | } 288 | 289 | func TestContextWithRootTraceContextMergeXrayTracesWithDatadogContext(t *testing.T) { 290 | ctx := mockLambdaXRayTraceContext(context.Background(), mockXRayTraceID, mockXRayEntityID, true) 291 | ev := loadRawJSON(t, "../testdata/apig-event-with-headers.json") 292 | 293 | newCTX, _ := contextWithRootTraceContext(ctx, *ev, true, DefaultTraceExtractor) 294 | traceContext, _ := newCTX.Value(traceContextKey).(TraceContext) 295 | 296 | expected := TraceContext{ 297 | traceIDHeader: "1231452342", 298 | parentIDHeader: convertedXRayEntityID, 299 | samplingPriorityHeader: "2", 300 | } 301 | assert.Equal(t, expected, traceContext) 302 | } 303 | -------------------------------------------------------------------------------- /internal/trace/listener.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Unless explicitly stated otherwise all files in this repository are licensed 3 | * under the Apache License Version 2.0. 4 | * 5 | * This product includes software developed at Datadog (https://www.datadoghq.com/). 6 | * Copyright 2021 Datadog, Inc. 7 | */ 8 | 9 | package trace 10 | 11 | import ( 12 | "context" 13 | "encoding/json" 14 | "fmt" 15 | "os" 16 | "strings" 17 | 18 | "github.com/DataDog/datadog-lambda-go/internal/extension" 19 | "github.com/DataDog/datadog-lambda-go/internal/logger" 20 | "github.com/DataDog/datadog-lambda-go/internal/version" 21 | "github.com/aws/aws-lambda-go/lambdacontext" 22 | "go.opentelemetry.io/otel" 23 | "gopkg.in/DataDog/dd-trace-go.v1/ddtrace" 24 | ddotel "gopkg.in/DataDog/dd-trace-go.v1/ddtrace/opentelemetry" 25 | "gopkg.in/DataDog/dd-trace-go.v1/ddtrace/tracer" 26 | ) 27 | 28 | type ( 29 | // Listener creates a function execution span and injects it into the context 30 | Listener struct { 31 | ddTraceEnabled bool 32 | mergeXrayTraces bool 33 | universalInstrumentation bool 34 | otelTracerEnabled bool 35 | extensionManager *extension.ExtensionManager 36 | traceContextExtractor ContextExtractor 37 | tracerOptions []tracer.StartOption 38 | } 39 | 40 | // Config gives options for how the Listener should work 41 | Config struct { 42 | DDTraceEnabled bool 43 | MergeXrayTraces bool 44 | UniversalInstrumentation bool 45 | OtelTracerEnabled bool 46 | TraceContextExtractor ContextExtractor 47 | TracerOptions []tracer.StartOption 48 | } 49 | ) 50 | 51 | // The function execution span is the top-level span representing the current Lambda function execution 52 | var functionExecutionSpan ddtrace.Span 53 | 54 | var tracerInitialized = false 55 | 56 | // MakeListener initializes a new trace lambda Listener 57 | func MakeListener(config Config, extensionManager *extension.ExtensionManager) Listener { 58 | 59 | return Listener{ 60 | ddTraceEnabled: config.DDTraceEnabled, 61 | mergeXrayTraces: config.MergeXrayTraces, 62 | universalInstrumentation: config.UniversalInstrumentation, 63 | otelTracerEnabled: config.OtelTracerEnabled, 64 | extensionManager: extensionManager, 65 | traceContextExtractor: config.TraceContextExtractor, 66 | tracerOptions: config.TracerOptions, 67 | } 68 | } 69 | 70 | // HandlerStarted sets up tracing and starts the function execution span if Datadog tracing is enabled 71 | func (l *Listener) HandlerStarted(ctx context.Context, msg json.RawMessage) context.Context { 72 | if !l.ddTraceEnabled { 73 | return ctx 74 | } 75 | 76 | if l.universalInstrumentation && l.extensionManager.IsExtensionRunning() { 77 | ctx = l.extensionManager.SendStartInvocationRequest(ctx, msg) 78 | } 79 | 80 | ctx, _ = contextWithRootTraceContext(ctx, msg, l.mergeXrayTraces, l.traceContextExtractor) 81 | 82 | if !tracerInitialized { 83 | serviceName := os.Getenv("DD_SERVICE") 84 | if serviceName == "" { 85 | serviceName = "aws.lambda" 86 | } 87 | extensionNotRunning := !l.extensionManager.IsExtensionRunning() 88 | opts := append([]tracer.StartOption{ 89 | tracer.WithService(serviceName), 90 | tracer.WithLambdaMode(extensionNotRunning), 91 | tracer.WithGlobalTag("_dd.origin", "lambda"), 92 | tracer.WithSendRetries(2), 93 | }, l.tracerOptions...) 94 | if l.otelTracerEnabled { 95 | provider := ddotel.NewTracerProvider( 96 | opts..., 97 | ) 98 | otel.SetTracerProvider(provider) 99 | } else { 100 | tracer.Start( 101 | opts..., 102 | ) 103 | } 104 | tracerInitialized = true 105 | } 106 | 107 | isDdServerlessSpan := l.universalInstrumentation && l.extensionManager.IsExtensionRunning() 108 | functionExecutionSpan, ctx = startFunctionExecutionSpan(ctx, l.mergeXrayTraces, isDdServerlessSpan) 109 | 110 | // Add the span to the context so the user can create child spans 111 | ctx = tracer.ContextWithSpan(ctx, functionExecutionSpan) 112 | 113 | return ctx 114 | } 115 | 116 | // HandlerFinished ends the function execution span and stops the tracer 117 | func (l *Listener) HandlerFinished(ctx context.Context, err error) { 118 | if functionExecutionSpan != nil { 119 | functionExecutionSpan.Finish(tracer.WithError(err)) 120 | 121 | finishConfig := ddtrace.FinishConfig{Error: err} 122 | 123 | if l.universalInstrumentation && l.extensionManager.IsExtensionRunning() { 124 | l.extensionManager.SendEndInvocationRequest(ctx, functionExecutionSpan, finishConfig) 125 | } 126 | } 127 | 128 | tracer.Flush() 129 | } 130 | 131 | // startFunctionExecutionSpan starts a span that represents the current Lambda function execution 132 | // and returns the span so that it can be finished when the function execution is complete 133 | func startFunctionExecutionSpan(ctx context.Context, mergeXrayTraces bool, isDdServerlessSpan bool) (tracer.Span, context.Context) { 134 | // Extract information from context 135 | lambdaCtx, _ := lambdacontext.FromContext(ctx) 136 | rootTraceContext, ok := ctx.Value(traceContextKey).(TraceContext) 137 | if !ok { 138 | logger.Error(fmt.Errorf("Error extracting trace context from context object")) 139 | } 140 | 141 | functionArn := lambdaCtx.InvokedFunctionArn 142 | functionArn = strings.ToLower(functionArn) 143 | functionArn, functionVersion := separateVersionFromFunctionArn(functionArn) 144 | 145 | // Set the root trace context as the parent of the function execution span 146 | var parentSpanContext ddtrace.SpanContext 147 | convertedSpanContext, err := ConvertTraceContextToSpanContext(rootTraceContext) 148 | if err == nil { 149 | parentSpanContext = convertedSpanContext 150 | } 151 | 152 | resourceName := lambdacontext.FunctionName 153 | if isDdServerlessSpan { 154 | // The extension will drop this span, prioritizing the execution span the extension creates 155 | resourceName = string(extension.DdSeverlessSpan) 156 | } 157 | 158 | span := tracer.StartSpan( 159 | "aws.lambda", // This operation name will be replaced with the value of the service tag by the Forwarder 160 | tracer.SpanType("serverless"), 161 | tracer.ChildOf(parentSpanContext), 162 | tracer.ResourceName(resourceName), 163 | tracer.Tag("cold_start", ctx.Value("cold_start")), 164 | tracer.Tag("function_arn", functionArn), 165 | tracer.Tag("function_version", functionVersion), 166 | tracer.Tag("request_id", lambdaCtx.AwsRequestID), 167 | tracer.Tag("resource_names", lambdacontext.FunctionName), 168 | tracer.Tag("functionname", strings.ToLower(lambdacontext.FunctionName)), 169 | tracer.Tag("datadog_lambda", version.DDLambdaVersion), 170 | tracer.Tag("dd_trace", version.DDTraceVersion), 171 | ) 172 | 173 | if parentSpanContext != nil && mergeXrayTraces { 174 | // This tag will cause the Forwarder to drop the span (to avoid redundancy with X-Ray) 175 | span.SetTag("_dd.parent_source", "xray") 176 | } 177 | 178 | ctx = context.WithValue(ctx, extension.DdSpanId, fmt.Sprint(span.Context().SpanID())) 179 | 180 | return span, ctx 181 | } 182 | 183 | func separateVersionFromFunctionArn(functionArn string) (arnWithoutVersion string, functionVersion string) { 184 | arnSegments := strings.Split(functionArn, ":") 185 | if cap(arnSegments) < 7 { 186 | return "", "" 187 | } 188 | functionVersion = "$LATEST" 189 | arnWithoutVersion = strings.Join(arnSegments[0:7], ":") 190 | if len(arnSegments) > 7 { 191 | functionVersion = arnSegments[7] 192 | } 193 | return arnWithoutVersion, functionVersion 194 | } 195 | -------------------------------------------------------------------------------- /internal/trace/listener_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Unless explicitly stated otherwise all files in this repository are licensed 3 | * under the Apache License Version 2.0. 4 | * 5 | * This product includes software developed at Datadog (https://www.datadoghq.com/). 6 | * Copyright 2021 Datadog, Inc. 7 | */ 8 | 9 | package trace 10 | 11 | import ( 12 | "context" 13 | "fmt" 14 | "testing" 15 | 16 | "github.com/DataDog/datadog-lambda-go/internal/extension" 17 | "github.com/aws/aws-lambda-go/lambdacontext" 18 | "github.com/stretchr/testify/assert" 19 | "gopkg.in/DataDog/dd-trace-go.v1/ddtrace/mocktracer" 20 | ) 21 | 22 | func TestSeparateVersionFromFunctionArnWithVersion(t *testing.T) { 23 | inputArn := "arn:aws:lambda:us-east-1:123456789012:function:my-function:9" 24 | 25 | arnWithoutVersion, functionVersion := separateVersionFromFunctionArn(inputArn) 26 | 27 | expectedArnWithoutVersion := "arn:aws:lambda:us-east-1:123456789012:function:my-function" 28 | expectedFunctionVersion := "9" 29 | assert.Equal(t, expectedArnWithoutVersion, arnWithoutVersion) 30 | assert.Equal(t, expectedFunctionVersion, functionVersion) 31 | } 32 | 33 | func TestSeparateVersionFromFunctionArnWithoutVersion(t *testing.T) { 34 | inputArn := "arn:aws:lambda:us-east-1:123456789012:function:my-function" 35 | 36 | arnWithoutVersion, functionVersion := separateVersionFromFunctionArn(inputArn) 37 | 38 | expectedArnWithoutVersion := "arn:aws:lambda:us-east-1:123456789012:function:my-function" 39 | expectedFunctionVersion := "$LATEST" 40 | assert.Equal(t, expectedArnWithoutVersion, arnWithoutVersion) 41 | assert.Equal(t, expectedFunctionVersion, functionVersion) 42 | } 43 | 44 | func TestSeparateVersionFromFunctionArnEmptyString(t *testing.T) { 45 | inputArn := "" 46 | 47 | arnWithoutVersion, functionVersion := separateVersionFromFunctionArn(inputArn) 48 | assert.Empty(t, arnWithoutVersion) 49 | assert.Empty(t, functionVersion) 50 | } 51 | 52 | var traceContextFromXray = TraceContext{ 53 | traceIDHeader: "1231452342", 54 | parentIDHeader: "45678910", 55 | } 56 | 57 | var traceContextFromEvent = TraceContext{ 58 | traceIDHeader: "1231452342", 59 | parentIDHeader: "45678910", 60 | } 61 | 62 | var mockLambdaContext = lambdacontext.LambdaContext{ 63 | AwsRequestID: "abcdefgh-1234-5678-1234-abcdefghijkl", 64 | InvokedFunctionArn: "arn:aws:lambda:us-east-1:123456789012:function:MyFunction:11", 65 | } 66 | 67 | func TestStartFunctionExecutionSpanFromXrayWithMergeEnabled(t *testing.T) { 68 | ctx := context.Background() 69 | 70 | lambdacontext.FunctionName = "MockFunctionName" 71 | ctx = lambdacontext.NewContext(ctx, &mockLambdaContext) 72 | ctx = context.WithValue(ctx, traceContextKey, traceContextFromXray) 73 | //nolint 74 | ctx = context.WithValue(ctx, "cold_start", true) 75 | 76 | mt := mocktracer.Start() 77 | defer mt.Stop() 78 | 79 | span, ctx := startFunctionExecutionSpan(ctx, true, false) 80 | span.Finish() 81 | finishedSpan := mt.FinishedSpans()[0] 82 | 83 | assert.Equal(t, "aws.lambda", finishedSpan.OperationName()) 84 | 85 | assert.Equal(t, true, finishedSpan.Tag("cold_start")) 86 | // We expect the function ARN to be lowercased, and the version removed 87 | assert.Equal(t, "arn:aws:lambda:us-east-1:123456789012:function:myfunction", finishedSpan.Tag("function_arn")) 88 | assert.Equal(t, "11", finishedSpan.Tag("function_version")) 89 | assert.Equal(t, "abcdefgh-1234-5678-1234-abcdefghijkl", finishedSpan.Tag("request_id")) 90 | assert.Equal(t, "MockFunctionName", finishedSpan.Tag("resource.name")) 91 | assert.Equal(t, "MockFunctionName", finishedSpan.Tag("resource_names")) 92 | assert.Equal(t, "mockfunctionname", finishedSpan.Tag("functionname")) 93 | assert.Equal(t, "serverless", finishedSpan.Tag("span.type")) 94 | assert.Equal(t, "xray", finishedSpan.Tag("_dd.parent_source")) 95 | assert.Equal(t, fmt.Sprint(span.Context().SpanID()), ctx.Value(extension.DdSpanId).(string)) 96 | } 97 | 98 | func TestStartFunctionExecutionSpanFromXrayWithMergeDisabled(t *testing.T) { 99 | ctx := context.Background() 100 | 101 | lambdacontext.FunctionName = "MockFunctionName" 102 | ctx = lambdacontext.NewContext(ctx, &mockLambdaContext) 103 | ctx = context.WithValue(ctx, traceContextKey, traceContextFromXray) 104 | //nolint 105 | ctx = context.WithValue(ctx, "cold_start", true) 106 | 107 | mt := mocktracer.Start() 108 | defer mt.Stop() 109 | 110 | span, ctx := startFunctionExecutionSpan(ctx, false, false) 111 | span.Finish() 112 | finishedSpan := mt.FinishedSpans()[0] 113 | 114 | assert.Equal(t, nil, finishedSpan.Tag("_dd.parent_source")) 115 | assert.Equal(t, fmt.Sprint(span.Context().SpanID()), ctx.Value(extension.DdSpanId).(string)) 116 | } 117 | 118 | func TestStartFunctionExecutionSpanFromEventWithMergeEnabled(t *testing.T) { 119 | ctx := context.Background() 120 | 121 | lambdacontext.FunctionName = "MockFunctionName" 122 | ctx = lambdacontext.NewContext(ctx, &mockLambdaContext) 123 | ctx = context.WithValue(ctx, traceContextKey, traceContextFromEvent) 124 | //nolint 125 | ctx = context.WithValue(ctx, "cold_start", true) 126 | 127 | mt := mocktracer.Start() 128 | defer mt.Stop() 129 | 130 | span, ctx := startFunctionExecutionSpan(ctx, true, false) 131 | span.Finish() 132 | finishedSpan := mt.FinishedSpans()[0] 133 | 134 | assert.Equal(t, "xray", finishedSpan.Tag("_dd.parent_source")) 135 | assert.Equal(t, fmt.Sprint(span.Context().SpanID()), ctx.Value(extension.DdSpanId).(string)) 136 | } 137 | 138 | func TestStartFunctionExecutionSpanFromEventWithMergeDisabled(t *testing.T) { 139 | ctx := context.Background() 140 | 141 | lambdacontext.FunctionName = "MockFunctionName" 142 | ctx = lambdacontext.NewContext(ctx, &mockLambdaContext) 143 | ctx = context.WithValue(ctx, traceContextKey, traceContextFromEvent) 144 | //nolint 145 | ctx = context.WithValue(ctx, "cold_start", true) 146 | 147 | mt := mocktracer.Start() 148 | defer mt.Stop() 149 | 150 | span, ctx := startFunctionExecutionSpan(ctx, false, false) 151 | span.Finish() 152 | finishedSpan := mt.FinishedSpans()[0] 153 | 154 | assert.Equal(t, nil, finishedSpan.Tag("_dd.parent_source")) 155 | assert.Equal(t, fmt.Sprint(span.Context().SpanID()), ctx.Value(extension.DdSpanId).(string)) 156 | } 157 | 158 | func TestStartFunctionExecutionSpanWithExtension(t *testing.T) { 159 | ctx := context.Background() 160 | 161 | lambdacontext.FunctionName = "MockFunctionName" 162 | ctx = lambdacontext.NewContext(ctx, &mockLambdaContext) 163 | ctx = context.WithValue(ctx, traceContextKey, traceContextFromEvent) 164 | //nolint 165 | ctx = context.WithValue(ctx, "cold_start", true) 166 | 167 | mt := mocktracer.Start() 168 | defer mt.Stop() 169 | 170 | span, ctx := startFunctionExecutionSpan(ctx, false, true) 171 | span.Finish() 172 | finishedSpan := mt.FinishedSpans()[0] 173 | 174 | assert.Equal(t, string(extension.DdSeverlessSpan), finishedSpan.Tag("resource.name")) 175 | assert.Equal(t, fmt.Sprint(span.Context().SpanID()), ctx.Value(extension.DdSpanId).(string)) 176 | } 177 | -------------------------------------------------------------------------------- /internal/version/version.go: -------------------------------------------------------------------------------- 1 | // Do not modify this file manually. It is modified by the release script. 2 | package version 3 | 4 | // DDLambdaVersion is the current version number of this library (datadog-lambda-go). 5 | // It is automatically updated by the release script. 6 | const DDLambdaVersion = "1.23.0" 7 | 8 | // DDTraceVersion is the version of dd-trace-go required by this libary. 9 | // This should match the version number specified in go.mod. 10 | // It is automatically updated by the release script 11 | const DDTraceVersion = "1.72.1" 12 | -------------------------------------------------------------------------------- /internal/wrapper/wrap_handler.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Unless explicitly stated otherwise all files in this repository are licensed 3 | * under the Apache License Version 2.0. 4 | * 5 | * This product includes software developed at Datadog (https://www.datadoghq.com/). 6 | * Copyright 2021 Datadog, Inc. 7 | */ 8 | 9 | package wrapper 10 | 11 | import ( 12 | "context" 13 | "encoding/json" 14 | "errors" 15 | "fmt" 16 | 17 | "github.com/DataDog/datadog-lambda-go/internal/extension" 18 | "github.com/DataDog/datadog-lambda-go/internal/logger" 19 | "github.com/aws/aws-lambda-go/lambda" 20 | 21 | "reflect" 22 | ) 23 | 24 | var ( 25 | // CurrentContext is the last create lambda context object. 26 | CurrentContext context.Context 27 | ) 28 | 29 | type ( 30 | // HandlerListener is a point where listener logic can be injected into a handler 31 | HandlerListener interface { 32 | HandlerStarted(ctx context.Context, msg json.RawMessage) context.Context 33 | HandlerFinished(ctx context.Context, err error) 34 | } 35 | 36 | DatadogHandler struct { 37 | coldStart bool 38 | handler lambda.Handler 39 | listeners []HandlerListener 40 | } 41 | ) 42 | 43 | // WrapHandlerWithListeners wraps a lambda handler, and calls listeners before and after every invocation. 44 | func WrapHandlerWithListeners(handler interface{}, listeners ...HandlerListener) interface{} { 45 | err := validateHandler(handler) 46 | if err != nil { 47 | // This wasn't a valid handler function, pass back to AWS SDK to let it handle the error. 48 | logger.Error(fmt.Errorf("handler function was in format ddlambda doesn't recognize: %v", err)) 49 | return handler 50 | } 51 | coldStart := true 52 | 53 | // Return custom handler, to be called once per invocation 54 | return func(ctx context.Context, msg json.RawMessage) (interface{}, error) { 55 | //nolint 56 | ctx = context.WithValue(ctx, "cold_start", coldStart) 57 | for _, listener := range listeners { 58 | ctx = listener.HandlerStarted(ctx, msg) 59 | } 60 | CurrentContext = ctx 61 | result, err := callHandler(ctx, msg, handler) 62 | for _, listener := range listeners { 63 | ctx = context.WithValue(ctx, extension.DdLambdaResponse, result) 64 | listener.HandlerFinished(ctx, err) 65 | } 66 | coldStart = false 67 | CurrentContext = nil 68 | return result, err 69 | } 70 | } 71 | 72 | func (h *DatadogHandler) Invoke(ctx context.Context, payload []byte) ([]byte, error) { 73 | //nolint 74 | ctx = context.WithValue(ctx, "cold_start", h.coldStart) 75 | msg := json.RawMessage{} 76 | err := msg.UnmarshalJSON(payload) 77 | if err != nil { 78 | logger.Error(fmt.Errorf("couldn't load handler payload: %v", err)) 79 | } 80 | 81 | for _, listener := range h.listeners { 82 | ctx = listener.HandlerStarted(ctx, msg) 83 | } 84 | 85 | CurrentContext = ctx 86 | result, err := h.handler.Invoke(ctx, payload) 87 | for _, listener := range h.listeners { 88 | listener.HandlerFinished(ctx, err) 89 | } 90 | h.coldStart = false 91 | CurrentContext = nil 92 | return result, err 93 | } 94 | 95 | func WrapHandlerInterfaceWithListeners(handler lambda.Handler, listeners ...HandlerListener) lambda.Handler { 96 | return &DatadogHandler{ 97 | coldStart: true, 98 | handler: handler, 99 | listeners: listeners, 100 | } 101 | } 102 | 103 | func validateHandler(handler interface{}) error { 104 | // Detect the handler follows the right format, based on the GO AWS SDK. 105 | // https://docs.aws.amazon.com/lambda/latest/dg/go-programming-model-handler-types.html 106 | handlerType := reflect.TypeOf(handler) 107 | if handlerType.Kind() != reflect.Func { 108 | return errors.New("handler is not a function") 109 | } 110 | 111 | if handlerType.NumIn() == 2 { 112 | contextType := reflect.TypeOf((*context.Context)(nil)).Elem() 113 | firstArgType := handlerType.In(0) 114 | if !firstArgType.Implements(contextType) { 115 | return errors.New("handler should take context as first argument") 116 | } 117 | } 118 | if handlerType.NumIn() > 2 { 119 | return errors.New("handler takes too many arguments") 120 | } 121 | 122 | errorType := reflect.TypeOf((*error)(nil)).Elem() 123 | 124 | if handlerType.NumOut() > 2 { 125 | return errors.New("handler returns more than two values") 126 | } 127 | if handlerType.NumOut() > 0 { 128 | rt := handlerType.Out(handlerType.NumOut() - 1) // Last returned value 129 | if !rt.Implements(errorType) { 130 | return errors.New("handler doesn't return error as it's last value") 131 | } 132 | } 133 | return nil 134 | } 135 | 136 | func callHandler(ctx context.Context, msg json.RawMessage, handler interface{}) (interface{}, error) { 137 | ev, err := unmarshalEventForHandler(msg, handler) 138 | if err != nil { 139 | return nil, err 140 | } 141 | handlerType := reflect.TypeOf(handler) 142 | 143 | args := []reflect.Value{} 144 | 145 | if handlerType.NumIn() == 1 { 146 | // When there is only one argument, argument is either the event payload, or the context. 147 | contextType := reflect.TypeOf((*context.Context)(nil)).Elem() 148 | firstArgType := handlerType.In(0) 149 | if firstArgType.Implements(contextType) { 150 | args = []reflect.Value{reflect.ValueOf(ctx)} 151 | } else { 152 | args = []reflect.Value{ev.Elem()} 153 | 154 | } 155 | } else if handlerType.NumIn() == 2 { 156 | // Or when there are two arguments, context is always first, followed by event payload. 157 | args = []reflect.Value{reflect.ValueOf(ctx), ev.Elem()} 158 | } 159 | 160 | handlerValue := reflect.ValueOf(handler) 161 | output := handlerValue.Call(args) 162 | 163 | var response interface{} 164 | var errResponse error 165 | 166 | if len(output) > 0 { 167 | // If there are any output values, the last should always be an error 168 | val := output[len(output)-1].Interface() 169 | if errVal, ok := val.(error); ok { 170 | errResponse = errVal 171 | } 172 | } 173 | 174 | if len(output) > 1 { 175 | // If there is more than one output value, the first should be the response payload. 176 | response = output[0].Interface() 177 | } 178 | 179 | return response, errResponse 180 | } 181 | 182 | func unmarshalEventForHandler(ev json.RawMessage, handler interface{}) (reflect.Value, error) { 183 | handlerType := reflect.TypeOf(handler) 184 | if handlerType.NumIn() == 0 { 185 | return reflect.ValueOf(nil), nil 186 | } 187 | 188 | messageType := handlerType.In(handlerType.NumIn() - 1) 189 | contextType := reflect.TypeOf((*context.Context)(nil)).Elem() 190 | firstArgType := handlerType.In(0) 191 | 192 | if handlerType.NumIn() == 1 && firstArgType.Implements(contextType) { 193 | return reflect.ValueOf(nil), nil 194 | } 195 | 196 | newMessage := reflect.New(messageType) 197 | err := json.Unmarshal(ev, newMessage.Interface()) 198 | if err != nil { 199 | return reflect.ValueOf(nil), err 200 | } 201 | return newMessage, err 202 | } 203 | -------------------------------------------------------------------------------- /internal/wrapper/wrap_handler_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Unless explicitly stated otherwise all files in this repository are licensed 3 | * under the Apache License Version 2.0. 4 | * 5 | * This product includes software developed at Datadog (https://www.datadoghq.com/). 6 | * Copyright 2021 Datadog, Inc. 7 | */ 8 | 9 | package wrapper 10 | 11 | import ( 12 | "context" 13 | "encoding/json" 14 | "errors" 15 | "os" 16 | "reflect" 17 | "testing" 18 | 19 | "github.com/aws/aws-lambda-go/events" 20 | "github.com/aws/aws-lambda-go/lambda" 21 | "github.com/stretchr/testify/assert" 22 | ) 23 | 24 | type ( 25 | mockHandlerListener struct { 26 | inputCTX context.Context 27 | inputMSG json.RawMessage 28 | outputCTX context.Context 29 | } 30 | 31 | mockNonProxyEvent struct { 32 | MyCustomEvent map[string]int `json:"my-custom-event"` 33 | FakeID string `json:"fake-id"` 34 | } 35 | ) 36 | 37 | func (mhl *mockHandlerListener) HandlerStarted(ctx context.Context, msg json.RawMessage) context.Context { 38 | mhl.inputCTX = ctx 39 | mhl.inputMSG = msg 40 | return ctx 41 | } 42 | 43 | func (mhl *mockHandlerListener) HandlerFinished(ctx context.Context, err error) { 44 | mhl.outputCTX = ctx 45 | } 46 | 47 | func runHandlerWithJSON(t *testing.T, filename string, handler interface{}) (*mockHandlerListener, interface{}, error) { 48 | ctx := context.Background() 49 | payload := loadRawJSON(t, filename) 50 | 51 | mhl := mockHandlerListener{} 52 | 53 | wrappedHandler := WrapHandlerWithListeners(handler, &mhl).(func(context.Context, json.RawMessage) (interface{}, error)) 54 | 55 | response, err := wrappedHandler(ctx, *payload) 56 | return &mhl, response, err 57 | } 58 | 59 | func runHandlerInterfaceWithJSON(t *testing.T, filename string, handler lambda.Handler) (*mockHandlerListener, []byte, error) { 60 | ctx := context.Background() 61 | payload, err := os.ReadFile(filename) 62 | if err != nil { 63 | assert.Fail(t, "Couldn't find JSON file") 64 | return nil, nil, nil 65 | } 66 | mhl := mockHandlerListener{} 67 | 68 | wrappedHandler := WrapHandlerInterfaceWithListeners(handler, &mhl) 69 | 70 | response, err := wrappedHandler.Invoke(ctx, payload) 71 | return &mhl, response, err 72 | } 73 | 74 | func loadRawJSON(t *testing.T, filename string) *json.RawMessage { 75 | bytes, err := os.ReadFile(filename) 76 | if err != nil { 77 | assert.Fail(t, "Couldn't find JSON file") 78 | return nil 79 | } 80 | msg := json.RawMessage{} 81 | err = msg.UnmarshalJSON(bytes) 82 | assert.NoError(t, err) 83 | return &msg 84 | } 85 | 86 | func TestValidateHandlerNotFunction(t *testing.T) { 87 | nonFunction := 1 88 | 89 | err := validateHandler(nonFunction) 90 | assert.EqualError(t, err, "handler is not a function") 91 | } 92 | func TestValidateHandlerToManyArguments(t *testing.T) { 93 | tooManyArgs := func(a, b, c int) { 94 | } 95 | 96 | err := validateHandler(tooManyArgs) 97 | assert.EqualError(t, err, "handler takes too many arguments") 98 | } 99 | 100 | func TestValidateHandlerContextIsNotFirstArgument(t *testing.T) { 101 | firstArgNotContext := func(arg1, arg2 int) { 102 | } 103 | 104 | err := validateHandler(firstArgNotContext) 105 | assert.EqualError(t, err, "handler should take context as first argument") 106 | } 107 | 108 | func TestValidateHandlerTwoArguments(t *testing.T) { 109 | twoArguments := func(arg1 context.Context, arg2 int) { 110 | } 111 | 112 | err := validateHandler(twoArguments) 113 | assert.NoError(t, err) 114 | } 115 | 116 | func TestValidateHandlerOneArgument(t *testing.T) { 117 | oneArgument := func(arg1 int) { 118 | } 119 | 120 | err := validateHandler(oneArgument) 121 | assert.NoError(t, err) 122 | } 123 | 124 | func TestValidateHandlerTooManyReturnValues(t *testing.T) { 125 | tooManyReturns := func() (int, int, error) { 126 | return 0, 0, nil 127 | } 128 | 129 | err := validateHandler(tooManyReturns) 130 | assert.EqualError(t, err, "handler returns more than two values") 131 | } 132 | func TestValidateHandlerLastReturnValueNotError(t *testing.T) { 133 | lastNotError := func() (int, int) { 134 | return 0, 0 135 | } 136 | 137 | err := validateHandler(lastNotError) 138 | assert.EqualError(t, err, "handler doesn't return error as it's last value") 139 | } 140 | func TestValidateHandlerCorrectFormat(t *testing.T) { 141 | correct := func(context context.Context) (int, error) { 142 | return 0, nil 143 | } 144 | 145 | err := validateHandler(correct) 146 | assert.NoError(t, err) 147 | } 148 | 149 | func TestWrapHandlerAPIGEvent(t *testing.T) { 150 | called := false 151 | 152 | handler := func(ctx context.Context, request events.APIGatewayProxyRequest) (int, error) { 153 | called = true 154 | assert.Equal(t, "c6af9ac6-7b61-11e6-9a41-93e8deadbeef", request.RequestContext.RequestID) 155 | return 5, nil 156 | } 157 | 158 | _, response, err := runHandlerWithJSON(t, "../testdata/apig-event-no-headers.json", handler) 159 | 160 | assert.True(t, called) 161 | assert.NoError(t, err) 162 | assert.Equal(t, 5, response) 163 | } 164 | 165 | func TestWrapHandlerNonProxyEvent(t *testing.T) { 166 | called := false 167 | 168 | handler := func(ctx context.Context, request mockNonProxyEvent) (int, error) { 169 | called = true 170 | assert.Equal(t, "12345678910", request.FakeID) 171 | return 5, nil 172 | } 173 | 174 | _, response, err := runHandlerWithJSON(t, "../testdata/non-proxy-no-headers.json", handler) 175 | 176 | assert.True(t, called) 177 | assert.NoError(t, err) 178 | assert.Equal(t, 5, response) 179 | } 180 | 181 | func TestWrapHandlerEventArgumentOnly(t *testing.T) { 182 | called := false 183 | 184 | handler := func(request mockNonProxyEvent) (int, error) { 185 | called = true 186 | assert.Equal(t, "12345678910", request.FakeID) 187 | return 5, nil 188 | } 189 | 190 | _, response, err := runHandlerWithJSON(t, "../testdata/non-proxy-no-headers.json", handler) 191 | 192 | assert.True(t, called) 193 | assert.NoError(t, err) 194 | assert.Equal(t, 5, response) 195 | } 196 | 197 | func TestWrapHandlerContextArgumentOnly(t *testing.T) { 198 | called := true 199 | var handler = func(ctx context.Context) (interface{}, error) { 200 | return nil, nil 201 | } 202 | 203 | mhl := mockHandlerListener{} 204 | wrappedHandler := WrapHandlerWithListeners(handler, &mhl).(func(context.Context, json.RawMessage) (interface{}, error)) 205 | 206 | _, err := wrappedHandler(context.Background(), nil) 207 | assert.NoError(t, err) 208 | assert.True(t, called) 209 | } 210 | 211 | func TestWrapHandlerNoArguments(t *testing.T) { 212 | called := false 213 | 214 | handler := func() (int, error) { 215 | called = true 216 | return 5, nil 217 | } 218 | 219 | _, response, err := runHandlerWithJSON(t, "../testdata/non-proxy-no-headers.json", handler) 220 | 221 | assert.True(t, called) 222 | assert.NoError(t, err) 223 | assert.Equal(t, 5, response) 224 | } 225 | 226 | func TestWrapHandlerInvalidData(t *testing.T) { 227 | called := false 228 | 229 | handler := func(request mockNonProxyEvent) (int, error) { 230 | called = true 231 | return 5, nil 232 | } 233 | 234 | _, response, err := runHandlerWithJSON(t, "../testdata/invalid.json", handler) 235 | 236 | assert.False(t, called) 237 | assert.Error(t, err) 238 | assert.Equal(t, nil, response) 239 | } 240 | 241 | func TestWrapHandlerReturnsError(t *testing.T) { 242 | called := false 243 | defaultErr := errors.New("Some error") 244 | 245 | handler := func(request mockNonProxyEvent) (int, error) { 246 | called = true 247 | return 5, defaultErr 248 | } 249 | 250 | _, response, err := runHandlerWithJSON(t, "../testdata/non-proxy-no-headers.json", handler) 251 | 252 | assert.True(t, called) 253 | assert.Equal(t, defaultErr, err) 254 | assert.Equal(t, 5, response) 255 | } 256 | 257 | func TestWrapHandlerReturnsErrorOnly(t *testing.T) { 258 | called := false 259 | defaultErr := errors.New("Some error") 260 | 261 | handler := func(request mockNonProxyEvent) error { 262 | called = true 263 | return defaultErr 264 | } 265 | 266 | _, response, err := runHandlerWithJSON(t, "../testdata/non-proxy-no-headers.json", handler) 267 | 268 | assert.True(t, called) 269 | assert.Equal(t, defaultErr, err) 270 | assert.Equal(t, nil, response) 271 | } 272 | 273 | func TestWrapHandlerReturnsOriginalHandlerIfInvalid(t *testing.T) { 274 | 275 | var handler interface{} = func(arg1, arg2, arg3 int) (int, error) { 276 | return 0, nil 277 | } 278 | mhl := mockHandlerListener{} 279 | 280 | wrappedHandler := WrapHandlerWithListeners(handler, &mhl) 281 | 282 | assert.Equal(t, reflect.ValueOf(handler).Pointer(), reflect.ValueOf(wrappedHandler).Pointer()) 283 | 284 | } 285 | 286 | func TestWrapHandlerInterfaceWithListeners(t *testing.T) { 287 | called := false 288 | 289 | handler := lambda.NewHandler(func(ctx context.Context, request events.APIGatewayProxyRequest) (int, error) { 290 | called = true 291 | assert.Equal(t, "c6af9ac6-7b61-11e6-9a41-93e8deadbeef", request.RequestContext.RequestID) 292 | return 5, nil 293 | }) 294 | 295 | _, response, err := runHandlerInterfaceWithJSON(t, "../testdata/apig-event-no-headers.json", handler) 296 | 297 | assert.True(t, called) 298 | assert.NoError(t, err) 299 | assert.Equal(t, uint8('5'), response[0]) 300 | } 301 | -------------------------------------------------------------------------------- /scripts/release.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Run from the root directory. 4 | # Use with `./release.sh ` 5 | 6 | set -e 7 | 8 | # Ensure on main, and pull the latest 9 | BRANCH=$(git rev-parse --abbrev-ref HEAD) 10 | if [ $BRANCH != "main" ]; then 11 | echo "Not on main, aborting" 12 | exit 1 13 | else 14 | echo "Updating main" 15 | git pull origin main 16 | fi 17 | 18 | # Ensure no uncommitted changes 19 | if [ -n "$(git status --porcelain)" ]; then 20 | echo "Detected uncommitted changes, aborting" 21 | exit 1 22 | fi 23 | 24 | # Check that the new desired version number was specified correctly 25 | if [ -z "$1" ]; then 26 | echo "Must specify a desired version number" 27 | exit 1 28 | elif [[ ! $1 =~ [0-9]+\.[0-9]+\.[0-9]+ ]]; then 29 | echo "Must use a semantic version, e.g., 3.1.4" 30 | exit 1 31 | else 32 | NEW_VERSION=$1 33 | fi 34 | 35 | CURRENT_DD_TRACE_VERSION="$(grep "const DDTraceVersion" internal/version/version.go | grep -o -E "[0-9]+\.[0-9]+\.[0-9]+")" 36 | NEW_DD_TRACE_VERSION="$(grep "dd-trace-go.v1" go.mod | grep -o -E "[0-9]+\.[0-9]+\.[0-9]+")" 37 | if [ "$CURRENT_DD_TRACE_VERSION" != "$NEW_DD_TRACE_VERSION" ]; then 38 | read -p "Confirm updating dd-trace-go version from $CURRENT_DD_TRACE_VERSION to $NEW_DD_TRACE_VERSION (y/n)?" CONT 39 | if [ "$CONT" != "y" ]; then 40 | echo "Exiting" 41 | exit 1 42 | fi 43 | fi 44 | 45 | CURRENT_VERSION="$(grep "const DDLambdaVersion" internal/version/version.go | grep -o -E "[0-9]+\.[0-9]+\.[0-9]+")" 46 | read -p "Ready to update the library version from $CURRENT_VERSION to $NEW_VERSION and release the library (y/n)?" CONT 47 | if [ "$CONT" != "y" ]; then 48 | echo "Exiting" 49 | exit 1 50 | fi 51 | 52 | # Replace version numbers in version.go 53 | sed -E -i '' "s/(DDLambdaVersion = \")[0-9]+\.[0-9]+\.[0-9]+/\1$NEW_VERSION/g" internal/version/version.go 54 | sed -E -i '' "s/(DDTraceVersion = \")[0-9]+\.[0-9]+\.[0-9]+/\1$NEW_DD_TRACE_VERSION/g" internal/version/version.go 55 | 56 | # # Commit change 57 | git commit internal/version/version.go -m "Bump version to ${NEW_VERSION}" 58 | git push origin main 59 | 60 | # # Tag new release 61 | git tag "v$NEW_VERSION" 62 | git push origin "refs/tags/v$NEW_VERSION" 63 | 64 | echo 65 | echo "Now create a new release with the tag v${NEW_VERSION}" 66 | echo "https://github.com/DataDog/datadog-lambda-go/releases/new?tag=v$NEW_VERSION&title=v$NEW_VERSION" 67 | 68 | 69 | -------------------------------------------------------------------------------- /tests/integration_tests/.gitignore: -------------------------------------------------------------------------------- 1 | vendor 2 | 3 | build/* 4 | -------------------------------------------------------------------------------- /tests/integration_tests/README.md: -------------------------------------------------------------------------------- 1 | # Integration tests 2 | 3 | 4 | ## Requirements 5 | 6 | - Node 7 | - Go 8 | - DD_API_KEY 9 | 10 | ## Running 11 | 12 | ```bash 13 | DD_API_KEY= aws-vault exec sandbox-account-admin -- ./run_integration_tests.sh 14 | ``` 15 | 16 | Use `UPDATE_SNAPSHOTS=true` to update snapshots 17 | -------------------------------------------------------------------------------- /tests/integration_tests/error/go.mod: -------------------------------------------------------------------------------- 1 | module github.com/DataDog/datadog-lambda-go/tests/integration_tests/bin/error 2 | 3 | go 1.21 4 | 5 | toolchain go1.21.5 6 | 7 | require ( 8 | github.com/DataDog/datadog-lambda-go v1.14.0 9 | github.com/aws/aws-lambda-go v1.46.0 10 | ) 11 | 12 | require ( 13 | github.com/DataDog/appsec-internal-go v1.6.0 // indirect 14 | github.com/DataDog/datadog-agent/pkg/obfuscate v0.50.2 // indirect 15 | github.com/DataDog/datadog-agent/pkg/remoteconfig/state v0.50.2 // indirect 16 | github.com/DataDog/datadog-go/v5 v5.5.0 // indirect 17 | github.com/DataDog/go-libddwaf/v3 v3.2.1 // indirect 18 | github.com/DataDog/go-sqllexer v0.0.10 // indirect 19 | github.com/DataDog/go-tuf v1.0.2-0.5.2 // indirect 20 | github.com/DataDog/sketches-go v1.4.5 // indirect 21 | github.com/Microsoft/go-winio v0.6.1 // indirect 22 | github.com/andybalholm/brotli v1.1.0 // indirect 23 | github.com/aws/aws-sdk-go v1.50.9 // indirect 24 | github.com/aws/aws-sdk-go-v2 v1.24.1 // indirect 25 | github.com/aws/aws-sdk-go-v2/config v1.26.6 // indirect 26 | github.com/aws/aws-sdk-go-v2/credentials v1.16.16 // indirect 27 | github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.11 // indirect 28 | github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.10 // indirect 29 | github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.10 // indirect 30 | github.com/aws/aws-sdk-go-v2/internal/ini v1.7.3 // indirect 31 | github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.4 // indirect 32 | github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.10 // indirect 33 | github.com/aws/aws-sdk-go-v2/service/kms v1.27.9 // indirect 34 | github.com/aws/aws-sdk-go-v2/service/sso v1.18.7 // indirect 35 | github.com/aws/aws-sdk-go-v2/service/ssooidc v1.21.7 // indirect 36 | github.com/aws/aws-sdk-go-v2/service/sts v1.26.7 // indirect 37 | github.com/aws/aws-xray-sdk-go v1.8.3 // indirect 38 | github.com/aws/smithy-go v1.19.0 // indirect 39 | github.com/cenkalti/backoff/v4 v4.2.1 // indirect 40 | github.com/cespare/xxhash/v2 v2.2.0 // indirect 41 | github.com/dustin/go-humanize v1.0.1 // indirect 42 | github.com/ebitengine/purego v0.6.0-alpha.5 // indirect 43 | github.com/go-logr/logr v1.4.1 // indirect 44 | github.com/go-logr/stdr v1.2.2 // indirect 45 | github.com/golang/protobuf v1.5.3 // indirect 46 | github.com/google/uuid v1.6.0 // indirect 47 | github.com/jmespath/go-jmespath v0.4.0 // indirect 48 | github.com/klauspost/compress v1.17.5 // indirect 49 | github.com/outcaste-io/ristretto v0.2.3 // indirect 50 | github.com/philhofer/fwd v1.1.2 // indirect 51 | github.com/pkg/errors v0.9.1 // indirect 52 | github.com/secure-systems-lab/go-securesystemslib v0.8.0 // indirect 53 | github.com/sony/gobreaker v0.5.0 // indirect 54 | github.com/tinylib/msgp v1.1.9 // indirect 55 | github.com/valyala/bytebufferpool v1.0.0 // indirect 56 | github.com/valyala/fasthttp v1.51.0 // indirect 57 | go.opentelemetry.io/otel v1.24.0 // indirect 58 | go.opentelemetry.io/otel/metric v1.24.0 // indirect 59 | go.opentelemetry.io/otel/trace v1.24.0 // indirect 60 | go.uber.org/atomic v1.11.0 // indirect 61 | golang.org/x/mod v0.14.0 // indirect 62 | golang.org/x/net v0.23.0 // indirect 63 | golang.org/x/sys v0.20.0 // indirect 64 | golang.org/x/text v0.14.0 // indirect 65 | golang.org/x/time v0.5.0 // indirect 66 | golang.org/x/tools v0.17.0 // indirect 67 | golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect 68 | google.golang.org/genproto/googleapis/rpc v0.0.0-20240125205218-1f4bbc51befe // indirect 69 | google.golang.org/grpc v1.61.0 // indirect 70 | google.golang.org/protobuf v1.33.0 // indirect 71 | gopkg.in/DataDog/dd-trace-go.v1 v1.65.1 // indirect 72 | ) 73 | 74 | replace github.com/DataDog/datadog-lambda-go => ../../../ 75 | -------------------------------------------------------------------------------- /tests/integration_tests/error/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | 7 | "github.com/aws/aws-lambda-go/lambda" 8 | 9 | ddlambda "github.com/DataDog/datadog-lambda-go" 10 | "github.com/aws/aws-lambda-go/events" 11 | ) 12 | 13 | func handleRequest(ctx context.Context, ev events.APIGatewayProxyRequest) (events.APIGatewayProxyResponse, error) { 14 | return events.APIGatewayProxyResponse{ 15 | StatusCode: 500, 16 | Body: "error", 17 | }, errors.New("something went wrong") 18 | } 19 | 20 | func main() { 21 | lambda.Start(ddlambda.WrapHandler(handleRequest, nil)) 22 | } 23 | -------------------------------------------------------------------------------- /tests/integration_tests/hello/go.mod: -------------------------------------------------------------------------------- 1 | module github.com/DataDog/datadog-lambda-go/tests/integration_tests/bin/hello 2 | 3 | go 1.21 4 | 5 | toolchain go1.21.5 6 | 7 | require ( 8 | github.com/DataDog/datadog-lambda-go v1.14.0 9 | github.com/aws/aws-lambda-go v1.46.0 10 | gopkg.in/DataDog/dd-trace-go.v1 v1.65.1 11 | ) 12 | 13 | require ( 14 | github.com/DataDog/appsec-internal-go v1.6.0 // indirect 15 | github.com/DataDog/datadog-agent/pkg/obfuscate v0.50.2 // indirect 16 | github.com/DataDog/datadog-agent/pkg/remoteconfig/state v0.50.2 // indirect 17 | github.com/DataDog/datadog-go/v5 v5.5.0 // indirect 18 | github.com/DataDog/go-libddwaf/v3 v3.2.1 // indirect 19 | github.com/DataDog/go-sqllexer v0.0.10 // indirect 20 | github.com/DataDog/go-tuf v1.0.2-0.5.2 // indirect 21 | github.com/DataDog/sketches-go v1.4.5 // indirect 22 | github.com/Microsoft/go-winio v0.6.1 // indirect 23 | github.com/andybalholm/brotli v1.1.0 // indirect 24 | github.com/aws/aws-sdk-go v1.50.9 // indirect 25 | github.com/aws/aws-sdk-go-v2 v1.24.1 // indirect 26 | github.com/aws/aws-sdk-go-v2/config v1.26.6 // indirect 27 | github.com/aws/aws-sdk-go-v2/credentials v1.16.16 // indirect 28 | github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.11 // indirect 29 | github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.10 // indirect 30 | github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.10 // indirect 31 | github.com/aws/aws-sdk-go-v2/internal/ini v1.7.3 // indirect 32 | github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.4 // indirect 33 | github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.10 // indirect 34 | github.com/aws/aws-sdk-go-v2/service/kms v1.27.9 // indirect 35 | github.com/aws/aws-sdk-go-v2/service/sso v1.18.7 // indirect 36 | github.com/aws/aws-sdk-go-v2/service/ssooidc v1.21.7 // indirect 37 | github.com/aws/aws-sdk-go-v2/service/sts v1.26.7 // indirect 38 | github.com/aws/aws-xray-sdk-go v1.8.3 // indirect 39 | github.com/aws/smithy-go v1.19.0 // indirect 40 | github.com/cenkalti/backoff/v4 v4.2.1 // indirect 41 | github.com/cespare/xxhash/v2 v2.2.0 // indirect 42 | github.com/dustin/go-humanize v1.0.1 // indirect 43 | github.com/eapache/queue/v2 v2.0.0-20230407133247-75960ed334e4 // indirect 44 | github.com/ebitengine/purego v0.6.0-alpha.5 // indirect 45 | github.com/go-logr/logr v1.4.1 // indirect 46 | github.com/go-logr/stdr v1.2.2 // indirect 47 | github.com/golang/protobuf v1.5.3 // indirect 48 | github.com/google/uuid v1.6.0 // indirect 49 | github.com/hashicorp/go-secure-stdlib/parseutil v0.1.7 // indirect 50 | github.com/hashicorp/go-secure-stdlib/strutil v0.1.2 // indirect 51 | github.com/hashicorp/go-sockaddr v1.0.2 // indirect 52 | github.com/jmespath/go-jmespath v0.4.0 // indirect 53 | github.com/klauspost/compress v1.17.5 // indirect 54 | github.com/mitchellh/mapstructure v1.5.0 // indirect 55 | github.com/outcaste-io/ristretto v0.2.3 // indirect 56 | github.com/philhofer/fwd v1.1.2 // indirect 57 | github.com/pkg/errors v0.9.1 // indirect 58 | github.com/ryanuber/go-glob v1.0.0 // indirect 59 | github.com/secure-systems-lab/go-securesystemslib v0.8.0 // indirect 60 | github.com/sony/gobreaker v0.5.0 // indirect 61 | github.com/tinylib/msgp v1.1.9 // indirect 62 | github.com/valyala/bytebufferpool v1.0.0 // indirect 63 | github.com/valyala/fasthttp v1.51.0 // indirect 64 | go.opentelemetry.io/otel v1.24.0 // indirect 65 | go.opentelemetry.io/otel/metric v1.24.0 // indirect 66 | go.opentelemetry.io/otel/trace v1.24.0 // indirect 67 | go.uber.org/atomic v1.11.0 // indirect 68 | golang.org/x/mod v0.14.0 // indirect 69 | golang.org/x/net v0.23.0 // indirect 70 | golang.org/x/sys v0.20.0 // indirect 71 | golang.org/x/text v0.14.0 // indirect 72 | golang.org/x/time v0.5.0 // indirect 73 | golang.org/x/tools v0.17.0 // indirect 74 | golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect 75 | google.golang.org/genproto/googleapis/rpc v0.0.0-20240125205218-1f4bbc51befe // indirect 76 | google.golang.org/grpc v1.61.0 // indirect 77 | google.golang.org/protobuf v1.33.0 // indirect 78 | ) 79 | 80 | replace github.com/DataDog/datadog-lambda-go => ../../../ 81 | -------------------------------------------------------------------------------- /tests/integration_tests/hello/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "net/http" 7 | "strconv" 8 | "time" 9 | 10 | "github.com/aws/aws-lambda-go/lambda" 11 | 12 | ddlambda "github.com/DataDog/datadog-lambda-go" 13 | "github.com/aws/aws-lambda-go/events" 14 | httptrace "gopkg.in/DataDog/dd-trace-go.v1/contrib/net/http" 15 | "gopkg.in/DataDog/dd-trace-go.v1/ddtrace/tracer" 16 | ) 17 | 18 | func handleRequest(ctx context.Context, ev events.APIGatewayProxyRequest) (events.APIGatewayProxyResponse, error) { 19 | currentSpan, _ := tracer.SpanFromContext(ctx) 20 | currentSpanContext := currentSpan.Context() 21 | fmt.Println("Current span ID: " + strconv.FormatUint(currentSpanContext.SpanID(), 10)) 22 | fmt.Println("Current trace ID: " + strconv.FormatUint(currentSpanContext.TraceID(), 10)) 23 | 24 | // HTTP request 25 | req, _ := http.NewRequestWithContext(ctx, "GET", "https://www.datadoghq.com", nil) 26 | client := http.Client{} 27 | client = *httptrace.WrapClient(&client) 28 | client.Do(req) 29 | 30 | // Metric 31 | ddlambda.Distribution("hello-go.dog", 1) 32 | 33 | // User-defined span 34 | for i := 0; i < 10; i++ { 35 | s, _ := tracer.StartSpanFromContext(ctx, "child.span") 36 | time.Sleep(100 * time.Millisecond) 37 | s.Finish() 38 | } 39 | 40 | return events.APIGatewayProxyResponse{ 41 | StatusCode: 200, 42 | Body: "hello, dog!", 43 | }, nil 44 | } 45 | 46 | func main() { 47 | lambda.Start(ddlambda.WrapHandler(handleRequest, nil)) 48 | } 49 | -------------------------------------------------------------------------------- /tests/integration_tests/input_events/api-gateway-get.json: -------------------------------------------------------------------------------- 1 | { 2 | "path": "/test/hello", 3 | "headers": { 4 | "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8", 5 | "Accept-Encoding": "gzip, deflate, lzma, sdch, br", 6 | "Accept-Language": "en-US,en;q=0.8", 7 | "CloudFront-Forwarded-Proto": "https", 8 | "CloudFront-Is-Desktop-Viewer": "true", 9 | "CloudFront-Is-Mobile-Viewer": "false", 10 | "CloudFront-Is-SmartTV-Viewer": "false", 11 | "CloudFront-Is-Tablet-Viewer": "false", 12 | "CloudFront-Viewer-Country": "US", 13 | "Host": "wt6mne2s9k.execute-api.us-west-2.amazonaws.com", 14 | "Upgrade-Insecure-Requests": "1", 15 | "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.82 Safari/537.36 OPR/39.0.2256.48", 16 | "Via": "1.1 fb7cca60f0ecd82ce07790c9c5eef16c.cloudfront.net (CloudFront)", 17 | "X-Amz-Cf-Id": "nBsWBOrSHMgnaROZJK1wGCZ9PcRcSpq_oSXZNQwQ10OTZL4cimZo3g==", 18 | "X-Forwarded-For": "192.168.100.1, 192.168.1.1", 19 | "X-Forwarded-Port": "443", 20 | "X-Forwarded-Proto": "https" 21 | }, 22 | "pathParameters": { 23 | "proxy": "hello" 24 | }, 25 | "requestContext": { 26 | "accountId": "123456789012", 27 | "resourceId": "us4z18", 28 | "stage": "test", 29 | "requestId": "41b45ea3-70b5-11e6-b7bd-69b5aaebc7d9", 30 | "identity": { 31 | "cognitoIdentityPoolId": "", 32 | "accountId": "", 33 | "cognitoIdentityId": "", 34 | "caller": "", 35 | "apiKey": "", 36 | "sourceIp": "192.168.100.1", 37 | "cognitoAuthenticationType": "", 38 | "cognitoAuthenticationProvider": "", 39 | "userArn": "", 40 | "userAgent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.82 Safari/537.36 OPR/39.0.2256.48", 41 | "user": "" 42 | }, 43 | "resourcePath": "/{proxy+}", 44 | "httpMethod": "GET", 45 | "apiId": "wt6mne2s9k" 46 | }, 47 | "resource": "/{proxy+}", 48 | "httpMethod": "GET", 49 | "queryStringParameters": { 50 | "name": "me" 51 | }, 52 | "stageVariables": { 53 | "stageVarName": "stageVarValue" 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /tests/integration_tests/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "lambda-sample-go", 3 | "version": "1.0.0" 4 | } 5 | -------------------------------------------------------------------------------- /tests/integration_tests/parse-json.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | var readline = require('readline'); 4 | var rl = readline.createInterface({ 5 | input: process.stdin, 6 | output: process.stdout, 7 | terminal: false 8 | }); 9 | 10 | rl.on('line', function(line){ 11 | try { 12 | const obj = JSON.parse(line) 13 | console.log(JSON.stringify(obj, Object.keys(obj).sort(), 2)) 14 | } catch (e) { 15 | console.log(line) 16 | } 17 | }) 18 | -------------------------------------------------------------------------------- /tests/integration_tests/run_integration_tests.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Usage - run commands from the /integration_tests directory: 4 | # To check if new changes to the library cause changes to any snapshots: 5 | # DD_API_KEY=XXXX aws-vault exec sandbox-account-admin -- ./run_integration_tests.sh 6 | # To regenerate snapshots: 7 | # UPDATE_SNAPSHOTS=true DD_API_KEY=XXXX aws-vault exec sandbox-account-admin -- ./run_integration_tests.sh 8 | 9 | set -e 10 | 11 | # Disable deprecation warnings. 12 | export SLS_DEPRECATION_DISABLE=* 13 | 14 | # These values need to be in sync with serverless.yml, where there needs to be a function 15 | # defined for every handler_runtime combination 16 | LAMBDA_HANDLERS=("hello" "error") 17 | 18 | LOGS_WAIT_SECONDS=20 19 | 20 | integration_tests_dir=$(cd `dirname $0` && pwd) 21 | echo $integration_tests_dir 22 | 23 | script_utc_start_time=$(date -u +"%Y%m%dT%H%M%S") 24 | 25 | mismatch_found=false 26 | 27 | if [ -z "$AWS_SECRET_ACCESS_KEY" ]; then 28 | echo "No AWS credentials were found in the environment." 29 | echo "Note that only Datadog employees can run these integration tests." 30 | exit 1 31 | fi 32 | 33 | if [ -z "$DD_API_KEY" ]; then 34 | echo "No DD_API_KEY env var set, exiting" 35 | exit 1 36 | fi 37 | 38 | if [ -n "$UPDATE_SNAPSHOTS" ]; then 39 | echo "Overwriting snapshots in this execution" 40 | fi 41 | 42 | echo "Building Go binaries" 43 | CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -ldflags="-s -w" -o build/hello/bootstrap hello/main.go 44 | CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -ldflags="-s -w" -o build/error/bootstrap error/main.go 45 | zip -j build/hello.zip build/hello/bootstrap 46 | zip -j build/error.zip build/error/bootstrap 47 | 48 | # Generate a random 8-character ID to avoid collisions with other runs 49 | run_id=$(xxd -l 4 -c 4 -p < /dev/random) 50 | 51 | # Always remove the stack before exiting, no matter what 52 | function remove_stack() { 53 | echo "Removing functions" 54 | serverless remove --stage $run_id 55 | } 56 | trap remove_stack EXIT 57 | 58 | sls --version 59 | 60 | echo "Deploying functions" 61 | sls deploy --stage $run_id 62 | 63 | cd $integration_tests_dir 64 | 65 | input_event_files=$(ls ./input_events) 66 | # Sort event files by name so that snapshots stay consistent 67 | input_event_files=($(for file_name in ${input_event_files[@]}; do echo $file_name; done | sort)) 68 | 69 | echo "Invoking functions" 70 | set +e # Don't exit this script if an invocation fails or there's a diff 71 | for input_event_file in "${input_event_files[@]}"; do 72 | for function_name in "${LAMBDA_HANDLERS[@]}"; do 73 | # Get event name without trailing ".json" so we can build the snapshot file name 74 | input_event_name=$(echo "$input_event_file" | sed "s/.json//") 75 | # Return value snapshot file format is snapshots/return_values/{handler}_{runtime}_{input-event} 76 | snapshot_path="$integration_tests_dir/snapshots/return_values/${function_name}_${input_event_name}.json" 77 | 78 | return_value=$(DD_API_KEY=$DD_API_KEY sls invoke --stage $run_id -f $function_name --path "$integration_tests_dir/input_events/$input_event_file") 79 | sls_invoke_exit_code=$? 80 | if [ $sls_invoke_exit_code -ne 0 ]; then 81 | return_value="Invocation failed" 82 | fi 83 | 84 | if [ ! -f $snapshot_path ]; then 85 | # If the snapshot file doesn't exist yet, we create it 86 | echo "Writing return value to $snapshot_path because no snapshot exists yet" 87 | echo "$return_value" >$snapshot_path 88 | elif [ -n "$UPDATE_SNAPSHOTS" ]; then 89 | # If $UPDATE_SNAPSHOTS is set to true, write the new logs over the current snapshot 90 | echo "Overwriting return value snapshot for $snapshot_path" 91 | echo "$return_value" >$snapshot_path 92 | else 93 | # Compare new return value to snapshot 94 | diff_output=$(echo "$return_value" | diff - $snapshot_path) 95 | if [ $? -eq 1 ]; then 96 | echo "Failed: Return value for $function_name does not match snapshot:" 97 | echo "$diff_output" 98 | mismatch_found=true 99 | else 100 | echo "Ok: Return value for $function_name with $input_event_name event matches snapshot" 101 | fi 102 | fi 103 | done 104 | done 105 | set -e 106 | 107 | echo "Sleeping $LOGS_WAIT_SECONDS seconds to wait for logs to appear in CloudWatch..." 108 | sleep $LOGS_WAIT_SECONDS 109 | 110 | set +e # Don't exit this script if there is a diff or the logs endpoint fails 111 | echo "Fetching logs for invocations and comparing to snapshots" 112 | for function_name in "${LAMBDA_HANDLERS[@]}"; do 113 | function_snapshot_path="./snapshots/logs/$function_name.log" 114 | 115 | # Fetch logs with serverless cli, retrying to avoid AWS account-wide rate limit error 116 | retry_counter=0 117 | while [ $retry_counter -lt 10 ]; do 118 | raw_logs=$(serverless logs -f $function_name --stage $run_id --startTime $script_utc_start_time) 119 | fetch_logs_exit_code=$? 120 | if [ $fetch_logs_exit_code -eq 1 ]; then 121 | echo "Retrying fetch logs for $function_name..." 122 | retry_counter=$(($retry_counter + 1)) 123 | sleep 10 124 | continue 125 | fi 126 | break 127 | done 128 | 129 | if [ $retry_counter -eq 9 ]; then 130 | echo "FAILURE: Could not retrieve logs for $function_name" 131 | echo "Error from final attempt to retrieve logs:" 132 | echo $raw_logs 133 | 134 | exit 1 135 | fi 136 | 137 | # Replace invocation-specific data like timestamps and IDs with XXXX to normalize logs across executions 138 | logs=$( 139 | echo "$raw_logs" | 140 | node parse-json.js | 141 | # Remove serverless cli errors 142 | sed '/Serverless: Recoverable error occurred/d' | 143 | # Remove dd-trace-go logs 144 | sed '/Datadog Tracer/d' | 145 | # Normalize Lambda runtime report logs 146 | perl -p -e 's/(RequestId|TraceId|init|SegmentId|Duration|Memory Used|"e"):( )?[a-z0-9\.\-]+/\1:\2XXXX/g' | 147 | # Normalize DD APM headers and AWS account ID 148 | perl -p -e "s/(Current span ID:|Current trace ID:|account_id:) ?[0-9]+/\1XXXX/g" | 149 | # Strip API key from logged requests 150 | perl -p -e "s/(api_key=|'api_key': ')[a-z0-9\.\-]+/\1XXXX/g" | 151 | # Normalize ISO combined date-time 152 | perl -p -e "s/[0-9]{4}\-[0-9]{2}\-[0-9]{2}(T?)[0-9]{2}:[0-9]{2}:[0-9]{2}(\.[0-9]+ \(\-?[0-9:]+\))?Z/XXXX-XX-XXTXX:XX:XX.XXXZ/" | 153 | # Normalize log timestamps 154 | perl -p -e "s/[0-9]{4}(\-|\/)[0-9]{2}(\-|\/)[0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2}(\.[0-9]+( \(\-?[0-9:]+\))?)?/XXXX-XX-XX XX:XX:XX.XXX/" | 155 | # Normalize DD trace ID injection 156 | perl -p -e "s/(dd\.trace_id=)[0-9]+ (dd\.span_id=)[0-9]+/\1XXXX \2XXXX/" | 157 | # Normalize execution ID in logs prefix 158 | perl -p -e $'s/[0-9a-z]+\-[0-9a-z]+\-[0-9a-z]+\-[0-9a-z]+\-[0-9a-z]+\t/XXXX-XXXX-XXXX-XXXX-XXXX\t/' | 159 | # Normalize layer version tag 160 | perl -p -e "s/(dd_lambda_layer:datadog-go)[0-9]+\.[0-9]+\.[0-9]+/\1X\.X\.X/g" | 161 | # Normalize package version tag 162 | perl -p -e "s/(datadog_lambda:v)[0-9]+\.[0-9]+\.[0-9]+/\1X\.X\.X/g" | 163 | # Normalize golang version tag 164 | perl -p -e "s/(go)[0-9]+\.[0-9]+\.[0-9]+/\1X\.X\.X/g" | 165 | # Normalize data in logged traces 166 | perl -p -e 's/"(span_id|apiid|runtime-id|record_ids|parent_id|trace_id|start|duration|tcp\.local\.address|tcp\.local\.port|dns\.address|request_id|function_arn|x-datadog-trace-id|x-datadog-parent-id|datadog_lambda|dd_trace)":\ ("?)[a-zA-Z0-9\.:\-]+("?)/"\1":\2XXXX\3/g' | 167 | # Remove metrics and metas in logged traces (their order is inconsistent) 168 | perl -p -e 's/"(meta|metrics)":{(.*?)}/"\1":{"XXXX": "XXXX"}/g' | 169 | # Strip out run ID (from function name, resource, etc.) 170 | perl -p -e "s/$run_id/XXXX/g" | 171 | # Normalize data in logged metrics 172 | perl -p -e 's/"(points\\\":\[\[)([0-9]+)/\1XXXX/g' | 173 | # Remove INIT_START log 174 | perl -p -e "s/INIT_START.*//g" 175 | ) 176 | 177 | if [ ! -f $function_snapshot_path ]; then 178 | # If no snapshot file exists yet, we create one 179 | echo "Writing logs to $function_snapshot_path because no snapshot exists yet" 180 | echo "$logs" >$function_snapshot_path 181 | elif [ -n "$UPDATE_SNAPSHOTS" ]; then 182 | # If $UPDATE_SNAPSHOTS is set to true write the new logs over the current snapshot 183 | echo "Overwriting log snapshot for $function_snapshot_path" 184 | echo "$logs" >$function_snapshot_path 185 | else 186 | # Compare new logs to snapshots 187 | diff_output=$(echo "$logs" | diff - $function_snapshot_path) 188 | if [ $? -eq 1 ]; then 189 | echo "Failed: Mismatch found between new $function_name logs (first) and snapshot (second):" 190 | echo "$diff_output" 191 | mismatch_found=true 192 | else 193 | echo "Ok: New logs for $function_name match snapshot" 194 | fi 195 | fi 196 | done 197 | set -e 198 | 199 | if [ "$mismatch_found" = true ]; then 200 | echo "FAILURE: A mismatch between new data and a snapshot was found and printed above." 201 | echo "If the change is expected, generate new snapshots by running 'UPDATE_SNAPSHOTS=true DD_API_KEY=XXXX ./run_integration_tests.sh'" 202 | exit 1 203 | fi 204 | 205 | if [ -n "$UPDATE_SNAPSHOTS" ]; then 206 | echo "SUCCESS: Wrote new snapshots for all functions" 207 | exit 0 208 | fi 209 | 210 | echo "SUCCESS: No difference found between snapshots and new return values or logs" 211 | -------------------------------------------------------------------------------- /tests/integration_tests/serverless.yml: -------------------------------------------------------------------------------- 1 | # IAM permissions require service name to begin with 'integration-tests' 2 | service: integration-tests-go 3 | 4 | package: 5 | individually: true # <- package each function individually, to prevent file name conflicts 6 | 7 | provider: 8 | name: aws 9 | region: eu-west-1 10 | tracing: 11 | lambda: true 12 | apiGateway: true 13 | memorySize: 128 14 | timeout: 30 15 | environment: 16 | DD_API_KEY: ${env:DD_API_KEY} 17 | DD_LOG_LEVEL: DEBUG 18 | DD_INTEGRATION_TEST: true 19 | DD_ENHANCED_METRICS: true 20 | DD_TRACE_ENABLED: true 21 | deploymentBucket: 22 | name: integration-tests-serververless-deployment-bucket 23 | iam: 24 | # IAM permissions require that all functions are deployed with this role 25 | role: "arn:aws:iam::425362996713:role/serverless-integration-test-lambda-role" 26 | 27 | functions: 28 | hello: 29 | runtime: provided.al2 30 | handler: bootstrap 31 | package: 32 | artifact: build/hello.zip 33 | error: 34 | runtime: provided.al2 35 | handler: bootstrap 36 | package: 37 | artifact: build/error.zip 38 | -------------------------------------------------------------------------------- /tests/integration_tests/snapshots/logs/error.log: -------------------------------------------------------------------------------- 1 | 2 | XXXX-XX-XX XX:XX:XX.XXX {"status":"debug","message":"datadog: Will use the API"} 3 | START 4 | XXXX-XX-XX XX:XX:XX.XXX {"status":"debug","message":"datadog: Merge X-Ray Traces is off, using trace context from Datadog only"} 5 | XXXX-XX-XX XX:XX:XX.XXX {"status":"debug","message":"datadog: Could not convert TraceContext to a SpanContext (most likely TraceContext was empty)"} 6 | XXXX-XX-XX XX:XX:XX.XXX {"status":"debug","message":"datadog: sending metric via log forwarder"} 7 | { 8 | "e": XXXX, 9 | "m": "aws.lambda.enhanced.invocations", 10 | "t": [ 11 | "functionname:integration-tests-go-XXXX-error", 12 | "region:eu-west-1", 13 | "account_id:XXXX", 14 | "memorysize:128", 15 | "cold_start:true", 16 | "datadog_lambda:vX.X.X", 17 | "resource:integration-tests-go-XXXX-error", 18 | "dd_lambda_layer:datadog-goX.X.X" 19 | ], 20 | "v": 1 21 | } 22 | { 23 | "traces": [ 24 | [ 25 | {} 26 | ] 27 | ] 28 | } 29 | XXXX-XX-XX XX:XX:XX.XXX {"status":"debug","message":"datadog: sending metric via log forwarder"} 30 | { 31 | "e": XXXX, 32 | "m": "aws.lambda.enhanced.errors", 33 | "t": [ 34 | "functionname:integration-tests-go-XXXX-error", 35 | "region:eu-west-1", 36 | "account_id:XXXX", 37 | "memorysize:128", 38 | "cold_start:true", 39 | "datadog_lambda:vX.X.X", 40 | "resource:integration-tests-go-XXXX-error", 41 | "dd_lambda_layer:datadog-goX.X.X" 42 | ], 43 | "v": 1 44 | } 45 | XXXX-XX-XX XX:XX:XX.XXX {"errorMessage":"something went wrong","errorType":"errorString"} 46 | END Duration: XXXX ms (init: XXXX ms) Memory Used: XXXX MB 47 | -------------------------------------------------------------------------------- /tests/integration_tests/snapshots/logs/hello.log: -------------------------------------------------------------------------------- 1 | 2 | XXXX-XX-XX XX:XX:XX.XXX {"status":"debug","message":"datadog: Will use the API"} 3 | START 4 | XXXX-XX-XX XX:XX:XX.XXX {"status":"debug","message":"datadog: Merge X-Ray Traces is off, using trace context from Datadog only"} 5 | XXXX-XX-XX XX:XX:XX.XXX {"status":"debug","message":"datadog: Could not convert TraceContext to a SpanContext (most likely TraceContext was empty)"} 6 | XXXX-XX-XX XX:XX:XX.XXX {"status":"debug","message":"datadog: sending metric via log forwarder"} 7 | { 8 | "e": XXXX, 9 | "m": "aws.lambda.enhanced.invocations", 10 | "t": [ 11 | "functionname:integration-tests-go-XXXX-hello", 12 | "region:eu-west-1", 13 | "account_id:XXXX", 14 | "memorysize:128", 15 | "cold_start:true", 16 | "datadog_lambda:vX.X.X", 17 | "resource:integration-tests-go-XXXX-hello", 18 | "dd_lambda_layer:datadog-goX.X.X" 19 | ], 20 | "v": 1 21 | } 22 | Current span ID:XXXX 23 | Current trace ID:XXXX 24 | XXXX-XX-XX XX:XX:XX.XXX {"status":"debug","message":"datadog: adding metric \"hello-go.dog\", with value 1.000000"} 25 | { 26 | "traces": [ 27 | [ 28 | {}, 29 | {}, 30 | {}, 31 | {}, 32 | {}, 33 | {}, 34 | {}, 35 | {}, 36 | {}, 37 | {}, 38 | {}, 39 | {} 40 | ] 41 | ] 42 | } 43 | XXXX-XX-XX XX:XX:XX.XXX {"status":"debug","message":"datadog: posting to url https://api.datadoghq.com/api/v1/distribution_points"} 44 | XXXX-XX-XX XX:XX:XX.XXX {"status":"debug","message":"datadog: Sending payload with body {\"series\":[{\"metric\":\"hello-go.dog\",\"tags\":[\"dd_lambda_layer:datadog-goX.X.X\"],\"type\":\"distribution\",\points\":[[XXXX,[1]]]}]}"} 45 | END Duration: XXXX ms (init: XXXX ms) Memory Used: XXXX MB 46 | -------------------------------------------------------------------------------- /tests/integration_tests/snapshots/return_values/error_api-gateway-get.json: -------------------------------------------------------------------------------- 1 | Invocation failed 2 | -------------------------------------------------------------------------------- /tests/integration_tests/snapshots/return_values/hello_api-gateway-get.json: -------------------------------------------------------------------------------- 1 | { 2 | "statusCode": 200, 3 | "headers": null, 4 | "multiValueHeaders": null, 5 | "body": "hello, dog!" 6 | } 7 | --------------------------------------------------------------------------------