├── .eslintrc.json ├── .gitignore ├── LICENSE ├── README.md ├── app ├── README.md ├── cloudwatch-publisher.js ├── collector-api.js ├── package.json ├── s3-publisher.js └── template.yaml ├── cloudwatch-publisher ├── package.json └── src │ └── lambda.js ├── collector-api ├── package.json ├── src │ ├── convert-from-desole.js │ ├── convert-from-sentry.js │ ├── create-generic-lambda-event.js │ ├── extract-keys.js │ ├── get-raven-stacktraces.js │ ├── lambda.js │ └── lowercase-keys.js └── tests │ ├── .eslintrc.json │ ├── convert-from-sentry.test.js │ └── test-events │ ├── lambda-proxy-event.json │ ├── raven-exception.json │ └── raven-manually-tracked.json ├── common ├── package.json └── src │ └── parse-sns-event.js ├── devtools ├── bin │ └── devtool.js └── package.json ├── elasticsearch-publisher ├── README.md ├── package.json ├── src │ └── lambda.js └── template.yaml ├── eventformat.md ├── pinpoint-publisher ├── README.md ├── package-lock.json ├── package.json ├── src │ ├── convert-analytics-event.js │ ├── convert-client-context.js │ ├── lambda.js │ └── store-single-event.js ├── template.yaml └── tests │ ├── .eslintrc.json │ ├── convert-analytics-event.test.js │ └── test-events │ └── sample-event.json ├── s3-publisher ├── package.json └── src │ └── lambda.js ├── test-events ├── README.md ├── from-browser-to-api.json ├── full-event.json ├── raven-manual-config.json └── raven.json └── todo.txt /.eslintrc.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "crockford", 3 | "env": { 4 | "node": true, 5 | "es6": true 6 | }, 7 | "parserOptions": { 8 | "ecmaVersion": 6 9 | }, 10 | "rules": { 11 | "semi": ["error", "always"], 12 | "no-unused-vars": "error", 13 | "indent": ["error", "tab", { "SwitchCase": 1, "MemberExpression": "off" } ], 14 | "no-const-assign": "error", 15 | "one-var": "error", 16 | "prefer-const": "error", 17 | "strict": ["error", "global"], 18 | "no-var": "error", 19 | "no-shadow": "error", 20 | "prefer-arrow-callback": "error", 21 | "no-plusplus": ["error", { "allowForLoopAfterthoughts": true }], 22 | "quotes": ["error", "single", {"avoidEscape": true, "allowTemplateLiterals": true}] 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | npm-debug.log 3 | output.yaml 4 | .vscode 5 | .idea 6 | .DS_Store 7 | app/desole.zip 8 | elasticsearch-publisher/es-publisher.zip 9 | pinpoint-publisher/pinpoint-publisher.zip 10 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | Copyright (c) 2018 Gojko Adzic, Slobodan Stojanovic, Aleksandar Simovic 3 | 4 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 5 | 6 | The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. 7 | 8 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 9 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Desole collector back-end 2 | 3 | Desole is an error-tracking system you can install in your AWS account, with just a few clicks. It enables organisations to track application exceptions and errors without having to choose between the convenience of software-as-a-service and the security of a self-hosted solution. You fully control the data, so it is easy to enforce compliance, encryption and data security requirements. At the same time, Desole uses highly-scalable AWS resources that can easily handle massive traffic, and auto-size on demand, so you do not have to worry about operating costs or administration. Check out for more information. 4 | 5 | ![](https://desole.io/images/desole-arch-2.png) 6 | 7 | This repository contains the back-end components for Desole: the event collector API and the standard publishers, including a Cloudformation template you can use to deploy the Desole back-end into your AWS account. Check out for more information on how to set up the client collectors. 8 | 9 | ## Deploy using the AWS Serverless App Repo 10 | 11 | Head over to the [Desole App](https://serverlessrepo.aws.amazon.com/applications/arn:aws:serverlessrepo:us-east-1:145266761615:applications~Desole) on the AWS Serverless App repo, click `Deploy` and follow the wizard to configure the app. 12 | 13 | ## Deploy the standard app using CloudFormation 14 | 15 | The standard app includes the CloudWatch and S3 publishers. 16 | 17 | Region | Launch 18 | -------|------- 19 | US East (N.Virginia) | [![Desole in us-east-1](http://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/images/cloudformation-launch-stack-button.png)](https://console.aws.amazon.com/cloudformation/home?region=us-east-1#/stacks/create/review?stackName=desole&templateURL=https://desole-packaging-us-east-1.s3.amazonaws.com/1.0.0/@desole/app.yaml) 20 | EU Central (Frankfurt) | [![Desole in eu-central-1](http://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/images/cloudformation-launch-stack-button.png)](https://console.aws.amazon.com/cloudformation/home?region=eu-central-1#/stacks/create/review?stackName=desole&templateURL=https://desole-packaging-eu-central-1.s3.amazonaws.com/1.0.0/@desole/app.yaml) 21 | US West (N. California) | [![Desole in us-west-1](http://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/images/cloudformation-launch-stack-button.png)](https://console.aws.amazon.com/cloudformation/home?region=us-west-1#/stacks/create/review?stackName=desole&templateURL=https://desole-packaging-us-west-1.s3.amazonaws.com/1.0.0/@desole/app.yaml) 22 | Asia Pacific (Sydney) | [![Desole in ap-southeast-2](http://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/images/cloudformation-launch-stack-button.png)](https://console.aws.amazon.com/cloudformation/home?region=ap-southeast-2#/stacks/create/review?stackName=desole&templateURL=https://desole-packaging-ap-southeast-2.s3.amazonaws.com/1.0.0/@desole/app.yaml) 23 | 24 | Check out the [app/README.md](app/README.md) for information on deploying your own custom bundle into other regions. 25 | 26 | ## Deploy the optional publishers 27 | 28 | * Deploy the [ElasticSearch publisher](elasticsearch-publisher/README.md) to enable easy querying and searching 29 | * Deploy the [PinPoint publisher](pinpoint-publisher/README.md) to enable automated engagement campaigns and dashboards to drill down into user demographics 30 | -------------------------------------------------------------------------------- /app/README.md: -------------------------------------------------------------------------------- 1 | # Cloudformation template for Desole back-end deployment 2 | 3 | This directory contains a Cloudformation template for the Desole collector API and standard publishers. Use this source code to create your own custom bundle for Desole backend. 4 | 5 | To deploy using a ready-made template, check out the parent directory [README.md](../README.md) 6 | 7 | ## Prerequisites 8 | 9 | * NPM 10 | * An S3 Bucket for Deployment, in the same region where you would like to deploy Desole 11 | * AWS CLI (command line tools), configured to use your account 12 | 13 | ## Deploying using AWS-CLI and CloudFormation 14 | 15 | For a detailed list of supported parameters, check out [`template.yaml`](template.yaml) 16 | 17 | 1. Install the dependencies 18 | ```bash 19 | npm install 20 | ``` 21 | 2. Prepare and pack your code 22 | ```bash 23 | npm run prepackage 24 | ``` 25 | 3. Package the template 26 | ```bash 27 | aws cloudformation package --template-file template.yaml --output-template-file output.yaml 28 | ``` 29 | 4. Deploy the packaged template 30 | ```bash 31 | aws cloudformation deploy --template-file output.yaml --capabilities CAPABILITY_IAM --stack-name 32 | ``` 33 | 34 | You can also override CloudFormation template parameters by using `--parameter-overrides =` after the `deploy` command. For a detailed list of supported parameters, check out [`template.yaml`](template.yaml) 35 | -------------------------------------------------------------------------------- /app/cloudwatch-publisher.js: -------------------------------------------------------------------------------- 1 | module.exports = require('@desole/cloudwatch-publisher'); 2 | -------------------------------------------------------------------------------- /app/collector-api.js: -------------------------------------------------------------------------------- 1 | module.exports = require('@desole/collector-api'); 2 | -------------------------------------------------------------------------------- /app/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@desole/app", 3 | "private": true, 4 | "version": "1.1.0", 5 | "description": "Shell package for cloudformation deployment, not intended to be used directly", 6 | "main": "lambda.js", 7 | "scripts": { 8 | "prepackage": "claudia pack --no-optional-dependencies --output desole.zip --force", 9 | "test": "sam validate", 10 | "package": "echo packaging $npm_package_config_region to $npm_package_config_bucket_name && aws cloudformation package --template-file template.yaml --output-template-file output.yaml --s3-bucket $npm_package_config_bucket_name --region $npm_package_config_region --s3-prefix $npm_package_version", 11 | "deploy": "aws cloudformation deploy --template-file output.yaml --stack-name $npm_package_config_cloudformation_stack --capabilities CAPABILITY_IAM --region $npm_package_config_region", 12 | "qd": "npm run package && npm run deploy && npm run get-deploy-outputs", 13 | "get-deploy-outputs": "aws cloudformation describe-stacks --region $npm_package_config_region --query 'Stacks[?StackName==`desoleStack`].Outputs' --output table", 14 | "upload": "aws s3 cp output.yaml s3://$npm_package_config_bucket_name/$npm_package_version/$npm_package_name.yaml --acl public-read", 15 | "make-public": "for c in $(aws s3api list-objects --bucket $npm_package_config_bucket_name --prefix $npm_package_version --output text --query Contents[].Key); do echo making $c public; aws s3api put-object-acl --acl public-read --bucket $npm_package_config_bucket_name --key $c; done", 16 | "release": "npm run package --$npm_package_name:region=$npm_package_config_region --$npm_package_name:bucket_name=$npm_package_config_bucket_name && npm run upload --$npm_package_name:region=$npm_package_config_region --$npm_package_name:bucket_name=$npm_package_config_bucket_name && npm run make-public --$npm_package_name:region=$npm_package_config_region --$npm_package_name:bucket_name=$npm_package_config_bucket_name", 17 | "regions": "for region in $(echo $npm_package_config_deployment_regions); do npm run release --$npm_package_name:region=$region --$npm_package_name:bucket_name=desole-packaging-$region; done;" 18 | }, 19 | "keywords": [], 20 | "author": "", 21 | "license": "MIT", 22 | "dependencies": { 23 | "@desole/s3-publisher": "file:../s3-publisher", 24 | "@desole/cloudwatch-publisher": "file:../cloudwatch-publisher", 25 | "@desole/collector-api": "file:../collector-api" 26 | }, 27 | "config": { 28 | "bucket_name": "desole-packaging-us-east-1", 29 | "deployment_regions": "us-east-1 eu-central-1 us-west-1 ap-southeast-2", 30 | "cloudformation_stack": "desoleStack", 31 | "region": "us-east-1" 32 | }, 33 | "devDependencies": { 34 | "claudia": "^5.0.0" 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /app/s3-publisher.js: -------------------------------------------------------------------------------- 1 | module.exports = require('@desole/s3-publisher'); 2 | -------------------------------------------------------------------------------- /app/template.yaml: -------------------------------------------------------------------------------- 1 | AWSTemplateFormatVersion: 2010-09-09 2 | Transform: AWS::Serverless-2016-10-31 3 | 4 | Parameters: 5 | BucketExpiryDays: 6 | Type: String 7 | Default: '' 8 | AllowedPattern: ^[0-9]*$ 9 | ConstraintDescription: Must be a number 10 | Description: Expiry time (in days) for events on S3 storage. The event details will be automatically removed from S3 after expiry. Leave empty for no expiration. 11 | BucketEncryption: 12 | Type: String 13 | Default: AES256 14 | AllowedValues: 15 | - None 16 | - AES256 17 | - aws:kms 18 | Description: Server-side encryption algorithm for your event S3 storage. Select AES256 for S3 managed keys, or AWS:KMS for KMS managed keys 19 | BucketEncryptionKMSKey: 20 | Type: String 21 | Default: '' 22 | Description: The AWS:KMS Master key for S3 server-side encryption. Leave blank if not using AWS:KMS bucket encryption. 23 | AllowedPattern: ^$|^arn:aws:kms:.+:.+:.+ 24 | ConstraintDescription: Must be an AWS KMS Key ARN (eg arn:aws:kms:us-east-1:123456789012:key/12345678-1234-1234-1234-123456789012) 25 | BucketPrefix: 26 | Type: String 27 | Default: archive 28 | Description: Storage prefix for S3 event storage 29 | MaxLength: 250 30 | MinLength: 1 31 | CorsOrigin: 32 | Type: String 33 | Default: "*" 34 | Description: Cross-origin resource sharing (CORS) restriction for the event submission Web Api. Restrict to a single web site domain to only allow submissions from that site, or leave as * for open submission. 35 | MaxLength: 250 36 | MinLength: 1 37 | CloudWatchNameSpace: 38 | Type: String 39 | Default: Desole 40 | Description: CloudWatch Metrics Namespace 41 | AllowedPattern: ^[0-9A-Za-z\._/-]+$ 42 | MaxLength: 250 43 | MinLength: 1 44 | ConstraintDescription: 'Up to 250 characters. Possible characters are: alphanumeric characters (0-9A-Za-z), period (.), hyphen (-), underscore (_), forward slash (/), hash (#), and colon (:)' 45 | Conditions: 46 | IsBucketExpiryDefined: !Not [ !Equals ['', !Ref BucketExpiryDays]] 47 | IsBucketEncryptionDefined: !Not [ !Equals ['None', !Ref BucketEncryption]] 48 | IsBucketEncryptionKMS: !Equals ['aws:kms', !Ref BucketEncryption] 49 | Resources: 50 | CollectorApi: 51 | Type: AWS::Serverless::Function 52 | Properties: 53 | CodeUri: desole.zip 54 | Handler: collector-api.handler 55 | Runtime: nodejs8.10 56 | Environment: 57 | Variables: 58 | TOPIC_ARN: !Ref CollectorTopic 59 | CORS_ORIGIN: !Ref CorsOrigin 60 | Policies: 61 | - SNSPublishMessagePolicy: 62 | TopicName: !GetAtt CollectorTopic.TopicName 63 | Events: 64 | PostResource: 65 | Type: Api 66 | Properties: 67 | Path: /desole 68 | Method: POST 69 | OptionsResource: 70 | Type: Api 71 | Properties: 72 | Path: /desole 73 | Method: OPTIONS 74 | PostResourceSentry: 75 | Type: Api 76 | Properties: 77 | Path: /api/raven/store 78 | Method: POST 79 | OptionsResourceSentry: 80 | Type: Api 81 | Properties: 82 | Path: /api/raven/store 83 | Method: OPTIONS 84 | CollectorTopic: 85 | Type: AWS::SNS::Topic 86 | S3Publisher: 87 | Type: AWS::Serverless::Function 88 | Properties: 89 | CodeUri: desole.zip 90 | Handler: s3-publisher.handler 91 | Runtime: nodejs8.10 92 | Environment: 93 | Variables: 94 | BUCKET_NAME: !Ref CollectedEventsStorage 95 | BUCKET_PREFIX: !Ref BucketPrefix 96 | Policies: 97 | - S3CrudPolicy: 98 | BucketName: !Ref CollectedEventsStorage 99 | Events: 100 | CollectorSubscribedTopic: 101 | Type: SNS 102 | Properties: 103 | Topic: !Ref CollectorTopic 104 | CloudwatchPublisher: 105 | Type: AWS::Serverless::Function 106 | Properties: 107 | CodeUri: desole.zip 108 | Handler: cloudwatch-publisher.handler 109 | Runtime: nodejs8.10 110 | Environment: 111 | Variables: 112 | CLOUDWATCH_NAMESPACE: !Ref CloudWatchNameSpace 113 | Policies: 114 | - CloudWatchPutMetricPolicy: {} 115 | Events: 116 | CollectorSubscribedTopic: 117 | Type: SNS 118 | Properties: 119 | Topic: !Ref CollectorTopic 120 | CollectedEventsStorage: 121 | Type: AWS::S3::Bucket 122 | Properties: 123 | BucketEncryption: 124 | !If 125 | - IsBucketEncryptionDefined 126 | - 127 | ServerSideEncryptionConfiguration: 128 | - ServerSideEncryptionByDefault: 129 | SSEAlgorithm: !Ref BucketEncryption 130 | KMSMasterKeyID: !If [IsBucketEncryptionKMS, !Ref BucketEncryptionKMSKey, !Ref "AWS::NoValue"] 131 | - !Ref AWS::NoValue 132 | LifecycleConfiguration: 133 | !If 134 | - IsBucketExpiryDefined 135 | - Rules: 136 | - ExpirationInDays: !Ref BucketExpiryDays 137 | Prefix: !Ref BucketPrefix 138 | Status: Enabled 139 | - !Ref AWS::NoValue 140 | Outputs: 141 | DesoleClientApiUrl: 142 | Description: The URL of your API endpoint -- use this to configure the Desole client 143 | Value: !Sub "https://${ServerlessRestApi}.execute-api.${AWS::Region}.amazonaws.com/${ServerlessRestApiProdStage}/desole" 144 | RavenClientApiUrl: 145 | Description: The URL of your API endpoint -- use this to configure the Raven client 146 | Value: !Sub "https://${ServerlessRestApi}.execute-api.${AWS::Region}.amazonaws.com/${ServerlessRestApiProdStage}/raven" 147 | S3Bucket: 148 | Value: !Ref CollectedEventsStorage 149 | Description: S3 bucket where the events will be archived 150 | CollectorSNSTopic: 151 | Value: !Ref CollectorTopic 152 | Description: SNS Collector Topic for custom integrations 153 | -------------------------------------------------------------------------------- /cloudwatch-publisher/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@desole/cloudwatch-publisher", 3 | "version": "1.0.0", 4 | "description": "Desole publisher saves cloudwatch metrics", 5 | "main": "src/lambda.js", 6 | "keywords": [], 7 | "files": [ 8 | "src" 9 | ], 10 | "private": true, 11 | "author": "", 12 | "license": "MIT", 13 | "devDependencies": { 14 | "@desole/devtools": "file:../devtools" 15 | }, 16 | "dependencies": { 17 | "@desole/common": "file:../common" 18 | }, 19 | "optionalDependencies": { 20 | "aws-sdk": "^2.224.1" 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /cloudwatch-publisher/src/lambda.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | const AWS = require('aws-sdk'), 3 | cloudWatch = new AWS.CloudWatch(), 4 | parseSNSEvent = require('@desole/common/src/parse-sns-event'), 5 | CLOUDWATCH_NAMESPACE = process.env.CLOUDWATCH_NAMESPACE, 6 | toDimension = (name, value) => ({Name: name, Value: value || '-'}), 7 | convertToSeverityMetricData = function (event) { 8 | return { 9 | MetricName: 'Count', 10 | Unit: 'Count', 11 | Value: 1.0, 12 | Dimensions: [ 13 | toDimension('App Name', event.app.name), 14 | toDimension('App Stage', event.app.stage), 15 | toDimension('Severity', event.severity) 16 | ], 17 | StorageResolution: 1, 18 | Timestamp: new Date(event.receivedAt) 19 | }; 20 | }, 21 | convertToTypeMetricData = function (event) { 22 | return { 23 | MetricName: 'Count', 24 | Unit: 'Count', 25 | Value: 1.0, 26 | Dimensions: [ 27 | toDimension('App Name', event.app.name), 28 | toDimension('App Stage', event.app.stage), 29 | toDimension('Type', event.type) 30 | ], 31 | StorageResolution: 1, 32 | Timestamp: new Date(event.receivedAt) 33 | }; 34 | }, 35 | storeSingleEvent = event => { 36 | const params = { 37 | MetricData: [convertToSeverityMetricData(event), convertToTypeMetricData(event)], 38 | Namespace: CLOUDWATCH_NAMESPACE 39 | }; 40 | return cloudWatch.putMetricData(params).promise(); 41 | }; 42 | 43 | exports.handler = (event) => { 44 | const records = parseSNSEvent(event); 45 | return Promise.all(records.map(storeSingleEvent)); 46 | }; 47 | -------------------------------------------------------------------------------- /collector-api/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@desole/collector-api", 3 | "version": "1.0.0", 4 | "description": "", 5 | "main": "src/lambda.js", 6 | "files": [ 7 | "src" 8 | ], 9 | "private": true, 10 | "license": "MIT", 11 | "devDependencies": { 12 | "@desole/devtools": "file:../devtools" 13 | }, 14 | "dependencies": { 15 | "useragent": "^2.3.0" 16 | }, 17 | "optionalDependencies": { 18 | "aws-sdk": "^2.224.1" 19 | }, 20 | "jest": { 21 | "roots": [ 22 | "/src/", 23 | "/tests/" 24 | ] 25 | }, 26 | "scripts": { 27 | "pretest": "devtool eslint src tests", 28 | "test": "devtool jest --color" 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /collector-api/src/convert-from-desole.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | const createGenericEventFromLambda = require('./create-generic-lambda-event'), 4 | extractKeys = require('./extract-keys'); 5 | 6 | module.exports = function convertFromDesole(lambdaProxyEvent, lambdaContext) { 7 | const body = JSON.parse(lambdaProxyEvent.body), 8 | genericEvent = createGenericEventFromLambda(lambdaProxyEvent), 9 | desoleEvent = Object.assign(genericEvent, extractKeys(body, ['severity', 'stack', 'type', 'timestamp', 'resource', 'tags'])); 10 | 11 | Object.assign(desoleEvent.endpoint, extractKeys(body.endpoint, ['id', 'platform', 'language'])); 12 | desoleEvent.app = extractKeys(body.app, ['name', 'version', 'stage']); 13 | desoleEvent.id = lambdaContext.awsRequestId; 14 | return desoleEvent; 15 | }; 16 | -------------------------------------------------------------------------------- /collector-api/src/convert-from-sentry.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | const createGenericLambdaEvent = require('./create-generic-lambda-event'), 4 | getRavenStacktraces = require('./get-raven-stacktraces'), 5 | extractBreadcrumbTimeStamp = breadcrumbs => { 6 | const unixTimestamp = breadcrumbs && Array.isArray(breadcrumbs.values) && breadcrumbs.values[breadcrumbs.values.length - 1].timestamp; 7 | return Math.floor(unixTimestamp * 1000); 8 | }; 9 | 10 | module.exports = function convertFromSentry(lambdaProxyEvent, lambdaContext) { 11 | const body = JSON.parse(lambdaProxyEvent.body), 12 | exception = (body.exception && body.exception.values[0]) || body, 13 | breadcrumbs = body.breadcrumbs, 14 | desoleEvent = Object.assign(createGenericLambdaEvent(lambdaProxyEvent), { 15 | severity: body.level || 'error', 16 | stack: getRavenStacktraces(exception), 17 | type: exception.type || 'RuntimeError', 18 | message: exception.value || body.message, 19 | timestamp: extractBreadcrumbTimeStamp(breadcrumbs), 20 | resource: body.request.url, 21 | tags: body.tags 22 | }); 23 | desoleEvent.endpoint.id = body.user && body.user.id; 24 | desoleEvent.endpoint.platform = body.platform; 25 | desoleEvent.app = { 26 | name: body.project, 27 | version: body.release, 28 | stage: body.environment 29 | }; 30 | desoleEvent.id = body.event_id || lambdaContext.awsRequestId; 31 | return desoleEvent; 32 | }; 33 | 34 | 35 | -------------------------------------------------------------------------------- /collector-api/src/create-generic-lambda-event.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | const uaParser = require('useragent'), 3 | lowercaseKeys = require('./lowercase-keys'), 4 | extractDeviceType = (headers) => { 5 | return ['SmartTV', 'Mobile', 'Tablet', 'Desktop'].find(type => headers[`cloudfront-is-${type.toLowerCase()}-viewer`] === 'true'); 6 | }; 7 | 8 | module.exports = function createGenericEventFromLambda(lambdaProxyEvent) { 9 | const desoleEvent = {}, 10 | normalizedHeaders = lowercaseKeys(lambdaProxyEvent.headers), 11 | userAgent = uaParser.parse(lambdaProxyEvent.requestContext.identity.userAgent); 12 | desoleEvent.receivedAt = Date.now(); 13 | desoleEvent.referrer = normalizedHeaders.referer; 14 | desoleEvent.endpoint = { 15 | country: normalizedHeaders['cloudfront-viewer-country'], 16 | userAgent: lambdaProxyEvent.requestContext.identity.userAgent, 17 | deviceType: extractDeviceType(normalizedHeaders), 18 | runtime: userAgent.family, 19 | runtimeVersion: userAgent.toVersion(), 20 | os: userAgent.os.family, 21 | osVersion: userAgent.os.toVersion() 22 | }; 23 | return desoleEvent; 24 | }; 25 | 26 | -------------------------------------------------------------------------------- /collector-api/src/extract-keys.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | module.exports = function extractKeys(pickFrom, keys) { 4 | 5 | if (!pickFrom || !pickFrom.hasOwnProperty || !keys || !Array.isArray(keys)) { 6 | throw 'invalid-args'; 7 | } 8 | const picked = {}; 9 | keys.forEach((key) => { 10 | if (pickFrom.hasOwnProperty(key)) { 11 | picked[key] = pickFrom[key]; 12 | } 13 | }); 14 | return picked; 15 | }; 16 | -------------------------------------------------------------------------------- /collector-api/src/get-raven-stacktraces.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | module.exports = function getRavenStacktraces(exception) { 4 | return exception && exception.stacktrace && Array.isArray(exception.stacktrace.frames) && 5 | exception.stacktrace.frames.map(err => { 6 | return `\t at ${err.function} (${err.filename}:${err.lineno})`; 7 | }).reverse().concat([`${exception.type} ${exception.value}`]).join('\n'); 8 | }; 9 | -------------------------------------------------------------------------------- /collector-api/src/lambda.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | const AWS = require('aws-sdk'), 3 | sns = new AWS.SNS(), 4 | TOPIC_ARN = process.env.TOPIC_ARN, 5 | converters = {'/desole': require('./convert-from-desole'), '/api/raven/store': require('./convert-from-sentry')}, 6 | 7 | htmlResponse = function (body, requestedCode) { 8 | const code = requestedCode || (body ? 200 : 204); 9 | return { 10 | statusCode: code, 11 | body: body || '', 12 | headers: { 13 | 'Access-Control-Allow-Headers': 'Content-Type,Authorization,X-Amz-Date,X-Api-Key,X-Amz-Security-Token', 14 | 'Access-Control-Allow-Methods': 'OPTIONS,POST', 15 | 'Access-Control-Allow-Origin': process.env.CORS_ORIGIN, 16 | 'Access-Control-Max-Age': '86400' 17 | } 18 | }; 19 | }, 20 | getConvertedEvent = (event, context) => { 21 | try { 22 | const converter = converters[event.requestContext.resourcePath]; 23 | return converter(event, context); 24 | } catch (e) { 25 | console.error(e); 26 | } 27 | }; 28 | 29 | exports.handler = (event, context) => { 30 | if (event.httpMethod === 'OPTIONS') { 31 | return Promise.resolve(htmlResponse()); 32 | } 33 | const desoleEvent = getConvertedEvent(event, context); 34 | if (!desoleEvent) { 35 | return Promise.resolve(htmlResponse('invalid-args', 400)); 36 | } 37 | return sns.publish({ 38 | Message: JSON.stringify(desoleEvent), 39 | TopicArn: TOPIC_ARN 40 | }) 41 | .promise() 42 | .then(() => htmlResponse()) 43 | .catch(e => { 44 | console.log(e); 45 | return htmlResponse('server-error', 500); 46 | }); 47 | }; 48 | -------------------------------------------------------------------------------- /collector-api/src/lowercase-keys.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | module.exports = function lowercaseKeys(object) { 3 | const result = {}; 4 | if (object && typeof object === 'object' && !Array.isArray(object)) { 5 | Object.keys(object).forEach(key => result[key.toLowerCase()] = object[key]); 6 | } 7 | return result; 8 | }; 9 | -------------------------------------------------------------------------------- /collector-api/tests/.eslintrc.json: -------------------------------------------------------------------------------- 1 | { 2 | "env": { 3 | "jest": true 4 | } 5 | } 6 | -------------------------------------------------------------------------------- /collector-api/tests/convert-from-sentry.test.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | const convertFromSentry = require('../src/convert-from-sentry'), 3 | lambdaProxyEvent = require('./test-events/lambda-proxy-event'), 4 | wrapEvent = (event) => { 5 | const result = JSON.parse(JSON.stringify(lambdaProxyEvent)); 6 | result.body = JSON.stringify(event); 7 | return result; 8 | }; 9 | 10 | describe('Convert from Sentry', () => { 11 | 12 | 13 | describe('Error type', () => { 14 | test('should pickup the error type if defined', () => { 15 | const event = wrapEvent(require('./test-events/raven-exception')); 16 | expect(convertFromSentry(event).type).toBe('TypeError'); 17 | }); 18 | 19 | test('should use runtime error type if undefined', () => { 20 | const event = wrapEvent(require('./test-events/raven-manually-tracked')); 21 | expect(convertFromSentry(event).type).toBe('RuntimeError'); 22 | }); 23 | }); 24 | 25 | describe('Error message', () => { 26 | test('should pickup the message if exception value defined', () => { 27 | const event = wrapEvent(require('./test-events/raven-exception')); 28 | expect(convertFromSentry(event).message).toBe(`Cannot read property 'captureException' of undefined`); 29 | }); 30 | 31 | test('should use body message if exception value undefined', () => { 32 | const event = wrapEvent(require('./test-events/raven-manually-tracked')); 33 | expect(convertFromSentry(event).message).toBe('error'); 34 | }); 35 | }); 36 | }); 37 | 38 | 39 | -------------------------------------------------------------------------------- /collector-api/tests/test-events/lambda-proxy-event.json: -------------------------------------------------------------------------------- 1 | { 2 | "resource": "/api/raven/store", 3 | "path": "/api/raven/store/", 4 | "httpMethod": "POST", 5 | "headers": { 6 | "Accept": "*/*", 7 | "Accept-Encoding": "gzip, deflate, br", 8 | "Accept-Language": "sr,en-US;q=0.9,en;q=0.8,hr;q=0.7,bs;q=0.6,fr;q=0.5", 9 | "cache-control": "no-cache", 10 | "CloudFront-Forwarded-Proto": "https", 11 | "CloudFront-Is-Desktop-Viewer": "true", 12 | "CloudFront-Is-Mobile-Viewer": "false", 13 | "CloudFront-Is-SmartTV-Viewer": "false", 14 | "CloudFront-Is-Tablet-Viewer": "false", 15 | "CloudFront-Viewer-Country": "RS", 16 | "content-type": "text/plain;charset=UTF-8", 17 | "Host": "gex57xa2id.execute-api.us-east-1.amazonaws.com", 18 | "origin": "http://localhost:4000", 19 | "pragma": "no-cache", 20 | "Referer": "http://localhost:4000/", 21 | "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.181 Safari/537.36", 22 | "Via": "2.0 3ccd008055d57b9960754b53f631671f.cloudfront.net (CloudFront)", 23 | "X-Amz-Cf-Id": "a6KjFsNZWJ1PJQNwclVWHAakQtWA2vjAtO50Fd57qY8y_fNkVS2EaQ==", 24 | "X-Amzn-Trace-Id": "Root=1-5b05b203-bfb8cca81abf27c4d2ec5b04", 25 | "X-Forwarded-For": "109.122.102.202, 54.182.255.98", 26 | "X-Forwarded-Port": "443", 27 | "X-Forwarded-Proto": "https" 28 | }, 29 | "queryStringParameters": { 30 | "sentry_version": "7", 31 | "sentry_client": "raven-js/3.25.2", 32 | "sentry_key": "" 33 | }, 34 | "pathParameters": null, 35 | "stageVariables": null, 36 | "requestContext": { 37 | "resourceId": "70pv2z", 38 | "resourcePath": "/api/raven/store", 39 | "httpMethod": "POST", 40 | "extendedRequestId": "HWjAgH50IAMF5EQ=", 41 | "requestTime": "23/May/2018:18:25:07 +0000", 42 | "path": "/Prod/api/raven/store/", 43 | "accountId": "145266761615", 44 | "protocol": "HTTP/1.1", 45 | "stage": "Prod", 46 | "requestTimeEpoch": 1527099907293, 47 | "requestId": "9e9cb9b8-5eb6-11e8-ae92-8d187890f3e4", 48 | "identity": { 49 | "cognitoIdentityPoolId": null, 50 | "accountId": null, 51 | "cognitoIdentityId": null, 52 | "caller": null, 53 | "sourceIp": "109.122.102.202", 54 | "accessKey": null, 55 | "cognitoAuthenticationType": null, 56 | "cognitoAuthenticationProvider": null, 57 | "userArn": null, 58 | "userAgent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.181 Safari/537.36", 59 | "user": null 60 | }, 61 | "apiId": "gex57xa2id" 62 | }, 63 | "body": "{\"project\":\"raven\",\"logger\":\"javascript\",\"platform\":\"javascript\",\"request\":{\"headers\":{\"User-Agent\":\"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.181 Safari/537.36\",\"Referer\":\"http://localhost:4000/\"},\"url\":\"http://localhost:4000/runtime-js-errors.html\"},\"exception\":{\"values\":[{\"type\":\"TypeError\",\"value\":\"Cannot read property 'captureException' of undefined\",\"stacktrace\":{\"frames\":[{\"filename\":\"http://localhost:4000/scripts/runtime-errors.js\",\"lineno\":33,\"colno\":18,\"function\":\"Promise.resolve.then.catch\",\"in_app\":true}]}}]},\"transaction\":\"http://localhost:4000/scripts/runtime-errors.js\",\"trimHeadFrames\":0,\"extra\":{\"unhandledPromiseRejection\":true,\"session:duration\":20244},\"tags\":{\"hello\":\"what\",\"sajkaca\":\"production\"},\"breadcrumbs\":{\"values\":[{\"timestamp\":1527099886.785,\"message\":\"Item added to shopping cart\",\"category\":\"action\",\"data\":{\"isbn\":\"978-1617290541\",\"cartSize\":\"3\"}},{\"timestamp\":1527099887.499,\"category\":\"ui.click\",\"message\":\"body > button#sentryException\"},{\"timestamp\":1527099887.505,\"category\":\"sentry\",\"message\":\"error\",\"event_id\":\"4c834a56bcb04587b18ccb9e898f08d1\",\"level\":\"info\"},{\"timestamp\":1527099904.369,\"category\":\"ui.click\",\"message\":\"body > button#sentryException\"},{\"timestamp\":1527099907.02,\"category\":\"ui.click\",\"message\":\"body > button#capturedPromise\"}]},\"user\":{\"email\":\"foo@example.com\"},\"environment\":\"production\",\"release\":\"1.3.0\",\"event_id\":\"230742b1d32b45de80d31e955e1853ee\"}", 64 | "isBase64Encoded": false 65 | } -------------------------------------------------------------------------------- /collector-api/tests/test-events/raven-exception.json: -------------------------------------------------------------------------------- 1 | { 2 | "project": "raven", 3 | "logger": "javascript", 4 | "platform": "javascript", 5 | "request": { 6 | "headers": { 7 | "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.181 Safari/537.36", 8 | "Referer": "http://localhost:4000/" 9 | }, 10 | "url": "http://localhost:4000/runtime-js-errors.html" 11 | }, 12 | "exception": { 13 | "values": [ 14 | { 15 | "type": "TypeError", 16 | "value": "Cannot read property 'captureException' of undefined", 17 | "stacktrace": { 18 | "frames": [ 19 | { 20 | "filename": "http://localhost:4000/scripts/runtime-errors.js", 21 | "lineno": 33, 22 | "colno": 18, 23 | "function": "Promise.resolve.then.catch", 24 | "in_app": true 25 | } 26 | ] 27 | } 28 | } 29 | ] 30 | }, 31 | "transaction": "http://localhost:4000/scripts/runtime-errors.js", 32 | "trimHeadFrames": 0, 33 | "extra": { 34 | "unhandledPromiseRejection": true, 35 | "session:duration": 66737 36 | }, 37 | "tags": { 38 | "hello": "what", 39 | "sajkaca": "production" 40 | }, 41 | "breadcrumbs": { 42 | "values": [ 43 | { 44 | "timestamp": 1527327248.232, 45 | "message": "Item added to shopping cart", 46 | "category": "action", 47 | "data": { 48 | "isbn": "978-1617290541", 49 | "cartSize": "3" 50 | } 51 | }, 52 | { 53 | "timestamp": 1527327249.255, 54 | "category": "ui.click", 55 | "message": "body > button#sentryException" 56 | }, 57 | { 58 | "timestamp": 1527327249.258, 59 | "category": "sentry", 60 | "message": "error", 61 | "event_id": "135ed56f23394505929fabbd53190277", 62 | "level": "info" 63 | }, 64 | { 65 | "timestamp": 1527327250.501, 66 | "category": "ui.click", 67 | "message": "body" 68 | }, 69 | { 70 | "timestamp": 1527327307.239, 71 | "category": "ui.click", 72 | "message": "body > button#uncaughtString" 73 | }, 74 | { 75 | "timestamp": 1527327307.241, 76 | "category": "sentry", 77 | "message": "uncaughtString", 78 | "event_id": "653c19cb791f42e4a5d76c97b2b79fc7", 79 | "level": "error" 80 | }, 81 | { 82 | "timestamp": 1527327309.632, 83 | "category": "ui.click", 84 | "message": "body > button#uncaughtPromise" 85 | }, 86 | { 87 | "timestamp": 1527327312.382, 88 | "category": "ui.click", 89 | "message": "body > button#uncaughtStrictError" 90 | }, 91 | { 92 | "timestamp": 1527327312.382, 93 | "category": "sentry", 94 | "message": "TypeError: Assignment to constant variable.", 95 | "event_id": "3b4f04eaff9f4a5791d18fe41a63df31", 96 | "level": "error" 97 | }, 98 | { 99 | "timestamp": 1527327314.962, 100 | "category": "ui.click", 101 | "message": "body > button#capturedPromise" 102 | } 103 | ] 104 | }, 105 | "user": { 106 | "email": "foo@example.com" 107 | }, 108 | "environment": "production", 109 | "release": "1.3.0", 110 | "event_id": "5786ffe689cc4f56b10d6b6cec6d98d9" 111 | } 112 | -------------------------------------------------------------------------------- /collector-api/tests/test-events/raven-manually-tracked.json: -------------------------------------------------------------------------------- 1 | { 2 | "project": "raven", 3 | "logger": "javascript", 4 | "platform": "javascript", 5 | "request": { 6 | "headers": { 7 | "User-Agent": "Mozilla\/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit\/537.36 (KHTML, like Gecko) Chrome\/66.0.3359.181 Safari\/537.36", 8 | "Referer": "http:\/\/localhost:4000\/" 9 | }, 10 | "url": "http:\/\/localhost:4000\/runtime-js-errors.html" 11 | }, 12 | "message": "error", 13 | "level": "info", 14 | "stacktrace": { 15 | "frames": [ 16 | { 17 | "filename": "https:\/\/cdn.ravenjs.com\/3.25.2\/raven.min.js", 18 | "lineno": 2, 19 | "colno": 4706, 20 | "function": "HTMLButtonElement.d", 21 | "in_app": false 22 | }, 23 | { 24 | "filename": "http:\/\/localhost:4000\/scripts\/runtime-errors.js", 25 | "lineno": 37, 26 | "colno": 9, 27 | "function": "HTMLButtonElement.", 28 | "in_app": true 29 | }, 30 | { 31 | "filename": "https:\/\/cdn.ravenjs.com\/3.25.2\/raven.min.js", 32 | "lineno": 2, 33 | "colno": 5793, 34 | "function": "f.captureException", 35 | "in_app": false 36 | }, 37 | { 38 | "filename": "https:\/\/cdn.ravenjs.com\/3.25.2\/raven.min.js", 39 | "lineno": 2, 40 | "colno": 6306, 41 | "function": "f.captureMessage", 42 | "in_app": false 43 | } 44 | ] 45 | }, 46 | "fingerprint": [ 47 | "error" 48 | ], 49 | "tags": { 50 | "hello": "what", 51 | "sajkaca": "production" 52 | }, 53 | "extra": { 54 | "session:duration": 1030 55 | }, 56 | "breadcrumbs": { 57 | "values": [ 58 | { 59 | "timestamp": 1527327248.232, 60 | "message": "Item added to shopping cart", 61 | "category": "action", 62 | "data": { 63 | "isbn": "978-1617290541", 64 | "cartSize": "3" 65 | } 66 | }, 67 | { 68 | "timestamp": 1527327249.255, 69 | "category": "ui.click", 70 | "message": "body > button#sentryException" 71 | } 72 | ] 73 | }, 74 | "user": { 75 | "email": "foo@example.com" 76 | }, 77 | "environment": "production", 78 | "release": "1.3.0", 79 | "event_id": "135ed56f23394505929fabbd53190277" 80 | } -------------------------------------------------------------------------------- /common/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@desole/common", 3 | "version": "1.0.0", 4 | "description": "common functions for all backend collector tools", 5 | "files": ["src"], 6 | "private": true, 7 | "author": "", 8 | "license": "MIT", 9 | "devDependencies": { 10 | "@desole/devtools": "file:../devtools" 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /common/src/parse-sns-event.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | module.exports = function parseSNSEvent(event) { 3 | const extractSns = record => record.Sns && record.Sns.Message && JSON.parse(record.Sns.Message); 4 | if (!event || !event.Records || !Array.isArray(event.Records)) { 5 | return []; 6 | } 7 | return event.Records.map(extractSns).filter(x => x); 8 | }; 9 | 10 | -------------------------------------------------------------------------------- /devtools/bin/devtool.js: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | 'use strict'; 3 | //workaround for npm5 incredibly daft decision to work with nested symlinks] 4 | const cp = require('child_process'), 5 | path = require('path'), 6 | script = process.argv[2], 7 | nodeExecutable = process.argv[0], 8 | executablePath = path.resolve(__dirname, '..', 'node_modules', '.bin', script), 9 | args = [executablePath].concat(process.argv.slice(3)), 10 | runCommand = function () { 11 | const subProcess = cp.spawn(nodeExecutable, args, {cwd: process.cwd(), env: process.env}); 12 | subProcess.stdout.pipe(process.stdout); 13 | subProcess.stderr.pipe(process.stderr); 14 | subProcess.on('close', (code) => { 15 | process.exit(code); 16 | }); 17 | subProcess.on('error', (err) => { 18 | console.error('error starting subprocess', err); 19 | process.exit(1); 20 | }); 21 | }; 22 | 23 | runCommand(); 24 | 25 | -------------------------------------------------------------------------------- /devtools/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@desole/devtools", 3 | "version": "1.0.0", 4 | "scripts": { 5 | "devtool": "./bin/devtool.js" 6 | }, 7 | "bin": { 8 | "devtool": "./bin/devtool.js" 9 | }, 10 | "description": "Common development tooling setup for desole projects", 11 | "private": true, 12 | "license": "MIT", 13 | "dependencies": { 14 | "eslint": "^4.19.1", 15 | "eslint-config-crockford": "^2.0.0", 16 | "eslint-config-defaults": "^9.0.0", 17 | "jest": "^23.0.0" 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /elasticsearch-publisher/README.md: -------------------------------------------------------------------------------- 1 | # Desole ElasticSearch publisher 2 | 3 | 4 | ![](https://desole.io/images/elasticsearch.png) 5 | 6 | This publisher submits Desole events to AWS ElasticSearch, where you can query and search them easily. The repository contains an example of a CloudFormation template which you can deploy with a single click, or modify to fit your needs. 7 | 8 | ## Deploy the ElasticSearch Publisher using CloudFormation 9 | 10 | The default template deploys a new AWS ElasticSearch domain on a t2.micro instance with a 10GB EBS volume capacity. This is good for small sites and for demonstration purposes, but it may be inadequate for high-traffic event collection. Check out the `Deploying Custom Bundle` section below to see how to deploy a modified template. 11 | 12 | 13 | Region | Launch 14 | -------|------- 15 | US East (N.Virginia) | [![Pinpoint publisher in us-east-1](http://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/images/cloudformation-launch-stack-button.png)](https://console.aws.amazon.com/cloudformation/home?region=us-east-1#/stacks/create/review?stackName=desole&templateURL=https://desole-packaging-us-east-1.s3.amazonaws.com/1.0.0/@desole/es-publisher.yaml) 16 | EU Central (Frankfurt) | [![Pinpoint publisher in eu-central-1](http://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/images/cloudformation-launch-stack-button.png)](https://console.aws.amazon.com/cloudformation/home?region=eu-central-1#/stacks/create/review?stackName=desole&templateURL=https://desole-packaging-eu-central-1.s3.amazonaws.com/1.0.0/@desole/es-publisher.yaml) 17 | US West (N. California) | [![Pinpoint publisher in us-west-1](http://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/images/cloudformation-launch-stack-button.png)](https://console.aws.amazon.com/cloudformation/home?region=us-west-1#/stacks/create/review?stackName=desole&templateURL=https://desole-packaging-us-west-1.s3.amazonaws.com/1.0.0/@desole/es-publisher.yaml) 18 | Asia Pacific (Sydney) | [![Pinpoint publisher in ap-southeast-2](http://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/images/cloudformation-launch-stack-button.png)](https://console.aws.amazon.com/cloudformation/home?region=ap-southeast-2#/stacks/create/review?stackName=desole&templateURL=https://desole-packaging-ap-southeast-2.s3.amazonaws.com/1.0.0/@desole/es-publisher.yaml) 19 | 20 | ## Deploy a custom bundle 21 | 22 | ## Prerequisites 23 | 24 | * NPM 25 | * An S3 Bucket for Deployment, in the same region where you would like to deploy Desole 26 | * AWS CLI (command line tools), configured to use your account 27 | 28 | ## Deployment process 29 | 30 | Change the `template.yaml` to modify the ElasticSearch instance size, volume and other parameters. For more information, check out the [ElasticSearch Domain CloudFormation Reference](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-elasticsearch-domain.html). 31 | 32 | 1. Install the dependencies 33 | ```bash 34 | npm install 35 | ``` 36 | 2. Prepare and pack your code 37 | ```bash 38 | npm run prepackage 39 | ``` 40 | 3. Package the template 41 | ```bash 42 | aws cloudformation package --template-file template.yaml --output-template-file output.yaml 43 | ``` 44 | 4. Deploy the packaged template 45 | ```bash 46 | aws cloudformation deploy --template-file output.yaml --capabilities CAPABILITY_IAM --stack-name --parameter-overrides CollectorSNSTopic= 47 | ``` 48 | 49 | For a detailed list of supported parameters, check out [`template.yaml`](template.yaml) 50 | 51 | 52 | 53 | -------------------------------------------------------------------------------- /elasticsearch-publisher/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@desole/es-publisher", 3 | "version": "1.0.0", 4 | "description": "Desole publisher saves AWS Pinpoint/Mobile analytics metrics", 5 | "main": "src/lambda.js", 6 | "keywords": [], 7 | "files": [ 8 | "src" 9 | ], 10 | "scripts": { 11 | "prepackage": "claudia pack --no-optional-dependencies --output es-publisher.zip --force", 12 | "test": "sam validate", 13 | "package": "echo packaging $npm_package_config_region to $npm_package_config_bucket_name && aws cloudformation package --template-file template.yaml --output-template-file output.yaml --s3-bucket $npm_package_config_bucket_name --region $npm_package_config_region --s3-prefix $npm_package_version", 14 | "deploy": "aws cloudformation deploy --template-file output.yaml --stack-name $npm_package_config_cloudformation_stack --capabilities CAPABILITY_IAM --region $npm_package_config_region --parameter-overrides PinpointApplicationId=$npm_package_config_pinpoint_id", 15 | "upload": "aws s3 cp output.yaml s3://$npm_package_config_bucket_name/$npm_package_version/$npm_package_name.yaml --acl public-read", 16 | "make-public": "for c in $(aws s3api list-objects --bucket $npm_package_config_bucket_name --prefix $npm_package_version --output text --query Contents[].Key); do echo making $c public; aws s3api put-object-acl --acl public-read --bucket $npm_package_config_bucket_name --key $c; done", 17 | "release": "npm run package --$npm_package_name:region=$npm_package_config_region --$npm_package_name:bucket_name=$npm_package_config_bucket_name && npm run upload --$npm_package_name:region=$npm_package_config_region --$npm_package_name:bucket_name=$npm_package_config_bucket_name && npm run make-public --$npm_package_name:region=$npm_package_config_region --$npm_package_name:bucket_name=$npm_package_config_bucket_name", 18 | "regions": "for region in $(echo $npm_package_config_deployment_regions); do npm run release --$npm_package_name:region=$region --$npm_package_name:bucket_name=desole-packaging-$region; done;" 19 | }, 20 | "private": true, 21 | "author": "", 22 | "license": "MIT", 23 | "devDependencies": { 24 | "@desole/devtools": "file:../devtools", 25 | "claudia": "^5.0.0" 26 | }, 27 | "dependencies": { 28 | "@desole/common": "file:../common", 29 | "elasticsearch": "^14.2.2", 30 | "http-aws-es": "^4.0.0" 31 | }, 32 | "optionalDependencies": { 33 | "aws-sdk": "^2.224.1" 34 | }, 35 | "config": { 36 | "bucket_name": "desole-packaging-us-east-1", 37 | "deployment_regions": "us-east-1 eu-central-1 us-west-1 ap-southeast-2", 38 | "cloudformation_stack": "desoleStack", 39 | "region": "us-east-1" 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /elasticsearch-publisher/src/lambda.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | const AWS = require('aws-sdk'), 3 | elasticsearch = require('elasticsearch'), 4 | awsES = require('http-aws-es'), 5 | util = require('util'), 6 | parseSNSEvent = require('@desole/common/src/parse-sns-event'), 7 | myCredentials = new AWS.EnvironmentCredentials('AWS'), 8 | index = process.env.ES_INDEX_NAME, 9 | documentType = process.env.ES_DOCUMENT_TYPE, 10 | client = new elasticsearch.Client({ 11 | host: process.env.ES_DOMAIN_NAME, 12 | connectionClass: awsES, 13 | amazonES: { 14 | region: process.env.AWS_REGION, 15 | credentials: myCredentials 16 | } 17 | }), 18 | storeSingleEvent = event => { 19 | const params = { 20 | index: index, 21 | type: documentType, 22 | id: event.id, 23 | body: event 24 | }; 25 | return new Promise((resolve, reject) => { 26 | client.create(params, (err, result) => { 27 | if (err) { 28 | return reject(err); 29 | } 30 | resolve(result); 31 | }); 32 | }); 33 | }; 34 | 35 | exports.handler = (event) => { 36 | const records = parseSNSEvent(event); 37 | return Promise.all(records.map(storeSingleEvent)); 38 | }; 39 | -------------------------------------------------------------------------------- /elasticsearch-publisher/template.yaml: -------------------------------------------------------------------------------- 1 | AWSTemplateFormatVersion: 2010-09-09 2 | Transform: AWS::Serverless-2016-10-31 3 | Parameters: 4 | CollectorSNSTopic: 5 | Type: String 6 | Description: Desole collector SNS Event topic ARN 7 | AllowedPattern: ^arn:aws:sns:.+:.+:.+$ 8 | ConstraintDescription: Must be an AWS SNS Topic ARN (eg arn:aws:sns:us-east-1:123456789012:my_corporate_topic) 9 | MaxLength: 250 10 | MinLength: 1 11 | IndexName: 12 | Type: String 13 | Default: desole 14 | Description: The ElasticSearch index name 15 | AllowedPattern: ^[a-z]+$ 16 | ConstraintDescription: 'Must be lowercase latin (ascii) string, no numbers or uppercase letters allowed' 17 | MaxLength: 250 18 | MinLength: 1 19 | DocumentType: 20 | Type: String 21 | Default: error 22 | AllowedPattern: ^[a-z]+$ 23 | Description: The ElasticSearch document type for Desole events 24 | ConstraintDescription: 'Must be lowercase latin (ascii) string, no numbers or uppercase letters allowed' 25 | MaxLength: 250 26 | MinLength: 1 27 | 28 | Resources: 29 | ESDomain: 30 | Type: AWS::Elasticsearch::Domain 31 | Properties: 32 | ElasticsearchClusterConfig: 33 | InstanceCount: 1 34 | InstanceType: t2.micro.elasticsearch 35 | EBSOptions: 36 | EBSEnabled: true 37 | Iops: 0 38 | VolumeSize: 10 39 | VolumeType: standard 40 | EsPublisher: 41 | Type: AWS::Serverless::Function 42 | Properties: 43 | CodeUri: es-publisher.zip 44 | Handler: src/lambda.handler 45 | Runtime: nodejs8.10 46 | Environment: 47 | Variables: 48 | ES_INDEX_NAME: !Ref IndexName 49 | ES_DOCUMENT_TYPE: !Ref DocumentType 50 | ES_DOMAIN_NAME: !GetAtt ESDomain.DomainEndpoint 51 | Policies: 52 | - Version: 2012-10-17 53 | Statement: 54 | - Effect: Allow 55 | Action: 56 | - es:ESHttpPost 57 | - es:ESHttpPut 58 | Resource: !Sub '${ESDomain.DomainArn}/*' 59 | Events: 60 | CollectorSubscribedTopic: 61 | Type: SNS 62 | Properties: 63 | Topic: !Ref CollectorSNSTopic 64 | Outputs: 65 | DomainArn: 66 | Value: !GetAtt ESDomain.DomainArn 67 | DomainEndpoint: 68 | Value: !GetAtt ESDomain.DomainEndpoint 69 | -------------------------------------------------------------------------------- /eventformat.md: -------------------------------------------------------------------------------- 1 | # Desole.io event format 2 | 3 | * All values are strings unless specified otherwise 4 | * All values are mandatory in the SNS Event unless specified otherwise 5 | * Check out [test events](/test-events) for examples 6 | 7 | ## Fields 8 | 9 | * id 10 | * purpose: unique event ID so you can trace the same event in multiple storage systems (eg from dashboard to logs) 11 | * collected at: API (lambdaContext.awsRequestId) 12 | * severity 13 | * purpose: error significance (eg fatal) 14 | * valid values: fatal, error, warning, info, debug 15 | * example: fatal 16 | * collected at: client 17 | * collector defaults to: error 18 | * stack 19 | * purpose: detailed error stack trace 20 | * example: info.js:10 21 | * collected at: client 22 | * type 23 | * purpose: error code classification 24 | * example: SyntaxError 25 | * collected at: client 26 | * timestamp 27 | * purpose: time filtering and reporting 28 | * type: javascript time stamp (milliseconds), number 29 | * collected at: client 30 | * receivedAt 31 | * purpose: time filtering and reporting 32 | * type: javascript time stamp (milliseconds), number 33 | * collected at: API 34 | * resource 35 | * purpose: identify the resource causing the exception, eg a page URL, a service or an executable app name 36 | * collected at: client 37 | * referrer 38 | * collected at: API (headers["Referrer"]) 39 | * app: JSON-key-value pairs 40 | * collected at: client 41 | * purpose: classify errors for individual apps in the dashboards 42 | * app.name 43 | * purpose: marketing name of the app 44 | * example: Pizzeria front-end 45 | * app.version 46 | * purpose: identify individual code versions 47 | * example: 3.0.1 48 | * app.stage 49 | * purpose: differentiate between production/testing/development 50 | * example: production 51 | * tags: optional 52 | * type: JSON key-value pairs 53 | * purpose: user-defined event categorisation 54 | * example: `{userType: 'freemium', trial: 'yes' }` 55 | * collected at: client 56 | * endpoint 57 | * id 58 | * collected at: client 59 | * purpose: uniquely identify a client device, such as a browser, especially for AWS pinpoint integration 60 | * example: 123-1455-1245-123466 61 | * collector defaults to: autogenerated UUID, stored to local storage or cookies 62 | * collected at: client 63 | * language 64 | * collected at: client (navigator.language) 65 | * platform 66 | * collected at: client (navigator.platform) 67 | * example: MacIntel 68 | * os 69 | * example: MacOS 70 | * osVersion 71 | * example: 10.12.1 72 | * runtime 73 | * example: NodeJs / Chrome 74 | * runtimeVersion 75 | * example: 8.10 76 | * country 77 | * collected at: API (headers["CloudFront-Viewer-Country"]) 78 | * userAgent 79 | * purpose: a vendor specific label identifying the endpoint technically 80 | * collected at: API (requestContext.identity.userAgent) 81 | * deviceType: 82 | * SmartTV | Mobile | Desktop | Tablet 83 | * collected at: API (headers(["CloudFront-Is-XXX-Viewer"]) 84 | -------------------------------------------------------------------------------- /pinpoint-publisher/README.md: -------------------------------------------------------------------------------- 1 | # Desole Pinpoint publisher 2 | 3 | 4 | ![](https://desole.io/images/pinpoint.png) 5 | 6 | This publisher submits Desole events to AWS Pinpoint, where you can use Pinpoint analytics dashboards (formerly AWS Mobile Analytics) to drill down into various dimensions of the events, and create automated customer engagement campaigns based on Desole events -- for example, send everyone who received a particular error an e-mail. 7 | 8 | To use this publisher, you'll need a Pinpoint (AWS Mobile Hub) Application ID. Create an app using the [AWS Pinpoint Console](https://console.aws.amazon.com/pinpoint/) or using `aws pinpoint create-app` from your command line. For example, 9 | 10 | ```bash 11 | aws pinpoint create-app --create-application-request '{"Name":"desole"}' --query ApplicationResponse.Id --output text 12 | ``` 13 | 14 | To link existing Pinpoint users to Desole events, just make sure to supply the corresponding device ID in the collector client. 15 | 16 | ## Deploy the Pinpoint Publisher using CloudFormation 17 | 18 | Region | Launch 19 | -------|------- 20 | US East (N.Virginia) | [![Pinpoint publisher in us-east-1](http://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/images/cloudformation-launch-stack-button.png)](https://console.aws.amazon.com/cloudformation/home?region=us-east-1#/stacks/create/review?stackName=desole&templateURL=https://desole-packaging-us-east-1.s3.amazonaws.com/1.0.0/@desole/pinpoint-publisher.yaml) 21 | EU Central (Frankfurt) | [![Pinpoint publisher in eu-central-1](http://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/images/cloudformation-launch-stack-button.png)](https://console.aws.amazon.com/cloudformation/home?region=eu-central-1#/stacks/create/review?stackName=desole&templateURL=https://desole-packaging-eu-central-1.s3.amazonaws.com/1.0.0/@desole/pinpoint-publisher.yaml) 22 | US West (N. California) | [![Pinpoint publisher in us-west-1](http://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/images/cloudformation-launch-stack-button.png)](https://console.aws.amazon.com/cloudformation/home?region=us-west-1#/stacks/create/review?stackName=desole&templateURL=https://desole-packaging-us-west-1.s3.amazonaws.com/1.0.0/@desole/pinpoint-publisher.yaml) 23 | Asia Pacific (Sydney) | [![Pinpoint publisher in ap-southeast-2](http://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/images/cloudformation-launch-stack-button.png)](https://console.aws.amazon.com/cloudformation/home?region=ap-southeast-2#/stacks/create/review?stackName=desole&templateURL=https://desole-packaging-ap-southeast-2.s3.amazonaws.com/1.0.0/@desole/pinpoint-publisher.yaml) 24 | 25 | ## Deploy a custom bundle 26 | 27 | ### Prerequisites 28 | 29 | * NPM 30 | * An S3 Bucket for Deployment, in the same region where you would like to deploy Desole 31 | * AWS CLI (command line tools), configured to use your account 32 | 33 | ### Deployment process 34 | 35 | 1. Install the dependencies 36 | ```bash 37 | npm install 38 | ``` 39 | 2. Prepare and pack your code 40 | ```bash 41 | npm run prepackage 42 | ``` 43 | 3. Package the template 44 | ```bash 45 | aws cloudformation package --template-file template.yaml --output-template-file output.yaml 46 | ``` 47 | 4. Deploy the packaged template 48 | ```bash 49 | aws cloudformation deploy --template-file output.yaml --capabilities CAPABILITY_IAM --stack-name --parameter-overrides PinpointApplicationId= CollectorSNSTopic= 50 | ``` 51 | 52 | For a detailed list of supported parameters, check out [`template.yaml`](template.yaml) 53 | 54 | -------------------------------------------------------------------------------- /pinpoint-publisher/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@desole/pinpoint-publisher", 3 | "version": "1.0.1", 4 | "description": "Desole publisher saves AWS Pinpoint/Mobile analytics metrics", 5 | "main": "src/lambda.js", 6 | "keywords": [], 7 | "files": [ 8 | "src" 9 | ], 10 | "scripts": { 11 | "prepackage": "claudia pack --no-optional-dependencies --output pinpoint-publisher.zip --force", 12 | "pretest": "devtool eslint src tests", 13 | "test": "devtool jest --color", 14 | "package": "echo packaging $npm_package_config_region to $npm_package_config_bucket_name && aws cloudformation package --template-file template.yaml --output-template-file output.yaml --s3-bucket $npm_package_config_bucket_name --region $npm_package_config_region --s3-prefix $npm_package_version", 15 | "deploy": "aws cloudformation deploy --template-file output.yaml --stack-name $npm_package_config_cloudformation_stack --capabilities CAPABILITY_IAM --region $npm_package_config_region --parameter-overrides PinpointApplicationId=$npm_package_config_pinpoint_id", 16 | "upload": "aws s3 cp output.yaml s3://$npm_package_config_bucket_name/$npm_package_version/$npm_package_name.yaml --acl public-read", 17 | "make-public": "for c in $(aws s3api list-objects --bucket $npm_package_config_bucket_name --prefix $npm_package_version --output text --query Contents[].Key); do echo making $c public; aws s3api put-object-acl --acl public-read --bucket $npm_package_config_bucket_name --key $c; done", 18 | "release": "npm run package --$npm_package_name:region=$npm_package_config_region --$npm_package_name:bucket_name=$npm_package_config_bucket_name && npm run upload --$npm_package_name:region=$npm_package_config_region --$npm_package_name:bucket_name=$npm_package_config_bucket_name && npm run make-public --$npm_package_name:region=$npm_package_config_region --$npm_package_name:bucket_name=$npm_package_config_bucket_name", 19 | "regions": "for region in $(echo $npm_package_config_deployment_regions); do npm run release --$npm_package_name:region=$region --$npm_package_name:bucket_name=desole-packaging-$region; done;" 20 | }, 21 | "private": true, 22 | "author": "", 23 | "license": "MIT", 24 | "devDependencies": { 25 | "@desole/devtools": "file:../devtools", 26 | "claudia": "^5.0.0" 27 | }, 28 | "dependencies": { 29 | "@desole/common": "file:../common", 30 | "flat": "^4.0.0" 31 | }, 32 | "optionalDependencies": { 33 | "aws-sdk": "^2.224.1" 34 | }, 35 | "jest": { 36 | "roots": [ 37 | "/src/", 38 | "/tests/" 39 | ] 40 | }, 41 | "config": { 42 | "bucket_name": "desole-packaging-us-east-1", 43 | "deployment_regions": "us-east-1 eu-central-1 us-west-1 ap-southeast-2", 44 | "cloudformation_stack": "desoleStack", 45 | "region": "us-east-1", 46 | "pinpoint_id": "" 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /pinpoint-publisher/src/convert-analytics-event.js: -------------------------------------------------------------------------------- 1 | 2 | 'use strict'; 3 | 4 | const flat = require('flat'); 5 | 6 | function parseEvent(event) { 7 | const clonedEvent = Object.assign({}, event); 8 | delete clonedEvent.tags; 9 | delete clonedEvent.timestamp; 10 | delete clonedEvent.receivedAt; 11 | return clonedEvent; 12 | } 13 | 14 | module.exports = function convertAnalyticsEvent(event, eventName) { 15 | const attributes = Object.assign({}, event.tags, flat(parseEvent(event), { delimiter: ' ' })); 16 | 17 | return { 18 | eventType: eventName, 19 | timestamp: new Date(event.receivedAt).toISOString(), 20 | attributes: attributes 21 | }; 22 | }; 23 | -------------------------------------------------------------------------------- /pinpoint-publisher/src/convert-client-context.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | module.exports = function convertClientContext(event, mobileHubApplication) { 4 | return { 5 | client: { 6 | client_id: event.endpoint.id 7 | }, 8 | services: { 9 | mobile_analytics: { 10 | app_id: mobileHubApplication 11 | } 12 | } 13 | }; 14 | }; 15 | -------------------------------------------------------------------------------- /pinpoint-publisher/src/lambda.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | const AWS = require('aws-sdk'), 3 | mobileAnalytics = new AWS.MobileAnalytics(), 4 | pinpoint = new AWS.Pinpoint(), 5 | parseSNSEvent = require('@desole/common/src/parse-sns-event'), 6 | storeSingleEvent = require('./store-single-event'); 7 | 8 | exports.handler = (event) => { 9 | if (!process.env.MOBILE_HUB_APPLICATION || !process.env.EVENT_NAME) { 10 | return Promise.resolve(); 11 | } 12 | const records = parseSNSEvent(event); 13 | return Promise.all(records.map(record => storeSingleEvent(record, mobileAnalytics, pinpoint, process.env))); 14 | }; 15 | -------------------------------------------------------------------------------- /pinpoint-publisher/src/store-single-event.js: -------------------------------------------------------------------------------- 1 | 2 | 'use strict'; 3 | 4 | const convertClientContext = require('./convert-client-context'), 5 | convertAnalyticsEvent = require('./convert-analytics-event'); 6 | 7 | module.exports = function storeSingleEvent(event, mobileAnalytics, pinpoint, env) { 8 | 9 | 10 | const clientContext = convertClientContext(event, env.MOBILE_HUB_APPLICATION), 11 | analyticsEvent = convertAnalyticsEvent(event, env.EVENT_NAME); 12 | 13 | return pinpoint.updateEndpoint({ 14 | ApplicationId: env.MOBILE_HUB_APPLICATION, 15 | EndpointId: event.endpoint.id, 16 | EndpointRequest: { 17 | Demographic: { 18 | AppVersion: event.app.version, 19 | Locale: event.endpoint.language, 20 | Make: event.endpoint.platform, 21 | Platform: event.endpoint.os, 22 | PlatformVersion: event.endpoint.osVersion, 23 | Model: event.endpoint.runtime, 24 | ModelVersion: event.endpoint.runtimeVersion 25 | }, 26 | Location: { 27 | Country: event.endpoint.country 28 | } 29 | } 30 | }).promise() 31 | .then(() => mobileAnalytics.putEvents({ 32 | clientContext: JSON.stringify(clientContext), 33 | events: [analyticsEvent] 34 | }).promise()); 35 | }; 36 | -------------------------------------------------------------------------------- /pinpoint-publisher/template.yaml: -------------------------------------------------------------------------------- 1 | AWSTemplateFormatVersion: 2010-09-09 2 | Transform: AWS::Serverless-2016-10-31 3 | Parameters: 4 | PinpointApplicationId: 5 | Type: String 6 | AllowedPattern: ^[0-9A-Za-z]+$ 7 | Description: Pinpoint (Mobile Hub) Application ID. 8 | ConstraintDescription: Only alphanumeric characters allowed 9 | MaxLength: 250 10 | MinLength: 1 11 | PinpointEventName: 12 | Type: String 13 | Default: Error 14 | Description: The name for custom events logged to Pinpoint analytics 15 | MaxLength: 250 16 | MinLength: 1 17 | CollectorSNSTopic: 18 | Type: String 19 | Description: Desole collector SNS Event topic ARN 20 | AllowedPattern: ^arn:aws:sns:.+:.+:.+$ 21 | ConstraintDescription: Must be an AWS SNS Topic ARN (eg arn:aws:sns:us-east-1:123456789012:my_corporate_topic) 22 | MaxLength: 250 23 | MinLength: 1 24 | Resources: 25 | PinpointPublisher: 26 | Type: AWS::Serverless::Function 27 | Properties: 28 | CodeUri: pinpoint-publisher.zip 29 | Handler: src/lambda.handler 30 | Runtime: nodejs8.10 31 | Environment: 32 | Variables: 33 | MOBILE_HUB_APPLICATION: !Ref PinpointApplicationId 34 | EVENT_NAME: !Ref PinpointEventName 35 | Policies: 36 | - Version: 2012-10-17 37 | Statement: 38 | - Effect: Allow 39 | Action: 'mobileanalytics:PutEvents' 40 | Resource: '*' 41 | - Effect: Allow 42 | Action: 'mobiletargeting:UpdateEndpoint' 43 | Resource: !Sub 'arn:aws:mobiletargeting:*:*:apps/${PinpointApplicationId}/endpoints/*' 44 | Events: 45 | CollectorSubscribedTopic: 46 | Type: SNS 47 | Properties: 48 | Topic: !Ref CollectorSNSTopic 49 | -------------------------------------------------------------------------------- /pinpoint-publisher/tests/.eslintrc.json: -------------------------------------------------------------------------------- 1 | { 2 | "env": { 3 | "jest": true 4 | } 5 | } 6 | -------------------------------------------------------------------------------- /pinpoint-publisher/tests/convert-analytics-event.test.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | const convertAnalyticsEvent = require('../src/convert-analytics-event'), 4 | eventTemplate = require('./test-events/sample-event.json'); 5 | 6 | describe('convertAnlyticsEvent', () => { 7 | 8 | let sampleEvent; 9 | 10 | beforeEach(() => { 11 | sampleEvent = JSON.parse(JSON.stringify(eventTemplate)); 12 | }); 13 | 14 | test('should flatten all event attributes', () => { 15 | const convertedEvent = convertAnalyticsEvent(sampleEvent, 'test event'); 16 | 17 | expect(convertedEvent).toEqual({ 18 | eventType: 'test event', 19 | timestamp: '2018-07-01T00:48:05.926Z', 20 | attributes: 21 | { 22 | severity: 'error', 23 | stack: 'com.gargoylesoftware.htmlunit.ScriptException: TypeError: Cannot find default value for object. (https://dashboard.vacationtracker.io:443/polyfills.569c7d225d020fab25d6.bundle.js#1)', 24 | resource: '', 25 | 'app name': 'STG Dashboard', 26 | 'app version': '1.0.0', 27 | 'app stage': 'prod', 28 | id: '6a9ff062-7cc8-11e8-ba99-d99efe337df0', 29 | referrer: '', 30 | 'endpoint id': '1ea0ce11-cf8b-4015-9124-d37709175e9a', 31 | 'endpoint platform': 'MacIntel', 32 | 'endpoint language': 'en-US', 33 | 'endpoint country': 'US', 34 | 'endpoint userAgent': 'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.76 Safari/537.36', 35 | 'endpoint deviceType': 'Desktop', 36 | 'endpoint runtime': 'Chrome', 37 | 'endpoint runtimeVersion': '56.0.2924', 38 | 'endpoint os': 'Windows', 39 | 'endpoint osVersion': '7.0.0' 40 | } 41 | }); 42 | }); 43 | 44 | test('should merge tags with attributes', () => { 45 | sampleEvent.tags = { UserType: 'free' }; 46 | const convertedEvent = convertAnalyticsEvent(sampleEvent, 'test event'); 47 | 48 | expect(convertedEvent).toEqual({ 49 | eventType: 'test event', 50 | timestamp: '2018-07-01T00:48:05.926Z', 51 | attributes: 52 | { 53 | 'UserType': 'free', 54 | severity: 'error', 55 | stack: 'com.gargoylesoftware.htmlunit.ScriptException: TypeError: Cannot find default value for object. (https://dashboard.vacationtracker.io:443/polyfills.569c7d225d020fab25d6.bundle.js#1)', 56 | resource: '', 57 | 'app name': 'STG Dashboard', 58 | 'app version': '1.0.0', 59 | 'app stage': 'prod', 60 | id: '6a9ff062-7cc8-11e8-ba99-d99efe337df0', 61 | referrer: '', 62 | 'endpoint id': '1ea0ce11-cf8b-4015-9124-d37709175e9a', 63 | 'endpoint platform': 'MacIntel', 64 | 'endpoint language': 'en-US', 65 | 'endpoint country': 'US', 66 | 'endpoint userAgent': 'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.76 Safari/537.36', 67 | 'endpoint deviceType': 'Desktop', 68 | 'endpoint runtime': 'Chrome', 69 | 'endpoint runtimeVersion': '56.0.2924', 70 | 'endpoint os': 'Windows', 71 | 'endpoint osVersion': '7.0.0' 72 | } 73 | }); 74 | }); 75 | 76 | test('should not override standard keys with tags', () => { 77 | sampleEvent.tags = { severity: 'free' }; 78 | const convertedEvent = convertAnalyticsEvent(sampleEvent, 'test event'); 79 | 80 | expect(convertedEvent).toEqual({ 81 | eventType: 'test event', 82 | timestamp: '2018-07-01T00:48:05.926Z', 83 | attributes: 84 | { 85 | severity: 'error', 86 | stack: 'com.gargoylesoftware.htmlunit.ScriptException: TypeError: Cannot find default value for object. (https://dashboard.vacationtracker.io:443/polyfills.569c7d225d020fab25d6.bundle.js#1)', 87 | resource: '', 88 | 'app name': 'STG Dashboard', 89 | 'app version': '1.0.0', 90 | 'app stage': 'prod', 91 | id: '6a9ff062-7cc8-11e8-ba99-d99efe337df0', 92 | referrer: '', 93 | 'endpoint id': '1ea0ce11-cf8b-4015-9124-d37709175e9a', 94 | 'endpoint platform': 'MacIntel', 95 | 'endpoint language': 'en-US', 96 | 'endpoint country': 'US', 97 | 'endpoint userAgent': 'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.76 Safari/537.36', 98 | 'endpoint deviceType': 'Desktop', 99 | 'endpoint runtime': 'Chrome', 100 | 'endpoint runtimeVersion': '56.0.2924', 101 | 'endpoint os': 'Windows', 102 | 'endpoint osVersion': '7.0.0' 103 | } 104 | }); 105 | }); 106 | 107 | test('should not mutate original even', () => { 108 | convertAnalyticsEvent(sampleEvent, 'test event'); 109 | expect(sampleEvent).toEqual(eventTemplate); 110 | }); 111 | 112 | test('should work when the event does not contain tags', () => { 113 | delete sampleEvent.tags; 114 | const convertedEvent = convertAnalyticsEvent(sampleEvent, 'test event'); 115 | 116 | expect(convertedEvent).toEqual({ 117 | eventType: 'test event', 118 | timestamp: '2018-07-01T00:48:05.926Z', 119 | attributes: 120 | { 121 | severity: 'error', 122 | stack: 'com.gargoylesoftware.htmlunit.ScriptException: TypeError: Cannot find default value for object. (https://dashboard.vacationtracker.io:443/polyfills.569c7d225d020fab25d6.bundle.js#1)', 123 | resource: '', 124 | 'app name': 'STG Dashboard', 125 | 'app version': '1.0.0', 126 | 'app stage': 'prod', 127 | id: '6a9ff062-7cc8-11e8-ba99-d99efe337df0', 128 | referrer: '', 129 | 'endpoint id': '1ea0ce11-cf8b-4015-9124-d37709175e9a', 130 | 'endpoint platform': 'MacIntel', 131 | 'endpoint language': 'en-US', 132 | 'endpoint country': 'US', 133 | 'endpoint userAgent': 'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.76 Safari/537.36', 134 | 'endpoint deviceType': 'Desktop', 135 | 'endpoint runtime': 'Chrome', 136 | 'endpoint runtimeVersion': '56.0.2924', 137 | 'endpoint os': 'Windows', 138 | 'endpoint osVersion': '7.0.0' 139 | } 140 | }); 141 | }); 142 | }); 143 | -------------------------------------------------------------------------------- /pinpoint-publisher/tests/test-events/sample-event.json: -------------------------------------------------------------------------------- 1 | { 2 | "severity": "error", 3 | "stack": "com.gargoylesoftware.htmlunit.ScriptException: TypeError: Cannot find default value for object. (https://dashboard.vacationtracker.io:443/polyfills.569c7d225d020fab25d6.bundle.js#1)", 4 | "timestamp": 1530403229851, 5 | "resource": "", 6 | "tags": {}, 7 | "app": { 8 | "name": "STG Dashboard", 9 | "version": "1.0.0", 10 | "stage": "prod" 11 | }, 12 | "id": "6a9ff062-7cc8-11e8-ba99-d99efe337df0", 13 | "receivedAt": 1530406085926, 14 | "referrer": "", 15 | "endpoint": { 16 | "id": "1ea0ce11-cf8b-4015-9124-d37709175e9a", 17 | "platform": "MacIntel", 18 | "language": "en-US", 19 | "country": "US", 20 | "userAgent": "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.76 Safari/537.36", 21 | "deviceType": "Desktop", 22 | "runtime": "Chrome", 23 | "runtimeVersion": "56.0.2924", 24 | "os": "Windows", 25 | "osVersion": "7.0.0" 26 | } 27 | } -------------------------------------------------------------------------------- /s3-publisher/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@desole/s3-publisher", 3 | "version": "1.0.0", 4 | "description": "Desole publisher that saves all events to a S3 Bucket", 5 | "main": "src/lambda.js", 6 | "keywords": [], 7 | "files": [ 8 | "src" 9 | ], 10 | "private": true, 11 | "author": "", 12 | "license": "MIT", 13 | "devDependencies": { 14 | "@desole/devtools": "file:../devtools" 15 | }, 16 | "dependencies": { 17 | "@desole/common": "file:../common" 18 | }, 19 | "optionalDependencies": { 20 | "aws-sdk": "^2.224.1" 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /s3-publisher/src/lambda.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | const AWS = require('aws-sdk'), 3 | parseSNSEvent = require('@desole/common/src/parse-sns-event'), 4 | s3 = new AWS.S3(), 5 | BUCKET_NAME = process.env.BUCKET_NAME, 6 | BUCKET_PREFIX = process.env.BUCKET_PREFIX || 'archive', 7 | calculateS3Key = function (event) { 8 | const date = new Date(event.receivedAt), 9 | year = date.getFullYear(), 10 | month = date.getMonth() + 1, 11 | day = date.getDate(); 12 | return [BUCKET_PREFIX, event.app.name, event.app.stage, year, month, day, event.severity, event.type, event.id].join('/'); 13 | }, 14 | storeSingleEvent = event => { 15 | return s3.putObject({ 16 | Key: calculateS3Key(event), 17 | Bucket: BUCKET_NAME, 18 | ContentType: 'application/json', 19 | Body: JSON.stringify(event, null, 2) 20 | }).promise(); 21 | }; 22 | 23 | exports.handler = (event) => { 24 | const records = parseSNSEvent(event); 25 | return Promise.all(records.map(storeSingleEvent)); 26 | }; 27 | -------------------------------------------------------------------------------- /test-events/README.md: -------------------------------------------------------------------------------- 1 | # Example Desole events 2 | 3 | This repository contains examples for two types of Desole events. 4 | 5 | * [from-browser-to-api.json](from-browser-to-api.json) is an example of an event format posted to the Collector API from client side apps, before enrichment 6 | * [full-event.json](full-event.json) is an example of an event posted from the Collector API to the Collector SNS topic, after enrichment 7 | 8 | ## Submitting using CURL 9 | 10 | ``` 11 | curl -X POST https:///prod/desole -H "Content-Type: application/json" -i --data @from-browser-to-api.json 12 | ``` 13 | 14 | -------------------------------------------------------------------------------- /test-events/from-browser-to-api.json: -------------------------------------------------------------------------------- 1 | { 2 | "type" : "Error", 3 | "timestamp" : 1526242077032, 4 | "resource" : "http://localhost:4000/runtime-js-errors.html", 5 | "app" : { 6 | "name" : "DesoleTest", 7 | "stage" : "dev", 8 | "version" : "4.5.6" 9 | }, 10 | "tags" : {}, 11 | "severity" : "error", 12 | "stack" : "Error: uncaughtException\n at HTMLButtonElement. (http://localhost:4000/scripts/runtime-errors.js:2:8)", 13 | "message" : "Uncaught Error: uncaughtException", 14 | "endpoint" : { 15 | "platform" : "MacIntel", 16 | "language" : "en-GB", 17 | "id" : "63def883-ad8c-4093-a496-8ea5ee5189d8" 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /test-events/full-event.json: -------------------------------------------------------------------------------- 1 | { 2 | "severity": "error", 3 | "stack": "ReferenceError: unknownFunction is not defined\n at HTMLButtonElement. (http://localhost:4000/scripts/runtime-errors.js:6:2)", 4 | "type": "ReferenceError", 5 | "timestamp": 1526242080425, 6 | "resource": "http://localhost:4000/runtime-js-errors.html", 7 | "tags": {}, 8 | "app": { 9 | "name": "DesoleTest", 10 | "version": "4.5.6", 11 | "stage": "dev" 12 | }, 13 | "id": "55fe2e61-56e9-11e8-819c-d55675ec3410", 14 | "receivedAt": 1526242080499, 15 | "referrer": "http://localhost:4000/runtime-js-errors.html", 16 | "endpoint": { 17 | "id": "51a246fa-f755-41b5-9fea-2b8c80baa8bf", 18 | "platform": "MacIntel", 19 | "language": "en-GB", 20 | "country": "PL", 21 | "userAgent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.139 Safari/537.36", 22 | "deviceType": "Desktop", 23 | "runtime": "Chrome", 24 | "runtimeVersion": "66.0.3359", 25 | "os": "Mac OS X", 26 | "osVersion": "10.13.4" 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /test-events/raven-manual-config.json: -------------------------------------------------------------------------------- 1 | { 2 | "release" : "1.3.0", 3 | "event_id" : "bbe521a9464e404384555e5254ff03a0", 4 | "extra" : { 5 | "session:duration" : 1084 6 | }, 7 | "logger" : "javascript", 8 | "request" : { 9 | "url" : "http://localhost:4000/runtime-js-errors.html", 10 | "headers" : { 11 | "Referer" : "http://localhost:4000/", 12 | "User-Agent" : "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.181 Safari/537.36" 13 | } 14 | }, 15 | "stacktrace" : { 16 | "frames" : [ 17 | { 18 | "lineno" : 2, 19 | "in_app" : false, 20 | "function" : "HTMLButtonElement.d", 21 | "colno" : 4706, 22 | "filename" : "https://cdn.ravenjs.com/3.25.2/raven.min.js" 23 | }, 24 | { 25 | "filename" : "http://localhost:4000/scripts/runtime-errors.js", 26 | "in_app" : true, 27 | "colno" : 9, 28 | "function" : "HTMLButtonElement.", 29 | "lineno" : 37 30 | }, 31 | { 32 | "filename" : "https://cdn.ravenjs.com/3.25.2/raven.min.js", 33 | "lineno" : 2, 34 | "in_app" : false, 35 | "colno" : 5793, 36 | "function" : "f.captureException" 37 | }, 38 | { 39 | "in_app" : false, 40 | "function" : "f.captureMessage", 41 | "colno" : 6306, 42 | "lineno" : 2, 43 | "filename" : "https://cdn.ravenjs.com/3.25.2/raven.min.js" 44 | } 45 | ] 46 | }, 47 | "tags" : { 48 | "sajkaca" : "production", 49 | "hello" : "what" 50 | }, 51 | "environment" : "production", 52 | "platform" : "javascript", 53 | "project" : "desole", 54 | "user" : { 55 | "email" : "foo@example.com" 56 | }, 57 | "message" : "error", 58 | "fingerprint" : [ 59 | "error" 60 | ], 61 | "level" : "info", 62 | "breadcrumbs" : { 63 | "values" : [ 64 | { 65 | "data" : { 66 | "isbn" : "978-1617290541", 67 | "cartSize" : "3" 68 | }, 69 | "message" : "Item added to shopping cart", 70 | "category" : "action", 71 | "timestamp" : 1527096468.864 72 | }, 73 | { 74 | "category" : "ui.click", 75 | "message" : "body > button#sentryException", 76 | "timestamp" : 1527096469.937 77 | } 78 | ] 79 | } 80 | } 81 | -------------------------------------------------------------------------------- /test-events/raven.json: -------------------------------------------------------------------------------- 1 | { 2 | "tags" : { 3 | "hello" : "what", 4 | "sajkaca" : "production" 5 | }, 6 | "release" : "1.3.0", 7 | "environment" : "production", 8 | "project" : "desole", 9 | "exception" : { 10 | "values" : [ 11 | { 12 | "stacktrace" : { 13 | "frames" : [ 14 | { 15 | "filename" : "https://cdn.ravenjs.com/3.25.2/raven.min.js", 16 | "colno" : 4706, 17 | "in_app" : false, 18 | "lineno" : 2, 19 | "function" : "HTMLButtonElement.d" 20 | }, 21 | { 22 | "colno" : 8, 23 | "in_app" : true, 24 | "lineno" : 2, 25 | "filename" : "http://localhost:4000/scripts/runtime-errors.js", 26 | "function" : "HTMLButtonElement." 27 | } 28 | ] 29 | }, 30 | "value" : "uncaughtException", 31 | "type" : "Error" 32 | } 33 | ] 34 | }, 35 | "event_id" : "9c8e250e645d4c1fb54346b1a1297664", 36 | "user" : { 37 | "email" : "foo@example.com" 38 | }, 39 | "extra" : { 40 | "session:duration" : 2129 41 | }, 42 | "platform" : "javascript", 43 | "request" : { 44 | "url" : "http://localhost:4000/runtime-js-errors.html", 45 | "headers" : { 46 | "User-Agent" : "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.181 Safari/537.36", 47 | "Referer" : "http://localhost:4000/" 48 | } 49 | }, 50 | "trimHeadFrames" : 0, 51 | "logger" : "javascript", 52 | "breadcrumbs" : { 53 | "values" : [ 54 | { 55 | "timestamp" : 1527095710.348, 56 | "data" : { 57 | "isbn" : "978-1617290541", 58 | "cartSize" : "3" 59 | }, 60 | "message" : "Item added to shopping cart", 61 | "category" : "action" 62 | }, 63 | { 64 | "message" : "body > button#uncaughtException", 65 | "category" : "ui.click", 66 | "timestamp" : 1527095712.465 67 | } 68 | ] 69 | }, 70 | "transaction" : "http://localhost:4000/scripts/runtime-errors.js" 71 | } 72 | -------------------------------------------------------------------------------- /todo.txt: -------------------------------------------------------------------------------- 1 | # Todo 2 | 3 | ## Collector API 4 | 5 | * support for the sentry format on /sentry (just the format) 6 | * enable npm 5 loading of @dependencies 7 | 8 | ## Pinpoint publisher 9 | 10 | * save just the first line of stack to pinpoint, not much point doing anything else 11 | --------------------------------------------------------------------------------