├── .gitignore ├── .npmignore ├── .travis.yml ├── Makefile ├── README.md ├── alert.go ├── alert_test.go ├── bin └── deepalert.ts ├── cdk.json ├── cdk └── deepalert-stack.ts ├── emitter ├── emitter.go └── emitter_test.go ├── examples ├── emitter │ ├── emitter_test.go │ └── main.go ├── inspector │ ├── main.go │ └── main_test.go └── reviewer │ ├── main.go │ └── main_test.go ├── go.mod ├── go.sum ├── inspector ├── inspector.go ├── inspector_test.go ├── mock.go ├── sqs.go ├── sqs_test.go └── test_utils.go ├── internal ├── adaptor │ ├── adaptor_test.go │ ├── repository.go │ ├── sfn.go │ └── sns.go ├── api │ ├── alert.go │ ├── api.go │ ├── api_test.go │ └── report.go ├── handler │ ├── arguments.go │ ├── arguments_test.go │ └── envvars.go ├── mock │ ├── no_test.go │ ├── repository.go │ ├── sfn.go │ └── sns.go ├── models │ ├── db.go │ └── db_test.go ├── repository │ ├── dynamodb.go │ └── no_test.go ├── service │ ├── repository.go │ ├── repository_test.go │ ├── sfn.go │ └── sns.go └── usecase │ ├── alert.go │ └── alert_test.go ├── jest.config.js ├── lambda ├── apiHandler │ ├── main.go │ └── main_test.go ├── compileReport │ ├── main.go │ └── main_test.go ├── dispatchInspection │ └── main.go ├── dummyReviewer │ └── main.go ├── feedbackAttribute │ └── main.go ├── publishReport │ ├── main.go │ └── main_test.go ├── receptAlert │ ├── main.go │ └── main_test.go ├── submitFinding │ └── main.go └── submitReport │ └── main.go ├── package-lock.json ├── package.json ├── report.go ├── report_test.go ├── task.go ├── test ├── stack │ └── deepalert.test.ts └── workflow │ ├── Makefile │ ├── README.md │ ├── bin │ └── stack.ts │ ├── cdk.json │ ├── emitter │ └── main.go │ ├── inspector │ └── main.go │ ├── jest.config.js │ ├── main_test.go │ ├── package-lock.json │ ├── package.json │ ├── result.go │ └── tsconfig.json └── tsconfig.json /.gitignore: -------------------------------------------------------------------------------- 1 | build 2 | 3 | *.js 4 | !jest.config.js 5 | *.d.ts 6 | node_modules 7 | 8 | # CDK asset staging directory 9 | .cdk.staging 10 | cdk.out 11 | **/apikey.json 12 | 13 | # Parcel default cache directory 14 | .parcel-cache 15 | 16 | -------------------------------------------------------------------------------- /.npmignore: -------------------------------------------------------------------------------- 1 | *.ts 2 | !*.d.ts 3 | 4 | # CDK asset staging directory 5 | .cdk.staging 6 | cdk.out 7 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | dist: bionic 2 | language: go 3 | sudo: false 4 | 5 | env: GO111MODULE=on 6 | 7 | go: 8 | - 1.13.x 9 | - 1.14.x 10 | 11 | git: 12 | depth: 1 13 | 14 | notifications: 15 | email: false 16 | 17 | before_script: 18 | - go install github.com/golangci/golangci-lint/cmd/golangci-lint 19 | 20 | script: 21 | - golangci-lint run 22 | - go test ./... 23 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | all: build 2 | 3 | CODE_DIR := $(shell dirname $(realpath $(lastword $(MAKEFILE_LIST)))) 4 | 5 | COMMON=$(CODE_DIR)/*.go $(CODE_DIR)/internal/*/*.go 6 | 7 | FUNCTIONS= \ 8 | $(CODE_DIR)/build/dummyReviewer \ 9 | $(CODE_DIR)/build/dispatchInspection \ 10 | $(CODE_DIR)/build/compileReport \ 11 | $(CODE_DIR)/build/receptAlert \ 12 | $(CODE_DIR)/build/submitReport \ 13 | $(CODE_DIR)/build/publishReport \ 14 | $(CODE_DIR)/build/submitFinding \ 15 | $(CODE_DIR)/build/apiHandler \ 16 | $(CODE_DIR)/build/feedbackAttribute 17 | 18 | GO_OPT=-ldflags="-s -w" -trimpath 19 | 20 | # Functions ------------------------ 21 | $(CODE_DIR)/build/dummyReviewer: $(CODE_DIR)/lambda/dummyReviewer/*.go $(COMMON) 22 | env GOARCH=amd64 GOOS=linux go build $(GO_OPT) -o $(CODE_DIR)/build/dummyReviewer ./lambda/dummyReviewer 23 | $(CODE_DIR)/build/dispatchInspection: $(CODE_DIR)/lambda/dispatchInspection/*.go $(COMMON) 24 | env GOARCH=amd64 GOOS=linux go build $(GO_OPT) -o $(CODE_DIR)/build/dispatchInspection ./lambda/dispatchInspection 25 | $(CODE_DIR)/build/compileReport: $(CODE_DIR)/lambda/compileReport/*.go $(COMMON) 26 | env GOARCH=amd64 GOOS=linux go build $(GO_OPT) -o $(CODE_DIR)/build/compileReport ./lambda/compileReport 27 | $(CODE_DIR)/build/receptAlert: $(CODE_DIR)/lambda/receptAlert/*.go $(COMMON) 28 | env GOARCH=amd64 GOOS=linux go build $(GO_OPT) -o $(CODE_DIR)/build/receptAlert ./lambda/receptAlert 29 | $(CODE_DIR)/build/publishReport: $(CODE_DIR)/lambda/publishReport/*.go $(COMMON) 30 | env GOARCH=amd64 GOOS=linux go build $(GO_OPT) -o $(CODE_DIR)/build/publishReport ./lambda/publishReport 31 | $(CODE_DIR)/build/submitReport: $(CODE_DIR)/lambda/submitReport/*.go $(COMMON) 32 | env GOARCH=amd64 GOOS=linux go build $(GO_OPT) -o $(CODE_DIR)/build/submitReport ./lambda/submitReport 33 | $(CODE_DIR)/build/submitFinding: $(CODE_DIR)/lambda/submitFinding/*.go $(COMMON) 34 | env GOARCH=amd64 GOOS=linux go build $(GO_OPT) -o $(CODE_DIR)/build/submitFinding ./lambda/submitFinding 35 | $(CODE_DIR)/build/apiHandler: $(CODE_DIR)/lambda/apiHandler/*.go $(COMMON) 36 | env GOARCH=amd64 GOOS=linux go build $(GO_OPT) -o $(CODE_DIR)/build/apiHandler ./lambda/apiHandler 37 | $(CODE_DIR)/build/feedbackAttribute: $(CODE_DIR)/lambda/feedbackAttribute/*.go $(COMMON) 38 | env GOARCH=amd64 GOOS=linux go build $(GO_OPT) -o $(CODE_DIR)/build/feedbackAttribute ./lambda/feedbackAttribute 39 | 40 | 41 | # Base Tasks ------------------------------------- 42 | # build: $(FUNCTIONS) $(CDK_STACK) 43 | build: $(FUNCTIONS) 44 | 45 | # Use in cdk deploy (bundling of lambda.Code.fromAsset) 46 | asset: build 47 | cp $(CODE_DIR)/build/* /asset-output 48 | 49 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # DeepAlert 2 | 3 | Serverless SOAR (Security Orchestration, Automation and Response) framework for automatic inspection and evaluation of security alert. 4 | 5 | ## Overview 6 | 7 | DeepAlert receives a security alert that is event of interest from security view point and responses the alert automatically. DeepAlert has 3 parts of automatic response. 8 | 9 | - **Inspector** investigates entities that are appeared in the alert including IP address, Domain name and store a result: reputation, history of malicious activities, associated cloud instance and etc. Following components are already provided to integrate with your DeepAlert environment. Also you can create own inspector to check logs that is stored into original log storage or log search system. 10 | - **Reviewer** receives the alert with result(s) of Inspector and evaluate severity of the alert. Reviewer should be written by each security operator/administrator of your organization because security policies are differ from organization to organization. 11 | - **Emitter** finally receives the alert with result of Reviewer's severity evaluation. After that, Emitter sends external integrated system. E.g. PagerDuty, Slack, Github Enterprise, etc. Also automatic quarantine can be configured by AWS Lambda function. 12 | 13 | ![Overview](https://user-images.githubusercontent.com/605953/76850323-80914100-688a-11ea-9c9a-96030094af2c.png) 14 | 15 | ## Deployment 16 | 17 | ### Prerequisite 18 | 19 | - Tools 20 | - `aws-cdk` >= 1.75.0 21 | - `go` >= 1.14 22 | - `node` >= 14.7.0 23 | - `npm` >= 6.14.9 24 | - Credential 25 | - AWS CLI credential to deploy CloudFormation. See [here](https://docs.aws.amazon.com/cli/latest/userguide/cli-chap-configure.html) for more detail. 26 | 27 | ### Configure your stack 28 | 29 | At first, you need to create AWS CDK repository and install deepalert as a npm module. 30 | 31 | ```bash 32 | $ mkdir your-stack 33 | $ cd your-stack 34 | $ cdk init --language typescript 35 | $ npm i @deepalert/deepalert 36 | ``` 37 | 38 | Then, edit `./bin/your-stack.ts` as following. 39 | 40 | ```ts 41 | #!/usr/bin/env node 42 | import 'source-map-support/register'; 43 | import * as cdk from '@aws-cdk/core'; 44 | import { DeepAlertStack } from '@deepalert/deepalert'; 45 | 46 | const app = new cdk.App(); 47 | new DeepAlertStack(app, 'YourDeepAlert', {}); 48 | ``` 49 | 50 | ### Deploy your stack 51 | 52 | ```bash 53 | $ cdk deploy 54 | ``` 55 | 56 | ## Alerting 57 | 58 | ### Alert data schema 59 | 60 | ```json 61 | { 62 | "detector": "your-anti-virus", 63 | "rule_name": "detected malware", 64 | "rule_id": "detect-malware-by-av", 65 | "alert_key": "xxxxxxxx", 66 | "timestamp": "2006-01-02T15:03:04Z", 67 | "attributes": [ 68 | { 69 | "type": "ipaddr", 70 | "key": "IP address of detected machine", 71 | "value": "10.2.3.4", 72 | "context": [ 73 | "local", 74 | "client" 75 | ], 76 | }, 77 | ] 78 | } 79 | ``` 80 | 81 | 82 | - `detector`: Subject name of monitoring system 83 | - `rule_id`: Machine readable rule identity 84 | - `timestamp`: Detected timestamp 85 | - `rule_name` (optional): Human readable rule name 86 | - `alert_key` (optional): Alert aggregation key if you need 87 | - `attributes` (optional): List of `attribute` 88 | - `type`: Choose from `ipaddr`, `domain`, `username`, `filehashvalue`, `json` and `url` 89 | - `key`: Label of the value 90 | - `value`: Actual value 91 | - `context`: One or multiple tags describe context of the attribute. See `AttrContext` in [alert.go](alert.go) 92 | 93 | ### Emit alert via API 94 | 95 | `apikey.json` is created in CWD when running `cdk deploy` and it has `X-API-KEY` to access deepalert API. 96 | 97 | ```bash 98 | $ export AWS_REGION=ap-northeast-1 # set your region 99 | $ export API_KEY=`cat apikey.json | jq '.["X-API-KEY"]' -r` 100 | $ export API_ID=`aws cloudformation describe-stack-resources --stack-name YourDeepAlert | jq -r '.StackResources[] | select(.ResourceType == "AWS::ApiGateway::RestApi") | .PhysicalResourceId'` 101 | $ curl -X POST \ 102 | -H "X-API-KEY: $API_KEY" \ 103 | https://$API_ID.execute-api.$AWS_REGION.amazonaws.com/prod/api/v1/alert \ 104 | -d '{ 105 | "detector": "your-anti-virus", 106 | "rule_name": "detected malware", 107 | "rule_id": "detect-malware-by-av", 108 | "alert_key": "xxxxxxxx" 109 | }' 110 | ``` 111 | 112 | 113 | ### Emit alert via SQS 114 | 115 | ```bash 116 | $ export QUEUE_URL=`aws cloudformation describe-stack-resources --stack-name YourDeepAlert | jq -r '.StackResources[] | select(.LogicalResourceId | startswith("alertQueue")) | .PhysicalResourceId'` 117 | $ aws sqs send-message --queue-url $QUEUE_URL --message-body '{ 118 | "detector": "your-anti-virus", 119 | "rule_name": "detected malware", 120 | "rule_id": "detect-malware-by-av", 121 | "alert_key": "xxxxxxxx" 122 | }' 123 | ``` 124 | 125 | 126 | ### Build and deploy Reviewer 127 | 128 | See examples and deploy it as Lambda Function. 129 | 130 | - Inspector example: [./examples/inspector](./examples/inspector) 131 | - Emitter example: [./examples/emitter](./examples/inspector) 132 | 133 | ## Development 134 | 135 | ### Architecture 136 | 137 | ![architecture overview](https://user-images.githubusercontent.com/605953/103391370-8677ba80-4b5c-11eb-8b96-d44e1d3263a5.png) 138 | 139 | 140 | ### Unit Test 141 | 142 | ``` 143 | $ go test ./... 144 | ``` 145 | 146 | ### Integration Test 147 | 148 | Move to `./test/workflow/` and run below. Then deploy test stack and execute integration test. 149 | 150 | ```bash 151 | $ npm i 152 | $ make deploy 153 | $ make test 154 | ``` 155 | 156 | ## License 157 | 158 | MIT License 159 | -------------------------------------------------------------------------------- /alert.go: -------------------------------------------------------------------------------- 1 | package deepalert 2 | 3 | import ( 4 | "crypto/sha256" 5 | "encoding/base64" 6 | "encoding/json" 7 | "fmt" 8 | "log" 9 | "sort" 10 | "strings" 11 | "time" 12 | 13 | "github.com/m-mizutani/golambda" 14 | ) 15 | 16 | // AttrType shows type of alert attribute. 17 | type AttrType string 18 | 19 | const ( 20 | // TypeIPAddr means IP address (v4/v6) 21 | TypeIPAddr AttrType = "ipaddr" 22 | // TypeDomainName means domain name 23 | TypeDomainName AttrType = "domain" 24 | // TypeUserName means user name on the system 25 | TypeUserName AttrType = "username" 26 | // TypeFileHashValue means hash value of a file. Hash algorithm is not specified for now. 27 | TypeFileHashValue AttrType = "filehashvalue" 28 | // TypeJSON means raw JSON data. It should be displayed as preformatted text 29 | TypeJSON AttrType = "json" 30 | // TypeURL means URL of some object 31 | TypeURL AttrType = "url" 32 | ) 33 | 34 | // AttrContext describes context of the attribute. 35 | type AttrContext string 36 | 37 | // AttrContexts is set of AttrContext 38 | type AttrContexts []AttrContext 39 | 40 | const ( 41 | // CtxRemote means an entity of the attribute is outside of your organization. 42 | // E.g. External Web site, Attacker Host 43 | CtxRemote AttrContext = "remote" 44 | 45 | // CtxLocal means an entity of the attribute is inside of your organization. 46 | // E.g. Staff's workstation, Owned cloud instance. 47 | CtxLocal = "local" 48 | 49 | // CtxSubject means an entity of the attribute is subject of the event. 50 | CtxSubject = "subject" 51 | 52 | // CtxObject means an entity of the attribute is target of the event. 53 | CtxObject = "object" 54 | 55 | // CtxClient means a network entity works as client (requester). 56 | CtxClient = "client" 57 | 58 | // CtxServer means a network entity works as server (responder). 59 | CtxServer = "server" 60 | 61 | // CtxFile means the attribute comes from file object. 62 | CtxFile = "file" 63 | 64 | // CtxAdditionalInfo means the attribute is meta contexts 65 | CtxAdditionalInfo = "additional" 66 | ) 67 | 68 | // Attribute is element of alert 69 | type Attribute struct { 70 | Type AttrType `json:"type"` 71 | 72 | // Key should be unique in alert.Attributes, but not must. 73 | Key string `json:"key"` 74 | 75 | // Value is main value of attribute. 76 | Value string `json:"value"` 77 | 78 | // Context explains background of the attribute value. 79 | Context AttrContexts `json:"context"` 80 | 81 | // Timestamp indicates observed time of the attribute. 82 | Timestamp *time.Time `json:"timestamp,omitempty"` 83 | } 84 | 85 | // Alert is extranted data from KinesisStream 86 | type Alert struct { 87 | Detector string `json:"detector"` 88 | RuleName string `json:"rule_name"` 89 | RuleID string `json:"rule_id"` 90 | AlertKey string `json:"alert_key"` 91 | Description string `json:"description"` 92 | 93 | Timestamp time.Time `json:"timestamp"` 94 | Attributes []Attribute `json:"attributes"` 95 | Body interface{} `json:"body,omitempty"` 96 | } 97 | 98 | // AddAttribute just appends the attribute to the Alert 99 | func (x *Alert) AddAttribute(attr Attribute) { 100 | x.Attributes = append(x.Attributes, attr) 101 | } 102 | 103 | // AddAttributes appends set of attribute to the Alert 104 | func (x *Alert) AddAttributes(attrs []Attribute) { 105 | x.Attributes = append(x.Attributes, attrs...) 106 | } 107 | 108 | // FindAttributes searches and returns matched attributes 109 | func (x *Alert) FindAttributes(key string) []Attribute { 110 | var attrs []Attribute 111 | for _, attr := range x.Attributes { 112 | if attr.Key == key { 113 | attrs = append(attrs, attr) 114 | } 115 | } 116 | 117 | return attrs 118 | } 119 | 120 | // AlertID calculate ID of the alert from Detector, RuleID and AlertKey. 121 | func (x *Alert) AlertID() string { 122 | key := strings.Join([]string{ 123 | base64.StdEncoding.EncodeToString([]byte(x.Detector)), 124 | base64.StdEncoding.EncodeToString([]byte(x.RuleID)), 125 | base64.StdEncoding.EncodeToString([]byte(x.AlertKey)), 126 | }, ":") 127 | 128 | hasher := sha256.New() 129 | _, err := hasher.Write([]byte(key)) 130 | if err != nil { 131 | log.Fatalf("Failed sha256.Write: %v", err) 132 | } 133 | return fmt.Sprintf("alert:%x", hasher.Sum(nil)) 134 | } 135 | 136 | var ( 137 | // ErrInvalidAlert represents not enough parameter(s) for Alert 138 | ErrInvalidAlert = golambda.NewError("Invalid Alert parameter") 139 | ) 140 | 141 | // Validate checks own parameters of Alert and returns error if something wrong 142 | func (x *Alert) Validate() error { 143 | if x.Detector == "" { 144 | return golambda.WrapError(ErrInvalidAlert, "Alert.Detector is required") 145 | } 146 | if x.RuleID == "" { 147 | return golambda.WrapError(ErrInvalidAlert, "Alert.RuleID is required") 148 | } 149 | return nil 150 | } 151 | 152 | // Match checks attribute type and context. 153 | func (x *Attribute) Match(context AttrContext, attrType AttrType) bool { 154 | if x.Type != attrType { 155 | return false 156 | } 157 | if x.Context == nil { 158 | return false 159 | } 160 | 161 | for _, ctx := range x.Context { 162 | if ctx == context { 163 | return true 164 | } 165 | } 166 | 167 | return false 168 | } 169 | 170 | // Hash provides an unique value for the Attribute. 171 | // Hash value must be same if it has same Type, Key, Value and Context. 172 | func (x Attribute) Hash() string { 173 | sort.Slice(x.Context, func(i, j int) bool { 174 | return x.Context[i] < x.Context[j] 175 | }) 176 | 177 | raw, err := json.Marshal(x) 178 | if err != nil { 179 | // Must marshal 180 | log.Fatalf("Fail to unmarshal attribute: %v %v", x, err) 181 | } 182 | 183 | hasher := sha256.New() 184 | if _, err := hasher.Write(raw); err != nil { 185 | log.Fatalf("Failed sha256.Write: %v", err) 186 | } 187 | sha := fmt.Sprintf("%x", hasher.Sum(nil)) 188 | 189 | return sha 190 | } 191 | 192 | // Have of AttrContexts checks if context is in AttrContexts 193 | func (x AttrContexts) Have(context AttrContext) bool { 194 | for _, ctx := range x { 195 | if ctx == context { 196 | return true 197 | } 198 | } 199 | 200 | return false 201 | } 202 | -------------------------------------------------------------------------------- /alert_test.go: -------------------------------------------------------------------------------- 1 | package deepalert_test 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | 8 | da "github.com/deepalert/deepalert" 9 | ) 10 | 11 | func TestLookupAttribute(t *testing.T) { 12 | alert := da.Alert{ 13 | Attributes: []da.Attribute{ 14 | { 15 | Key: "myaddr", 16 | Type: da.TypeIPAddr, 17 | Context: []da.AttrContext{ 18 | da.CtxRemote, 19 | da.CtxSubject, 20 | }, 21 | }, 22 | }, 23 | } 24 | 25 | attrs1 := alert.FindAttributes("myaddr") 26 | assert.Equal(t, 1, len(attrs1)) 27 | assert.True(t, attrs1[0].Match(da.CtxRemote, da.TypeIPAddr)) 28 | assert.False(t, attrs1[0].Match(da.CtxLocal, da.TypeIPAddr)) 29 | assert.False(t, attrs1[0].Match(da.CtxSubject, da.TypeUserName)) 30 | 31 | attrs2 := alert.FindAttributes("invalid key") 32 | assert.Equal(t, 0, len(attrs2)) 33 | } 34 | 35 | func TestAttributeHash(t *testing.T) { 36 | a1 := da.Attribute{ 37 | Key: "myaddr", 38 | Type: da.TypeIPAddr, 39 | Context: []da.AttrContext{ 40 | da.CtxRemote, 41 | da.CtxSubject, 42 | }, 43 | } 44 | 45 | a2 := da.Attribute{ 46 | Key: "hoge", 47 | Type: da.TypeIPAddr, 48 | Context: []da.AttrContext{ 49 | da.CtxRemote, 50 | da.CtxSubject, 51 | }, 52 | } 53 | 54 | a3 := da.Attribute{ 55 | Key: "myaddr", 56 | Type: da.TypeIPAddr, 57 | Context: []da.AttrContext{ 58 | // Reversed 59 | da.CtxSubject, 60 | da.CtxRemote, 61 | }, 62 | } 63 | assert.NotEqual(t, a1.Hash(), "") 64 | assert.NotEqual(t, a1.Hash(), a2.Hash()) 65 | assert.Equal(t, a1.Hash(), a3.Hash()) 66 | } 67 | -------------------------------------------------------------------------------- /bin/deepalert.ts: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | import "source-map-support/register"; 3 | import * as cdk from "@aws-cdk/core"; 4 | import { DeepAlertStack } from "../cdk/deepalert-stack"; 5 | 6 | const app = new cdk.App(); 7 | new DeepAlertStack(app, "DeepAlertStack", {}); 8 | -------------------------------------------------------------------------------- /cdk.json: -------------------------------------------------------------------------------- 1 | { 2 | "app": "npx ts-node bin/deepalert.ts", 3 | "context": { 4 | "@aws-cdk/core:enableStackNameDuplicates": "true", 5 | "aws-cdk:enableDiffNoFail": "true" 6 | } 7 | } 8 | -------------------------------------------------------------------------------- /cdk/deepalert-stack.ts: -------------------------------------------------------------------------------- 1 | import * as cdk from '@aws-cdk/core'; 2 | import * as lambda from '@aws-cdk/aws-lambda'; 3 | import * as iam from '@aws-cdk/aws-iam'; 4 | import * as sns from '@aws-cdk/aws-sns'; 5 | import * as sqs from '@aws-cdk/aws-sqs'; 6 | import * as dynamodb from '@aws-cdk/aws-dynamodb'; 7 | import * as sfn from '@aws-cdk/aws-stepfunctions'; 8 | import * as tasks from '@aws-cdk/aws-stepfunctions-tasks'; 9 | import * as apigateway from '@aws-cdk/aws-apigateway'; 10 | import { 11 | SqsEventSource, 12 | DynamoEventSource, 13 | } from '@aws-cdk/aws-lambda-event-sources'; 14 | import { SqsSubscription } from '@aws-cdk/aws-sns-subscriptions'; 15 | 16 | import * as path from 'path'; 17 | import * as fs from 'fs'; 18 | 19 | // import { SqsSubscription } from "@aws-cdk/aws-sns-subscriptions"; 20 | 21 | export interface Property extends cdk.StackProps { 22 | lambdaRoleARN?: string; 23 | sfnRoleARN?: string; 24 | reviewer?: lambda.Function; 25 | inspectDelay?: cdk.Duration; 26 | reviewDelay?: cdk.Duration; 27 | 28 | enableAPI?: boolean; 29 | apiKeyPath?: string; 30 | sentryDsn?: string; 31 | sentryEnv?: string; 32 | logLevel?: string; 33 | alertTopicARN?: string; 34 | } 35 | 36 | export class DeepAlertStack extends cdk.Stack { 37 | readonly cacheTable: dynamodb.Table; 38 | // Messaging 39 | readonly taskTopic: sns.Topic; 40 | readonly attributeTopic: sns.Topic; 41 | readonly reportTopic: sns.Topic; 42 | readonly alertQueue: sqs.Queue; 43 | readonly findingQueue: sqs.Queue; 44 | readonly attributeQueue: sqs.Queue; 45 | readonly deadLetterQueue: sqs.Queue; 46 | 47 | // Lambda 48 | receptAlert: lambda.Function; 49 | dispatchInspection: lambda.Function; 50 | submitFinding: lambda.Function; 51 | feedbackAttribute: lambda.Function; 52 | compileReport: lambda.Function; 53 | dummyReviewer: lambda.Function; 54 | submitReport: lambda.Function; 55 | publishReport: lambda.Function; 56 | apiHandler: lambda.Function; 57 | 58 | // StepFunctions 59 | readonly inspectionMachine: sfn.StateMachine; 60 | readonly reviewMachine: sfn.StateMachine; 61 | 62 | constructor(scope: cdk.Construct, id: string, props: Property) { 63 | super(scope, id, props); 64 | 65 | const lambdaRole = props.lambdaRoleARN 66 | ? iam.Role.fromRoleArn(this, "LambdaRole", props.lambdaRoleARN, { 67 | mutable: false, 68 | }) 69 | : undefined; 70 | const sfnRole = props.sfnRoleARN 71 | ? iam.Role.fromRoleArn(this, "SfnRole", props.sfnRoleARN, { 72 | mutable: false, 73 | }) 74 | : undefined; 75 | 76 | this.cacheTable = new dynamodb.Table(this, "cacheTable", { 77 | billingMode: dynamodb.BillingMode.PAY_PER_REQUEST, 78 | partitionKey: { name: "pk", type: dynamodb.AttributeType.STRING }, 79 | sortKey: { name: "sk", type: dynamodb.AttributeType.STRING }, 80 | timeToLiveAttribute: "expires_at", 81 | stream: dynamodb.StreamViewType.NEW_IMAGE, 82 | }); 83 | 84 | // ---------------------------------------------------------------- 85 | // Messaging Channels 86 | this.taskTopic = new sns.Topic(this, "taskTopic"); 87 | this.attributeTopic = new sns.Topic(this, "attributeTopic"); 88 | this.reportTopic = new sns.Topic(this, "reportTopic"); 89 | 90 | this.deadLetterQueue = new sqs.Queue(this, "deadLetterQueue"); 91 | 92 | const alertQueueTimeout = cdk.Duration.seconds(30); 93 | this.alertQueue = new sqs.Queue(this, "alertQueue", { 94 | visibilityTimeout: alertQueueTimeout, 95 | deadLetterQueue: { 96 | maxReceiveCount: 5, 97 | queue: this.deadLetterQueue, 98 | }, 99 | }); 100 | 101 | if (props.alertTopicARN !== undefined) { 102 | const alertTopic = sns.Topic.fromTopicArn(this, 'AlertTopic', props.alertTopicARN); 103 | alertTopic.addSubscription(new SqsSubscription(this.alertQueue)); 104 | } 105 | 106 | const findingQueueTimeout = cdk.Duration.seconds(30); 107 | this.findingQueue = new sqs.Queue(this, "findingQueue", { 108 | visibilityTimeout: findingQueueTimeout, 109 | deadLetterQueue: { 110 | maxReceiveCount: 5, 111 | queue: this.deadLetterQueue, 112 | }, 113 | }); 114 | 115 | const attributeQueueTimeout = cdk.Duration.seconds(30); 116 | this.attributeQueue = new sqs.Queue(this, "attributeQueue", { 117 | visibilityTimeout: attributeQueueTimeout, 118 | deadLetterQueue: { 119 | maxReceiveCount: 5, 120 | queue: this.deadLetterQueue, 121 | }, 122 | }); 123 | 124 | 125 | // ---------------------------------------------------------------- 126 | // Lambda Functions 127 | const baseEnvVars = { 128 | TASK_TOPIC: this.taskTopic.topicArn, 129 | REPORT_TOPIC: this.reportTopic.topicArn, 130 | CACHE_TABLE: this.cacheTable.tableName, 131 | 132 | SENTRY_DSN: props.sentryDsn || "", 133 | SENTRY_ENVIRONMENT: props.sentryEnv || "", 134 | LOG_LEVEL: props.logLevel || "", 135 | }; 136 | 137 | interface LambdaConfig { 138 | funcName: string; 139 | events?: lambda.IEventSource[]; 140 | timeout?: cdk.Duration; 141 | environment?: { [key: string]: string; }; 142 | setToStack: { 143 | (f :lambda.Function):void; 144 | }; 145 | } 146 | 147 | const rootPath = path.resolve(__dirname, '..'); 148 | const asset = lambda.Code.fromAsset(rootPath, { 149 | bundling: { 150 | image: lambda.Runtime.GO_1_X.bundlingDockerImage, 151 | user: 'root', 152 | command: ['make', 'asset'], 153 | }, 154 | exclude: ['*/node_modules', '*/cdk.out'], 155 | }); 156 | 157 | const buildLambdaFunction = (config: LambdaConfig) => { 158 | const f = new lambda.Function(this, config.funcName, { 159 | runtime: lambda.Runtime.GO_1_X, 160 | handler: config.funcName, 161 | code: asset, 162 | role: lambdaRole, 163 | events: config.events, 164 | timeout: config.timeout, 165 | environment: config.environment || baseEnvVars, 166 | deadLetterQueue: this.deadLetterQueue, 167 | }); 168 | config.setToStack(f); 169 | }; 170 | 171 | // receptAlert and apiHandler is configured later because they requires StepFunctions 172 | // in environment variables. 173 | const lambdaConfigs :LambdaConfig[] = [ 174 | { 175 | funcName: 'submitFinding', 176 | events: [new SqsEventSource(this.findingQueue)], 177 | setToStack: (f :lambda.Function) => { this.submitFinding = f; } 178 | }, 179 | { 180 | funcName: 'feedbackAttribute', 181 | events: [new SqsEventSource(this.attributeQueue)], 182 | timeout: attributeQueueTimeout, 183 | setToStack: (f :lambda.Function) => { this.feedbackAttribute = f; } 184 | }, 185 | { 186 | funcName: 'dispatchInspection', 187 | setToStack: (f :lambda.Function) => { this.dispatchInspection = f; } 188 | }, 189 | { 190 | funcName: 'compileReport', 191 | setToStack: (f :lambda.Function) => { this.compileReport = f; }, 192 | }, 193 | { 194 | funcName: 'dummyReviewer', 195 | setToStack: (f :lambda.Function) => { this.dummyReviewer = f; }, 196 | }, 197 | { 198 | funcName: 'submitReport', 199 | setToStack: (f :lambda.Function) => { this.submitReport = f; }, 200 | }, 201 | { 202 | funcName: 'publishReport', 203 | events: [ 204 | new DynamoEventSource(this.cacheTable, { 205 | startingPosition: lambda.StartingPosition.LATEST, 206 | batchSize: 1, 207 | }), 208 | ], 209 | setToStack: (f :lambda.Function) => { this.publishReport = f; }, 210 | }, 211 | ]; 212 | 213 | lambdaConfigs.forEach(buildLambdaFunction); 214 | 215 | this.inspectionMachine = buildInspectionMachine( 216 | this, id, 217 | this.dispatchInspection, 218 | props.inspectDelay, 219 | sfnRole 220 | ); 221 | 222 | this.reviewMachine = buildReviewMachine( 223 | this, id, 224 | this.compileReport, 225 | props.reviewer || this.dummyReviewer, 226 | this.submitReport, 227 | props.reviewDelay, 228 | sfnRole 229 | ); 230 | 231 | const envVarsWithSF = Object.assign(baseEnvVars, { 232 | INSPECTOR_MACHINE: this.inspectionMachine.stateMachineArn, 233 | REVIEW_MACHINE: this.reviewMachine.stateMachineArn, 234 | }); 235 | buildLambdaFunction({ 236 | funcName: 'receptAlert', 237 | timeout: alertQueueTimeout, 238 | events: [new SqsEventSource(this.alertQueue)], 239 | environment: envVarsWithSF, 240 | setToStack: (f :lambda.Function) => { this.receptAlert = f; }, 241 | }) 242 | 243 | if (props.enableAPI) { 244 | buildLambdaFunction({ 245 | funcName: 'apiHandler', 246 | environment: envVarsWithSF, 247 | setToStack: (f :lambda.Function) => { this.apiHandler = f; }, 248 | }) 249 | 250 | const api = new apigateway.LambdaRestApi(this, 'deepalertAPI', { 251 | handler: this.apiHandler, 252 | proxy: false, 253 | cloudWatchRole: false, 254 | endpointTypes: [apigateway.EndpointType.REGIONAL], 255 | policy: new iam.PolicyDocument({ 256 | statements: [ 257 | new iam.PolicyStatement({ 258 | actions: ['execute-api:Invoke'], 259 | resources: ['execute-api:/*/*'], 260 | effect: iam.Effect.ALLOW, 261 | principals: [new iam.AnyPrincipal()], 262 | }), 263 | ], 264 | }), 265 | }); 266 | const apiKey = api.addApiKey('APIKey', { 267 | value: getAPIKey(props.apiKeyPath), 268 | }) 269 | api.addUsagePlan('UsagePlan', { 270 | apiKey, 271 | }).addApiStage({ 272 | stage: api.deploymentStage, 273 | }) 274 | 275 | const apiOpt = { apiKeyRequired: true}; 276 | const v1 = api.root.addResource('api').addResource('v1',); 277 | const alertAPI = v1.addResource('alert'); 278 | alertAPI.addMethod('POST', undefined, apiOpt); 279 | alertAPI.addResource('{alert_id}').addResource('report').addMethod('GET', undefined, apiOpt); 280 | 281 | const reportAPI = v1.addResource('report'); 282 | const reportAPIwithID = reportAPI.addResource('{report_id}'); 283 | reportAPIwithID.addMethod('GET', undefined, apiOpt); 284 | reportAPIwithID.addResource('alert').addMethod('GET', undefined, apiOpt); 285 | reportAPIwithID.addResource('attribute').addMethod('GET', undefined, apiOpt); 286 | reportAPIwithID.addResource('section').addMethod('GET', undefined, apiOpt); 287 | } 288 | 289 | if (lambdaRole === undefined) { 290 | this.inspectionMachine.grantStartExecution(this.receptAlert); 291 | this.inspectionMachine.grantStartExecution(this.apiHandler); 292 | this.reviewMachine.grantStartExecution(this.receptAlert); 293 | this.reviewMachine.grantStartExecution(this.apiHandler); 294 | this.taskTopic.grantPublish(this.dispatchInspection); 295 | this.reportTopic.grantPublish(this.publishReport); 296 | 297 | // DynamoDB 298 | this.cacheTable.grantReadWriteData(this.receptAlert); 299 | this.cacheTable.grantReadWriteData(this.dispatchInspection); 300 | this.cacheTable.grantReadWriteData(this.feedbackAttribute); 301 | this.cacheTable.grantReadWriteData(this.submitFinding); 302 | this.cacheTable.grantReadWriteData(this.compileReport); 303 | this.cacheTable.grantReadWriteData(this.submitReport); 304 | this.cacheTable.grantReadWriteData(this.publishReport); 305 | this.cacheTable.grantReadWriteData(this.apiHandler); 306 | } 307 | } 308 | } 309 | 310 | function buildInspectionMachine( 311 | scope: cdk.Construct, 312 | stackID: string, 313 | dispatchInspection: lambda.Function, 314 | delay?: cdk.Duration, 315 | sfnRole?: iam.IRole 316 | ): sfn.StateMachine { 317 | const waitTime = delay || cdk.Duration.minutes(5); 318 | 319 | const wait = new sfn.Wait(scope, 'WaitDispatch', { 320 | time: sfn.WaitTime.duration(waitTime), 321 | }); 322 | const invokeDispatcher = new tasks.LambdaInvoke( 323 | scope, 324 | 'InvokeDispatchInspection', 325 | { lambdaFunction: dispatchInspection } 326 | ); 327 | 328 | const definition = wait.next(invokeDispatcher); 329 | 330 | return new sfn.StateMachine(scope, 'InspectionMachine', { 331 | stateMachineName: stackID + '-InspectionMachine', 332 | definition, 333 | role: sfnRole, 334 | }); 335 | } 336 | 337 | function buildReviewMachine( 338 | scope: cdk.Construct, 339 | stackID: string, 340 | compileReport: lambda.Function, 341 | reviewer: lambda.Function, 342 | submitReport: lambda.Function, 343 | delay?: cdk.Duration, 344 | sfnRole?: iam.IRole 345 | ): sfn.StateMachine { 346 | const waitTime = delay || cdk.Duration.minutes(10); 347 | 348 | const wait = new sfn.Wait(scope, 'WaitCompile', { 349 | time: sfn.WaitTime.duration(waitTime), 350 | }); 351 | 352 | const definition = wait 353 | .next( 354 | new tasks.LambdaInvoke(scope, 'invokeCompileReport', { 355 | lambdaFunction: compileReport, 356 | outputPath: '$', 357 | payloadResponseOnly: true, 358 | }) 359 | ) 360 | .next( 361 | new tasks.LambdaInvoke(scope, 'invokeReviewer', { 362 | lambdaFunction: reviewer, 363 | resultPath: '$.result', 364 | outputPath: '$', 365 | payloadResponseOnly: true, 366 | }) 367 | ) 368 | .next( 369 | new tasks.LambdaInvoke(scope, 'invokeSubmitReport', { 370 | lambdaFunction: submitReport, 371 | }) 372 | ); 373 | 374 | return new sfn.StateMachine(scope, 'ReviewMachine', { 375 | stateMachineName: stackID + '-ReviewMachine', 376 | definition, 377 | role: sfnRole, 378 | }); 379 | } 380 | 381 | function getAPIKey(apiKeyPath?: string): string { 382 | if (apiKeyPath === undefined) { 383 | apiKeyPath = path.join(process.cwd(), 'apikey.json'); 384 | } 385 | 386 | if (fs.existsSync(apiKeyPath)) { 387 | console.log('Read API key from: ', apiKeyPath); 388 | const buf = fs.readFileSync(apiKeyPath) 389 | const keyData = JSON.parse(buf.toString()); 390 | return keyData['X-API-KEY']; 391 | } else { 392 | const literals = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789'; 393 | const length = 32; 394 | const apiKey = Array.from(Array(length)).map(()=>literals[Math.floor(Math.random()*literals.length)]).join(''); 395 | fs.writeFileSync(apiKeyPath, JSON.stringify({'X-API-KEY': apiKey})) 396 | console.log('Generated and wrote API key to: ', apiKeyPath); 397 | return apiKey; 398 | } 399 | } 400 | -------------------------------------------------------------------------------- /emitter/emitter.go: -------------------------------------------------------------------------------- 1 | package emitter 2 | 3 | import ( 4 | "encoding/json" 5 | 6 | "github.com/aws/aws-lambda-go/events" 7 | "github.com/deepalert/deepalert" 8 | "github.com/m-mizutani/golambda" 9 | ) 10 | 11 | // SNSEventToReport extracts set of deepalert.Report from events.SNSEvent 12 | func SNSEventToReport(event events.SNSEvent) ([]*deepalert.Report, error) { 13 | var reports []*deepalert.Report 14 | for _, record := range event.Records { 15 | var report deepalert.Report 16 | msg := record.SNS.Message 17 | if err := json.Unmarshal([]byte(msg), &report); err != nil { 18 | return nil, golambda.WrapError(err, "Fail to unmarshal report").With("msg", msg) 19 | } 20 | 21 | reports = append(reports, &report) 22 | } 23 | 24 | return reports, nil 25 | } 26 | -------------------------------------------------------------------------------- /emitter/emitter_test.go: -------------------------------------------------------------------------------- 1 | package emitter_test 2 | 3 | import ( 4 | "encoding/json" 5 | "testing" 6 | 7 | "github.com/aws/aws-lambda-go/events" 8 | "github.com/deepalert/deepalert" 9 | "github.com/deepalert/deepalert/emitter" 10 | "github.com/stretchr/testify/assert" 11 | "github.com/stretchr/testify/require" 12 | ) 13 | 14 | func TestEmitter(t *testing.T) { 15 | t.Run("Valid SNS event", func(tt *testing.T) { 16 | report := deepalert.Report{ 17 | ID: deepalert.ReportID("t1"), 18 | Result: deepalert.ReportResult{ 19 | Severity: deepalert.SevUrgent, 20 | }, 21 | } 22 | 23 | raw, err := json.Marshal(report) 24 | require.NoError(tt, err) 25 | event := events.SNSEvent{ 26 | Records: []events.SNSEventRecord{ 27 | { 28 | SNS: events.SNSEntity{ 29 | Message: string(raw), 30 | }, 31 | }, 32 | }, 33 | } 34 | 35 | reports, err := emitter.SNSEventToReport(event) 36 | require.NoError(t, err) 37 | require.Equal(tt, 1, len(reports)) 38 | assert.Equal(tt, deepalert.ReportID("t1"), reports[0].ID) 39 | }) 40 | 41 | t.Run("Invalid SNS event (report)", func(tt *testing.T) { 42 | event := events.SNSEvent{ 43 | Records: []events.SNSEventRecord{ 44 | { 45 | SNS: events.SNSEntity{ 46 | Message: `{"id":"hoge"`, // missing a bracket of end 47 | }, 48 | }, 49 | }, 50 | } 51 | 52 | reports, err := emitter.SNSEventToReport(event) 53 | require.Error(t, err) 54 | assert.Equal(t, 0, len(reports)) 55 | }) 56 | } 57 | -------------------------------------------------------------------------------- /examples/emitter/emitter_test.go: -------------------------------------------------------------------------------- 1 | package main_test 2 | 3 | // No test required 4 | -------------------------------------------------------------------------------- /examples/emitter/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "log" 6 | 7 | "github.com/aws/aws-lambda-go/events" 8 | "github.com/aws/aws-lambda-go/lambda" 9 | "github.com/deepalert/deepalert" 10 | "github.com/deepalert/deepalert/emitter" 11 | ) 12 | 13 | func handler(ctx context.Context, report deepalert.Report) error { 14 | log.Println(report.Result.Severity) 15 | // Or do appropriate action according to report content and severity 16 | 17 | return nil 18 | } 19 | 20 | func main() { 21 | 22 | lambda.Start(func(ctx context.Context, event events.SNSEvent) error { 23 | reports, err := emitter.SNSEventToReport(event) 24 | if err != nil { 25 | return err 26 | } 27 | 28 | for _, report := range reports { 29 | log.Println(report.Result.Severity) 30 | // Or do appropriate action according to report content and severity 31 | } 32 | 33 | return nil 34 | }) 35 | } 36 | -------------------------------------------------------------------------------- /examples/inspector/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "os" 6 | 7 | "github.com/aws/aws-lambda-go/events" 8 | "github.com/aws/aws-lambda-go/lambda" 9 | "github.com/deepalert/deepalert" 10 | "github.com/deepalert/deepalert/inspector" 11 | ) 12 | 13 | func lookupHostname(value string) *string { 14 | response := "resolved.hostname.example.com" // It's jsut example, OK? 15 | return &response 16 | } 17 | 18 | // Handler is exported for main_test 19 | func Handler(ctx context.Context, attr deepalert.Attribute) (*deepalert.TaskResult, error) { 20 | // Check type of the attribute 21 | if attr.Type != deepalert.TypeIPAddr { 22 | return nil, nil 23 | } 24 | 25 | // Example 26 | resp := lookupHostname(attr.Value) 27 | if resp == nil { 28 | return nil, nil 29 | } 30 | 31 | result := deepalert.TaskResult{ 32 | Contents: []deepalert.ReportContent{ 33 | &deepalert.ContentHost{ 34 | HostName: []string{*resp}, 35 | }, 36 | }, 37 | } 38 | 39 | return &result, nil 40 | } 41 | 42 | func main() { 43 | lambda.Start(func(ctx context.Context, event events.SNSEvent) error { 44 | tasks, err := inspector.SNSEventToTasks(event) 45 | if err != nil { 46 | return err 47 | } 48 | 49 | inspector.Start(inspector.Arguments{ 50 | Context: ctx, 51 | Tasks: tasks, 52 | Handler: Handler, 53 | Author: "testInspector", 54 | FindingQueueURL: os.Getenv("FINDING_QUEUE"), 55 | AttrQueueURL: os.Getenv("ATTRIBUTE_QUEUE"), 56 | }) 57 | 58 | return nil 59 | }) 60 | } 61 | -------------------------------------------------------------------------------- /examples/inspector/main_test.go: -------------------------------------------------------------------------------- 1 | package main_test 2 | 3 | import ( 4 | "context" 5 | "testing" 6 | 7 | "github.com/deepalert/deepalert" 8 | main "github.com/deepalert/deepalert/examples/inspector" 9 | "github.com/deepalert/deepalert/inspector" 10 | "github.com/google/uuid" 11 | "github.com/stretchr/testify/require" 12 | ) 13 | 14 | func TestInspectorExample(t *testing.T) { 15 | attrURL := "https://sqs.ap-northeast-1.amazonaws.com/123456789xxx/attribute-queue" 16 | contentURL := "https://sqs.ap-northeast-1.amazonaws.com/123456789xxx/content-queue" 17 | 18 | args := inspector.Arguments{ 19 | Context: context.Background(), 20 | Handler: main.Handler, 21 | Author: "blue", 22 | AttrQueueURL: attrURL, 23 | FindingQueueURL: contentURL, 24 | } 25 | 26 | t.Run("With IPaddr attribute", func(tt *testing.T) { 27 | mock, newSQS := inspector.NewSQSMock() 28 | args.NewSQS = newSQS 29 | 30 | task := &deepalert.Task{ 31 | ReportID: deepalert.ReportID(uuid.New().String()), 32 | Attribute: &deepalert.Attribute{ 33 | Type: deepalert.TypeIPAddr, 34 | Key: "dst", 35 | Value: "192.10.0.1", 36 | }, 37 | } 38 | 39 | err := inspector.HandleTask(context.Background(), task, args) 40 | require.NoError(tt, err) 41 | sections, err := mock.GetSections(contentURL) 42 | require.NoError(tt, err) 43 | require.Equal(tt, 1, len(sections)) 44 | }) 45 | 46 | t.Run("With not IPaddr attribute", func(tt *testing.T) { 47 | mock, newSQS := inspector.NewSQSMock() 48 | args.NewSQS = newSQS 49 | 50 | task := &deepalert.Task{ 51 | ReportID: deepalert.ReportID(uuid.New().String()), 52 | Attribute: &deepalert.Attribute{ 53 | Type: deepalert.TypeUserName, 54 | Key: "login-name", 55 | Value: "mizutani", 56 | }, 57 | } 58 | 59 | err := inspector.HandleTask(context.Background(), task, args) 60 | require.NoError(tt, err) 61 | sections, err := mock.GetSections(contentURL) 62 | require.NoError(tt, err) 63 | require.Equal(tt, 0, len(sections)) 64 | }) 65 | } 66 | -------------------------------------------------------------------------------- /examples/reviewer/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/aws/aws-lambda-go/lambda" 7 | "github.com/deepalert/deepalert" 8 | ) 9 | 10 | func main() { 11 | lambda.Start(evaluate) 12 | } 13 | 14 | // Example to evaluate security alert of suspicious activity on AWS 15 | func evaluate(ctx context.Context, report deepalert.Report) (*deepalert.ReportResult, error) { 16 | for _, alert := range report.Alerts { 17 | // Skip if alert ruleID is not matched 18 | if alert.RuleID != "your_alert_rule_id" { 19 | return nil, nil 20 | } 21 | } 22 | 23 | // Extract results of Inspector 24 | for _, section := range report.Sections { 25 | for _, host := range section.Hosts { 26 | for _, owner := range host.Owner { 27 | // If source host is owned by your company 28 | if owner == "YOUR_COMPANY" { 29 | return &deepalert.ReportResult{ 30 | // Evaluate the alert as safe (no action required) 31 | Severity: deepalert.SevSafe, 32 | Reason: "The device accessing to G Suite is owned by YOUR_COMPANY.", 33 | }, nil 34 | } 35 | } 36 | } 37 | } 38 | 39 | return nil, nil 40 | } 41 | -------------------------------------------------------------------------------- /examples/reviewer/main_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "testing" 6 | 7 | "github.com/deepalert/deepalert" 8 | "github.com/stretchr/testify/assert" 9 | "github.com/stretchr/testify/require" 10 | ) 11 | 12 | func TestEvaluateAlert(t *testing.T) { 13 | t.Run("As SevSafe with your_alert_rule_id amd YOUR_COMPANY PC", func(tt *testing.T) { 14 | report := deepalert.Report{ 15 | Alerts: []*deepalert.Alert{{RuleID: "your_alert_rule_id"}}, 16 | Sections: []*deepalert.Section{ 17 | { 18 | Hosts: []*deepalert.ContentHost{ 19 | {Owner: []string{"YOUR_COMPANY"}}, 20 | }, 21 | }, 22 | }, 23 | } 24 | 25 | result, err := evaluate(context.Background(), report) 26 | require.NoError(tt, err) 27 | require.NotNil(tt, result) 28 | assert.NotEqual(tt, "", result.Reason) 29 | assert.Equal(tt, deepalert.SevSafe, result.Severity) 30 | }) 31 | 32 | t.Run("Return nil for an alert with your_alert_rule_id but not YOUR_COMPANY PC", func(tt *testing.T) { 33 | report := deepalert.Report{ 34 | Alerts: []*deepalert.Alert{{RuleID: "your_alert_rule_id"}}, 35 | Sections: []*deepalert.Section{ 36 | { 37 | Hosts: []*deepalert.ContentHost{}, 38 | }, 39 | }, 40 | } 41 | 42 | result, err := evaluate(context.Background(), report) 43 | require.NoError(tt, err) 44 | require.Nil(tt, result) 45 | }) 46 | 47 | t.Run("Return nil for an alert with YOUR_COMPANY PC but not your_alert_rule_id", func(tt *testing.T) { 48 | report := deepalert.Report{ 49 | Alerts: []*deepalert.Alert{{RuleID: "some_rule"}}, 50 | Sections: []*deepalert.Section{ 51 | { 52 | Hosts: []*deepalert.ContentHost{}, 53 | }, 54 | }, 55 | } 56 | 57 | result, err := evaluate(context.Background(), report) 58 | require.NoError(tt, err) 59 | require.Nil(tt, result) 60 | }) 61 | } 62 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/deepalert/deepalert 2 | 3 | go 1.15 4 | 5 | require ( 6 | github.com/Netflix/go-env v0.0.0-20200803161858-92715955ff70 7 | github.com/aws/aws-lambda-go v1.20.0 8 | github.com/aws/aws-sdk-go v1.36.12 9 | github.com/awslabs/aws-lambda-go-api-proxy v0.8.0 10 | github.com/cenkalti/backoff v2.2.1+incompatible // indirect 11 | github.com/gin-gonic/gin v1.6.3 12 | github.com/google/uuid v1.1.2 13 | github.com/guregu/dynamo v1.6.1 14 | github.com/m-mizutani/golambda v1.0.2 15 | github.com/stretchr/testify v1.6.1 16 | ) 17 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/AndreasBriese/bbloom v0.0.0-20190306092124-e2d15f34fcf9/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8= 2 | github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= 3 | github.com/CloudyKit/fastprinter v0.0.0-20200109182630-33d98a066a53/go.mod h1:+3IMCy2vIlbG1XG/0ggNQv0SvxCAIpPM5b1nCz56Xno= 4 | github.com/CloudyKit/jet/v3 v3.0.0/go.mod h1:HKQPgSJmdK8hdoAbKUUWajkHyHo4RaU5rMdUywE7VMo= 5 | github.com/Joker/hpp v1.0.0/go.mod h1:8x5n+M1Hp5hC0g8okX3sR3vFQwynaX/UgSOM9MeBKzY= 6 | github.com/Netflix/go-env v0.0.0-20200803161858-92715955ff70 h1:BiU1Of79Vu9jJub01I49Ny7G+ehHqLJQYeUonukTbaI= 7 | github.com/Netflix/go-env v0.0.0-20200803161858-92715955ff70/go.mod h1:9XMFaCeRyW7fC9XJOWQ+NdAv8VLG7ys7l3x4ozEGLUQ= 8 | github.com/Shopify/goreferrer v0.0.0-20181106222321-ec9c9a553398/go.mod h1:a1uqRtAwp2Xwc6WNPJEufxJ7fx3npB4UV/JOLmbu5I0= 9 | github.com/ajg/form v1.5.1/go.mod h1:uL1WgH+h2mgNtvBq0339dVnzXdBETtL2LeUXaIv25UY= 10 | github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= 11 | github.com/aws/aws-lambda-go v1.18.0 h1:13AfxzFoPlFjOzXHbRnKuTbteCzHbu4YQgKONNhWcmo= 12 | github.com/aws/aws-lambda-go v1.18.0/go.mod h1:FEwgPLE6+8wcGBTe5cJN3JWurd1Ztm9zN4jsXsjzKKw= 13 | github.com/aws/aws-lambda-go v1.20.0 h1:ZSweJx/Hy9BoIDXKBEh16vbHH0t0dehnF8MKpMiOWc0= 14 | github.com/aws/aws-lambda-go v1.20.0/go.mod h1:jJmlefzPfGnckuHdXX7/80O3BvUUi12XOkbv4w9SGLU= 15 | github.com/aws/aws-sdk-go v1.19.18/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= 16 | github.com/aws/aws-sdk-go v1.36.12 h1:YJpKFEMbqEoo+incs5qMe61n1JH3o4O1IMkMexLzJG8= 17 | github.com/aws/aws-sdk-go v1.36.12/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= 18 | github.com/awslabs/aws-lambda-go-api-proxy v0.8.0 h1:XUx+5PMwtZEIWc7oyMduXUfAhumHFU/xbSPwB2csYx0= 19 | github.com/awslabs/aws-lambda-go-api-proxy v0.8.0/go.mod h1:V3jj7BZnRY8y2QTKSABIwBc+dTjPkX7vLxz61Id7vsQ= 20 | github.com/aymerick/raymond v2.0.3-0.20180322193309-b565731e1464+incompatible/go.mod h1:osfaiScAUVup+UC9Nfq76eWqDhXlp+4UYaA8uhTBO6g= 21 | github.com/cenkalti/backoff v2.1.1+incompatible h1:tKJnvO2kl0zmb/jA5UKAt4VoEVw1qxKWjE/Bpp46npY= 22 | github.com/cenkalti/backoff v2.1.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= 23 | github.com/cenkalti/backoff v2.2.1+incompatible h1:tNowT99t7UNflLxfYYSlKYsBpXdEet03Pg2g16Swow4= 24 | github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= 25 | github.com/codegangsta/inject v0.0.0-20150114235600-33e0aa1cb7c0/go.mod h1:4Zcjuz89kmFXt9morQgcfYZAYZ5n8WHjt81YYWIwtTM= 26 | github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= 27 | github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= 28 | github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= 29 | github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= 30 | github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= 31 | github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= 32 | github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= 33 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 34 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 35 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 36 | github.com/dgraph-io/badger v1.6.0/go.mod h1:zwt7syl517jmP8s94KqSxTlM6IMsdhYy6psNgSztDR4= 37 | github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= 38 | github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= 39 | github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= 40 | github.com/eknkc/amber v0.0.0-20171010120322-cdade1c07385/go.mod h1:0vRUJqYpeSZifjYj7uP3BG/gKcuzL9xWVV/Y+cK33KM= 41 | github.com/etcd-io/bbolt v1.3.3/go.mod h1:ZF2nL25h33cCyBtcyWeZ2/I3HQOfTP+0PIEvHjkjCrw= 42 | github.com/fasthttp-contrib/websocket v0.0.0-20160511215533-1f3b11f56072/go.mod h1:duJ4Jxv5lDcvg4QuQr0oowTf7dz4/CR8NtyCooz9HL8= 43 | github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= 44 | github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= 45 | github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= 46 | github.com/gavv/httpexpect v2.0.0+incompatible/go.mod h1:x+9tiU1YnrOvnB725RkpoLv1M62hOWzwo5OXotisrKc= 47 | github.com/getsentry/sentry-go v0.9.0 h1:KIfpY/D9hX3gWAEd3d8z6ImuHNWtqEsjlpdF8zXFsHM= 48 | github.com/getsentry/sentry-go v0.9.0/go.mod h1:kELm/9iCblqUYh+ZRML7PNdCvEuw24wBvJPYyi86cws= 49 | github.com/gin-contrib/sse v0.0.0-20190301062529-5545eab6dad3/go.mod h1:VJ0WA2NBN22VlZ2dKZQPAPnyWw5XTlK1KymzLKsr59s= 50 | github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE= 51 | github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= 52 | github.com/gin-gonic/gin v1.4.0/go.mod h1:OW2EZn3DO8Ln9oIKOvM++LBO+5UPHJJDH72/q/3rZdM= 53 | github.com/gin-gonic/gin v1.6.3 h1:ahKqKTFpO5KTPHxWZjEdPScmYaGtLo8Y4DMHoEsnp14= 54 | github.com/gin-gonic/gin v1.6.3/go.mod h1:75u5sXoLsGZoRN5Sgbi1eraJ4GU3++wFwWzhwvtwp4M= 55 | github.com/go-check/check v0.0.0-20180628173108-788fd7840127/go.mod h1:9ES+weclKsC9YodN5RgxqK/VD9HM9JsCSh7rNhMZE98= 56 | github.com/go-chi/chi v4.1.2+incompatible/go.mod h1:eB3wogJHnLi3x/kFX2A+IbTBlXxmMeXJVKy9tTv1XzQ= 57 | github.com/go-errors/errors v1.0.1 h1:LUHzmkK3GUKUrL/1gfBUxAHzcev3apQlezX/+O7ma6w= 58 | github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q= 59 | github.com/go-martini/martini v0.0.0-20170121215854-22fa46961aab/go.mod h1:/P9AEU963A2AYjv4d1V5eVL1CQbEJq6aCNHDDjibzu8= 60 | github.com/go-playground/assert/v2 v2.0.1 h1:MsBgLAaY856+nPRTKrp3/OZK38U/wa0CcBYNjji3q3A= 61 | github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= 62 | github.com/go-playground/locales v0.13.0 h1:HyWk6mgj5qFqCT5fjGBuRArbVDfE4hi8+e8ceBS/t7Q= 63 | github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8= 64 | github.com/go-playground/universal-translator v0.17.0 h1:icxd5fm+REJzpZx7ZfpaD876Lmtgy7VtROAbHHXk8no= 65 | github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA= 66 | github.com/go-playground/validator/v10 v10.2.0 h1:KgJ0snyC2R9VXYN2rneOtQcw5aHQB1Vv0sFl1UcHBOY= 67 | github.com/go-playground/validator/v10 v10.2.0/go.mod h1:uOYAAleCW8F/7oMFd6aG0GOhaH6EGOAJShg8Id5JGkI= 68 | github.com/gobwas/httphead v0.0.0-20180130184737-2c6c146eadee/go.mod h1:L0fX3K22YWvt/FAX9NnzrNzcI4wNYi9Yku4O0LKYflo= 69 | github.com/gobwas/pool v0.2.0/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw= 70 | github.com/gobwas/ws v1.0.2/go.mod h1:szmBTxLgaFppYjEmNtny/v3w89xOydFnnZMcgRRu/EM= 71 | github.com/gofrs/uuid v3.2.0+incompatible h1:y12jRkkFxsd7GpqdSZ+/KCs/fJbqpEXSGd4+jfEaewE= 72 | github.com/gofrs/uuid v3.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= 73 | github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 74 | github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 75 | github.com/golang/protobuf v1.3.3 h1:gyjaxf+svBWX08ZjK86iN9geUJF0H6gp2IRKX6Nf6/I= 76 | github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= 77 | github.com/gomodule/redigo v1.7.1-0.20190724094224-574c33c3df38/go.mod h1:B4C85qUVwatsJoIUNIfCRsp7qO0iAmpGFZ4EELWSbC4= 78 | github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4= 79 | github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 80 | github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= 81 | github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= 82 | github.com/google/uuid v1.1.2 h1:EVhdT+1Kseyi1/pUmXKaFxYsDNy9RQYkMWRH68J/W7Y= 83 | github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= 84 | github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= 85 | github.com/gorilla/mux v1.7.4/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= 86 | github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= 87 | github.com/guregu/dynamo v1.6.1 h1:yp6nX5eaqXhcfZwE9B6nVP9/b3KMogzW4yLkmc5I5tM= 88 | github.com/guregu/dynamo v1.6.1/go.mod h1:rhVS0QFu0uAGzNfi8k8LzDrcfw/y335eTCtMzZ2DpEo= 89 | github.com/hashicorp/go-version v1.2.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= 90 | github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= 91 | github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= 92 | github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= 93 | github.com/imkira/go-interpol v1.1.0/go.mod h1:z0h2/2T3XF8kyEPpRgJ3kmNv+C43p+I/CoI+jC3w2iA= 94 | github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= 95 | github.com/iris-contrib/blackfriday v2.0.0+incompatible/go.mod h1:UzZ2bDEoaSGPbkg6SAB4att1aAwTmVIx/5gCVqeyUdI= 96 | github.com/iris-contrib/go.uuid v2.0.0+incompatible/go.mod h1:iz2lgM/1UnEf1kP0L/+fafWORmlnuysV2EMP8MW+qe0= 97 | github.com/iris-contrib/jade v1.1.3/go.mod h1:H/geBymxJhShH5kecoiOCSssPX7QWYH7UaeZTSWddIk= 98 | github.com/iris-contrib/pongo2 v0.0.1/go.mod h1:Ssh+00+3GAZqSQb30AvBRNxBx7rf0GqwkjqxNd0u65g= 99 | github.com/iris-contrib/schema v0.0.1/go.mod h1:urYA3uvUNG1TIIjOSCzHr9/LmbQo8LrOcOqfqxa4hXw= 100 | github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5imkbgOkpRUYLnmbU7UEFbjtDA2hxJ1ichM= 101 | github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= 102 | github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= 103 | github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= 104 | github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= 105 | github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= 106 | github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= 107 | github.com/json-iterator/go v1.1.9 h1:9yzud/Ht36ygwatGx56VwCZtlI/2AD15T1X2sjSuGns= 108 | github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= 109 | github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= 110 | github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88/go.mod h1:3w7q1U84EfirKl04SVQ/s7nPm1ZPhiXd34z40TNz36k= 111 | github.com/kataras/golog v0.0.10/go.mod h1:yJ8YKCmyL+nWjERB90Qwn+bdyBZsaQwU3bTVFgkFIp8= 112 | github.com/kataras/iris/v12 v12.1.8/go.mod h1:LMYy4VlP67TQ3Zgriz8RE2h2kMZV2SgMYbq3UhfoFmE= 113 | github.com/kataras/neffos v0.0.14/go.mod h1:8lqADm8PnbeFfL7CLXh1WHw53dG27MC3pgi2R1rmoTE= 114 | github.com/kataras/pio v0.0.2/go.mod h1:hAoW0t9UmXi4R5Oyq5Z4irTbaTsOemSrDGUtaTl7Dro= 115 | github.com/kataras/sitemap v0.0.5/go.mod h1:KY2eugMKiPwsJgx7+U103YZehfvNGOXURubcGyk0Bz8= 116 | github.com/klauspost/compress v1.8.2/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= 117 | github.com/klauspost/compress v1.9.7/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= 118 | github.com/klauspost/cpuid v1.2.1/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= 119 | github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= 120 | github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= 121 | github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= 122 | github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= 123 | github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= 124 | github.com/labstack/echo/v4 v4.1.11/go.mod h1:i541M3Fj6f76NZtHSj7TXnyM8n2gaodfvfxNnFqi74g= 125 | github.com/labstack/echo/v4 v4.1.16/go.mod h1:awO+5TzAjvL8XpibdsfXxPgHr+orhtXZJZIQCVjogKI= 126 | github.com/labstack/gommon v0.3.0/go.mod h1:MULnywXg0yavhxWKc+lOruYdAhDwPK9wf0OL7NoOu+k= 127 | github.com/leodido/go-urn v1.2.0 h1:hpXL4XnriNwQ/ABnpepYM/1vCLWNDfUNts8dX3xTG6Y= 128 | github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII= 129 | github.com/m-mizutani/golambda v1.0.2 h1:ulZfJmxSRP6jBvJgZ9QokXgxEHtGy608fNgMn6YLWts= 130 | github.com/m-mizutani/golambda v1.0.2/go.mod h1:qUqQ4qX270xOxkhqH/sut5QDtBOy3n+4150FaYlwg4Q= 131 | github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= 132 | github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= 133 | github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= 134 | github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= 135 | github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= 136 | github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ= 137 | github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY= 138 | github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= 139 | github.com/mattn/goveralls v0.0.2/go.mod h1:8d1ZMHsd7fW6IRPKQh46F2WRpyib5/X4FOpevwGNQEw= 140 | github.com/mediocregopher/radix/v3 v3.4.2/go.mod h1:8FL3F6UQRXHXIBSPUs5h0RybMF8i4n7wVopoX3x7Bv8= 141 | github.com/microcosm-cc/bluemonday v1.0.2/go.mod h1:iVP4YcDBq+n/5fb23BhYFvIMq/leAFZyRl6bYmGDlGc= 142 | github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= 143 | github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= 144 | github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= 145 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= 146 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= 147 | github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= 148 | github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= 149 | github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= 150 | github.com/moul/http2curl v1.0.0/go.mod h1:8UbvGypXm98wA/IqH45anm5Y2Z6ep6O31QGOAZ3H0fQ= 151 | github.com/nats-io/jwt v0.3.0/go.mod h1:fRYCDE99xlTsqUzISS1Bi75UBJ6ljOJQOAAu5VglpSg= 152 | github.com/nats-io/nats.go v1.9.1/go.mod h1:ZjDU1L/7fJ09jvUSRVBR2e7+RnLiiIQyqyzEE/Zbp4w= 153 | github.com/nats-io/nkeys v0.1.0/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= 154 | github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= 155 | github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= 156 | github.com/onsi/ginkgo v1.10.3 h1:OoxbjfXVZyod1fmWYhI7SEyaD8B00ynP3T+D5GiyHOY= 157 | github.com/onsi/ginkgo v1.10.3/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= 158 | github.com/onsi/gomega v1.7.1 h1:K0jcRCwNQM3vFGh1ppMtDh/+7ApJrjldlX8fA0jDTLQ= 159 | github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= 160 | github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= 161 | github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4= 162 | github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= 163 | github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 164 | github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= 165 | github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 166 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 167 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 168 | github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= 169 | github.com/rs/zerolog v1.20.0 h1:38k9hgtUBdxFwE34yS8rTHmHBa4eN16E4DJlv177LNs= 170 | github.com/rs/zerolog v1.20.0/go.mod h1:IzD0RJ65iWH0w97OQQebJEvTZYvsCUm9WVLWBQrJRjo= 171 | github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= 172 | github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0RK8m9o+Q= 173 | github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= 174 | github.com/ryanuber/columnize v2.1.0+incompatible/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= 175 | github.com/schollz/closestmatch v2.1.0+incompatible/go.mod h1:RtP1ddjLong6gTkbtmuhtR2uUrrJOpYzYRvbcPAid+g= 176 | github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= 177 | github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo= 178 | github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= 179 | github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= 180 | github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= 181 | github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= 182 | github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= 183 | github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= 184 | github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= 185 | github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= 186 | github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= 187 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 188 | github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= 189 | github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= 190 | github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= 191 | github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= 192 | github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= 193 | github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= 194 | github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 195 | github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= 196 | github.com/ugorji/go v1.1.7 h1:/68gy2h+1mWMrwZFeD1kQialdSzAb432dtpeJ42ovdo= 197 | github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw= 198 | github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= 199 | github.com/ugorji/go/codec v1.1.7 h1:2SvQaVZ1ouYrrKKwoSk2pzd4A9evlKJb9oTL+OaLUSs= 200 | github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY= 201 | github.com/urfave/cli/v2 v2.1.1/go.mod h1:SE9GqnLQmjVa0iPEY0f1w3ygNIYcIJ0OKPMoW2caLfQ= 202 | github.com/urfave/cli/v2 v2.2.0/go.mod h1:SE9GqnLQmjVa0iPEY0f1w3ygNIYcIJ0OKPMoW2caLfQ= 203 | github.com/urfave/negroni v1.0.0/go.mod h1:Meg73S6kFm/4PpbYdq35yYWoCZ9mS/YSx+lKnmiohz4= 204 | github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= 205 | github.com/valyala/fasthttp v1.6.0/go.mod h1:FstJa9V+Pj9vQ7OJie2qMHdwemEDaDiSdBnvPM1Su9w= 206 | github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8= 207 | github.com/valyala/fasttemplate v1.1.0/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8= 208 | github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio= 209 | github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= 210 | github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= 211 | github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= 212 | github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= 213 | github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0/go.mod h1:/LWChgwKmvncFJFHJ7Gvn9wZArjbV5/FppcK2fKk/tI= 214 | github.com/yudai/gojsondiff v1.0.0/go.mod h1:AY32+k2cwILAkW1fbgxQ5mUmMiZFgLIV+FBNExI05xg= 215 | github.com/yudai/golcs v0.0.0-20170316035057-ecda9a501e82/go.mod h1:lgjkn3NuSvDfVJdfcVVdX+jpBxNmX4rDAzaS45IcYoM= 216 | github.com/yudai/pp v2.0.1+incompatible/go.mod h1:PuxR/8QJ7cyCkFp/aUDS+JY727OFEZkTdatxwunjIkc= 217 | golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= 218 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 219 | golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= 220 | golang.org/x/crypto v0.0.0-20191227163750-53104e6ec876/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= 221 | golang.org/x/crypto v0.0.0-20200221231518-2aa609cf4a9d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= 222 | golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= 223 | golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 224 | golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 225 | golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 226 | golang.org/x/net v0.0.0-20190318221613-d196dffd7c2b h1:ZWpVMTsK0ey5WJCu+vVdfMldWq7/ezaOcjnKWIHWVkE= 227 | golang.org/x/net v0.0.0-20190318221613-d196dffd7c2b/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 228 | golang.org/x/net v0.0.0-20190327091125-710a502c58a2/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 229 | golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 230 | golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 231 | golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 232 | golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 233 | golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 234 | golang.org/x/net v0.0.0-20200226121028-0de0cce0169b h1:0mm1VjtFUOIlE1SbDlwjYaDxZVDP2S5ou6y0gSgXHu8= 235 | golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 236 | golang.org/x/net v0.0.0-20201110031124-69a78807bb2b h1:uwuIcX0g4Yl1NC5XAz37xsr2lTtcqevgzYNVt49waME= 237 | golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= 238 | golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 239 | golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 240 | golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 241 | golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 242 | golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 243 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a h1:1BGLXjeY4akVXGgbC9HugT3Jv3hCI0z56oJR5vAMgBU= 244 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 245 | golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 246 | golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 247 | golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 248 | golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a h1:aYOabOQFp6Vj6W1F80affTUvO9UxmJRx8K0gsfABByQ= 249 | golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 250 | golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 251 | golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae h1:/WDfKMnPU+m5M4xB+6x4kaepxRw6jWvR5iDRdvjHgy8= 252 | golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 253 | golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f h1:+Nyd8tzPX9R7BWHguqsrbFdRx3WQ/1ib8I44HXV5yTA= 254 | golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 255 | golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= 256 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 257 | golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= 258 | golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= 259 | golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k= 260 | golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= 261 | golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 262 | golang.org/x/tools v0.0.0-20181221001348-537d06c36207/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 263 | golang.org/x/tools v0.0.0-20190327201419-c70d86f8b7cf/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= 264 | golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= 265 | golang.org/x/tools v0.0.0-20190828213141-aed303cbaa74/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 266 | golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 267 | golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 268 | golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= 269 | golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 270 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= 271 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 272 | gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= 273 | gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 274 | gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= 275 | gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= 276 | gopkg.in/go-playground/assert.v1 v1.2.1 h1:xoYuJVE7KT85PYWrN730RguIQO0ePzVRfFMXadIrXTM= 277 | gopkg.in/go-playground/assert.v1 v1.2.1/go.mod h1:9RXL0bg/zibRAgZUYszZSwO/z8Y/a8bDuhia5mkpMnE= 278 | gopkg.in/go-playground/validator.v8 v8.18.2/go.mod h1:RX2a/7Ha8BgOhfk7j780h4/u/RRjR0eouCJSH80/M2Y= 279 | gopkg.in/ini.v1 v1.51.1/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= 280 | gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA= 281 | gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= 282 | gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= 283 | gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 284 | gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 285 | gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= 286 | gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 287 | gopkg.in/yaml.v3 v3.0.0-20191120175047-4206685974f2/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 288 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 289 | gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776 h1:tQIYjPdBoyREyB9XMu+nnTclpTYkz2zFM+lzLJFO4gQ= 290 | gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 291 | -------------------------------------------------------------------------------- /inspector/inspector.go: -------------------------------------------------------------------------------- 1 | package inspector 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "fmt" 7 | 8 | "github.com/aws/aws-lambda-go/events" 9 | "github.com/deepalert/deepalert" 10 | "github.com/m-mizutani/golambda" 11 | ) 12 | 13 | // InspectHandler is a function type of callback of inspector. 14 | type InspectHandler func(ctx context.Context, attr deepalert.Attribute) (*deepalert.TaskResult, error) 15 | 16 | // Logger is github.com/m-mizutani/golambda logger and exported to be controlled from external module. 17 | var Logger = golambda.Logger 18 | 19 | type reportIDKey struct{} 20 | 21 | var contextKey = &reportIDKey{} 22 | 23 | // ReportIDFromCtx extracts ReportID from context. The function is available in handler called by Start 24 | func ReportIDFromCtx(ctx context.Context) (*deepalert.ReportID, bool) { 25 | lc, ok := ctx.Value(contextKey).(*deepalert.ReportID) 26 | return lc, ok 27 | } 28 | 29 | // Arguments is parameters to invoke Start(). 30 | type Arguments struct { 31 | Context context.Context 32 | 33 | Tasks []*deepalert.Task 34 | 35 | // Handler is callback function of Start(). Handler may be called multiply. (Required) 36 | Handler InspectHandler 37 | 38 | // HandlerData is data for Handler. deepalert/inspector never access HandlerData and set additional argument if you need in Handler (optional) 39 | HandlerData interface{} 40 | 41 | // Author indicates owner of new attributes and contents. It does not require explicit unique name, but unique name helps your debugging and troubleshooting. (Required) 42 | Author string 43 | 44 | // AttrQueueURL is URL to send new attributes discovered inspector (e.g. a new related IP address). It should be exported CloudFormation value and can be imported by Fn::ImportValue: + YOU_STACK_NAME-AttributeQueue to your inspector CloudFormation stack. (Required) 45 | AttrQueueURL string 46 | 47 | // FindingQueueURL is also URL to send contents generated inspector (e.g. IP address is blacklisted or not). It should be exported CloudFormation value and can be imported by Fn::ImportValue: + YOU_STACK_NAME-ContentQueue to your inspector CloudFormation stack. (Required) 48 | FindingQueueURL string 49 | 50 | // NewSQS is constructor of SQSClient that is interface of AWS SDK. This function is to set stub for testing. If NewSQS is nil, use default constructor, newAwsSQSClient. (Optional) 51 | NewSQS SQSClientFactory 52 | } 53 | 54 | // Start is a wrapper of Inspector. 55 | func Start(args Arguments) error { 56 | for _, task := range args.Tasks { 57 | if err := HandleTask(args.Context, task, args); err != nil { 58 | return err 59 | } 60 | } 61 | 62 | return nil 63 | } 64 | 65 | // SNSEventToTasks extracts deepalert.Task from SNS Event 66 | func SNSEventToTasks(event events.SNSEvent) ([]*deepalert.Task, error) { 67 | var results []*deepalert.Task 68 | 69 | for _, record := range event.Records { 70 | var task deepalert.Task 71 | msg := record.SNS.Message 72 | if err := json.Unmarshal([]byte(msg), &task); err != nil { 73 | return nil, golambda.WrapError(err, "Fail to unmarshal task").With("msg", msg) 74 | } 75 | 76 | results = append(results, &task) 77 | } 78 | 79 | return results, nil 80 | } 81 | 82 | // HandleTask is called with task by task. It's exported for testing 83 | func HandleTask(ctx context.Context, task *deepalert.Task, args Arguments) error { 84 | Logger. 85 | With("task", task). 86 | With("ctx", ctx). 87 | With("Author", args.Author). 88 | With("AttrQueueURL", args.AttrQueueURL). 89 | With("FindingQueueURL", args.FindingQueueURL). 90 | Info("Start inspector") 91 | 92 | // Check Arguments 93 | if args.Handler == nil { 94 | return fmt.Errorf("Handler is not set in inspector.Argument") 95 | } 96 | if args.Author == "" { 97 | return fmt.Errorf("Author is not set in inspector.Argument") 98 | } 99 | if args.AttrQueueURL == "" { 100 | return fmt.Errorf("AttrQueueURL is not set in inspector.Argument") 101 | } 102 | if args.FindingQueueURL == "" { 103 | return fmt.Errorf("FindingQueueURL is not set in inspector.Argument") 104 | } 105 | if task == nil { 106 | return fmt.Errorf("Task is nil") 107 | } 108 | 109 | if args.NewSQS == nil { 110 | args.NewSQS = newAwsSQSClient 111 | } 112 | 113 | newCtx := context.WithValue(ctx, contextKey, &task.ReportID) 114 | 115 | result, err := args.Handler(newCtx, *task.Attribute) 116 | if err != nil { 117 | return golambda.WrapError(err, "Fail to handle task").With("task", task) 118 | } 119 | 120 | if result == nil { 121 | return nil 122 | } 123 | 124 | // Sending entities 125 | for _, entity := range result.Contents { 126 | finding := deepalert.Finding{ 127 | ReportID: task.ReportID, 128 | Attribute: *task.Attribute, 129 | Author: args.Author, 130 | Type: entity.Type(), 131 | Content: entity, 132 | } 133 | Logger.With("finding", finding).Trace("Sending finding") 134 | 135 | if err := sendSQS(args.NewSQS, finding, args.FindingQueueURL); err != nil { 136 | return golambda.WrapError(err, "Fail to publish ReportContent").With("url", args.FindingQueueURL).With("finding", finding) 137 | } 138 | } 139 | 140 | var newAttrs []*deepalert.Attribute 141 | for _, attr := range result.NewAttributes { 142 | if attr.Timestamp == nil { 143 | attr.Timestamp = task.Attribute.Timestamp 144 | } 145 | newAttrs = append(newAttrs, attr) 146 | } 147 | 148 | // Sending new attributes 149 | if len(result.NewAttributes) > 0 { 150 | attrReport := deepalert.ReportAttribute{ 151 | ReportID: task.ReportID, 152 | OriginAttr: *task.Attribute, 153 | Attributes: newAttrs, 154 | Author: args.Author, 155 | } 156 | 157 | Logger.With("ReportAttribute", attrReport).Trace("Sending new attributes") 158 | if err := sendSQS(args.NewSQS, attrReport, args.AttrQueueURL); err != nil { 159 | return golambda.WrapError(err, "Fail to publish ReportAttribute").With("url", args.AttrQueueURL).With("report", attrReport) 160 | } 161 | } 162 | 163 | Logger.Trace("Exit handler normally") 164 | return nil 165 | } 166 | -------------------------------------------------------------------------------- /inspector/inspector_test.go: -------------------------------------------------------------------------------- 1 | package inspector_test 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "testing" 7 | 8 | "github.com/aws/aws-sdk-go/aws" 9 | "github.com/deepalert/deepalert" 10 | "github.com/deepalert/deepalert/inspector" 11 | "github.com/google/uuid" 12 | "github.com/stretchr/testify/assert" 13 | "github.com/stretchr/testify/require" 14 | ) 15 | 16 | func dummyInspector(ctx context.Context, attr deepalert.Attribute) (*deepalert.TaskResult, error) { 17 | // tableName := os.Getenv("RESULT_TABLE") 18 | // reportID, _ := deepalert.ReportIDFromCtx(ctx) 19 | 20 | hostReport := deepalert.ContentHost{ 21 | IPAddr: []string{"10.1.2.3"}, 22 | Owner: []string{"superman"}, 23 | } 24 | 25 | newAttr := deepalert.Attribute{ 26 | Key: "username", 27 | Value: "mizutani", 28 | Type: deepalert.TypeUserName, 29 | } 30 | 31 | return &deepalert.TaskResult{ 32 | Contents: []deepalert.ReportContent{&hostReport}, 33 | NewAttributes: []*deepalert.Attribute{&newAttr}, 34 | }, nil 35 | } 36 | 37 | func TestInspectorHandler(t *testing.T) { 38 | result, err := inspector.StartTest(inspector.Arguments{ 39 | Handler: dummyInspector, 40 | Author: "dummyInspector", 41 | }, deepalert.Attribute{ 42 | Type: deepalert.TypeIPAddr, 43 | Key: "SrcIP", 44 | Value: "10.0.0.1", 45 | }) 46 | 47 | assert.NoError(t, err) 48 | assert.NotNil(t, result) 49 | assert.Equal(t, 1, len(result.Contents)) 50 | assert.Equal(t, 1, len(result.NewAttributes)) 51 | } 52 | 53 | func convert(src interface{}, dst interface{}) error { 54 | raw, err := json.Marshal(src) 55 | if err != nil { 56 | return err 57 | } 58 | 59 | return json.Unmarshal(raw, dst) 60 | } 61 | 62 | func TestSQS(t *testing.T) { 63 | mock, newSQS := inspector.NewSQSMock() 64 | 65 | attrURL := "https://sqs.ap-northeast-1.amazonaws.com/123456789xxx/attribute-queue" 66 | contentURL := "https://sqs.ap-northeast-1.amazonaws.com/123456789xxx/content-queue" 67 | args := inspector.Arguments{ 68 | Handler: dummyInspector, 69 | Author: "blue", 70 | AttrQueueURL: attrURL, 71 | FindingQueueURL: contentURL, 72 | NewSQS: newSQS, 73 | } 74 | 75 | task := deepalert.Task{ 76 | ReportID: deepalert.ReportID(uuid.New().String()), 77 | Attribute: &deepalert.Attribute{ 78 | Type: deepalert.TypeIPAddr, 79 | Key: "dst", 80 | Value: "192.10.0.1", 81 | }, 82 | } 83 | 84 | err := inspector.HandleTask(context.Background(), &task, args) 85 | require.NoError(t, err) 86 | assert.Equal(t, 2, len(mock.InputMap)) 87 | require.Equal(t, 1, len(mock.InputMap[attrURL])) 88 | require.Equal(t, 1, len(mock.InputMap[contentURL])) 89 | 90 | cq := mock.InputMap[contentURL][0] 91 | aq := mock.InputMap[attrURL][0] 92 | 93 | var req1 deepalert.Finding 94 | err = json.Unmarshal([]byte(*cq.MessageBody), &req1) 95 | require.NoError(t, err) 96 | assert.Equal(t, contentURL, aws.StringValue(cq.QueueUrl)) 97 | assert.Equal(t, attrURL, aws.StringValue(aq.QueueUrl)) 98 | 99 | var host deepalert.ContentHost 100 | require.NoError(t, convert(req1.Content, &host)) 101 | assert.Equal(t, "10.1.2.3", host.IPAddr[0]) 102 | assert.Equal(t, "superman", host.Owner[0]) 103 | } 104 | -------------------------------------------------------------------------------- /inspector/mock.go: -------------------------------------------------------------------------------- 1 | package inspector 2 | 3 | import ( 4 | "encoding/json" 5 | 6 | "github.com/aws/aws-sdk-go/aws" 7 | "github.com/aws/aws-sdk-go/service/sqs" 8 | "github.com/deepalert/deepalert" 9 | "github.com/m-mizutani/golambda" 10 | ) 11 | 12 | // MockSQSClient is for testing. Just storing sqs.SendMessageInput 13 | type MockSQSClient struct { 14 | InputMap map[string][]*sqs.SendMessageInput 15 | Region string 16 | } 17 | 18 | func newMockSQSClient() *MockSQSClient { 19 | return &MockSQSClient{ 20 | InputMap: make(map[string][]*sqs.SendMessageInput), 21 | } 22 | } 23 | 24 | // NewSQSMock creates a pair of MockSQSClient and constructor that returns the MockSQSClient 25 | func NewSQSMock() (*MockSQSClient, SQSClientFactory) { 26 | client := newMockSQSClient() 27 | return client, func(_ string) (SQSClient, error) { 28 | return client, nil 29 | } 30 | } 31 | 32 | // SendMessage stores input to own struct, not sending. Just for test. 33 | func (x *MockSQSClient) SendMessage(input *sqs.SendMessageInput) (*sqs.SendMessageOutput, error) { 34 | url := aws.StringValue(input.QueueUrl) 35 | if _, ok := x.InputMap[url]; !ok { 36 | x.InputMap[url] = []*sqs.SendMessageInput{} 37 | } 38 | x.InputMap[url] = append(x.InputMap[url], input) 39 | return &sqs.SendMessageOutput{}, nil 40 | } 41 | 42 | func (x *MockSQSClient) GetSections(url string) ([]*deepalert.Section, error) { 43 | queues, ok := x.InputMap[url] 44 | if !ok { 45 | return nil, nil 46 | } 47 | 48 | var output []*deepalert.Section 49 | for _, q := range queues { 50 | var section deepalert.Section 51 | if err := json.Unmarshal([]byte(*q.MessageBody), §ion); err != nil { 52 | return nil, golambda.WrapError(err, "Failed to parse section queue") 53 | } 54 | 55 | output = append(output, §ion) 56 | } 57 | 58 | return output, nil 59 | } 60 | 61 | func (x *MockSQSClient) GetAttributes(url string) ([]*deepalert.ReportAttribute, error) { 62 | queues, ok := x.InputMap[url] 63 | if !ok { 64 | return nil, nil 65 | } 66 | 67 | var output []*deepalert.ReportAttribute 68 | for _, q := range queues { 69 | var attr deepalert.ReportAttribute 70 | if err := json.Unmarshal([]byte(*q.MessageBody), &attr); err != nil { 71 | return nil, golambda.WrapError(err, "Failed to parse attribute queue") 72 | } 73 | 74 | output = append(output, &attr) 75 | } 76 | 77 | return output, nil 78 | } 79 | -------------------------------------------------------------------------------- /inspector/sqs.go: -------------------------------------------------------------------------------- 1 | package inspector 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "regexp" 7 | 8 | "github.com/aws/aws-sdk-go/aws" 9 | "github.com/aws/aws-sdk-go/aws/session" 10 | "github.com/aws/aws-sdk-go/service/sqs" 11 | "github.com/m-mizutani/golambda" 12 | ) 13 | 14 | // SQSClient is interface of AWS SDK SQS. Need to have only SendMessage() 15 | type SQSClient interface { 16 | SendMessage(*sqs.SendMessageInput) (*sqs.SendMessageOutput, error) 17 | } 18 | 19 | // SQSClientFactory is constructor of SQSClient with region 20 | type SQSClientFactory func(region string) (SQSClient, error) 21 | 22 | func newAwsSQSClient(region string) (SQSClient, error) { 23 | ssn, err := session.NewSession(&aws.Config{Region: aws.String(region)}) 24 | if err != nil { 25 | return nil, err 26 | } 27 | client := sqs.New(ssn) 28 | return client, nil 29 | } 30 | 31 | // Sample: https://sqs.ap-northeast-1.amazonaws.com/123456789xxx/some-queue-name 32 | var regexSqsURL = regexp.MustCompile(`https://sqs.([a-z0-9-]+).amazonaws.com`) 33 | 34 | func extractRegionFromURL(url string) (*string, error) { 35 | if m := regexSqsURL.FindStringSubmatch(url); len(m) == 2 { 36 | return &m[1], nil 37 | } 38 | return nil, fmt.Errorf("Invalid SQS URL foramt: %v", url) 39 | } 40 | 41 | func sendSQS(newSQS SQSClientFactory, msg interface{}, targetURL string) error { 42 | region, err := extractRegionFromURL(targetURL) 43 | if err != nil { 44 | return err 45 | } 46 | 47 | client, err := newSQS(*region) 48 | if err != nil { 49 | return golambda.WrapError(err, "Failed to create a new SQS client") 50 | } 51 | 52 | raw, err := json.Marshal(msg) 53 | if err != nil { 54 | return golambda.WrapError(err, "Failed to marshal message").With("msg", msg) 55 | } 56 | 57 | input := sqs.SendMessageInput{ 58 | QueueUrl: &targetURL, 59 | MessageBody: aws.String(string(raw)), 60 | } 61 | resp, err := client.SendMessage(&input) 62 | 63 | if err != nil { 64 | return golambda.WrapError(err, "Failed to send SQS message").With("input", input) 65 | } 66 | 67 | Logger.With("resp", resp).Trace("Sent SQS message") 68 | 69 | return nil 70 | } 71 | -------------------------------------------------------------------------------- /inspector/sqs_test.go: -------------------------------------------------------------------------------- 1 | package inspector 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | ) 8 | 9 | func TestExtractRegionFromURL(t *testing.T) { 10 | var region *string 11 | var err error 12 | 13 | // Valid case 14 | region, err = extractRegionFromURL("https://sqs.ap-northeast-1.amazonaws.com/123456789xxx/attribute-queue") 15 | assert.NoError(t, err) 16 | assert.Equal(t, "ap-northeast-1", *region) 17 | 18 | // Invalid cases 19 | region, err = extractRegionFromURL("https://sns.ap-northeast-1.amazonaws.com/123456789xxx/attribute-queue") 20 | assert.Error(t, err) 21 | assert.Nil(t, region) 22 | 23 | region, err = extractRegionFromURL("https://sqs.ap-northeast-1.xxx.amazonaws.com/123456789xxx/attribute-queue") 24 | assert.Error(t, err) 25 | assert.Nil(t, region) 26 | } 27 | -------------------------------------------------------------------------------- /inspector/test_utils.go: -------------------------------------------------------------------------------- 1 | package inspector 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | 7 | "github.com/deepalert/deepalert" 8 | "github.com/google/uuid" 9 | "github.com/m-mizutani/golambda" 10 | ) 11 | 12 | // StartTest emulates inspector.Start, but 13 | func StartTest(args Arguments, attr deepalert.Attribute) (*deepalert.TaskResult, error) { 14 | if args.Handler == nil { 15 | return nil, fmt.Errorf("Handler is not set in emitter.Argument") 16 | } 17 | if args.Author == "" { 18 | return nil, fmt.Errorf("Author is not set in emitter.Argument") 19 | } 20 | 21 | reportID := uuid.New().String() 22 | ctx := context.WithValue(context.Background(), contextKey, &reportID) 23 | 24 | result, err := args.Handler(ctx, attr) 25 | if err != nil { 26 | return nil, golambda.WrapError(err, "Fail to run Handler") 27 | } 28 | 29 | return result, nil 30 | } 31 | -------------------------------------------------------------------------------- /internal/adaptor/adaptor_test.go: -------------------------------------------------------------------------------- 1 | package adaptor_test 2 | 3 | // No test required 4 | -------------------------------------------------------------------------------- /internal/adaptor/repository.go: -------------------------------------------------------------------------------- 1 | package adaptor 2 | 3 | import ( 4 | "time" 5 | 6 | "github.com/deepalert/deepalert" 7 | "github.com/deepalert/deepalert/internal/models" 8 | ) 9 | 10 | // RepositoryFactory is interface Repository constructor 11 | type RepositoryFactory func(region, tableName string) Repository 12 | 13 | // Repository is interface of AWS SDK SQS 14 | type Repository interface { 15 | PutAlertEntry(entry *models.AlertEntry, ts time.Time) error 16 | GetAlertEntry(pk, sk string) (*models.AlertEntry, error) 17 | PutAlertCache(cache *models.AlertCache) error 18 | GetAlertCaches(pk string) ([]*models.AlertCache, error) 19 | PutInspectorReport(record *models.InspectorReportRecord) error 20 | GetInspectorReports(pk string) ([]*models.InspectorReportRecord, error) 21 | PutAttributeCache(attr *models.AttributeCache, ts time.Time) error 22 | GetAttributeCaches(pk string) ([]*models.AttributeCache, error) 23 | PutReport(pk string, report *deepalert.Report) error 24 | GetReport(pk string) (*deepalert.Report, error) 25 | 26 | IsConditionalCheckErr(err error) bool 27 | } 28 | 29 | // NewRepository creates actual AWS SFn SDK client 30 | func NewRepository(region, tableName string) Repository { 31 | return nil 32 | } 33 | -------------------------------------------------------------------------------- /internal/adaptor/sfn.go: -------------------------------------------------------------------------------- 1 | package adaptor 2 | 3 | import ( 4 | "github.com/aws/aws-sdk-go/aws" 5 | "github.com/aws/aws-sdk-go/aws/session" 6 | "github.com/aws/aws-sdk-go/service/sfn" 7 | ) 8 | 9 | // SFnClientFactory is interface SFnClient constructor 10 | type SFnClientFactory func(region string) (SFnClient, error) 11 | 12 | // SFnClient is interface of AWS SDK SQS 13 | type SFnClient interface { 14 | StartExecution(*sfn.StartExecutionInput) (*sfn.StartExecutionOutput, error) 15 | } 16 | 17 | // NewSFnClient creates actual AWS SFn SDK client 18 | func NewSFnClient(region string) (SFnClient, error) { 19 | ssn, err := session.NewSession(&aws.Config{Region: aws.String(region)}) 20 | if err != nil { 21 | return nil, err 22 | } 23 | return sfn.New(ssn), nil 24 | } 25 | -------------------------------------------------------------------------------- /internal/adaptor/sns.go: -------------------------------------------------------------------------------- 1 | package adaptor 2 | 3 | import ( 4 | "github.com/aws/aws-sdk-go/aws" 5 | "github.com/aws/aws-sdk-go/aws/session" 6 | "github.com/aws/aws-sdk-go/service/sns" 7 | ) 8 | 9 | // SNSClientFactory is interface SNSClient constructor 10 | type SNSClientFactory func(region string) (SNSClient, error) 11 | 12 | // SNSClient is interface of AWS SDK SQS 13 | type SNSClient interface { 14 | Publish(*sns.PublishInput) (*sns.PublishOutput, error) 15 | } 16 | 17 | // NewSNSClient creates actual AWS SNS SDK client 18 | func NewSNSClient(region string) (SNSClient, error) { 19 | ssn, err := session.NewSession(&aws.Config{Region: aws.String(region)}) 20 | if err != nil { 21 | return nil, err 22 | } 23 | return sns.New(ssn), nil 24 | } 25 | -------------------------------------------------------------------------------- /internal/api/alert.go: -------------------------------------------------------------------------------- 1 | package api 2 | 3 | import ( 4 | "net/http" 5 | "time" 6 | 7 | "github.com/deepalert/deepalert" 8 | "github.com/deepalert/deepalert/internal/usecase" 9 | "github.com/gin-gonic/gin" 10 | "github.com/m-mizutani/golambda" 11 | ) 12 | 13 | func postAlert(c *gin.Context) { 14 | args := getArguments(c) 15 | now := time.Now().UTC() 16 | 17 | var alert deepalert.Alert 18 | if err := c.BindJSON(&alert); err != nil { 19 | resp(c, http.StatusBadRequest, err) 20 | return 21 | } 22 | 23 | report, err := usecase.HandleAlert(args, &alert, now) 24 | if err != nil { 25 | resp(c, http.StatusInternalServerError, err) 26 | return 27 | } 28 | 29 | resp(c, http.StatusOK, report) 30 | } 31 | 32 | func getReportByAlertID(c *gin.Context) { 33 | args := getArguments(c) 34 | repo, err := args.Repository() 35 | if err != nil { 36 | resp(c, http.StatusInternalServerError, err) 37 | return 38 | } 39 | 40 | alertID := c.Param(paramAlertID) 41 | reportID, err := repo.GetReportID(alertID) 42 | if err != nil { 43 | resp(c, http.StatusInternalServerError, err) 44 | return 45 | } 46 | if reportID == deepalert.NullReportID { 47 | resp(c, http.StatusNotFound, golambda.NewError("No such alert").With("alert_id", alertID)) 48 | return 49 | } 50 | 51 | report, err := repo.GetReport(reportID) 52 | if err != nil { 53 | resp(c, http.StatusInternalServerError, err) 54 | return 55 | } 56 | if report == nil { 57 | resp(c, http.StatusNotFound, golambda.NewError("No such report").With("alert_id", alertID).With("reportID", reportID)) 58 | return 59 | } 60 | 61 | resp(c, http.StatusOK, report) 62 | } 63 | -------------------------------------------------------------------------------- /internal/api/api.go: -------------------------------------------------------------------------------- 1 | package api 2 | 3 | import ( 4 | "errors" 5 | 6 | "github.com/deepalert/deepalert/internal/handler" 7 | "github.com/gin-gonic/gin" 8 | "github.com/google/uuid" 9 | "github.com/m-mizutani/golambda" 10 | ) 11 | 12 | const ( 13 | contextArgumentKey = "handler.arguments" 14 | contextRequestID = "request.id" 15 | paramReportID = "report_id" 16 | paramAlertID = "alert_id" 17 | ) 18 | 19 | var logger = golambda.Logger 20 | 21 | func getArguments(c *gin.Context) *handler.Arguments { 22 | // In API code, handler.Arguments must be retrieved. If failed, the process must fail 23 | ptr, ok := c.Get(contextArgumentKey) 24 | if !ok { 25 | logger.With("key", contextArgumentKey).Error("Config is not set in API") 26 | panic("Config is not set in API") 27 | } 28 | 29 | args, ok := ptr.(*handler.Arguments) 30 | if !ok { 31 | logger.With("key", contextArgumentKey).Error("Config data can not be casted") 32 | panic("Config data can not be casted") 33 | } 34 | 35 | return args 36 | } 37 | 38 | func getRequestID(c *gin.Context) string { 39 | // In API code, requestID must be retrieved. If failed, the process must fail 40 | ptr, ok := c.Get(contextRequestID) 41 | if !ok { 42 | logger.With("contextRequestID", contextRequestID).Error("RequestID is not set in API") 43 | panic("RequestID is not set in API") 44 | } 45 | 46 | reqID, ok := ptr.(string) 47 | if !ok { 48 | logger.With("contextRequestID", contextRequestID).Error("RequestID can not be casted") 49 | panic("RequestID can not be casted") 50 | } 51 | 52 | return reqID 53 | } 54 | 55 | func wrapErr(msg string) map[string]string { 56 | return map[string]string{ 57 | "error": msg, 58 | } 59 | } 60 | 61 | func resp(c *gin.Context, status int, data interface{}) { 62 | reqID := getRequestID(c) 63 | c.Header("DeepAlert-Request-ID", reqID) 64 | 65 | if err, ok := data.(error); ok { 66 | entry := golambda.Logger.Entry() 67 | 68 | var e *golambda.Error 69 | if errors.As(err, &e) { 70 | entry.With("error.values", e.Values()) 71 | entry.With("error.stacktrace", e.Stacks()) 72 | } 73 | 74 | entry.Error(err.Error()) 75 | data = wrapErr(err.Error()) 76 | } 77 | 78 | c.JSON(status, data) 79 | } 80 | 81 | // SetupRoute binds route of gin and API 82 | func SetupRoute(r *gin.RouterGroup, args *handler.Arguments) { 83 | r.Use(func(c *gin.Context) { 84 | reqID := uuid.New().String() 85 | logger. 86 | With("path", c.FullPath()). 87 | With("params", c.Params). 88 | With("request_id", reqID). 89 | With("remote", c.ClientIP()). 90 | With("ua", c.Request.UserAgent()). 91 | Info("API request") 92 | 93 | c.Set(contextRequestID, reqID) 94 | c.Set(contextArgumentKey, args) 95 | c.Next() 96 | }) 97 | 98 | r.POST("/alert", postAlert) 99 | r.GET("/alert/:"+paramAlertID+"/report", getReportByAlertID) 100 | r.GET("/report/:"+paramReportID, getReport) 101 | r.GET("/report/:"+paramReportID+"/alert", getReportAlerts) 102 | r.GET("/report/:"+paramReportID+"/section", getSections) 103 | r.GET("/report/:"+paramReportID+"/attribute", getReportAttributes) 104 | } 105 | -------------------------------------------------------------------------------- /internal/api/api_test.go: -------------------------------------------------------------------------------- 1 | package api_test 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "log" 7 | "testing" 8 | 9 | "github.com/aws/aws-lambda-go/events" 10 | ginadapter "github.com/awslabs/aws-lambda-go-api-proxy/gin" 11 | "github.com/deepalert/deepalert" 12 | "github.com/deepalert/deepalert/internal/api" 13 | "github.com/deepalert/deepalert/internal/handler" 14 | "github.com/deepalert/deepalert/internal/mock" 15 | "github.com/gin-gonic/gin" 16 | "github.com/stretchr/testify/assert" 17 | "github.com/stretchr/testify/require" 18 | ) 19 | 20 | func handleRequest(args *handler.Arguments, req events.APIGatewayProxyRequest) (events.APIGatewayProxyResponse, error) { 21 | route := gin.New() 22 | v1 := route.Group("/api/v1") 23 | api.SetupRoute(v1, args) 24 | return ginadapter.New(route).Proxy(req) 25 | } 26 | 27 | func toBody(v interface{}) string { 28 | raw, err := json.Marshal(v) 29 | if err != nil { 30 | log.Fatalf("Failed to marshal: %v", v) 31 | } 32 | return string(raw) 33 | } 34 | 35 | func TestCreateReport(t *testing.T) { 36 | t.Run("Get report after creating", func(tt *testing.T) { 37 | _, repoFactory := mock.NewMockRepositorySet() 38 | args := &handler.Arguments{ 39 | NewRepository: repoFactory, 40 | NewSFn: mock.NewSFnClient, 41 | NewSNS: mock.NewSNSClient, 42 | EnvVars: handler.EnvVars{ 43 | InspectorMashine: "arn:aws:states:us-east-1:111122223333:stateMachine:inspect", 44 | ReviewMachine: "arn:aws:states:us-east-1:111122223333:stateMachine:review", 45 | ReportTopic: "arn:aws:sns:us-east-1:111122223333:report", 46 | }, 47 | } 48 | alert := &deepalert.Alert{ 49 | Detector: "testDetector", 50 | RuleID: "r1", 51 | RuleName: "testRule", 52 | } 53 | 54 | postResp, err := handleRequest(args, events.APIGatewayProxyRequest{ 55 | Path: "/api/v1/alert", 56 | HTTPMethod: "POST", 57 | Body: toBody(alert), 58 | }) 59 | require.NoError(tt, err) 60 | assert.Equal(tt, 200, postResp.StatusCode) 61 | 62 | var report deepalert.Report 63 | require.NoError(tt, json.Unmarshal([]byte(postResp.Body), &report)) 64 | assert.NotEqual(tt, deepalert.ReportID(""), report.ID) 65 | 66 | getReportResp, err := handleRequest(args, events.APIGatewayProxyRequest{ 67 | Path: fmt.Sprintf("/api/v1/report/%s", report.ID), 68 | HTTPMethod: "GET", 69 | }) 70 | require.NoError(tt, err) 71 | var getReport deepalert.Report 72 | require.NoError(tt, json.Unmarshal([]byte(getReportResp.Body), &getReport)) 73 | assert.Equal(tt, report.ID, getReport.ID) 74 | require.Equal(tt, 1, len(getReport.Alerts)) 75 | assert.Equal(tt, alert, getReport.Alerts[0]) 76 | 77 | getResp, err := handleRequest(args, events.APIGatewayProxyRequest{ 78 | Path: fmt.Sprintf("/api/v1/report/%s/alert", report.ID), 79 | HTTPMethod: "GET", 80 | }) 81 | require.NoError(tt, err) 82 | assert.Equal(tt, 200, getResp.StatusCode) 83 | var alerts []*deepalert.Alert 84 | require.NoError(tt, json.Unmarshal([]byte(getResp.Body), &alerts)) 85 | require.Equal(tt, 1, len(alerts)) 86 | assert.Equal(tt, "testRule", alerts[0].RuleName) 87 | }) 88 | } 89 | 90 | func TestErrorCase(t *testing.T) { 91 | t.Run("Invalid path", func(tt *testing.T) { 92 | _, repoFactory := mock.NewMockRepositorySet() 93 | args := &handler.Arguments{NewRepository: repoFactory} 94 | 95 | req := events.APIGatewayProxyRequest{ 96 | Path: "/api/v0/report", 97 | HTTPMethod: "POST", 98 | } 99 | 100 | resp, err := handleRequest(args, req) 101 | require.NoError(tt, err) 102 | assert.Equal(tt, 404, resp.StatusCode) 103 | }) 104 | } 105 | -------------------------------------------------------------------------------- /internal/api/report.go: -------------------------------------------------------------------------------- 1 | package api 2 | 3 | import ( 4 | "net/http" 5 | 6 | "github.com/deepalert/deepalert" 7 | "github.com/gin-gonic/gin" 8 | "github.com/m-mizutani/golambda" 9 | ) 10 | 11 | func getReport(c *gin.Context) { 12 | args := getArguments(c) 13 | reportID := deepalert.ReportID(c.Param(paramReportID)) 14 | 15 | repo, err := args.Repository() 16 | if err != nil { 17 | resp(c, http.StatusInternalServerError, err) 18 | return 19 | } 20 | 21 | report, err := repo.GetReport(reportID) 22 | if err != nil { 23 | resp(c, http.StatusInternalServerError, golambda.WrapError(err, "Failed GetReport")) 24 | return 25 | } 26 | 27 | resp(c, http.StatusOK, report) 28 | } 29 | 30 | func getReportAlerts(c *gin.Context) { 31 | args := getArguments(c) 32 | reportID := deepalert.ReportID(c.Param(paramReportID)) 33 | 34 | repo, err := args.Repository() 35 | if err != nil { 36 | resp(c, http.StatusInternalServerError, err) 37 | return 38 | } 39 | 40 | alerts, err := repo.FetchAlertCache(reportID) 41 | if err != nil { 42 | resp(c, http.StatusInternalServerError, err) 43 | return 44 | } 45 | if alerts == nil { 46 | resp(c, http.StatusNotFound, golambda.NewError("alerts not found")) 47 | return 48 | } 49 | 50 | resp(c, http.StatusOK, alerts) 51 | } 52 | 53 | func getSections(c *gin.Context) { 54 | args := getArguments(c) 55 | reportID := deepalert.ReportID(c.Param(paramReportID)) 56 | 57 | repo, err := args.Repository() 58 | if err != nil { 59 | resp(c, http.StatusInternalServerError, err) 60 | return 61 | } 62 | 63 | sections, err := repo.FetchSection(reportID) 64 | if err != nil { 65 | resp(c, http.StatusInternalServerError, err) 66 | return 67 | } 68 | if sections == nil { 69 | resp(c, http.StatusNotFound, golambda.NewError("sections not found")) 70 | return 71 | } 72 | 73 | resp(c, http.StatusOK, sections) 74 | } 75 | 76 | func getReportAttributes(c *gin.Context) { 77 | args := getArguments(c) 78 | reportID := deepalert.ReportID(c.Param(paramReportID)) 79 | 80 | repo, err := args.Repository() 81 | if err != nil { 82 | resp(c, http.StatusInternalServerError, err) 83 | return 84 | } 85 | 86 | attributes, err := repo.FetchAttributeCache(reportID) 87 | if err != nil { 88 | resp(c, http.StatusInternalServerError, err) 89 | return 90 | } 91 | if attributes == nil { 92 | resp(c, http.StatusNotFound, golambda.NewError("attributes not found")) 93 | return 94 | } 95 | 96 | resp(c, http.StatusOK, attributes) 97 | } 98 | -------------------------------------------------------------------------------- /internal/handler/arguments.go: -------------------------------------------------------------------------------- 1 | package handler 2 | 3 | import ( 4 | "github.com/deepalert/deepalert/internal/adaptor" 5 | "github.com/deepalert/deepalert/internal/repository" 6 | "github.com/deepalert/deepalert/internal/service" 7 | ) 8 | 9 | // Arguments has environment variables, Event record and adaptor 10 | type Arguments struct { 11 | EnvVars 12 | 13 | NewSNS adaptor.SNSClientFactory `json:"-"` 14 | NewSFn adaptor.SFnClientFactory `json:"-"` 15 | NewRepository adaptor.RepositoryFactory `json:"-"` 16 | } 17 | 18 | // NewArguments is constructor of Arguments 19 | func NewArguments() *Arguments { 20 | return &Arguments{} 21 | } 22 | 23 | // SNSService provides service.SNSService with SQS adaptor 24 | func (x *Arguments) SNSService() *service.SNSService { 25 | if x.NewSNS != nil { 26 | return service.NewSNSService(x.NewSNS) 27 | } 28 | return service.NewSNSService(adaptor.NewSNSClient) 29 | } 30 | 31 | // SFnService provides service.SFnService with SQS adaptor 32 | func (x *Arguments) SFnService() *service.SFnService { 33 | if x.NewSFn != nil { 34 | return service.NewSFnService(x.NewSFn) 35 | } 36 | return service.NewSFnService(adaptor.NewSFnClient) 37 | } 38 | 39 | // Repository provides data store accessor created by NewDynamoDB. If Arguments.NewRepository is set, this function returns repository object created by NewRepository. 40 | func (x *Arguments) Repository() (*service.RepositoryService, error) { 41 | var ttl int64 = 1800 42 | var repo adaptor.Repository 43 | 44 | if x.NewRepository != nil { 45 | repo = x.NewRepository(x.AwsRegion, x.CacheTable) 46 | } else { 47 | dynamodb, err := repository.NewDynamoDB(x.AwsRegion, x.CacheTable) 48 | if err != nil { 49 | return nil, err 50 | } 51 | repo = dynamodb 52 | } 53 | 54 | return service.NewRepositoryService(repo, ttl), nil 55 | } 56 | -------------------------------------------------------------------------------- /internal/handler/arguments_test.go: -------------------------------------------------------------------------------- 1 | package handler_test 2 | -------------------------------------------------------------------------------- /internal/handler/envvars.go: -------------------------------------------------------------------------------- 1 | package handler 2 | 3 | import ( 4 | "github.com/Netflix/go-env" 5 | "github.com/m-mizutani/golambda" 6 | ) 7 | 8 | // EnvVars has all environment variables that should be given to Lambda function 9 | type EnvVars struct { 10 | // From arguments 11 | TaskTopic string `env:"TASK_TOPIC"` 12 | ReportTopic string `env:"REPORT_TOPIC"` 13 | CacheTable string `env:"CACHE_TABLE"` 14 | 15 | // Only recvAlert can use because of dependency 16 | InspectorMashine string `env:"INSPECTOR_MACHINE"` 17 | ReviewMachine string `env:"REVIEW_MACHINE"` 18 | 19 | // Utilities 20 | SentryDSN string `env:"SENTRY_DSN"` 21 | SentryEnv string `env:"SENTRY_ENVIRONMENT"` 22 | LogLevel string `env:"LOG_LEVEL"` 23 | 24 | // From AWS Lambda 25 | AwsRegion string `env:"AWS_REGION"` 26 | } 27 | 28 | // BindEnvVars loads environments variables and set them to EnvVars 29 | func (x *EnvVars) BindEnvVars() error { 30 | if _, err := env.UnmarshalFromEnviron(x); err != nil { 31 | return golambda.WrapError(err) 32 | } 33 | 34 | return nil 35 | } 36 | -------------------------------------------------------------------------------- /internal/mock/no_test.go: -------------------------------------------------------------------------------- 1 | package mock_test 2 | 3 | // No test for now. Mock test shoud be done in internal/service 4 | -------------------------------------------------------------------------------- /internal/mock/repository.go: -------------------------------------------------------------------------------- 1 | package mock 2 | 3 | import ( 4 | "fmt" 5 | "time" 6 | 7 | "github.com/deepalert/deepalert" 8 | "github.com/deepalert/deepalert/internal/adaptor" 9 | "github.com/deepalert/deepalert/internal/models" 10 | ) 11 | 12 | // Repository is mock data store. Behaviour of the mock.Repository must be same with repository.DynamoDBRepositry 13 | type Repository struct { 14 | region string 15 | tableName string 16 | data map[string]map[string]interface{} 17 | } 18 | 19 | func NewRepository(region, tableName string) adaptor.Repository { 20 | return newMockRepository(region, tableName) 21 | } 22 | 23 | func newMockRepository(region, tableName string) *Repository { 24 | return &Repository{ 25 | region: region, 26 | tableName: tableName, 27 | data: make(map[string]map[string]interface{}), 28 | } 29 | } 30 | 31 | // NewMockRepositorySet provides a pair of mock.Repository and RepositoryFactory of the mock.Repository. Saved data can be accessed via mock.Repository. 32 | func NewMockRepositorySet() (*Repository, adaptor.RepositoryFactory) { 33 | repo := newMockRepository("test-region", "test-table") 34 | return repo, func(_, _ string) adaptor.Repository { 35 | return repo 36 | } 37 | } 38 | 39 | var errCondition = fmt.Errorf("condition error") 40 | 41 | func (x *Repository) put(pk, sk string, v interface{}) { 42 | m, ok := x.data[pk] 43 | if !ok { 44 | m = make(map[string]interface{}) 45 | x.data[pk] = m 46 | } 47 | 48 | m[sk] = v 49 | } 50 | 51 | func (x *Repository) get(pk, sk string) interface{} { 52 | m, ok := x.data[pk] 53 | if !ok { 54 | return nil 55 | } 56 | 57 | return m[sk] 58 | } 59 | 60 | func (x *Repository) getAll(pk string) []interface{} { 61 | m, ok := x.data[pk] 62 | if !ok { 63 | return nil 64 | } 65 | 66 | var out []interface{} 67 | for _, v := range m { 68 | out = append(out, v) 69 | } 70 | return out 71 | } 72 | 73 | func (x *Repository) PutAlertEntry(entry *models.AlertEntry, ts time.Time) error { 74 | v := x.get(entry.PKey, entry.SKey) 75 | if e, ok := v.(*models.AlertEntry); ok && ts.UTC().Unix() <= e.ExpiresAt { 76 | return errCondition 77 | } 78 | x.put(entry.PKey, entry.SKey, entry) 79 | 80 | return nil 81 | } 82 | 83 | func (x *Repository) GetAlertEntry(pk, sk string) (*models.AlertEntry, error) { 84 | v := x.get(pk, sk) 85 | if d, ok := v.(*models.AlertEntry); ok { 86 | return d, nil 87 | } 88 | return nil, nil 89 | } 90 | 91 | func (x *Repository) PutAlertCache(cache *models.AlertCache) error { 92 | x.put(cache.PKey, cache.SKey, cache) 93 | return nil 94 | } 95 | 96 | func (x *Repository) GetAlertCaches(pk string) ([]*models.AlertCache, error) { 97 | var out []*models.AlertCache 98 | for _, v := range x.getAll(pk) { 99 | out = append(out, v.(*models.AlertCache)) 100 | } 101 | return out, nil 102 | } 103 | 104 | func (x *Repository) PutInspectorReport(record *models.InspectorReportRecord) error { 105 | x.put(record.PKey, record.SKey, record) 106 | return nil 107 | } 108 | 109 | func (x *Repository) GetInspectorReports(pk string) ([]*models.InspectorReportRecord, error) { 110 | var out []*models.InspectorReportRecord 111 | for _, v := range x.getAll(pk) { 112 | out = append(out, v.(*models.InspectorReportRecord)) 113 | } 114 | return out, nil 115 | } 116 | 117 | func (x *Repository) PutAttributeCache(attr *models.AttributeCache, ts time.Time) error { 118 | v := x.get(attr.PKey, attr.SKey) 119 | if e, ok := v.(*models.AttributeCache); ok && ts.UTC().Unix() <= e.ExpiresAt { 120 | return errCondition 121 | } 122 | x.put(attr.PKey, attr.SKey, attr) 123 | 124 | return nil 125 | } 126 | func (x *Repository) GetAttributeCaches(pk string) ([]*models.AttributeCache, error) { 127 | var out []*models.AttributeCache 128 | for _, v := range x.getAll(pk) { 129 | out = append(out, v.(*models.AttributeCache)) 130 | } 131 | return out, nil 132 | } 133 | 134 | func (x *Repository) PutReport(pk string, report *deepalert.Report) error { 135 | x.data[pk] = map[string]interface{}{"-": report} 136 | return nil 137 | } 138 | 139 | func (x *Repository) GetReport(pk string) (*deepalert.Report, error) { 140 | report, ok := x.data[pk] 141 | if !ok { 142 | return nil, nil 143 | } 144 | return report["-"].(*deepalert.Report), nil 145 | } 146 | 147 | func (x *Repository) IsConditionalCheckErr(err error) bool { 148 | return err == errCondition 149 | } 150 | -------------------------------------------------------------------------------- /internal/mock/sfn.go: -------------------------------------------------------------------------------- 1 | package mock 2 | 3 | import ( 4 | "github.com/aws/aws-sdk-go/service/sfn" 5 | "github.com/deepalert/deepalert/internal/adaptor" 6 | ) 7 | 8 | // NewSFnClient creates mock SNS client 9 | func NewSFnClient(region string) (adaptor.SFnClient, error) { 10 | return &SFnClient{region: region}, nil 11 | } 12 | 13 | // SFnClient is mock 14 | type SFnClient struct { 15 | region string 16 | Input []*sfn.StartExecutionInput 17 | } 18 | 19 | // StartExecution of mock SFnClient only stores sfn.StartExecutionInput 20 | func (x *SFnClient) StartExecution(input *sfn.StartExecutionInput) (*sfn.StartExecutionOutput, error) { 21 | x.Input = append(x.Input, input) 22 | return &sfn.StartExecutionOutput{}, nil 23 | } 24 | -------------------------------------------------------------------------------- /internal/mock/sns.go: -------------------------------------------------------------------------------- 1 | package mock 2 | 3 | import ( 4 | "github.com/aws/aws-sdk-go/service/sns" 5 | "github.com/deepalert/deepalert/internal/adaptor" 6 | ) 7 | 8 | // NewSNSClient creates mock SNS client 9 | func NewSNSClient(region string) (adaptor.SNSClient, error) { 10 | return &SNSClient{Region: region}, nil 11 | } 12 | 13 | // SNSClient is mock 14 | type SNSClient struct { 15 | Region string 16 | Input []*sns.PublishInput 17 | } 18 | 19 | // Publish of mock SNSClient only stores sns.PublishInput 20 | func (x *SNSClient) Publish(input *sns.PublishInput) (*sns.PublishOutput, error) { 21 | x.Input = append(x.Input, input) 22 | return &sns.PublishOutput{}, nil 23 | } 24 | 25 | // NewMockSNSClientSet returns a pair of SNSClient and SNSClientFactory 26 | func NewMockSNSClientSet() (*SNSClient, adaptor.SNSClientFactory) { 27 | client := &SNSClient{} 28 | return client, func(region string) (adaptor.SNSClient, error) { 29 | client.Region = region 30 | return client, nil 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /internal/models/db.go: -------------------------------------------------------------------------------- 1 | package models 2 | 3 | import ( 4 | "encoding/json" 5 | "strconv" 6 | "time" 7 | 8 | "github.com/aws/aws-lambda-go/events" 9 | "github.com/deepalert/deepalert" 10 | "github.com/m-mizutani/golambda" 11 | ) 12 | 13 | const ( 14 | // DynamoPKeyName is common name of PartitionKey (HashKey) of DynamoDB 15 | DynamoPKeyName = "pk" 16 | // DynamoSKeyName = "sk" 17 | ) 18 | 19 | type RecordBase struct { 20 | PKey string `dynamo:"pk"` 21 | SKey string `dynamo:"sk"` 22 | ExpiresAt int64 `dynamo:"expires_at"` 23 | CreatedAt int64 `dynamo:"created_at,omitempty"` 24 | } 25 | 26 | type AlertEntry struct { 27 | RecordBase 28 | ReportID deepalert.ReportID `dynamo:"report_id"` 29 | } 30 | 31 | type AlertCache struct { 32 | RecordBase 33 | AlertData []byte `dynamo:"alert_data"` 34 | } 35 | 36 | type InspectorReportRecord struct { 37 | RecordBase 38 | Data []byte `dynamo:"data"` 39 | } 40 | 41 | type AttributeCache struct { 42 | RecordBase 43 | Timestamp time.Time `dynamo:"timestamp"` 44 | AttrKey string `dynamo:"attr_key"` 45 | AttrType string `dynamo:"attr_type"` 46 | AttrValue string `dynamo:"attr_value"` 47 | AttrContext deepalert.AttrContexts `dynamo:"attr_context"` 48 | } 49 | 50 | type ReportEntry struct { 51 | RecordBase 52 | ID string `dynamo:"id"` 53 | Result string `dynamo:"result"` 54 | Status string `dynamo:"status"` 55 | } 56 | 57 | // ErrRecordIsNotReport means DynamoDB record is not event of add/modify report. 58 | var ErrRecordIsNotReport = golambda.NewError("Record is not report") 59 | 60 | // ImportDynamoRecord copies values from record data in DynamoDB stream 61 | func (x *ReportEntry) ImportDynamoRecord(record *events.DynamoDBEventRecord) error { 62 | if record.Change.NewImage == nil { 63 | return ErrRecordIsNotReport 64 | } 65 | 66 | getString := func(key string) string { 67 | value, ok := record.Change.NewImage[key] 68 | if !ok { 69 | return "" 70 | } 71 | return value.String() 72 | } 73 | 74 | x.ID = getString("id") 75 | x.Result = getString("result") 76 | x.Status = getString("status") 77 | 78 | createdAtValue, ok := record.Change.NewImage["created_at"] 79 | if !ok { 80 | return golambda.NewError("created_at is not available in DynamoDB event").With("record", record) 81 | } 82 | 83 | v, err := strconv.ParseInt(createdAtValue.Number(), 10, 64) 84 | if err != nil { 85 | return golambda.WrapError(err, "Failed to parse createdAt of DynamoRecord"). 86 | With("record", record) 87 | } 88 | x.CreatedAt = v 89 | 90 | return nil 91 | } 92 | 93 | // Import copies values from deepalert.Report to own 94 | func (x *ReportEntry) Import(report *deepalert.Report) error { 95 | x.ID = string(report.ID) 96 | 97 | raw, err := json.Marshal(report.Result) 98 | if err != nil { 99 | return golambda.WrapError(err, "Failed to marshal report.Result").With("report", report) 100 | } 101 | x.Result = string(raw) 102 | 103 | x.Status = string(report.Status) 104 | x.CreatedAt = report.CreatedAt.UTC().Unix() 105 | 106 | return nil 107 | } 108 | 109 | // Export creates a new deepalert.Report from own values 110 | func (x *ReportEntry) Export() (*deepalert.Report, error) { 111 | var report deepalert.Report 112 | 113 | report.ID = deepalert.ReportID(x.ID) 114 | if err := json.Unmarshal([]byte(x.Result), &report.Result); err != nil { 115 | return nil, golambda.WrapError(err, "Failed to unmarshal reprot.Result").With("entry", *x) 116 | } 117 | 118 | report.Status = deepalert.ReportStatus(x.Status) 119 | report.CreatedAt = time.Unix(x.CreatedAt, 0) 120 | 121 | return &report, nil 122 | } 123 | -------------------------------------------------------------------------------- /internal/models/db_test.go: -------------------------------------------------------------------------------- 1 | package models_test 2 | 3 | import ( 4 | "encoding/json" 5 | "testing" 6 | "time" 7 | 8 | "github.com/aws/aws-lambda-go/events" 9 | "github.com/deepalert/deepalert" 10 | "github.com/deepalert/deepalert/internal/models" 11 | "github.com/stretchr/testify/assert" 12 | "github.com/stretchr/testify/require" 13 | ) 14 | 15 | func TestReportEntry(t *testing.T) { 16 | t.Run("Import and Export", func(tt *testing.T) { 17 | r1 := &deepalert.Report{ 18 | ID: "xba123", 19 | Alerts: []*deepalert.Alert{ 20 | { 21 | AlertKey: "cxz", 22 | Detector: "saber", 23 | RuleID: "s1", 24 | Attributes: []deepalert.Attribute{ 25 | { 26 | Type: deepalert.TypeIPAddr, 27 | Context: deepalert.AttrContexts{ 28 | deepalert.CtxRemote, 29 | }, 30 | Key: "srcAddr", 31 | Value: "10.1.2.3", 32 | }, 33 | }, 34 | }, 35 | { 36 | AlertKey: "bnc", 37 | Detector: "archer", 38 | RuleID: "a1", 39 | Attributes: []deepalert.Attribute{ 40 | { 41 | Type: deepalert.TypeIPAddr, 42 | Context: deepalert.AttrContexts{ 43 | deepalert.CtxLocal, 44 | }, 45 | Key: "dstAddr", 46 | Value: "192.168.2.3", 47 | }, 48 | }, 49 | }, 50 | }, 51 | Attributes: []*deepalert.Attribute{ 52 | { 53 | Type: deepalert.TypeIPAddr, 54 | Context: deepalert.AttrContexts{ 55 | deepalert.CtxRemote, 56 | }, 57 | Key: "srcAddr", 58 | Value: "10.1.2.3", 59 | }, 60 | { 61 | Type: deepalert.TypeIPAddr, 62 | Context: deepalert.AttrContexts{ 63 | deepalert.CtxLocal, 64 | }, 65 | Key: "dstAddr", 66 | Value: "192.168.2.3", 67 | }, 68 | }, 69 | Sections: []*deepalert.Section{ 70 | { 71 | Attr: deepalert.Attribute{ 72 | Type: deepalert.TypeIPAddr, 73 | Context: deepalert.AttrContexts{ 74 | deepalert.CtxLocal, 75 | }, 76 | Key: "dstAddr", 77 | Value: "192.168.2.3", 78 | }, 79 | 80 | Users: []*deepalert.ContentUser{ 81 | { 82 | Activities: []deepalert.EntityActivity{ 83 | { 84 | Action: "hoge", 85 | RemoteAddr: "10.5.6.7", 86 | }, 87 | }, 88 | }, 89 | }, 90 | }, 91 | }, 92 | 93 | Result: deepalert.ReportResult{ 94 | Severity: deepalert.SevSafe, 95 | Reason: "no reason", 96 | }, 97 | Status: deepalert.StatusPublished, 98 | CreatedAt: time.Now(), 99 | } 100 | 101 | var entry models.ReportEntry 102 | err := entry.Import(r1) 103 | require.NoError(tt, err) 104 | r2, err := entry.Export() 105 | require.NoError(tt, err) 106 | assert.Equal(tt, r1.ID, r2.ID) 107 | 108 | tt.Run("Fetched report does not have Alert, Attribute and Section", func(ttt *testing.T) { 109 | // Because they should be fetched by FetchAlertCache, FetchAttributeCache and FetchSection 110 | assert.Equal(tt, 0, len(r2.Alerts)) 111 | assert.Equal(tt, 0, len(r2.Attributes)) 112 | assert.Equal(tt, 0, len(r2.Sections)) 113 | }) 114 | 115 | tt.Run("Result, Status and CreatedAt should be matched with original report", func(ttt *testing.T) { 116 | // Result, status, createdAt 117 | assert.Equal(tt, r1.Result, r2.Result) 118 | assert.Equal(tt, r1.Status, r2.Status) 119 | assert.Equal(tt, r1.CreatedAt.UTC().Unix(), r2.CreatedAt.Unix()) 120 | }) 121 | }) 122 | } 123 | 124 | func TestImportDynamoRecord(t *testing.T) { 125 | sample := `{ 126 | "awsRegion": "ap-northeast-1", 127 | "dynamodb": { 128 | "ApproximateCreationDateTime": 1604111356, 129 | "Keys": { 130 | "pk": { 131 | "S": "report/20c62a1d-99a2-45b5-bca1-2f6949b6ee61" 132 | }, 133 | "sk": { 134 | "S": "-" 135 | } 136 | }, 137 | "NewImage": { 138 | "created_at": { 139 | "N": "1604111355" 140 | }, 141 | "expires_at": { 142 | "N": "0" 143 | }, 144 | "id": { 145 | "S": "20c62a1d-99a2-45b5-bca1-2f6949b6ee61" 146 | }, 147 | "pk": { 148 | "S": "report/20c62a1d-99a2-45b5-bca1-2f6949b6ee61" 149 | }, 150 | "result": { 151 | "S": "{\"severity\":\"safe\",\"reason\":\"not sane\"}" 152 | }, 153 | "sk": { 154 | "S": "-" 155 | }, 156 | "status": { 157 | "S": "new" 158 | } 159 | }, 160 | "SequenceNumber": "366860700000000000755927238", 161 | "SizeBytes": 203, 162 | "StreamViewType": "NEW_IMAGE" 163 | }, 164 | "eventID": "f4963c472adeda5d90748601e6affbc8", 165 | "eventName": "INSERT", 166 | "eventSource": "aws:dynamodb", 167 | "eventSourceARN": "arn:aws:dynamodb:ap-northeast-1:783957204773:table/DeepAlertTestStack-cacheTable730E8AED-1FEYS10RXIN14/stream/2020-10-12T13:48:05.842", 168 | "eventVersion": "1.1" 169 | }` 170 | var record events.DynamoDBEventRecord 171 | require.NoError(t, json.Unmarshal([]byte(sample), &record)) 172 | 173 | t.Run("Normal case", func(t *testing.T) { 174 | var entry models.ReportEntry 175 | require.NoError(t, entry.ImportDynamoRecord(&record)) 176 | report, err := entry.Export() 177 | require.NoError(t, err) 178 | assert.Equal(t, int64(1604111355), report.CreatedAt.Unix()) 179 | assert.Equal(t, deepalert.ReportID("20c62a1d-99a2-45b5-bca1-2f6949b6ee61"), report.ID) 180 | assert.Equal(t, deepalert.SevSafe, report.Result.Severity) 181 | assert.Equal(t, deepalert.StatusNew, report.Status) 182 | }) 183 | } 184 | -------------------------------------------------------------------------------- /internal/repository/dynamodb.go: -------------------------------------------------------------------------------- 1 | package repository 2 | 3 | import ( 4 | "time" 5 | 6 | "github.com/aws/aws-sdk-go/aws" 7 | "github.com/aws/aws-sdk-go/aws/awserr" 8 | "github.com/aws/aws-sdk-go/aws/session" 9 | "github.com/deepalert/deepalert" 10 | "github.com/deepalert/deepalert/internal/adaptor" 11 | "github.com/deepalert/deepalert/internal/models" 12 | "github.com/guregu/dynamo" 13 | "github.com/m-mizutani/golambda" 14 | ) 15 | 16 | type DynamoDBRepositry struct { 17 | tableName string 18 | region string 19 | table dynamo.Table 20 | timeToLive time.Duration 21 | } 22 | 23 | // NewDynamoDB is constructor of DynamoDBRepositry 24 | func NewDynamoDB(region, tableName string) (adaptor.Repository, error) { 25 | ssn, err := session.NewSession(&aws.Config{Region: aws.String(region)}) 26 | if err != nil { 27 | return nil, golambda.WrapError(err, "Failed session.NewSession for DynamoDB").With("region", region) 28 | } 29 | db := dynamo.New(ssn) 30 | x := &DynamoDBRepositry{ 31 | tableName: tableName, 32 | region: region, 33 | table: db.Table(tableName), 34 | timeToLive: time.Hour * 3, 35 | } 36 | 37 | return x, nil 38 | } 39 | 40 | func (x *DynamoDBRepositry) PutAlertEntry(entry *models.AlertEntry, ts time.Time) error { 41 | cond := "(attribute_not_exists(pk) AND attribute_not_exists(sk)) OR expires_at < ?" 42 | if err := x.table.Put(entry).If(cond, ts.UTC().Unix()).Run(); err != nil { 43 | return err 44 | } 45 | 46 | return nil 47 | } 48 | 49 | func (x *DynamoDBRepositry) GetAlertEntry(pk, sk string) (*models.AlertEntry, error) { 50 | var output models.AlertEntry 51 | if err := x.table.Get("pk", pk).Range("sk", dynamo.Equal, sk).One(&output); err != nil { 52 | if err == dynamo.ErrNotFound { 53 | return nil, nil 54 | } 55 | 56 | return nil, err 57 | } 58 | 59 | return &output, nil 60 | } 61 | 62 | func (x *DynamoDBRepositry) PutAlertCache(cache *models.AlertCache) error { 63 | if err := x.table.Put(cache).Run(); err != nil { 64 | return golambda.WrapError(err, "Failed PutAlertCache").With("cache", cache) 65 | } 66 | 67 | return nil 68 | } 69 | 70 | func (x *DynamoDBRepositry) GetAlertCaches(pk string) ([]*models.AlertCache, error) { 71 | var caches []*models.AlertCache 72 | 73 | if err := x.table.Get("pk", pk).All(&caches); err != nil { 74 | if err == dynamo.ErrNotFound { 75 | return nil, nil 76 | } 77 | 78 | return nil, golambda.WrapError(err, "Failed GetAlertCaches").With("pk", pk) 79 | } 80 | 81 | return caches, nil 82 | } 83 | 84 | func (x *DynamoDBRepositry) PutInspectorReport(record *models.InspectorReportRecord) error { 85 | if err := x.table.Put(record).Run(); err != nil { 86 | return golambda.WrapError(err, "Failed PutInspectorReport").With("record", record) 87 | } 88 | 89 | return nil 90 | } 91 | 92 | func (x *DynamoDBRepositry) GetInspectorReports(pk string) ([]*models.InspectorReportRecord, error) { 93 | var records []*models.InspectorReportRecord 94 | 95 | if err := x.table.Get("pk", pk).All(&records); err != nil { 96 | return nil, golambda.WrapError(err, "Failed GetInspectorReports").With("pk", pk) 97 | } 98 | 99 | return records, nil 100 | } 101 | 102 | func (x *DynamoDBRepositry) PutAttributeCache(attr *models.AttributeCache, ts time.Time) error { 103 | if err := x.table.Put(attr).If("(attribute_not_exists(pk) AND attribute_not_exists(sk)) OR expires_at < ?", ts.UTC().Unix()).Run(); err != nil { 104 | return err 105 | } 106 | 107 | return nil 108 | } 109 | 110 | func (x *DynamoDBRepositry) GetAttributeCaches(pk string) ([]*models.AttributeCache, error) { 111 | var attrs []*models.AttributeCache 112 | 113 | if err := x.table.Get("pk", pk).All(&attrs); err != nil { 114 | return nil, golambda.WrapError(err, "Failed GetAttributeCaches").With("pk", pk) 115 | } 116 | 117 | return attrs, nil 118 | } 119 | 120 | func (x *DynamoDBRepositry) PutReport(pk string, report *deepalert.Report) error { 121 | var entry models.ReportEntry 122 | if err := entry.Import(report); err != nil { 123 | return err 124 | } 125 | entry.PKey = pk 126 | entry.SKey = "-" 127 | 128 | if err := x.table.Put(&entry).Run(); err != nil { 129 | return err 130 | } 131 | return nil 132 | } 133 | 134 | func (x *DynamoDBRepositry) GetReport(pk string) (*deepalert.Report, error) { 135 | var entry models.ReportEntry 136 | if err := x.table.Get("pk", pk).Range("sk", dynamo.Equal, "-").One(&entry); err != nil { 137 | if err == dynamo.ErrNotFound { 138 | return nil, nil 139 | } 140 | return nil, golambda.WrapError(err, "Failed to get report").With("pk", pk) 141 | } 142 | 143 | report, err := entry.Export() 144 | if err != nil { 145 | return nil, err 146 | } 147 | return report, nil 148 | } 149 | 150 | // Error handling 151 | 152 | func (x *DynamoDBRepositry) IsConditionalCheckErr(err error) bool { 153 | if ae, ok := err.(awserr.RequestFailure); ok { 154 | return ae.Code() == "ConditionalCheckFailedException" 155 | } 156 | return false 157 | } 158 | -------------------------------------------------------------------------------- /internal/repository/no_test.go: -------------------------------------------------------------------------------- 1 | package repository_test 2 | 3 | // internal/repository test should be done in internal/service 4 | -------------------------------------------------------------------------------- /internal/service/repository.go: -------------------------------------------------------------------------------- 1 | package service 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "strings" 7 | "time" 8 | 9 | "github.com/aws/aws-lambda-go/events" 10 | "github.com/deepalert/deepalert" 11 | "github.com/deepalert/deepalert/internal/adaptor" 12 | "github.com/deepalert/deepalert/internal/models" 13 | "github.com/google/uuid" 14 | "github.com/m-mizutani/golambda" 15 | ) 16 | 17 | /* 18 | DynamoDB Design 19 | 20 | Data models 21 | - Alert : Generated by a security monitoring device. It has attribute(s). 22 | - Attribute : Values appeared in an Alert (e.g. IP address, domain name, user name, etc.) 23 | - Content : A result of attribute inspection by Inspector 24 | - Report : All results. It consists of Alert(S), Content(S) and a result of Reviewer. 25 | 26 | 27 | Keys 28 | - AlertID : Generated by Alert.Detector, Alert.RuneName and Alert.AlertKey. 29 | - ReportID : Assigned to unique AlertKey and time range. Same AlertID can have multiple 30 | ReportID if timestamps of alert are distant from each other. 31 | - AttrHash: Hashed value of an attribute, generated by all fields of Attribute. 32 | 33 | Primary/secondary key design (in "pk", "sk" field and stored data) 34 | - alertmap/{AlertID}, fixedkey -> ReportID 35 | - alert/{ReportID}, cache/{random} -> Alert(s) 36 | - content/{ReportID}, {AttrHash}/{Random} -> Content(S) 37 | - attribute/{ReportID}, {AttrHash} -> Attribute (for caching) 38 | */ 39 | 40 | const ( 41 | alertMapfixedKey = "-" 42 | ) 43 | 44 | // RepositoryService is interface of data repository. This is designed to be used with DynamoDB, but adaptor.Repository can be replaced with other repository. (e.g. mock.Repository) 45 | type RepositoryService struct { 46 | repo adaptor.Repository 47 | ttl time.Duration 48 | } 49 | 50 | // NewRepositoryService is constructor of RepositoryService. ttl is used to calculate ExpiresAt by now + ttl * time.Second 51 | func NewRepositoryService(repo adaptor.Repository, ttl int64) *RepositoryService { 52 | return &RepositoryService{ 53 | repo: repo, 54 | ttl: time.Duration(ttl) * time.Second, 55 | } 56 | } 57 | 58 | // ----------------------------------------------------------- 59 | // Control alertEntry to manage AlertID to ReportID mapping 60 | // 61 | 62 | func newReportID() deepalert.ReportID { 63 | return deepalert.ReportID(uuid.New().String()) 64 | } 65 | 66 | func toAlertMapPKey(alertID string) string { 67 | return "alertmap/" + alertID 68 | } 69 | 70 | func (x *RepositoryService) TakeReport(alert deepalert.Alert, now time.Time) (*deepalert.Report, error) { 71 | alertID := alert.AlertID() 72 | 73 | entry := models.AlertEntry{ 74 | RecordBase: models.RecordBase{ 75 | PKey: toAlertMapPKey(alertID), 76 | SKey: alertMapfixedKey, 77 | ExpiresAt: now.UTC().Add(x.ttl).Unix(), 78 | CreatedAt: now.UTC().Unix(), 79 | }, 80 | ReportID: newReportID(), 81 | } 82 | 83 | if err := x.repo.PutAlertEntry(&entry, now); err != nil { 84 | if x.repo.IsConditionalCheckErr(err) { 85 | existedEntry, err := x.repo.GetAlertEntry(entry.PKey, entry.SKey) 86 | if err != nil { 87 | return nil, golambda.WrapError(err, "Fail to get cached reportID").With("AlertID", alertID) 88 | } 89 | 90 | return &deepalert.Report{ 91 | ID: existedEntry.ReportID, 92 | Status: deepalert.StatusMore, 93 | CreatedAt: time.Unix(existedEntry.CreatedAt, 0), 94 | }, nil 95 | } 96 | 97 | return nil, golambda.WrapError(err, "Fail to create new alert entry"). 98 | With("AlertID", alertID).With("repo", x.repo) 99 | } 100 | 101 | return &deepalert.Report{ 102 | ID: entry.ReportID, 103 | Status: deepalert.StatusNew, 104 | CreatedAt: now, 105 | }, nil 106 | } 107 | 108 | func (x *RepositoryService) GetReportID(alertID string) (deepalert.ReportID, error) { 109 | 110 | existedEntry, err := x.repo.GetAlertEntry(toAlertMapPKey(alertID), alertMapfixedKey) 111 | if err != nil { 112 | return deepalert.NullReportID, golambda.WrapError(err, "Fail to get cached reportID").With("AlertID", alertID) 113 | } else if existedEntry == nil { 114 | return deepalert.NullReportID, nil 115 | } 116 | 117 | return existedEntry.ReportID, nil 118 | } 119 | 120 | // ----------------------------------------------------------- 121 | // Control alertCache to manage published alert data 122 | // 123 | 124 | func toAlertCacheKey(reportID deepalert.ReportID) (string, string) { 125 | return fmt.Sprintf("alert/%s", reportID), "cache/" + uuid.New().String() 126 | } 127 | 128 | func (x *RepositoryService) SaveAlertCache(reportID deepalert.ReportID, alert deepalert.Alert, now time.Time) error { 129 | raw, err := json.Marshal(alert) 130 | if err != nil { 131 | return golambda.WrapError(err, "Fail to marshal alert").With("alert", alert) 132 | } 133 | 134 | pk, sk := toAlertCacheKey(reportID) 135 | cache := &models.AlertCache{ 136 | RecordBase: models.RecordBase{ 137 | PKey: pk, 138 | SKey: sk, 139 | ExpiresAt: now.UTC().Add(x.ttl).Unix(), 140 | }, 141 | AlertData: raw, 142 | } 143 | 144 | if err := x.repo.PutAlertCache(cache); err != nil { 145 | return err 146 | } 147 | 148 | return nil 149 | } 150 | 151 | func (x *RepositoryService) FetchAlertCache(reportID deepalert.ReportID) ([]*deepalert.Alert, error) { 152 | pk, _ := toAlertCacheKey(reportID) 153 | var alerts []*deepalert.Alert 154 | 155 | caches, err := x.repo.GetAlertCaches(pk) 156 | if err != nil { 157 | return nil, golambda.WrapError(err, "GetAlertCaches").With("reportID", reportID) 158 | } 159 | 160 | for _, cache := range caches { 161 | var alert deepalert.Alert 162 | if err := json.Unmarshal(cache.AlertData, &alert); err != nil { 163 | return nil, golambda.WrapError(err, "Fail to unmarshal alert").With("data", string(cache.AlertData)) 164 | } 165 | alerts = append(alerts, &alert) 166 | } 167 | 168 | return alerts, nil 169 | } 170 | 171 | // ----------------------------------------------------------- 172 | // Control reportRecord to manage report contents by inspector 173 | // 174 | 175 | func toFindingKeys(reportID deepalert.ReportID, inspect *deepalert.Finding) (string, string) { 176 | pk := fmt.Sprintf("content/%s", reportID) 177 | sk := "" 178 | if inspect != nil { 179 | sk = fmt.Sprintf("%s/%s", inspect.Attribute.Hash(), uuid.New().String()) 180 | } 181 | return pk, sk 182 | } 183 | 184 | func (x *RepositoryService) SaveFinding(section deepalert.Finding, now time.Time) error { 185 | raw, err := json.Marshal(section) 186 | if err != nil { 187 | return golambda.WrapError(err, "Fail to marshal Section").With("section", section) 188 | } 189 | 190 | pk, sk := toFindingKeys(section.ReportID, §ion) 191 | record := &models.InspectorReportRecord{ 192 | RecordBase: models.RecordBase{ 193 | PKey: pk, 194 | SKey: sk, 195 | ExpiresAt: now.UTC().Add(x.ttl).Unix(), 196 | }, 197 | Data: raw, 198 | } 199 | 200 | if err := x.repo.PutInspectorReport(record); err != nil { 201 | return golambda.WrapError(err, "Fail to put report record") 202 | } 203 | 204 | return nil 205 | } 206 | 207 | func (x *RepositoryService) FetchSection(reportID deepalert.ReportID) ([]*deepalert.Section, error) { 208 | pk, _ := toFindingKeys(reportID, nil) 209 | 210 | records, err := x.repo.GetInspectorReports(pk) 211 | if err != nil { 212 | return nil, err 213 | } 214 | 215 | var reports []*deepalert.Finding 216 | for _, record := range records { 217 | var section deepalert.Finding 218 | if err := json.Unmarshal(record.Data, §ion); err != nil { 219 | return nil, golambda.WrapError(err, "Fail to unmarshal report content"). 220 | With("record", record). 221 | With("data", string(record.Data)) 222 | } 223 | 224 | reports = append(reports, §ion) 225 | } 226 | 227 | sections, err := remapSection(reports) 228 | if err != nil { 229 | return nil, golambda.WrapError(err, "Failed to remap Finding") 230 | } 231 | return sections, nil 232 | } 233 | 234 | func rebuildCotent(src interface{}, dst interface{}) error { 235 | raw, err := json.Marshal(src) 236 | if err != nil { 237 | return golambda.NewError("Failed to marshal content") 238 | } 239 | if err := json.Unmarshal(raw, dst); err != nil { 240 | return golambda.WrapError(err, "Failed to unmarshal marshaled content").With("raw", string(raw)) 241 | } 242 | 243 | return nil 244 | } 245 | 246 | func remapSection(inspectReports []*deepalert.Finding) ([]*deepalert.Section, error) { 247 | sections := map[string]*deepalert.Section{} 248 | 249 | for _, ir := range inspectReports { 250 | hv := ir.Attribute.Hash() 251 | section, ok := sections[hv] 252 | if !ok { 253 | section = &deepalert.Section{ 254 | Attr: ir.Attribute, 255 | } 256 | sections[hv] = section 257 | } 258 | switch ir.Type { 259 | case deepalert.ContentTypeHost: 260 | var c deepalert.ContentHost 261 | if err := rebuildCotent(ir.Content, &c); err != nil { 262 | return nil, golambda.WrapError(err, "Invalid deepalert.ContentHost data") 263 | } 264 | section.Hosts = append(section.Hosts, &c) 265 | 266 | case deepalert.ContentTypeUser: 267 | var c deepalert.ContentUser 268 | if err := rebuildCotent(ir.Content, &c); err != nil { 269 | return nil, golambda.WrapError(err, "Invalid deepalert.ContentUser data") 270 | } 271 | section.Users = append(section.Users, &c) 272 | 273 | case deepalert.ContentTypeBinary: 274 | var c deepalert.ContentBinary 275 | if err := rebuildCotent(ir.Content, &c); err != nil { 276 | return nil, golambda.WrapError(err, "Invalid deepalert.ContentBinary data") 277 | } 278 | section.Binaries = append(section.Binaries, &c) 279 | } 280 | } 281 | 282 | var sectionList []*deepalert.Section 283 | for _, section := range sections { 284 | sectionList = append(sectionList, section) 285 | } 286 | return sectionList, nil 287 | } 288 | 289 | // ----------------------------------------------------------- 290 | // Control attribute cache to prevent duplicated invocation of Inspector with same attribute 291 | // 292 | 293 | func toAttributeCacheKey(reportID deepalert.ReportID) string { 294 | return fmt.Sprintf("attribute/%s", reportID) 295 | } 296 | 297 | func toReportKey(reportID deepalert.ReportID) string { 298 | return fmt.Sprintf("report/%s", reportID) 299 | } 300 | 301 | // IsReportStreamEvent checks if the record has reportKey 302 | func IsReportStreamEvent(record *events.DynamoDBEventRecord) bool { 303 | pk, ok := record.Change.Keys[models.DynamoPKeyName] 304 | if !ok { 305 | return false 306 | } 307 | return strings.HasPrefix(pk.String(), "report/") 308 | } 309 | 310 | // PutAttributeCache puts attributeCache to DB and returns true. If the attribute alrady exists, 311 | // it returns false. 312 | func (x *RepositoryService) PutAttributeCache(reportID deepalert.ReportID, attr deepalert.Attribute, now time.Time) (bool, error) { 313 | var ts time.Time 314 | if attr.Timestamp != nil { 315 | ts = *attr.Timestamp 316 | } else { 317 | ts = now 318 | } 319 | 320 | cache := &models.AttributeCache{ 321 | RecordBase: models.RecordBase{ 322 | PKey: toAttributeCacheKey(reportID), 323 | SKey: attr.Hash(), 324 | ExpiresAt: now.Add(x.ttl).Unix(), 325 | }, 326 | Timestamp: ts, 327 | AttrKey: attr.Key, 328 | AttrType: string(attr.Type), 329 | AttrValue: attr.Value, 330 | AttrContext: attr.Context, 331 | } 332 | 333 | if err := x.repo.PutAttributeCache(cache, now); err != nil { 334 | if x.repo.IsConditionalCheckErr(err) { 335 | // The attribute already exists 336 | return false, nil 337 | } 338 | 339 | return false, golambda.WrapError(err, "Fail to put attr cache"). 340 | With("reportID", reportID). 341 | With("attr", attr) 342 | } 343 | 344 | return true, nil 345 | } 346 | 347 | // FetchAttributeCache retrieves all cached attribute from DB. 348 | func (x *RepositoryService) FetchAttributeCache(reportID deepalert.ReportID) ([]*deepalert.Attribute, error) { 349 | pk := toAttributeCacheKey(reportID) 350 | 351 | caches, err := x.repo.GetAttributeCaches(pk) 352 | if err != nil { 353 | return nil, golambda.WrapError(err, "Fail to retrieve attributeCache").With("reportID", reportID) 354 | } 355 | 356 | var attrs []*deepalert.Attribute 357 | for _, cache := range caches { 358 | attr := deepalert.Attribute{ 359 | Type: deepalert.AttrType(cache.AttrType), 360 | Key: cache.AttrKey, 361 | Value: cache.AttrValue, 362 | Context: cache.AttrContext, 363 | Timestamp: &cache.Timestamp, 364 | } 365 | 366 | attrs = append(attrs, &attr) 367 | } 368 | 369 | return attrs, nil 370 | } 371 | 372 | // PutReport puts report with a key based on report.ID and DOES NOT save attributes and alerts because attributes and alerts management must not be depended to report management. 373 | func (x *RepositoryService) PutReport(report *deepalert.Report) error { 374 | pk := toReportKey(report.ID) 375 | if err := x.repo.PutReport(pk, report); err != nil { 376 | return err 377 | } 378 | return nil 379 | } 380 | 381 | // GetReport gets a report by a key based on report.ID with attributes, alerts and sections. 382 | func (x *RepositoryService) GetReport(reportID deepalert.ReportID) (*deepalert.Report, error) { 383 | pk := toReportKey(reportID) 384 | report, err := x.repo.GetReport(pk) 385 | if err != nil { 386 | return nil, err 387 | } 388 | if report == nil { 389 | return nil, nil 390 | } 391 | 392 | sections, err := x.FetchSection(report.ID) 393 | if err != nil { 394 | return nil, golambda.WrapError(err, "FetchSection").With("report", report) 395 | } 396 | 397 | alerts, err := x.FetchAlertCache(report.ID) 398 | if err != nil { 399 | return nil, golambda.WrapError(err, "FetchAlertCache").With("report", report) 400 | } 401 | 402 | attrs, err := x.FetchAttributeCache(report.ID) 403 | if err != nil { 404 | return nil, golambda.WrapError(err, "FetchAttributeCache").With("report", report) 405 | } 406 | 407 | report.Alerts = alerts 408 | report.Attributes = attrs 409 | report.Sections = sections 410 | 411 | return report, nil 412 | } 413 | -------------------------------------------------------------------------------- /internal/service/repository_test.go: -------------------------------------------------------------------------------- 1 | package service_test 2 | 3 | import ( 4 | "os" 5 | "testing" 6 | "time" 7 | 8 | "github.com/deepalert/deepalert" 9 | "github.com/deepalert/deepalert/internal/mock" 10 | "github.com/deepalert/deepalert/internal/repository" 11 | "github.com/deepalert/deepalert/internal/service" 12 | "github.com/google/uuid" 13 | "github.com/stretchr/testify/assert" 14 | "github.com/stretchr/testify/require" 15 | ) 16 | 17 | const commonTTL = int64(10) 18 | 19 | func testRepositoryService(t *testing.T, svc *service.RepositoryService) { 20 | t.Run("TakeReport", func(tt *testing.T) { 21 | testTakeReport(tt, svc) 22 | }) 23 | t.Run("AlertCache", func(tt *testing.T) { 24 | testAlertCache(tt, svc) 25 | }) 26 | t.Run("Finding", func(tt *testing.T) { 27 | testFinding(tt, svc) 28 | }) 29 | t.Run("AttributeCache", func(tt *testing.T) { 30 | testAttributeCache(tt, svc) 31 | }) 32 | t.Run("Report", func(tt *testing.T) { 33 | testRpoert(tt, svc) 34 | }) 35 | } 36 | 37 | func testTakeReport(t *testing.T, svc *service.RepositoryService) { 38 | now := time.Now() 39 | 40 | t.Run("Take new reports with diffent keys", func(tt *testing.T) { 41 | r1, err := svc.TakeReport(deepalert.Alert{ 42 | AlertKey: "x1", 43 | RuleID: "r1", 44 | Description: "d1", 45 | }, now) 46 | require.NoError(tt, err) 47 | require.NotNil(tt, r1) 48 | 49 | r2, err := svc.TakeReport(deepalert.Alert{ 50 | AlertKey: "x2", 51 | RuleID: "r1", 52 | Description: "d1", 53 | }, now) 54 | require.NoError(tt, err) 55 | require.NotNil(tt, r2) 56 | 57 | assert.NotEqual(tt, r1.ID, r2.ID) 58 | }) 59 | 60 | t.Run("Take reports with same key", func(tt *testing.T) { 61 | r1, err := svc.TakeReport(deepalert.Alert{ 62 | AlertKey: "x1", 63 | RuleID: "r1", 64 | Description: "d1", 65 | }, now) 66 | require.NoError(tt, err) 67 | require.NotNil(tt, r1) 68 | 69 | r2, err := svc.TakeReport(deepalert.Alert{ 70 | AlertKey: "x1", 71 | RuleID: "r1", 72 | Description: "d1", 73 | }, now.Add(1*time.Second)) 74 | require.NoError(tt, err) 75 | require.NotNil(tt, r2) 76 | 77 | assert.Equal(tt, r1.ID, r2.ID) 78 | }) 79 | } 80 | 81 | func testAlertCache(t *testing.T, svc *service.RepositoryService) { 82 | t.Run("Save and fetch alert cache", func(tt *testing.T) { 83 | id1 := deepalert.ReportID(uuid.New().String()) 84 | id2 := deepalert.ReportID(uuid.New().String()) 85 | 86 | alert1 := deepalert.Alert{ 87 | AlertKey: "k1", 88 | RuleID: "r1", 89 | RuleName: "n1", 90 | } 91 | alert2 := deepalert.Alert{ 92 | AlertKey: "k2", 93 | RuleID: "r2", 94 | RuleName: "n2", 95 | } 96 | alert3 := deepalert.Alert{ 97 | AlertKey: "k3", 98 | RuleID: "r3", 99 | RuleName: "n3", 100 | } 101 | now := time.Now() 102 | require.NoError(tt, svc.SaveAlertCache(id1, alert1, now)) 103 | require.NoError(tt, svc.SaveAlertCache(id1, alert2, now)) 104 | require.NoError(tt, svc.SaveAlertCache(id2, alert3, now)) 105 | 106 | cache, err := svc.FetchAlertCache(id1) 107 | require.NoError(tt, err) 108 | assert.Contains(tt, cache, &alert1) 109 | assert.Contains(tt, cache, &alert2) 110 | assert.NotContains(tt, cache, &alert3) 111 | }) 112 | } 113 | 114 | func testFinding(t *testing.T, svc *service.RepositoryService) { 115 | t.Run("Savea and Fetch report section", func(tt *testing.T) { 116 | id1 := deepalert.ReportID(uuid.New().String()) 117 | id2 := deepalert.ReportID(uuid.New().String()) 118 | now := time.Now() 119 | s1 := deepalert.Finding{ 120 | ReportID: id1, 121 | Author: "a1", 122 | Attribute: deepalert.Attribute{ 123 | Type: deepalert.TypeIPAddr, 124 | Value: "10.0.0.1", 125 | }, 126 | Type: deepalert.ContentTypeHost, 127 | Content: deepalert.ContentHost{ 128 | HostName: []string{"h1"}, 129 | }, 130 | } 131 | s2 := deepalert.Finding{ 132 | ReportID: id1, 133 | Author: "a2", 134 | Attribute: deepalert.Attribute{ 135 | Type: deepalert.TypeIPAddr, 136 | Value: "10.0.0.2", 137 | }, 138 | Type: deepalert.ContentTypeHost, 139 | Content: deepalert.ContentHost{ 140 | HostName: []string{"h2"}, 141 | }, 142 | } 143 | s3 := deepalert.Finding{ 144 | ReportID: id2, 145 | Author: "a3", 146 | Attribute: deepalert.Attribute{ 147 | Type: deepalert.TypeIPAddr, 148 | Value: "10.0.0.3", 149 | }, 150 | Type: deepalert.ContentTypeHost, 151 | Content: deepalert.ContentHost{ 152 | HostName: []string{"h3"}, 153 | }, 154 | } 155 | 156 | attrs := []deepalert.Attribute{s1.Attribute, s2.Attribute, s3.Attribute} 157 | require.NoError(tt, svc.SaveFinding(s1, now)) 158 | require.NoError(tt, svc.SaveFinding(s2, now)) 159 | require.NoError(tt, svc.SaveFinding(s3, now)) 160 | 161 | sections, err := svc.FetchSection(id1) 162 | require.NoError(tt, err) 163 | require.Equal(tt, 2, len(sections)) 164 | assert.Contains(tt, attrs, sections[0].Attr) 165 | assert.Contains(tt, attrs, sections[1].Attr) 166 | }) 167 | } 168 | 169 | func testAttributeCache(t *testing.T, svc *service.RepositoryService) { 170 | t.Run("Put and Fetch attributes", func(t *testing.T) { 171 | id1 := deepalert.ReportID(uuid.New().String()) 172 | id2 := deepalert.ReportID(uuid.New().String()) 173 | now := time.Now() 174 | 175 | attr1 := deepalert.Attribute{ 176 | Type: deepalert.TypeIPAddr, 177 | Context: deepalert.AttrContexts{deepalert.CtxRemote}, 178 | Key: "dst", 179 | Value: "10.0.0.2", 180 | } 181 | attr2 := deepalert.Attribute{ 182 | Type: deepalert.TypeIPAddr, 183 | Context: deepalert.AttrContexts{deepalert.CtxRemote}, 184 | Key: "dst", 185 | Value: "10.0.0.3", 186 | } 187 | attr3 := deepalert.Attribute{ 188 | Type: deepalert.TypeIPAddr, 189 | Context: deepalert.AttrContexts{deepalert.CtxRemote}, 190 | Key: "dst", 191 | Value: "10.0.0.4", 192 | } 193 | 194 | b1, err := svc.PutAttributeCache(id1, attr1, now) 195 | require.NoError(t, err) 196 | assert.True(t, b1) 197 | 198 | b2, err := svc.PutAttributeCache(id1, attr2, now) 199 | require.NoError(t, err) 200 | assert.True(t, b2) 201 | 202 | b3, err := svc.PutAttributeCache(id2, attr3, now) 203 | require.NoError(t, err) 204 | assert.True(t, b3) 205 | 206 | attrs, err := svc.FetchAttributeCache(id1) 207 | require.NoError(t, err) 208 | 209 | var attrList []*deepalert.Attribute 210 | for _, attr := range attrs { 211 | a := attr 212 | a.Timestamp = nil 213 | attrList = append(attrList, a) 214 | } 215 | assert.Contains(t, attrList, &attr1) 216 | assert.Contains(t, attrList, &attr2) 217 | assert.NotContains(t, attrList, &attr3) 218 | }) 219 | 220 | t.Run("Duplicated attribute", func(t *testing.T) { 221 | id1 := deepalert.ReportID(uuid.New().String()) 222 | now := time.Now() 223 | 224 | attr1 := deepalert.Attribute{ 225 | Type: deepalert.TypeIPAddr, 226 | Context: deepalert.AttrContexts{deepalert.CtxRemote}, 227 | Key: "dst", 228 | Value: "10.0.0.2", 229 | } 230 | b1, err := svc.PutAttributeCache(id1, attr1, now) 231 | require.NoError(t, err) 232 | assert.True(t, b1) 233 | 234 | // No error by second PutAttributeCache. But returns false to indicate the attribute already exists 235 | b1d, err := svc.PutAttributeCache(id1, attr1, now) 236 | require.NoError(t, err) 237 | assert.False(t, b1d) 238 | 239 | attrs, err := svc.FetchAttributeCache(id1) 240 | require.NoError(t, err) 241 | assert.Equal(t, 1, len(attrs)) 242 | attrs[0].Timestamp = nil 243 | assert.Equal(t, attr1, *attrs[0]) 244 | }) 245 | } 246 | 247 | func testRpoert(t *testing.T, svc *service.RepositoryService) { 248 | t.Run("Put and Get", func(tt *testing.T) { 249 | r1 := &deepalert.Report{ 250 | ID: deepalert.ReportID(uuid.New().String()), 251 | Alerts: []*deepalert.Alert{ 252 | { 253 | AlertKey: "cxz", 254 | Detector: "saber", 255 | RuleID: "s1", 256 | Attributes: []deepalert.Attribute{ 257 | { 258 | Type: deepalert.TypeIPAddr, 259 | Context: deepalert.AttrContexts{ 260 | deepalert.CtxRemote, 261 | }, 262 | Key: "srcAddr", 263 | Value: "10.1.2.3", 264 | }, 265 | }, 266 | }, 267 | { 268 | AlertKey: "bnc", 269 | Detector: "archer", 270 | RuleID: "a1", 271 | Attributes: []deepalert.Attribute{ 272 | { 273 | Type: deepalert.TypeIPAddr, 274 | Context: deepalert.AttrContexts{ 275 | deepalert.CtxLocal, 276 | }, 277 | Key: "dstAddr", 278 | Value: "192.168.2.3", 279 | }, 280 | }, 281 | }, 282 | }, 283 | Attributes: []*deepalert.Attribute{ 284 | { 285 | Type: deepalert.TypeIPAddr, 286 | Context: deepalert.AttrContexts{ 287 | deepalert.CtxRemote, 288 | }, 289 | Key: "srcAddr", 290 | Value: "10.1.2.3", 291 | }, 292 | { 293 | Type: deepalert.TypeIPAddr, 294 | Context: deepalert.AttrContexts{ 295 | deepalert.CtxLocal, 296 | }, 297 | Key: "dstAddr", 298 | Value: "192.168.2.3", 299 | }, 300 | }, 301 | Sections: []*deepalert.Section{ 302 | { 303 | Attr: deepalert.Attribute{ 304 | Type: deepalert.TypeIPAddr, 305 | Context: deepalert.AttrContexts{ 306 | deepalert.CtxLocal, 307 | }, 308 | Key: "dstAddr", 309 | Value: "192.168.2.3", 310 | }, 311 | Users: []*deepalert.ContentUser{ 312 | { 313 | Activities: []deepalert.EntityActivity{ 314 | { 315 | Action: "hoge", 316 | RemoteAddr: "10.5.6.7", 317 | }, 318 | }, 319 | }, 320 | }, 321 | }, 322 | }, 323 | Result: deepalert.ReportResult{ 324 | Severity: deepalert.SevSafe, 325 | Reason: "no reason", 326 | }, 327 | Status: deepalert.StatusPublished, 328 | CreatedAt: time.Now(), 329 | } 330 | 331 | err := svc.PutReport(r1) 332 | require.NoError(tt, err) 333 | r2, err := svc.GetReport(r1.ID) 334 | require.NoError(tt, err) 335 | assert.Equal(tt, r1.ID, r2.ID) 336 | require.Equal(tt, 0, len(r2.Attributes)) // PutReport does not save attributes 337 | }) 338 | 339 | t.Run("Not found", func(tt *testing.T) { 340 | id := deepalert.ReportID(uuid.New().String()) 341 | r0, err := svc.GetReport(id) 342 | require.NoError(tt, err) 343 | assert.Nil(tt, r0) 344 | }) 345 | } 346 | 347 | func TestDynamoDBRepository(t *testing.T) { 348 | region, tableName := os.Getenv("DEEPALERT_TEST_REGION"), os.Getenv("DEEPALERT_TEST_TABLE") 349 | if region == "" || tableName == "" { 350 | t.Skip("Either of DEEPALERT_TEST_REGION and DEEPALERT_TEST_TABLE are not set") 351 | } 352 | 353 | repo, err := repository.NewDynamoDB(region, tableName) 354 | require.NoError(t, err) 355 | svc := service.NewRepositoryService(repo, commonTTL) 356 | 357 | testRepositoryService(t, svc) 358 | } 359 | 360 | func TestMockRepository(t *testing.T) { 361 | repo := mock.NewRepository("test-region", "test-table") 362 | svc := service.NewRepositoryService(repo, commonTTL) 363 | 364 | testRepositoryService(t, svc) 365 | } 366 | -------------------------------------------------------------------------------- /internal/service/sfn.go: -------------------------------------------------------------------------------- 1 | package service 2 | 3 | import ( 4 | "encoding/json" 5 | "strings" 6 | 7 | "github.com/aws/aws-sdk-go/aws" 8 | "github.com/aws/aws-sdk-go/service/sfn" 9 | "github.com/deepalert/deepalert/internal/adaptor" 10 | "github.com/m-mizutani/golambda" 11 | ) 12 | 13 | // SFnService is utility to use AWS StepFunctions 14 | type SFnService struct { 15 | newSFn adaptor.SFnClientFactory 16 | } 17 | 18 | // NewSFnService is constructor of SFnService 19 | func NewSFnService(newSFn adaptor.SFnClientFactory) *SFnService { 20 | return &SFnService{ 21 | newSFn: newSFn, 22 | } 23 | } 24 | 25 | // Exec invokes sfn.StartExecution with data 26 | func (x *SFnService) Exec(arn string, data interface{}) error { 27 | raw, err := json.Marshal(data) 28 | if err != nil { 29 | return golambda.WrapError(err, "Fail to marshal report data") 30 | } 31 | 32 | region, daErr := extractSFnRegion(arn) 33 | if daErr != nil { 34 | return daErr 35 | } 36 | 37 | svc, err := x.newSFn(region) 38 | if err != nil { 39 | return golambda.WrapError(err, "Failed to create new SFn adaptor") 40 | } 41 | 42 | input := sfn.StartExecutionInput{ 43 | Input: aws.String(string(raw)), 44 | StateMachineArn: aws.String(arn), 45 | } 46 | 47 | if _, err := svc.StartExecution(&input); err != nil { 48 | return golambda.WrapError(err, "Fail to execute state machine").With("arn", arn).With("data", string(raw)) 49 | } 50 | 51 | return nil 52 | } 53 | 54 | func extractSFnRegion(arn string) (string, error) { 55 | // arn sample: arn:aws:states:us-east-1:111122223333:stateMachine:machine-name 56 | arnParts := strings.Split(arn, ":") 57 | 58 | if len(arnParts) != 7 { 59 | return "", golambda.NewError("Invalid state machine ARN").With("ARN", arn) 60 | } 61 | 62 | return arnParts[3], nil 63 | } 64 | -------------------------------------------------------------------------------- /internal/service/sns.go: -------------------------------------------------------------------------------- 1 | package service 2 | 3 | import ( 4 | "encoding/json" 5 | "strings" 6 | 7 | "github.com/aws/aws-sdk-go/aws" 8 | "github.com/aws/aws-sdk-go/service/sns" 9 | "github.com/deepalert/deepalert/internal/adaptor" 10 | "github.com/m-mizutani/golambda" 11 | ) 12 | 13 | var logger = golambda.Logger 14 | 15 | // SNSService is accessor to SQS 16 | type SNSService struct { 17 | newSNS adaptor.SNSClientFactory 18 | } 19 | 20 | // NewSNSService is constructor of 21 | func NewSNSService(newSNS adaptor.SNSClientFactory) *SNSService { 22 | return &SNSService{ 23 | newSNS: newSNS, 24 | } 25 | } 26 | 27 | func extractSNSRegion(topicARN string) (string, error) { 28 | // topicARN sample: arn:aws:sns:us-east-1:111122223333:my-topic 29 | arnParts := strings.Split(topicARN, ":") 30 | 31 | if len(arnParts) != 6 { 32 | return "", golambda.NewError("Invalid SNS topic ARN").With("ARN", topicARN) 33 | } 34 | 35 | return arnParts[3], nil 36 | } 37 | 38 | // Publish is wrapper of sns:Publish of AWS 39 | func (x *SNSService) Publish(topicARN string, msg interface{}) error { 40 | region, daErr := extractSNSRegion(topicARN) 41 | if daErr != nil { 42 | return daErr 43 | } 44 | 45 | client, err := x.newSNS(region) 46 | if err != nil { 47 | return err 48 | } 49 | 50 | raw, err := json.Marshal(msg) 51 | if err != nil { 52 | return golambda.WrapError(err, "Fail to marshal message").With("msg", msg) 53 | } 54 | 55 | input := sns.PublishInput{ 56 | TopicArn: aws.String(topicARN), 57 | Message: aws.String(string(raw)), 58 | } 59 | resp, err := client.Publish(&input) 60 | 61 | if err != nil { 62 | return golambda.WrapError(err, "Fail to send SQS message").With("input", input) 63 | } 64 | 65 | logger.With("resp", resp).Trace("Sent SQS message") 66 | 67 | return nil 68 | } 69 | -------------------------------------------------------------------------------- /internal/usecase/alert.go: -------------------------------------------------------------------------------- 1 | package usecase 2 | 3 | import ( 4 | "time" 5 | 6 | "github.com/deepalert/deepalert" 7 | "github.com/deepalert/deepalert/internal/handler" 8 | "github.com/m-mizutani/golambda" 9 | ) 10 | 11 | var logger = golambda.Logger 12 | 13 | // HandleAlert creates a report from alert and invoke delay machines 14 | func HandleAlert(args *handler.Arguments, alert *deepalert.Alert, now time.Time) (*deepalert.Report, error) { 15 | logger.With("alert", alert).Info("Taking report") 16 | 17 | if err := alert.Validate(); err != nil { 18 | return nil, golambda.WrapError(err, "Invalid alert format") 19 | } 20 | 21 | sfnSvc := args.SFnService() 22 | repo, err := args.Repository() 23 | if err != nil { 24 | return nil, err 25 | } 26 | 27 | report, err := repo.TakeReport(*alert, now) 28 | if err != nil { 29 | return nil, golambda.WrapError(err, "Fail to take reportID for alert").With("alert", alert) 30 | } 31 | if report == nil { 32 | return nil, golambda.WrapError(err, "No report in cache"). 33 | With("alert", alert) 34 | 35 | } 36 | 37 | logger. 38 | With("ReportID", report.ID). 39 | With("Status", report.Status). 40 | With("Error", err). 41 | With("AlertID", alert.AlertID()). 42 | Info("ReportID has been retrieved") 43 | 44 | report.Alerts = []*deepalert.Alert{alert} 45 | 46 | if err := repo.SaveAlertCache(report.ID, *alert, now); err != nil { 47 | return nil, golambda.WrapError(err, "Fail to save alert cache") 48 | 49 | } 50 | 51 | if err := sfnSvc.Exec(args.InspectorMashine, &report); err != nil { 52 | return nil, golambda.WrapError(err, "Fail to execute InspectorDelayMachine") 53 | } 54 | 55 | if report.IsNew() { 56 | if err := sfnSvc.Exec(args.ReviewMachine, &report); err != nil { 57 | return nil, golambda.WrapError(err, "Fail to execute ReviewerDelayMachine") 58 | } 59 | } 60 | 61 | if err := repo.PutReport(report); err != nil { 62 | return nil, golambda.WrapError(err, "Fail PutReport") 63 | 64 | } 65 | 66 | return report, nil 67 | } 68 | -------------------------------------------------------------------------------- /internal/usecase/alert_test.go: -------------------------------------------------------------------------------- 1 | package usecase_test 2 | 3 | import ( 4 | "encoding/json" 5 | "errors" 6 | "testing" 7 | "time" 8 | 9 | "github.com/deepalert/deepalert" 10 | "github.com/deepalert/deepalert/internal/adaptor" 11 | "github.com/deepalert/deepalert/internal/handler" 12 | "github.com/deepalert/deepalert/internal/mock" 13 | "github.com/deepalert/deepalert/internal/service" 14 | "github.com/deepalert/deepalert/internal/usecase" 15 | "github.com/stretchr/testify/assert" 16 | "github.com/stretchr/testify/require" 17 | ) 18 | 19 | func TestHandleAlert(t *testing.T) { 20 | basicSetup := func() (*handler.Arguments, adaptor.SFnClient, adaptor.Repository) { 21 | dummySFn, _ := mock.NewSFnClient("") 22 | dummyRepo := mock.NewRepository("", "") 23 | args := &handler.Arguments{ 24 | NewRepository: func(string, string) adaptor.Repository { return dummyRepo }, 25 | NewSFn: func(string) (adaptor.SFnClient, error) { return dummySFn, nil }, 26 | EnvVars: handler.EnvVars{ 27 | InspectorMashine: "arn:aws:states:us-east-1:111122223333:stateMachine:blue", 28 | ReviewMachine: "arn:aws:states:us-east-1:111122223333:stateMachine:orange", 29 | }, 30 | } 31 | return args, dummySFn, dummyRepo 32 | } 33 | 34 | t.Run("Recept single alert", func(t *testing.T) { 35 | alert := &deepalert.Alert{ 36 | AlertKey: "5", 37 | RuleID: "five", 38 | RuleName: "fifth", 39 | Detector: "ao", 40 | } 41 | 42 | args, dummySFn, dummyRepo := basicSetup() 43 | 44 | report, err := usecase.HandleAlert(args, alert, time.Now()) 45 | require.NoError(t, err) 46 | assert.NotNil(t, report) 47 | assert.NotEqual(t, "", report.ID) 48 | 49 | repoSvc := service.NewRepositoryService(dummyRepo, 10) 50 | 51 | t.Run("StepFunctions should be executed", func(t *testing.T) { 52 | sfn, ok := dummySFn.(*mock.SFnClient) 53 | require.True(t, ok) 54 | require.Equal(t, 2, len(sfn.Input)) 55 | assert.Equal(t, "arn:aws:states:us-east-1:111122223333:stateMachine:blue", *sfn.Input[0].StateMachineArn) 56 | assert.Equal(t, "arn:aws:states:us-east-1:111122223333:stateMachine:orange", *sfn.Input[1].StateMachineArn) 57 | 58 | var report1, report2 deepalert.Report 59 | require.NoError(t, json.Unmarshal([]byte(*sfn.Input[0].Input), &report1)) 60 | require.Equal(t, 1, len(report1.Alerts)) 61 | assert.Equal(t, alert, report1.Alerts[0]) 62 | 63 | require.NoError(t, json.Unmarshal([]byte(*sfn.Input[1].Input), &report2)) 64 | require.Equal(t, 1, len(report2.Alerts)) 65 | assert.Equal(t, alert, report2.Alerts[0]) 66 | 67 | assert.Equal(t, report1, report2) 68 | }) 69 | 70 | t.Run("AlertCachce should be stored in repository", func(t *testing.T) { 71 | alertCache, err := repoSvc.FetchAlertCache(report.ID) 72 | require.NoError(t, err) 73 | require.Equal(t, 1, len(alertCache)) 74 | assert.Equal(t, alert, alertCache[0]) 75 | }) 76 | 77 | t.Run("Report should be stored in repository", func(t *testing.T) { 78 | report, err := repoSvc.GetReport(report.ID) 79 | require.NoError(t, err) 80 | require.Equal(t, 1, len(report.Alerts)) 81 | assert.Equal(t, alert, report.Alerts[0]) 82 | }) 83 | 84 | }) 85 | 86 | t.Run("Error cases", func(t *testing.T) { 87 | t.Run("Alert without Detector field is not allowed", func(t *testing.T) { 88 | alert := &deepalert.Alert{ 89 | AlertKey: "5", 90 | RuleID: "five", 91 | RuleName: "fifth", 92 | Detector: "", 93 | } 94 | 95 | args, dummySFn, _ := basicSetup() 96 | 97 | report, err := usecase.HandleAlert(args, alert, time.Now()) 98 | require.Error(t, err) 99 | assert.True(t, errors.Is(err, deepalert.ErrInvalidAlert)) 100 | assert.Nil(t, report) 101 | 102 | sfn, ok := dummySFn.(*mock.SFnClient) 103 | require.True(t, ok) 104 | require.Equal(t, 0, len(sfn.Input)) 105 | }) 106 | 107 | t.Run("Alert without RuleID field is not allowed", func(t *testing.T) { 108 | alert := &deepalert.Alert{ 109 | AlertKey: "5", 110 | RuleID: "", 111 | RuleName: "fifth", 112 | Detector: "ao", 113 | } 114 | 115 | args, dummySFn, _ := basicSetup() 116 | 117 | report, err := usecase.HandleAlert(args, alert, time.Now()) 118 | require.Error(t, err) 119 | assert.True(t, errors.Is(err, deepalert.ErrInvalidAlert)) 120 | assert.Nil(t, report) 121 | 122 | sfn, ok := dummySFn.(*mock.SFnClient) 123 | require.True(t, ok) 124 | require.Equal(t, 0, len(sfn.Input)) 125 | }) 126 | }) 127 | 128 | t.Run("Recept alerts with same AlertID", func(t *testing.T) { 129 | // AlertID is calculated by AlertKey, RuleID and Detector 130 | alert1 := &deepalert.Alert{ 131 | AlertKey: "123", 132 | RuleID: "blue", 133 | RuleName: "fifth", 134 | Detector: "ao", 135 | } 136 | alert2 := &deepalert.Alert{ 137 | AlertKey: "123", 138 | RuleID: "blue", 139 | RuleName: "five", 140 | Detector: "ao", 141 | } 142 | args, _, _ := basicSetup() 143 | 144 | report1, err := usecase.HandleAlert(args, alert1, time.Now()) 145 | require.NoError(t, err) 146 | assert.NotNil(t, report1) 147 | assert.NotEqual(t, "", report1.ID) 148 | 149 | report2, err := usecase.HandleAlert(args, alert2, time.Now()) 150 | require.NoError(t, err) 151 | assert.NotNil(t, report2) 152 | assert.NotEqual(t, "", report2.ID) 153 | 154 | t.Run("ReportIDs should be same", func(t *testing.T) { 155 | assert.Equal(t, report1.ID, report2.ID) 156 | }) 157 | }) 158 | 159 | t.Run("ReportIDs should be different if AlertID is not same", func(t *testing.T) { 160 | // AlertID is calculated by AlertKey, RuleID and Detector 161 | args, _, _ := basicSetup() 162 | 163 | t.Run("Different AlertKey", func(t *testing.T) { 164 | alert1 := &deepalert.Alert{ 165 | AlertKey: "234", 166 | RuleID: "blue", 167 | RuleName: "fifth", 168 | Detector: "ao", 169 | } 170 | alert2 := &deepalert.Alert{ 171 | AlertKey: "123", 172 | RuleID: "blue", 173 | RuleName: "five", 174 | Detector: "ao", 175 | } 176 | 177 | report1, err := usecase.HandleAlert(args, alert1, time.Now()) 178 | require.NoError(t, err) 179 | report2, err := usecase.HandleAlert(args, alert2, time.Now()) 180 | require.NoError(t, err) 181 | assert.NotEqual(t, report1.ID, report2.ID) 182 | }) 183 | 184 | t.Run("Different RuleID", func(t *testing.T) { 185 | alert1 := &deepalert.Alert{ 186 | AlertKey: "123", 187 | RuleID: "blue", 188 | RuleName: "fifth", 189 | Detector: "ao", 190 | } 191 | alert2 := &deepalert.Alert{ 192 | AlertKey: "123", 193 | RuleID: "orange", 194 | RuleName: "five", 195 | Detector: "ao", 196 | } 197 | 198 | report1, err := usecase.HandleAlert(args, alert1, time.Now()) 199 | require.NoError(t, err) 200 | report2, err := usecase.HandleAlert(args, alert2, time.Now()) 201 | require.NoError(t, err) 202 | assert.NotEqual(t, report1.ID, report2.ID) 203 | }) 204 | 205 | t.Run("Different Detector", func(t *testing.T) { 206 | alert1 := &deepalert.Alert{ 207 | AlertKey: "123", 208 | RuleID: "blue", 209 | RuleName: "fifth", 210 | Detector: "ao", 211 | } 212 | alert2 := &deepalert.Alert{ 213 | AlertKey: "123", 214 | RuleID: "blue", 215 | RuleName: "five", 216 | Detector: "tou", 217 | } 218 | 219 | report1, err := usecase.HandleAlert(args, alert1, time.Now()) 220 | require.NoError(t, err) 221 | report2, err := usecase.HandleAlert(args, alert2, time.Now()) 222 | require.NoError(t, err) 223 | assert.NotEqual(t, report1.ID, report2.ID) 224 | }) 225 | }) 226 | } 227 | -------------------------------------------------------------------------------- /jest.config.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | roots: ['/test'], 3 | testMatch: ['**/*.test.ts'], 4 | transform: { 5 | '^.+\\.tsx?$': 'ts-jest' 6 | } 7 | }; 8 | -------------------------------------------------------------------------------- /lambda/apiHandler/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "github.com/aws/aws-lambda-go/events" 5 | ginadapter "github.com/awslabs/aws-lambda-go-api-proxy/gin" 6 | "github.com/gin-gonic/gin" 7 | "github.com/m-mizutani/golambda" 8 | 9 | "github.com/deepalert/deepalert/internal/api" 10 | "github.com/deepalert/deepalert/internal/handler" 11 | ) 12 | 13 | var logger = golambda.Logger 14 | 15 | func main() { 16 | golambda.Start(func(event golambda.Event) (interface{}, error) { 17 | args := handler.NewArguments() 18 | if err := args.BindEnvVars(); err != nil { 19 | return nil, err 20 | } 21 | 22 | return handleRequest(args, event) 23 | }) 24 | } 25 | 26 | func handleRequest(args *handler.Arguments, event golambda.Event) (interface{}, error) { 27 | var req events.APIGatewayProxyRequest 28 | if err := event.Bind(&req); err != nil { 29 | return nil, err 30 | } 31 | 32 | logger.With("request", req).Info("HTTP request") 33 | gin.SetMode(gin.ReleaseMode) 34 | r := gin.Default() 35 | 36 | v1 := r.Group("/api/v1") 37 | api.SetupRoute(v1, args) 38 | 39 | return ginadapter.New(r).Proxy(req) 40 | } 41 | -------------------------------------------------------------------------------- /lambda/apiHandler/main_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "encoding/json" 5 | "testing" 6 | 7 | "github.com/aws/aws-lambda-go/events" 8 | "github.com/deepalert/deepalert" 9 | "github.com/deepalert/deepalert/internal/adaptor" 10 | "github.com/deepalert/deepalert/internal/handler" 11 | "github.com/deepalert/deepalert/internal/mock" 12 | "github.com/m-mizutani/golambda" 13 | "github.com/stretchr/testify/assert" 14 | "github.com/stretchr/testify/require" 15 | ) 16 | 17 | func TestReceptAlert(t *testing.T) { 18 | t.Run("Recept single alert", func(tt *testing.T) { 19 | alert := &deepalert.Alert{ 20 | AlertKey: "5", 21 | RuleID: "five", 22 | RuleName: "fifth", 23 | Detector: "ao", 24 | } 25 | raw, err := json.Marshal(alert) 26 | require.NoError(tt, err) 27 | 28 | dummySFn, _ := mock.NewSFnClient("") 29 | dummyRepo := mock.NewRepository("", "") 30 | args := &handler.Arguments{ 31 | NewRepository: func(string, string) adaptor.Repository { return dummyRepo }, 32 | NewSFn: func(string) (adaptor.SFnClient, error) { return dummySFn, nil }, 33 | EnvVars: handler.EnvVars{ 34 | InspectorMashine: "arn:aws:states:us-east-1:111122223333:stateMachine:blue", 35 | ReviewMachine: "arn:aws:states:us-east-1:111122223333:stateMachine:orange", 36 | }, 37 | } 38 | 39 | event := events.APIGatewayProxyRequest{ 40 | HTTPMethod: "POST", 41 | Path: "/api/v1/alert", 42 | Body: string(raw), 43 | } 44 | resp, err := handleRequest(args, golambda.Event{Origin: event}) 45 | require.NoError(tt, err) 46 | assert.NotNil(tt, resp) 47 | httpResp, ok := resp.(events.APIGatewayProxyResponse) 48 | require.True(tt, ok) 49 | assert.Equal(tt, 200, httpResp.StatusCode) 50 | 51 | // Check only execution of StepFunctions. More detailed test are in internal/usecase 52 | sfn, ok := dummySFn.(*mock.SFnClient) 53 | require.True(tt, ok) 54 | require.Equal(tt, 2, len(sfn.Input)) 55 | }) 56 | } 57 | -------------------------------------------------------------------------------- /lambda/compileReport/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "github.com/deepalert/deepalert" 5 | "github.com/deepalert/deepalert/internal/handler" 6 | "github.com/m-mizutani/golambda" 7 | ) 8 | 9 | func main() { 10 | golambda.Start(func(event golambda.Event) (interface{}, error) { 11 | args := handler.NewArguments() 12 | if err := args.BindEnvVars(); err != nil { 13 | return nil, err 14 | } 15 | 16 | return handleRequest(args, event) 17 | }) 18 | } 19 | 20 | func handleRequest(args *handler.Arguments, event golambda.Event) (interface{}, error) { 21 | var report deepalert.Report 22 | if err := event.Bind(&report); err != nil { 23 | return nil, err 24 | } 25 | 26 | svc, err := args.Repository() 27 | if err != nil { 28 | return nil, err 29 | } 30 | 31 | compiledReport, err := svc.GetReport(report.ID) 32 | if err != nil { 33 | return nil, err 34 | } 35 | golambda.Logger.With("report", compiledReport).Info("Compiled report") 36 | 37 | return compiledReport, nil 38 | } 39 | -------------------------------------------------------------------------------- /lambda/compileReport/main_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "testing" 5 | "time" 6 | 7 | "github.com/deepalert/deepalert" 8 | "github.com/deepalert/deepalert/internal/handler" 9 | "github.com/deepalert/deepalert/internal/mock" 10 | "github.com/deepalert/deepalert/internal/service" 11 | "github.com/google/uuid" 12 | "github.com/m-mizutani/golambda" 13 | "github.com/stretchr/testify/assert" 14 | "github.com/stretchr/testify/require" 15 | ) 16 | 17 | func TestHandleReport(t *testing.T) { 18 | // Setup dummy repository 19 | mockRepo, newMockRepo := mock.NewMockRepositorySet() 20 | repo := service.NewRepositoryService(mockRepo, 10) 21 | now := time.Now() 22 | 23 | reportID := deepalert.ReportID(uuid.New().String()) 24 | attr := deepalert.Attribute{ 25 | Type: deepalert.TypeUserName, 26 | Key: "username", 27 | Value: "blue", 28 | } 29 | finding := deepalert.Finding{ 30 | ReportID: reportID, 31 | Attribute: attr, 32 | Author: "tester", 33 | Type: deepalert.ContentTypeHost, 34 | Content: &deepalert.ContentHost{ 35 | HostName: []string{"h1"}, 36 | }, 37 | } 38 | 39 | alert := deepalert.Alert{ 40 | Detector: "tester", 41 | RuleName: "testRule", 42 | RuleID: "testID", 43 | } 44 | 45 | report := &deepalert.Report{ 46 | ID: reportID, 47 | } 48 | 49 | require.NoError(t, repo.PutReport(report)) 50 | require.NoError(t, repo.SaveFinding(finding, now)) 51 | require.NoError(t, repo.SaveAlertCache(reportID, alert, now)) 52 | _, err := repo.PutAttributeCache(reportID, attr, now) 53 | require.NoError(t, err) 54 | 55 | args := &handler.Arguments{ 56 | NewRepository: newMockRepo, 57 | } 58 | 59 | resp, err := handleRequest(args, golambda.Event{Origin: report}) 60 | require.NoError(t, err) 61 | updatedReport := resp.(*deepalert.Report) 62 | require.NotNil(t, updatedReport) 63 | assert.Equal(t, len(updatedReport.Sections), 1) 64 | assert.Equal(t, len(updatedReport.Alerts), 1) 65 | assert.Equal(t, len(updatedReport.Attributes), 1) 66 | } 67 | -------------------------------------------------------------------------------- /lambda/dispatchInspection/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "time" 5 | 6 | "github.com/deepalert/deepalert" 7 | "github.com/deepalert/deepalert/internal/handler" 8 | "github.com/m-mizutani/golambda" 9 | ) 10 | 11 | func main() { 12 | golambda.Start(func(event golambda.Event) (interface{}, error) { 13 | args := handler.NewArguments() 14 | if err := args.BindEnvVars(); err != nil { 15 | return nil, err 16 | } 17 | 18 | return handleRequest(args, event) 19 | }) 20 | } 21 | 22 | func handleRequest(args *handler.Arguments, event golambda.Event) (interface{}, error) { 23 | var report deepalert.Report 24 | if err := event.Bind(&report); err != nil { 25 | return nil, err 26 | } 27 | 28 | now := time.Now() 29 | repo, err := args.Repository() 30 | if err != nil { 31 | return nil, err 32 | } 33 | snsSvc := args.SNSService() 34 | for _, alert := range report.Alerts { 35 | for _, attr := range alert.Attributes { 36 | sendable, err := repo.PutAttributeCache(report.ID, attr, now) 37 | if err != nil { 38 | return nil, golambda.WrapError(err, "Fail to manage attribute cache").With("attr", attr) 39 | } 40 | 41 | if !sendable { 42 | continue 43 | } 44 | 45 | if attr.Timestamp == nil { 46 | attr.Timestamp = &alert.Timestamp 47 | } 48 | 49 | task := deepalert.Task{ 50 | ReportID: report.ID, 51 | Attribute: &attr, 52 | } 53 | 54 | if err := snsSvc.Publish(args.TaskTopic, &task); err != nil { 55 | return nil, golambda.WrapError(err, "Fail to publish task notification").With("task", task) 56 | } 57 | 58 | golambda.Logger.With("task", task).Debug("Dispatched event") 59 | } 60 | } 61 | 62 | return nil, nil 63 | } 64 | -------------------------------------------------------------------------------- /lambda/dummyReviewer/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "github.com/m-mizutani/golambda" 5 | 6 | "github.com/deepalert/deepalert" 7 | ) 8 | 9 | func main() { 10 | golambda.Start(func(event golambda.Event) (interface{}, error) { 11 | return deepalert.ReportResult{ 12 | Severity: deepalert.SevUnclassified, 13 | Reason: "I'm novice", 14 | }, nil 15 | }) 16 | } 17 | -------------------------------------------------------------------------------- /lambda/feedbackAttribute/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "encoding/json" 5 | "time" 6 | 7 | "github.com/m-mizutani/golambda" 8 | 9 | "github.com/deepalert/deepalert" 10 | "github.com/deepalert/deepalert/internal/handler" 11 | ) 12 | 13 | var logger = golambda.Logger 14 | 15 | func main() { 16 | golambda.Start(func(event golambda.Event) (interface{}, error) { 17 | args := handler.NewArguments() 18 | if err := args.BindEnvVars(); err != nil { 19 | return nil, err 20 | } 21 | 22 | return handleRequest(args, event) 23 | }) 24 | } 25 | 26 | func handleRequest(args *handler.Arguments, event golambda.Event) (interface{}, error) { 27 | snsSvc := args.SNSService() 28 | repo, err := args.Repository() 29 | if err != nil { 30 | return nil, err 31 | } 32 | 33 | now := time.Now() 34 | 35 | sqsMessages, err := event.DecapSQSBody() 36 | if err != nil { 37 | return nil, err 38 | } 39 | 40 | for _, msg := range sqsMessages { 41 | var reportedAttr deepalert.ReportAttribute 42 | if err := json.Unmarshal(msg, &reportedAttr); err != nil { 43 | return nil, golambda.WrapError(err, "Unmarshal ReportAttribute").With("msg", string(msg)) 44 | } 45 | 46 | logger.With("reportedAttr", reportedAttr).Info("unmarshaled reported attribute") 47 | 48 | for _, attr := range reportedAttr.Attributes { 49 | sendable, err := repo.PutAttributeCache(reportedAttr.ReportID, *attr, now) 50 | if err != nil { 51 | return nil, golambda.WrapError(err, "Fail to manage attribute cache").With("attr", attr) 52 | } 53 | 54 | logger.With("sendable", sendable).With("attr", attr).Info("attribute") 55 | if !sendable { 56 | continue 57 | } 58 | 59 | task := deepalert.Task{ 60 | ReportID: reportedAttr.ReportID, 61 | Attribute: attr, 62 | } 63 | 64 | if err := snsSvc.Publish(args.TaskTopic, &task); err != nil { 65 | return nil, err 66 | } 67 | } 68 | } 69 | 70 | return nil, nil 71 | } 72 | -------------------------------------------------------------------------------- /lambda/publishReport/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "github.com/aws/aws-lambda-go/events" 5 | "github.com/deepalert/deepalert" 6 | "github.com/deepalert/deepalert/internal/handler" 7 | "github.com/deepalert/deepalert/internal/models" 8 | "github.com/deepalert/deepalert/internal/service" 9 | "github.com/m-mizutani/golambda" 10 | ) 11 | 12 | var logger = golambda.Logger 13 | 14 | func main() { 15 | golambda.Start(func(event golambda.Event) (interface{}, error) { 16 | args := handler.NewArguments() 17 | if err := args.BindEnvVars(); err != nil { 18 | return nil, err 19 | } 20 | 21 | return handleRequest(args, event) 22 | }) 23 | } 24 | 25 | func handleRequest(args *handler.Arguments, event golambda.Event) (interface{}, error) { 26 | repo, err := args.Repository() 27 | if err != nil { 28 | return nil, err 29 | } 30 | 31 | var dynamoEvent events.DynamoDBEvent 32 | if err := event.Bind(&dynamoEvent); err != nil { 33 | return nil, err 34 | } 35 | 36 | for _, record := range dynamoEvent.Records { 37 | logger.With("event", event).Info("Recv DynamoDB event") 38 | 39 | if !service.IsReportStreamEvent(&record) { 40 | continue 41 | } 42 | 43 | var reportEntry models.ReportEntry 44 | if err := reportEntry.ImportDynamoRecord(&record); err != nil { 45 | if err != models.ErrRecordIsNotReport { 46 | return nil, err 47 | } 48 | continue 49 | } 50 | 51 | report, err := repo.GetReport(deepalert.ReportID(reportEntry.ID)) 52 | if err != nil { 53 | return nil, err 54 | } 55 | 56 | logger.With("report", report).Info("Publishing report") 57 | 58 | if err := args.SNSService().Publish(args.ReportTopic, &report); err != nil { 59 | return nil, golambda.WrapError(err, "Fail to publish report") 60 | } 61 | } 62 | 63 | return nil, nil 64 | } 65 | -------------------------------------------------------------------------------- /lambda/publishReport/main_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "encoding/json" 5 | "testing" 6 | "time" 7 | 8 | "github.com/aws/aws-lambda-go/events" 9 | "github.com/deepalert/deepalert" 10 | "github.com/deepalert/deepalert/internal/handler" 11 | "github.com/deepalert/deepalert/internal/mock" 12 | "github.com/deepalert/deepalert/internal/models" 13 | "github.com/deepalert/deepalert/internal/service" 14 | "github.com/google/uuid" 15 | "github.com/m-mizutani/golambda" 16 | "github.com/stretchr/testify/assert" 17 | "github.com/stretchr/testify/require" 18 | ) 19 | 20 | func TestHandleReport(t *testing.T) { 21 | // Setup dummy repository 22 | mockRepo, newMockRepo := mock.NewMockRepositorySet() 23 | mockSNS, newMockSNS := mock.NewMockSNSClientSet() 24 | repo := service.NewRepositoryService(mockRepo, 10) 25 | now := time.Now() 26 | 27 | reportID := deepalert.ReportID(uuid.New().String()) 28 | attr := deepalert.Attribute{ 29 | Type: deepalert.TypeUserName, 30 | Key: "username", 31 | Value: "blue", 32 | } 33 | finding := deepalert.Finding{ 34 | ReportID: reportID, 35 | Attribute: attr, 36 | Author: "tester", 37 | Type: deepalert.ContentTypeHost, 38 | Content: &deepalert.ContentHost{ 39 | HostName: []string{"h1"}, 40 | }, 41 | } 42 | 43 | alert := deepalert.Alert{ 44 | Detector: "tester", 45 | RuleName: "testRule", 46 | RuleID: "testID", 47 | } 48 | 49 | require.NoError(t, repo.PutReport(&deepalert.Report{ 50 | ID: reportID, 51 | })) 52 | require.NoError(t, repo.SaveFinding(finding, now)) 53 | require.NoError(t, repo.SaveAlertCache(reportID, alert, now)) 54 | _, err := repo.PutAttributeCache(reportID, attr, now) 55 | require.NoError(t, err) 56 | 57 | args := &handler.Arguments{ 58 | NewRepository: newMockRepo, 59 | NewSNS: newMockSNS, 60 | EnvVars: handler.EnvVars{ 61 | ReportTopic: "arn:aws:sns:us-east-1:111122223333:my-topic", 62 | }, 63 | } 64 | 65 | dynamoEvent := events.DynamoDBEvent{ 66 | Records: []events.DynamoDBEventRecord{ 67 | { 68 | Change: events.DynamoDBStreamRecord{ 69 | Keys: map[string]events.DynamoDBAttributeValue{ 70 | models.DynamoPKeyName: events.NewStringAttribute("report/" + string(reportID)), 71 | }, 72 | NewImage: map[string]events.DynamoDBAttributeValue{ 73 | "id": events.NewStringAttribute(string(reportID)), 74 | "result": events.NewStringAttribute(`{}`), 75 | "status": events.NewStringAttribute(string(deepalert.StatusPublished)), 76 | "created_at": events.NewNumberAttribute("1612167325"), 77 | }, 78 | }, 79 | }, 80 | }, 81 | } 82 | 83 | _, err = handleRequest(args, golambda.Event{Origin: dynamoEvent}) 84 | require.NoError(t, err) 85 | 86 | require.Equal(t, 1, len(mockSNS.Input)) 87 | assert.Equal(t, "arn:aws:sns:us-east-1:111122223333:my-topic", *(mockSNS.Input[0].TopicArn)) 88 | 89 | var report deepalert.Report 90 | 91 | require.NoError(t, json.Unmarshal([]byte(*mockSNS.Input[0].Message), &report)) 92 | assert.Equal(t, 1, len(report.Sections)) 93 | assert.Contains(t, report.Alerts, &alert) 94 | require.Equal(t, 1, len(report.Attributes)) 95 | assert.Equal(t, report.Attributes[0].Value, attr.Value) 96 | } 97 | -------------------------------------------------------------------------------- /lambda/receptAlert/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "encoding/json" 5 | "time" 6 | 7 | "github.com/deepalert/deepalert" 8 | "github.com/deepalert/deepalert/internal/handler" 9 | "github.com/deepalert/deepalert/internal/usecase" 10 | "github.com/m-mizutani/golambda" 11 | ) 12 | 13 | var logger = golambda.Logger 14 | 15 | func main() { 16 | golambda.Start(func(event golambda.Event) (interface{}, error) { 17 | args := handler.NewArguments() 18 | if err := args.BindEnvVars(); err != nil { 19 | return nil, err 20 | } 21 | 22 | return HandleRequest(args, event) 23 | }) 24 | } 25 | 26 | // HandleRequest is main logic of ReceptAlert 27 | func HandleRequest(args *handler.Arguments, event golambda.Event) (interface{}, error) { 28 | messages, err := event.DecapSQSBody() 29 | if err != nil { 30 | return nil, err 31 | } 32 | 33 | now := time.Now().UTC() 34 | 35 | for _, msg := range messages { 36 | var ev map[string]interface{} 37 | if err := msg.Bind(&ev); err != nil { 38 | return nil, golambda.WrapError(err, "Failed to bind event").With("msg", msg) 39 | } 40 | 41 | var data []byte 42 | if v, ok := ev["Message"]; ok { 43 | data = []byte(v.(string)) 44 | } else { 45 | data = msg 46 | } 47 | 48 | logger.With("data", string(data)).Debug("Start handle alert") 49 | 50 | var alert deepalert.Alert 51 | if err := json.Unmarshal(data, &alert); err != nil { 52 | return nil, golambda.WrapError(err, "Fail to unmarshal alert").With("alert", string(msg)) 53 | } 54 | 55 | _, err = usecase.HandleAlert(args, &alert, now) 56 | if err != nil { 57 | return nil, err 58 | } 59 | } 60 | 61 | return nil, nil 62 | } 63 | -------------------------------------------------------------------------------- /lambda/receptAlert/main_test.go: -------------------------------------------------------------------------------- 1 | package main_test 2 | 3 | import ( 4 | "encoding/json" 5 | "testing" 6 | 7 | "github.com/aws/aws-lambda-go/events" 8 | "github.com/deepalert/deepalert" 9 | "github.com/deepalert/deepalert/internal/adaptor" 10 | "github.com/deepalert/deepalert/internal/handler" 11 | "github.com/deepalert/deepalert/internal/mock" 12 | "github.com/google/uuid" 13 | "github.com/m-mizutani/golambda" 14 | "github.com/stretchr/testify/assert" 15 | "github.com/stretchr/testify/require" 16 | 17 | main "github.com/deepalert/deepalert/lambda/receptAlert" 18 | ) 19 | 20 | func TestReceptAlert(t *testing.T) { 21 | t.Run("Recept single alert via SQS", func(tt *testing.T) { 22 | alert := &deepalert.Alert{ 23 | AlertKey: uuid.New().String(), 24 | RuleID: "five", 25 | RuleName: "fifth", 26 | Detector: "ao", 27 | } 28 | 29 | dummySFn, _ := mock.NewSFnClient("") 30 | dummyRepo := mock.NewRepository("", "") 31 | args := &handler.Arguments{ 32 | NewRepository: func(string, string) adaptor.Repository { return dummyRepo }, 33 | NewSFn: func(string) (adaptor.SFnClient, error) { return dummySFn, nil }, 34 | EnvVars: handler.EnvVars{ 35 | InspectorMashine: "arn:aws:states:us-east-1:111122223333:stateMachine:blue", 36 | ReviewMachine: "arn:aws:states:us-east-1:111122223333:stateMachine:orange", 37 | }, 38 | } 39 | 40 | var event golambda.Event 41 | require.NoError(t, event.EncapSQS(alert)) 42 | 43 | resp, err := main.HandleRequest(args, event) 44 | require.NoError(tt, err) 45 | assert.Nil(tt, resp) 46 | 47 | // Check only execution of StepFunctions. More detailed test are in internal/usecase 48 | sfn, ok := dummySFn.(*mock.SFnClient) 49 | require.True(tt, ok) 50 | require.Equal(tt, 2, len(sfn.Input)) 51 | }) 52 | 53 | t.Run("Recept single alert via SQS+SNS", func(tt *testing.T) { 54 | alert := &deepalert.Alert{ 55 | AlertKey: uuid.New().String(), 56 | RuleID: "five", 57 | RuleName: "fifth", 58 | Detector: "ao", 59 | } 60 | raw, err := json.Marshal(alert) 61 | require.NoError(tt, err) 62 | 63 | snsEntity := &events.SNSEntity{Message: string(raw)} 64 | 65 | var event golambda.Event 66 | require.NoError(t, event.EncapSQS(snsEntity)) 67 | 68 | dummySFn, _ := mock.NewSFnClient("") 69 | dummyRepo := mock.NewRepository("", "") 70 | args := &handler.Arguments{ 71 | NewRepository: func(string, string) adaptor.Repository { return dummyRepo }, 72 | NewSFn: func(string) (adaptor.SFnClient, error) { return dummySFn, nil }, 73 | EnvVars: handler.EnvVars{ 74 | InspectorMashine: "arn:aws:states:us-east-1:111122223333:stateMachine:blue", 75 | ReviewMachine: "arn:aws:states:us-east-1:111122223333:stateMachine:orange", 76 | }, 77 | } 78 | 79 | resp, err := main.HandleRequest(args, event) 80 | require.NoError(tt, err) 81 | assert.Nil(tt, resp) 82 | 83 | // Check only execution of StepFunctions. More detailed test are in internal/usecase 84 | sfn, ok := dummySFn.(*mock.SFnClient) 85 | require.True(tt, ok) 86 | require.Equal(tt, 2, len(sfn.Input)) 87 | }) 88 | 89 | } 90 | -------------------------------------------------------------------------------- /lambda/submitFinding/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "encoding/json" 5 | "time" 6 | 7 | "github.com/deepalert/deepalert" 8 | "github.com/deepalert/deepalert/internal/handler" 9 | "github.com/m-mizutani/golambda" 10 | ) 11 | 12 | var logger = golambda.Logger 13 | 14 | func main() { 15 | golambda.Start(func(event golambda.Event) (interface{}, error) { 16 | args := handler.NewArguments() 17 | if err := args.BindEnvVars(); err != nil { 18 | return nil, err 19 | } 20 | 21 | if err := handleRequest(args, event); err != nil { 22 | return nil, err 23 | } 24 | return nil, nil 25 | }) 26 | } 27 | 28 | func handleRequest(args *handler.Arguments, event golambda.Event) error { 29 | messages, err := event.DecapSQSBody() 30 | if err != nil { 31 | return err 32 | } 33 | 34 | repo, err := args.Repository() 35 | if err != nil { 36 | return err 37 | } 38 | now := time.Now() 39 | 40 | for _, msg := range messages { 41 | var ir deepalert.Finding 42 | if err := json.Unmarshal(msg, &ir); err != nil { 43 | return golambda.WrapError(err, "Fail to unmarshal Finding from SubmitNotification").With("msg", string(msg)) 44 | } 45 | logger.With("inspectReport", ir).Debug("Handling inspect report") 46 | 47 | if err := repo.SaveFinding(ir, now); err != nil { 48 | return golambda.WrapError(err, "Fail to save Finding").With("report", ir) 49 | } 50 | logger.With("section", ir).Info("Saved content") 51 | } 52 | 53 | return nil 54 | } 55 | -------------------------------------------------------------------------------- /lambda/submitReport/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "github.com/deepalert/deepalert" 5 | "github.com/deepalert/deepalert/internal/handler" 6 | "github.com/m-mizutani/golambda" 7 | ) 8 | 9 | var logger = golambda.Logger 10 | 11 | func main() { 12 | golambda.Start(func(event golambda.Event) (interface{}, error) { 13 | args := handler.NewArguments() 14 | if err := args.BindEnvVars(); err != nil { 15 | return nil, err 16 | } 17 | 18 | if err := handleRequest(args, event); err != nil { 19 | return nil, err 20 | } 21 | return nil, nil 22 | }) 23 | } 24 | 25 | func handleRequest(args *handler.Arguments, event golambda.Event) error { 26 | var report deepalert.Report 27 | if err := event.Bind(&report); err != nil { 28 | return err 29 | } 30 | 31 | report.Status = deepalert.StatusPublished 32 | if report.Result.Severity == "" { 33 | report.Result.Severity = deepalert.SevUnclassified 34 | } 35 | 36 | repo, err := args.Repository() 37 | if err != nil { 38 | return err 39 | } 40 | 41 | logger.With("report", report).Info("Publishing report") 42 | if err := repo.PutReport(&report); err != nil { 43 | return golambda.WrapError(err, "Fail to submit report") 44 | } 45 | 46 | return nil 47 | } 48 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@deepalert/deepalert", 3 | "version": "1.1.0-alpha", 4 | "scripts": { 5 | "watch": "tsc -w", 6 | "prepare": "tsc" 7 | }, 8 | "main": "./cdk/deepalert-stack.js", 9 | "type": "./cdk/deepalert-stack.d.ts", 10 | "files": [ 11 | "Makefile", 12 | "go.*", 13 | "*.go", 14 | "internal", 15 | "lambda", 16 | "cdk", 17 | "tsconfig.json" 18 | ], 19 | "devDependencies": { 20 | "@types/node": "10.17.5", 21 | "aws-cdk": "1.75.0", 22 | "ts-node": "^8.1.0", 23 | "typescript": "~3.7.2" 24 | }, 25 | "dependencies": { 26 | "@aws-cdk/aws-dynamodb": "1.75.0", 27 | "@aws-cdk/aws-iam": "1.75.0", 28 | "@aws-cdk/aws-lambda": "1.75.0", 29 | "@aws-cdk/aws-lambda-event-sources": "1.75.0", 30 | "@aws-cdk/aws-lambda-nodejs": "1.75.0", 31 | "@aws-cdk/aws-sns": "1.75.0", 32 | "@aws-cdk/aws-sns-subscriptions": "1.75.0", 33 | "@aws-cdk/aws-sqs": "1.75.0", 34 | "@aws-cdk/aws-stepfunctions": "1.75.0", 35 | "@aws-cdk/aws-stepfunctions-tasks": "1.75.0", 36 | "@aws-cdk/core": "1.75.0", 37 | "source-map-support": "^0.5.16" 38 | }, 39 | "cdk-lambda": "/asset-output/index.js", 40 | "targets": { 41 | "cdk-lambda": { 42 | "context": "node", 43 | "includeNodeModules": { 44 | "aws-sdk": false 45 | }, 46 | "sourceMap": false, 47 | "minify": false, 48 | "engines": { 49 | "node": ">= 12" 50 | } 51 | } 52 | }, 53 | "description": "Serverless SOAR (Security Orchestration, Automation and Response) framework for automatic inspection and evaluation of security alert.", 54 | "directories": { 55 | "example": "examples", 56 | "test": "test" 57 | }, 58 | "repository": { 59 | "type": "git", 60 | "url": "git+https://github.com/deepalert/deepalert.git" 61 | }, 62 | "author": "Masayoshi Mizutani", 63 | "license": "MIT", 64 | "bugs": { 65 | "url": "https://github.com/deepalert/deepalert/issues" 66 | }, 67 | "homepage": "https://github.com/deepalert/deepalert" 68 | } 69 | -------------------------------------------------------------------------------- /report.go: -------------------------------------------------------------------------------- 1 | package deepalert 2 | 3 | import ( 4 | "time" 5 | ) 6 | 7 | // ReportID is a unique ID of a report. Multiple alerts can be aggregated to 8 | // one report by same Detector, RuleName and AlertKey. 9 | type ReportID string 10 | 11 | // NullReportID means not available ID 12 | const NullReportID ReportID = "" 13 | 14 | // ReportStatus shows "new" or "published". "new" means that the report have 15 | // not been reviewed by Reviewer and inspection may be still ongoing. 16 | // "publihsed" means that the report is already submitted to ReportNotification 17 | // as a reviwed report. 18 | type ReportStatus string 19 | 20 | const ( 21 | // StatusNew means the report is newly created with a new alert. "New alert" means 22 | // "CacheTable does not have same alert key in expiration time slot (now 3h)" 23 | StatusNew ReportStatus = "new" 24 | // StatusMore means there is an existing report for a key of the alert. 25 | StatusMore ReportStatus = "more" 26 | // StatusPublished means the report has all alert data and inspection results. 27 | StatusPublished ReportStatus = "published" 28 | ) 29 | 30 | // ReportSeverity has three statuses: "safe", "unclassified", "urgent". 31 | // - "safe": Reviewer determined the alert has no or minimal risk. 32 | // E.g. Win32 malware is detected in a host, but the host's OS is MacOS. 33 | // - "unclassified": Reviewer has no suitable policy or can not determine risk. 34 | // - "urgent": The alert has a big impact and a security operator must 35 | // respond it immediately. 36 | type ReportSeverity string 37 | 38 | const ( 39 | // SevSafe : Reviewer determined the alert has no or minimal risk. 40 | // E.g. Win32 malware is detected in a host, but the host's OS is MacOS. 41 | SevSafe ReportSeverity = "safe" 42 | // SevUnclassified : Reviewer has no suitable policy or can not determine risk. 43 | SevUnclassified ReportSeverity = "unclassified" 44 | // SevUrgent : The alert has a big impact and a security operator must respond it immediately. 45 | SevUrgent ReportSeverity = "urgent" 46 | ) 47 | 48 | // ReportContentType shows "user", "host" or "binary". It helps to parse 49 | // Content field in ReportContnet. 50 | type ReportContentType string 51 | 52 | // Report is a container to deliver contents and inspection results of the alert. 53 | type Report struct { 54 | ID ReportID `json:"id"` 55 | Alerts []*Alert `json:"alerts"` 56 | Attributes []*Attribute `json:"attributes"` 57 | Sections []*Section `json:"sections"` 58 | Result ReportResult `json:"result"` 59 | Status ReportStatus `json:"status"` 60 | CreatedAt time.Time `json:"created_at"` 61 | } 62 | 63 | // Section is set of Report content (user, host and binary) 64 | type Section struct { 65 | Attr Attribute `json:"attr"` 66 | Users []*ContentUser `json:"users,omitempty"` 67 | Hosts []*ContentHost `json:"hosts,omitempty"` 68 | Binaries []*ContentBinary `json:"binaries,omitempty"` 69 | } 70 | 71 | // Finding is a result of inspector. a Finding has one Content and metadata. 72 | type Finding struct { 73 | ReportID ReportID `json:"report_id"` 74 | Author string `json:"author"` 75 | Attribute Attribute `json:"attribute"` 76 | Type ReportContentType `json:"type"` 77 | Content interface{} `json:"content"` 78 | } 79 | 80 | const ( 81 | // ContentTypeUser means Content field is ContentUser. 82 | ContentTypeUser ReportContentType = "user" 83 | // ContentTypeHost means Content field is ContentHost. 84 | ContentTypeHost ReportContentType = "host" 85 | // ContentTypeBinary means Content field is ContentBinary. 86 | ContentTypeBinary ReportContentType = "binary" 87 | ) 88 | 89 | // ReportResult shows output of Reviewer invoked to evaluate risk of the alert. 90 | type ReportResult struct { 91 | Severity ReportSeverity `json:"severity"` 92 | Reason string `json:"reason"` 93 | } 94 | 95 | // IsNew returns status of the report 96 | func (x *Report) IsNew() bool { return x.Status == StatusNew } 97 | 98 | // IsMore returns status of the report 99 | func (x *Report) IsMore() bool { return x.Status == StatusMore } 100 | 101 | // IsPublished returns status of the report 102 | func (x *Report) IsPublished() bool { return x.Status == StatusPublished } 103 | 104 | // ----------------------------------------------- 105 | // Entities 106 | 107 | // ReportContent is interface of report entity. 108 | type ReportContent interface { 109 | Type() ReportContentType 110 | } 111 | 112 | // ContentUser describes a user indicator on remote services. 113 | type ContentUser struct { 114 | Activities []EntityActivity `json:"activities"` 115 | } 116 | 117 | // Type of ContentUser returns ContentTypeUser always 118 | func (x *ContentUser) Type() ReportContentType { 119 | return ContentTypeUser 120 | } 121 | 122 | // ContentBinary describes a binary file indicator including executable format. 123 | type ContentBinary struct { 124 | RelatedMalware []EntityMalware `json:"related_malware,omitempty"` 125 | Software []string `json:"software,omitempty"` 126 | OS []string `json:"os,omitempty"` 127 | Activities []EntityActivity `json:"activities,omitempty"` 128 | } 129 | 130 | // Type of ContentBinary returns ContentTypeBinary always 131 | func (x *ContentBinary) Type() ReportContentType { 132 | return ContentTypeBinary 133 | } 134 | 135 | // ContentHost describes a host indicator binding IP address, domain name 136 | type ContentHost struct { 137 | // Network related entities 138 | IPAddr []string `json:"ipaddr,omitempty"` 139 | Country []string `json:"country,omitempty"` 140 | ASOwner []string `json:"as_owner,omitempty"` 141 | RelatedDomains []EntityDomain `json:"related_domains,omitempty"` 142 | RelatedURLs []EntityURL `json:"related_urls,omitempty"` 143 | RelatedMalware []EntityMalware `json:"related_malware,omitempty"` 144 | Activities []EntityActivity `json:"activities,omitempty"` 145 | 146 | // Internal environment 147 | UserName []string `json:"username,omitempty"` 148 | Owner []string `json:"owner,omitempty"` 149 | OS []string `json:"os,omitempty"` 150 | MACAddr []string `json:"macaddr,omitempty"` 151 | HostName []string `json:"hostname,omitempty"` 152 | Software []EntitySoftware `json:"software,omitempty"` 153 | } 154 | 155 | // Type of ContentHost returns ContentTypeHost always 156 | func (x *ContentHost) Type() ReportContentType { 157 | return ContentTypeHost 158 | } 159 | 160 | // ----------------------------------------------- 161 | // Entity Objects 162 | 163 | // EntityActivity shows history of user/host activity such as accessing a cloud service. 164 | type EntityActivity struct { 165 | ServiceName string `json:"service_name"` 166 | RemoteAddr string `json:"remote_addr"` 167 | Principal string `json:"principal"` 168 | Action string `json:"action"` 169 | Target string `json:"target"` 170 | LastSeen time.Time `json:"last_seen"` 171 | } 172 | 173 | // EntityMalware shows set of malware scan result by AntiVirus software. 174 | type EntityMalware struct { 175 | SHA256 string `json:"sha256"` 176 | Timestamp time.Time `json:"timestamp"` 177 | Scans []EntityMalwareScan `json:"scans"` 178 | Relation string `json:"relation"` 179 | } 180 | 181 | // EntityMalwareScan shows a result of malware scan. 182 | type EntityMalwareScan struct { 183 | Vendor string `json:"vendor"` 184 | Name string `json:"name"` 185 | Positive bool `json:"positive"` 186 | Source string `json:"source"` 187 | } 188 | 189 | // EntityDomain shows a related domain to the host. 190 | type EntityDomain struct { 191 | Name string `json:"name"` 192 | Timestamp time.Time `json:"timestamp"` 193 | Source string `json:"source"` 194 | } 195 | 196 | // EntityURL shows a related URL to the host. 197 | type EntityURL struct { 198 | URL string `json:"url"` 199 | Reference string `json:"reference"` 200 | Timestamp time.Time `json:"timestamp"` 201 | Source string `json:"source"` 202 | } 203 | 204 | // EntitySoftware shows installed software to the host. 205 | type EntitySoftware struct { 206 | Name string `json:"name"` 207 | Location string `json:"location"` 208 | LastSeen time.Time `json:"last_seen"` 209 | } 210 | 211 | // ReportAttribute has attribute(S) that are found newly by inspector. 212 | type ReportAttribute struct { 213 | ReportID ReportID `json:"report_id"` 214 | Author string `json:"author"` 215 | OriginAttr Attribute `json:"origin_attr"` 216 | Attributes []*Attribute `json:"attributes"` 217 | } 218 | -------------------------------------------------------------------------------- /report_test.go: -------------------------------------------------------------------------------- 1 | package deepalert_test 2 | -------------------------------------------------------------------------------- /task.go: -------------------------------------------------------------------------------- 1 | package deepalert 2 | 3 | // Task is invoke argument of inspectors 4 | type Task struct { 5 | ReportID ReportID `json:"report_id"` 6 | Attribute *Attribute `json:"attribute"` 7 | } 8 | 9 | // TaskResult is generated by Inspector and can have both of contents and 10 | // new atributes because of efficiecy. 11 | type TaskResult struct { 12 | Contents []ReportContent 13 | NewAttributes []*Attribute 14 | } 15 | -------------------------------------------------------------------------------- /test/stack/deepalert.test.ts: -------------------------------------------------------------------------------- 1 | /* 2 | import { 3 | expect as expectCDK, 4 | matchTemplate, 5 | MatchStyle, 6 | } from "@aws-cdk/assert"; 7 | import * as cdk from "@aws-cdk/core"; 8 | import * as Deepalert from "../../cdk/deepalert-stack"; 9 | 10 | test("Empty Stack", () => { 11 | const app = new cdk.App(); 12 | // WHEN 13 | const stack = new Deepalert.DeepAlertStack(app, "MyTestStack", {}); 14 | // THEN 15 | expectCDK(stack).to( 16 | matchTemplate( 17 | { 18 | Resources: {}, 19 | }, 20 | MatchStyle.EXACT 21 | ) 22 | ); 23 | }); 24 | */ -------------------------------------------------------------------------------- /test/workflow/Makefile: -------------------------------------------------------------------------------- 1 | BASE_DIR := $(shell dirname $(realpath $(lastword $(MAKEFILE_LIST)))) 2 | 3 | COMMON=result.go 4 | ASSETS=build/inspector build/emitter 5 | CDKOUT=cdk.out/cdk.out 6 | DEEPALERT_CODES=../../*.go ../../internal/**/*.go ../../lambda/**/*.go ../../cdk/*.ts 7 | 8 | all: build 9 | 10 | build/inspector: inspector/* $(COMMON) 11 | env GOARCH=amd64 GOOS=linux go build -o ./build/inspector ./inspector 12 | build/emitter: emitter/* $(COMMON) 13 | env GOARCH=amd64 GOOS=linux go build -o ./build/emitter ./emitter 14 | 15 | # Base Tasks ------------------------------------- 16 | build: $(ASSETS) 17 | 18 | clean: 19 | rm $(ASSETS) 20 | 21 | $(CDKOUT): build $(DEEPALERT_CODES) $(ASSETS) 22 | cd ../.. && tsc && cd $(BASE_DIR) 23 | cdk deploy "*" 24 | 25 | deploy: $(DEEPALERT_CODES) $(ASSETS) 26 | cd ../.. && tsc && cd $(BASE_DIR) 27 | cdk deploy "*" 28 | 29 | test: $(CDKOUT) 30 | go test -count=1 -v . 31 | -------------------------------------------------------------------------------- /test/workflow/README.md: -------------------------------------------------------------------------------- 1 | # Workflow test (integration test) 2 | 3 | ## setup 4 | 5 | ```bash 6 | % export DEEPALERT_TEST_STACK_NAME="YourStackName" # Optional 7 | % export DEEPALERT_WORKFLOW_STACK_NAME="YourWorflowStackName" # Optional 8 | % npm run deploy 9 | ``` 10 | 11 | ## test 12 | 13 | ```bash 14 | % go test -count 1 15 | ``` 16 | -------------------------------------------------------------------------------- /test/workflow/bin/stack.ts: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | import "source-map-support/register"; 3 | import * as cdk from "@aws-cdk/core"; 4 | import * as dynamodb from "@aws-cdk/aws-dynamodb"; 5 | import * as lambda from "@aws-cdk/aws-lambda"; 6 | 7 | import { DeepAlertStack } from "../../../cdk/deepalert-stack"; 8 | import { SnsEventSource } from "@aws-cdk/aws-lambda-event-sources"; 9 | 10 | import 'process'; 11 | 12 | interface properties extends cdk.StackProps { 13 | readonly deepalert: DeepAlertStack; 14 | } 15 | 16 | export class WorkflowStack extends cdk.Stack { 17 | constructor(scope: cdk.Construct, id: string, props: properties) { 18 | super(scope, id, props); 19 | 20 | const table = new dynamodb.Table(this, "resultTable", { 21 | billingMode: dynamodb.BillingMode.PAY_PER_REQUEST, 22 | partitionKey: { name: "pk", type: dynamodb.AttributeType.STRING }, 23 | sortKey: { name: "sk", type: dynamodb.AttributeType.STRING }, 24 | }); 25 | 26 | const buildPath = lambda.Code.fromAsset("./build"); 27 | 28 | const testInspector = new lambda.Function(this, "testInspector", { 29 | runtime: lambda.Runtime.GO_1_X, 30 | handler: "inspector", 31 | timeout: cdk.Duration.seconds(30), 32 | code: buildPath, 33 | events: [new SnsEventSource(props.deepalert.taskTopic)], 34 | environment: { 35 | RESULT_TABLE: table.tableName, 36 | FINDING_QUEUE: props.deepalert.findingQueue.queueUrl, 37 | ATTRIBUTE_QUEUE: props.deepalert.attributeQueue.queueUrl, 38 | }, 39 | }); 40 | 41 | const testEmitter = new lambda.Function(this, "testEmitter", { 42 | runtime: lambda.Runtime.GO_1_X, 43 | handler: "emitter", 44 | timeout: cdk.Duration.seconds(30), 45 | code: buildPath, 46 | events: [new SnsEventSource(props.deepalert.reportTopic)], 47 | environment: { 48 | RESULT_TABLE: table.tableName, 49 | }, 50 | }); 51 | 52 | table.grantReadWriteData(testInspector); 53 | table.grantReadWriteData(testEmitter); 54 | props.deepalert.findingQueue.grantSendMessages(testInspector); 55 | props.deepalert.attributeQueue.grantSendMessages(testInspector); 56 | } 57 | } 58 | 59 | const deepalertStackName = 60 | process.env.DEEPALERT_TEST_STACK_NAME || "DeepAlertTestStack"; 61 | const workflowStackName = 62 | process.env.DEEPALERT_WORKFLOW_STACK_NAME || "DeepAlertTestWorkflowStack"; 63 | 64 | const app = new cdk.App(); 65 | const deepalert = new DeepAlertStack(app, deepalertStackName, { 66 | stackName: deepalertStackName, 67 | apiKeyPath: "./apikey.json", 68 | inspectDelay: cdk.Duration.seconds(1), 69 | reviewDelay: cdk.Duration.seconds(5), 70 | logLevel: "DEBUG", 71 | }); 72 | new WorkflowStack(app, workflowStackName, { 73 | deepalert: deepalert, 74 | stackName: workflowStackName, 75 | }); 76 | -------------------------------------------------------------------------------- /test/workflow/cdk.json: -------------------------------------------------------------------------------- 1 | { 2 | "app": "npx ts-node bin/stack.ts", 3 | "context": { 4 | "@aws-cdk/core:enableStackNameDuplicates": "true", 5 | "aws-cdk:enableDiffNoFail": "true" 6 | } 7 | } 8 | -------------------------------------------------------------------------------- /test/workflow/emitter/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "os" 5 | 6 | "github.com/deepalert/deepalert" 7 | "github.com/deepalert/deepalert/test/workflow" 8 | "github.com/m-mizutani/golambda" 9 | ) 10 | 11 | var logger = golambda.Logger 12 | 13 | func main() { 14 | golambda.Start(func(event golambda.Event) (interface{}, error) { 15 | messages, err := event.DecapSNSMessage() 16 | if err != nil { 17 | return nil, err 18 | } 19 | 20 | awsRegion := os.Getenv("AWS_REGION") 21 | tableName := os.Getenv("RESULT_TABLE") 22 | repo, err := workflow.NewRepository(awsRegion, tableName) 23 | if err != nil { 24 | return nil, golambda.WrapError(err).With("region", awsRegion).With("table", tableName) 25 | } 26 | 27 | for _, msg := range messages { 28 | var report deepalert.Report 29 | if err := msg.Bind(&report); err != nil { 30 | return nil, err 31 | } 32 | 33 | logger.With("report", report).Debug("Start emitter") 34 | if err := repo.PutEmitterResult(&report); err != nil { 35 | return nil, err 36 | } 37 | } 38 | 39 | return nil, nil 40 | }) 41 | } 42 | -------------------------------------------------------------------------------- /test/workflow/inspector/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "os" 6 | 7 | "github.com/aws/aws-lambda-go/events" 8 | "github.com/aws/aws-lambda-go/lambda" 9 | "github.com/deepalert/deepalert" 10 | "github.com/deepalert/deepalert/inspector" 11 | ) 12 | 13 | func dummyInspector(ctx context.Context, attr deepalert.Attribute) (*deepalert.TaskResult, error) { 14 | // tableName := os.Getenv("RESULT_TABLE") 15 | // reportID, _ := deepalert.ReportIDFromCtx(ctx) 16 | 17 | hostReport := deepalert.ContentHost{ 18 | IPAddr: []string{"198.51.100.2"}, 19 | Owner: []string{"superman"}, 20 | } 21 | 22 | newAttr := deepalert.Attribute{ 23 | Key: "username", 24 | Value: "mizutani", 25 | Type: deepalert.TypeUserName, 26 | } 27 | 28 | return &deepalert.TaskResult{ 29 | Contents: []deepalert.ReportContent{&hostReport}, 30 | NewAttributes: []*deepalert.Attribute{&newAttr}, 31 | }, nil 32 | } 33 | 34 | func main() { 35 | lambda.Start(func(ctx context.Context, event events.SNSEvent) error { 36 | tasks, err := inspector.SNSEventToTasks(event) 37 | if err != nil { 38 | return err 39 | } 40 | 41 | return inspector.Start(inspector.Arguments{ 42 | Context: ctx, 43 | Tasks: tasks, 44 | Handler: dummyInspector, 45 | Author: "dummyInspector", 46 | AttrQueueURL: os.Getenv("ATTRIBUTE_QUEUE"), 47 | FindingQueueURL: os.Getenv("FINDING_QUEUE"), 48 | }) 49 | }) 50 | } 51 | -------------------------------------------------------------------------------- /test/workflow/jest.config.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | roots: ['/test'], 3 | testMatch: ['**/*.test.ts'], 4 | transform: { 5 | '^.+\\.tsx?$': 'ts-jest' 6 | } 7 | }; 8 | -------------------------------------------------------------------------------- /test/workflow/main_test.go: -------------------------------------------------------------------------------- 1 | package workflow_test 2 | 3 | import ( 4 | "bytes" 5 | "encoding/json" 6 | "fmt" 7 | "io" 8 | "io/ioutil" 9 | "math" 10 | "net/http" 11 | "os" 12 | "path" 13 | "strings" 14 | "testing" 15 | "time" 16 | 17 | "github.com/aws/aws-sdk-go/aws" 18 | "github.com/aws/aws-sdk-go/aws/session" 19 | "github.com/aws/aws-sdk-go/service/cloudformation" 20 | "github.com/aws/aws-sdk-go/service/sqs" 21 | "github.com/deepalert/deepalert" 22 | "github.com/deepalert/deepalert/test/workflow" 23 | "github.com/google/uuid" 24 | "github.com/m-mizutani/golambda" 25 | "github.com/stretchr/testify/assert" 26 | "github.com/stretchr/testify/require" 27 | ) 28 | 29 | const ( 30 | apiKeyFile = "apikey.json" 31 | ) 32 | 33 | var ( 34 | logger = golambda.Logger 35 | mainStackName = "DeepAlertTestStack" 36 | testStackName = "DeepAlertTestWorkflowStack" 37 | ) 38 | 39 | func init() { 40 | if v, ok := os.LookupEnv("DEEPALERT_TEST_STACK_NAME"); ok { 41 | mainStackName = v 42 | } 43 | if v, ok := os.LookupEnv("DEEPALERT_WORKFLOW_STACK_NAME"); ok { 44 | testStackName = v 45 | } 46 | } 47 | 48 | func TestWorkflow(t *testing.T) { 49 | region := os.Getenv("AWS_REGION") 50 | if region == "" { 51 | t.Skip("AWS_REGION is not set") 52 | } 53 | client, err := newDAClient(region) 54 | require.NoError(t, err) 55 | 56 | repo, err := newRepository(region) 57 | require.NoError(t, err) 58 | 59 | testResults := func(t *testing.T, report *deepalert.Report) { 60 | require.NoError(t, expBackOff(10, func(_ uint) bool { 61 | respReport, err := client.Request("GET", fmt.Sprintf("report/%s", report.ID), nil) 62 | require.NoError(t, err) 63 | 64 | if respReport.StatusCode == 200 { 65 | var gotReport deepalert.Report 66 | require.NoError(t, unmarshal(respReport.Body, &gotReport)) 67 | assert.Equal(t, report.ID, gotReport.ID) 68 | return true 69 | } 70 | 71 | return false 72 | })) 73 | 74 | require.NoError(t, expBackOff(10, func(_ uint) bool { 75 | results, err := repo.GetEmitterResult(report.ID) 76 | require.NoError(t, err) 77 | 78 | if len(results) == 2 { 79 | for _, res := range results { 80 | var tmp deepalert.Report 81 | require.NoError(t, json.Unmarshal([]byte(res.Data), &tmp)) 82 | assert.Equal(t, report.ID, tmp.ID) 83 | t.Log(tmp) 84 | if tmp.Status == deepalert.StatusPublished { 85 | t.Log(tmp) 86 | return true 87 | } 88 | } 89 | } 90 | return false 91 | })) 92 | } 93 | 94 | t.Run("Sends an alert via API GW", func(t *testing.T) { 95 | alert := deepalert.Alert{ 96 | AlertKey: uuid.New().String(), 97 | RuleID: "five", 98 | Detector: "blue", 99 | Attributes: []deepalert.Attribute{ 100 | { 101 | Type: deepalert.TypeIPAddr, 102 | Value: "198.51.100.1", 103 | }, 104 | }, 105 | } 106 | 107 | resp, err := client.Request("POST", "alert", alert) 108 | require.NoError(t, err) 109 | 110 | var report deepalert.Report 111 | respRaw, err := ioutil.ReadAll(resp.Body) 112 | require.NoError(t, err) 113 | 114 | // t.Log(string(respRaw)) 115 | require.Equal(t, http.StatusOK, resp.StatusCode) 116 | 117 | require.NoError(t, json.Unmarshal(respRaw, &report)) 118 | assert.NotEmpty(t, report.ID) 119 | 120 | testResults(t, &report) 121 | }) 122 | 123 | t.Run("Sends an alert via SQS", func(t *testing.T) { 124 | ssn := session.Must(session.NewSession()) 125 | sqsClient := sqs.New(ssn, aws.NewConfig().WithRegion(region)) 126 | 127 | stacks, err := getStackResources(region, mainStackName) 128 | require.NoError(t, err) 129 | 130 | queues := stacks. 131 | filterByResourceType("AWS::SQS::Queue").filterByLogicalResourceId("alertQueue9836D344") 132 | require.Equal(t, 1, len(queues)) 133 | 134 | alert := deepalert.Alert{ 135 | AlertKey: uuid.New().String(), 136 | RuleID: "six", 137 | Detector: "blue", 138 | Attributes: []deepalert.Attribute{ 139 | { 140 | Type: deepalert.TypeIPAddr, 141 | Value: "198.51.100.1", 142 | }, 143 | }, 144 | } 145 | raw, err := json.Marshal(alert) 146 | require.NoError(t, err) 147 | 148 | input := &sqs.SendMessageInput{ 149 | QueueUrl: aws.String(*queues[0].PhysicalResourceId), 150 | MessageBody: aws.String(string(raw)), 151 | } 152 | 153 | _, err = sqsClient.SendMessage(input) 154 | require.NoError(t, err) 155 | 156 | var report deepalert.Report 157 | require.NoError(t, expBackOff(10, func(_ uint) bool { 158 | 159 | resp, err := client.Request("GET", "alert/"+alert.AlertID()+"/report", nil) 160 | require.NoError(t, err) 161 | 162 | respRaw, err := ioutil.ReadAll(resp.Body) 163 | require.NoError(t, err) 164 | 165 | if resp.StatusCode == http.StatusOK { 166 | t.Log("resp", string(respRaw)) 167 | require.NoError(t, json.Unmarshal(respRaw, &report)) 168 | assert.NotEmpty(t, report.ID) 169 | return true 170 | } 171 | return false 172 | })) 173 | 174 | testResults(t, &report) 175 | }) 176 | } 177 | 178 | func unmarshal(reader io.Reader, data interface{}) error { 179 | raw, err := ioutil.ReadAll(reader) 180 | if err != nil { 181 | return err 182 | } 183 | 184 | if err := json.Unmarshal(raw, data); err != nil { 185 | return err 186 | } 187 | return nil 188 | } 189 | 190 | type stackResources []*cloudformation.StackResource 191 | 192 | func (x stackResources) filterByResourceType(resourceType string) stackResources { 193 | var result stackResources 194 | for _, resource := range x { 195 | if aws.StringValue(resource.ResourceType) == resourceType { 196 | result = append(result, resource) 197 | } 198 | } 199 | return result 200 | } 201 | 202 | func (x stackResources) filterByLogicalResourceId(LogicalResourceId string) stackResources { 203 | var result stackResources 204 | for _, resource := range x { 205 | if strings.Contains(aws.StringValue(resource.LogicalResourceId), LogicalResourceId) { 206 | result = append(result, resource) 207 | } 208 | } 209 | return result 210 | } 211 | 212 | func getStackResources(region, stackName string) (stackResources, error) { 213 | ssn := session.Must(session.NewSession()) 214 | cfn := cloudformation.New(ssn, aws.NewConfig().WithRegion(region)) 215 | 216 | req := cloudformation.DescribeStackResourcesInput{ 217 | StackName: aws.String(stackName), 218 | } 219 | resp, err := cfn.DescribeStackResources(&req) 220 | if err != nil { 221 | return nil, err 222 | } 223 | 224 | return resp.StackResources, nil 225 | } 226 | 227 | type httpHeader map[string]string 228 | 229 | func loadAPIKey(path string) (httpHeader, error) { 230 | hdr := make(httpHeader) 231 | raw, err := ioutil.ReadFile(path) 232 | if err != nil { 233 | return nil, err 234 | } 235 | if err := json.Unmarshal(raw, &hdr); err != nil { 236 | return nil, err 237 | } 238 | 239 | return hdr, nil 240 | } 241 | 242 | type daClient struct { 243 | BaseURL string 244 | header map[string][]string 245 | } 246 | 247 | func newDAClient(region string) (*daClient, error) { 248 | stacks, err := getStackResources(region, mainStackName) 249 | if err != nil { 250 | return nil, err 251 | } 252 | 253 | restAPIs := stacks.filterByResourceType("AWS::ApiGateway::RestApi") 254 | if len(restAPIs) != 1 { 255 | return nil, fmt.Errorf("Invalid number of AWS::ApiGateway::RestApi") 256 | } 257 | 258 | restAPI := restAPIs[0] 259 | baseURL := fmt.Sprintf("https://%s.execute-api.%s.amazonaws.com/prod/api/v1/", aws.StringValue(restAPI.PhysicalResourceId), region) 260 | 261 | cwd, err := os.Getwd() 262 | if err != nil { 263 | return nil, err 264 | } 265 | apiKeyPath := path.Join(cwd, apiKeyFile) 266 | apiKey, err := loadAPIKey(apiKeyPath) 267 | if err != nil { 268 | return nil, err 269 | } 270 | 271 | client := &daClient{ 272 | BaseURL: baseURL, 273 | header: http.Header{ 274 | "X-API-KEY": []string{apiKey["X-API-KEY"]}, 275 | "content-type": []string{"application/json"}, 276 | }, 277 | } 278 | 279 | return client, nil 280 | } 281 | 282 | func (x *daClient) Request(method, path string, data interface{}) (*http.Response, error) { 283 | var reader io.Reader 284 | if data != nil { 285 | raw, err := json.Marshal(data) 286 | if err != nil { 287 | return nil, err 288 | } 289 | reader = bytes.NewReader(raw) 290 | } 291 | 292 | url := x.BaseURL + path 293 | req, err := http.NewRequest(method, url, reader) 294 | if err != nil { 295 | return nil, err 296 | } 297 | 298 | client := &http.Client{} 299 | for key, values := range x.header { 300 | for _, value := range values { 301 | req.Header.Add(key, value) 302 | } 303 | } 304 | 305 | return client.Do(req) 306 | } 307 | 308 | // Exponential backoff timer 309 | var errRetryMaxExceeded = fmt.Errorf("RetryMax is exceeded") 310 | 311 | func expBackOff(retryMax uint, callback func(count uint) bool) error { 312 | 313 | for i := uint(0); i < retryMax; i++ { 314 | logger.With("callback", callback).With("i", i).Trace("Callback") 315 | if exit := callback(i); exit { 316 | return nil 317 | } 318 | 319 | if i+1 < retryMax { 320 | wait := math.Pow(float64(i)/3.14, 2) 321 | if wait > 10 { 322 | wait = 10 323 | } 324 | time.Sleep(time.Duration(wait * float64(time.Second))) 325 | } 326 | } 327 | 328 | return errRetryMaxExceeded 329 | } 330 | 331 | func newRepository(region string) (*workflow.Repository, error) { 332 | stacks, err := getStackResources(region, testStackName) 333 | if err != nil { 334 | return nil, err 335 | } 336 | 337 | tables := stacks.filterByResourceType("AWS::DynamoDB::Table") 338 | if len(tables) != 1 { 339 | return nil, fmt.Errorf("Invalid number of AWS::DynamoDB::Table") 340 | } 341 | 342 | logger.With("table", *tables[0]).Debug("Test table") 343 | return workflow.NewRepository(region, *tables[0].PhysicalResourceId) 344 | } 345 | -------------------------------------------------------------------------------- /test/workflow/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "workflow", 3 | "version": "0.1.0", 4 | "bin": { 5 | "workflow": "bin/workflow.js" 6 | }, 7 | "scripts": { 8 | "build": "tsc", 9 | "watch": "tsc -w", 10 | "test": "jest", 11 | "deploy": "make; make -f ../../Makefile; cdk deploy '*'", 12 | "cdk": "cdk" 13 | }, 14 | "devDependencies": { 15 | "@types/node": "^10.17.39", 16 | "aws-cdk": "1.75.0", 17 | "ts-node": "^8.10.2", 18 | "typescript": "~3.7.2" 19 | }, 20 | "dependencies": { 21 | "@aws-cdk/aws-dynamodb": "1.75.0", 22 | "@aws-cdk/core": "1.75.0", 23 | "@aws-cdk/aws-lambda": "1.75.0", 24 | "@aws-cdk/aws-lambda-event-sources": "1.75.0", 25 | "source-map-support": "^0.5.16" 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /test/workflow/result.go: -------------------------------------------------------------------------------- 1 | package workflow 2 | 3 | import ( 4 | "encoding/json" 5 | "math" 6 | "time" 7 | 8 | "github.com/aws/aws-sdk-go/aws" 9 | "github.com/aws/aws-sdk-go/aws/session" 10 | "github.com/deepalert/deepalert" 11 | "github.com/google/uuid" 12 | "github.com/guregu/dynamo" 13 | "github.com/m-mizutani/golambda" 14 | ) 15 | 16 | // Repository is accessor to DynamoDB 17 | type Repository struct { 18 | table dynamo.Table 19 | } 20 | 21 | type baseResult struct { 22 | PKey string `dynamo:"pk"` 23 | SKey string `dynamo:"sk"` 24 | Data string `dynamo:"data"` 25 | } 26 | 27 | // NewRepository is constructor of repository to access DynamoDB 28 | func NewRepository(region, tableName string) (*Repository, error) { 29 | ssn, err := session.NewSession(&aws.Config{Region: aws.String(region)}) 30 | if err != nil { 31 | return nil, golambda.WrapError(err, "Failed session.NewSession") 32 | } 33 | 34 | db := dynamo.New(ssn, &aws.Config{Region: aws.String(region)}) 35 | 36 | return &Repository{ 37 | table: db.Table(tableName), 38 | }, nil 39 | } 40 | 41 | // EmitterResult is record of Emitter test 42 | type EmitterResult struct { 43 | baseResult 44 | Timestamp time.Time `dynamo:"timestamp"` 45 | } 46 | 47 | // PutEmitterResult puts EmitterResult to DynamoDB 48 | func (x *Repository) PutEmitterResult(report *deepalert.Report) error { 49 | raw, err := json.Marshal(report) 50 | if err != nil { 51 | return golambda.WrapError(err, "Failed to marshal report").With("report", report) 52 | } 53 | 54 | value := EmitterResult{ 55 | Timestamp: time.Now().UTC(), 56 | baseResult: baseResult{ 57 | PKey: "emitter/" + string(report.ID), 58 | SKey: uuid.New().String(), 59 | Data: string(raw), 60 | }, 61 | } 62 | 63 | if err := x.table.Put(value).Run(); err != nil { 64 | return golambda.WrapError(err, "Fail to put result").With("value", value) 65 | } 66 | 67 | return nil 68 | } 69 | 70 | // GetEmitterResult looks up EmitterResult from DynamoDB 71 | func (x *Repository) GetEmitterResult(reportID deepalert.ReportID) ([]*EmitterResult, error) { 72 | var values []*EmitterResult 73 | 74 | const maxRetry = 30 75 | start := time.Now() 76 | pk := "emitter/" + string(reportID) 77 | 78 | for i := 0; i < maxRetry; i++ { 79 | if err := x.table.Get("pk", pk).All(&values); err != nil { 80 | if err != dynamo.ErrNotFound { 81 | return nil, golambda.WrapError(err, "Fail to get result").With("pk", pk) 82 | } 83 | 84 | sleep := math.Pow(1.1, float64(i)) 85 | if sleep > 20 { 86 | sleep = 20 87 | } 88 | time.Sleep(time.Second * time.Duration(sleep)) 89 | } else { 90 | return values, nil 91 | } 92 | } 93 | 94 | end := time.Now() 95 | sec := end.Sub(start).Seconds() 96 | return nil, golambda.NewError("Timeout to get value").With("waited", sec).With("pk", pk) 97 | } 98 | -------------------------------------------------------------------------------- /test/workflow/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "target": "ES2018", 4 | "module": "commonjs", 5 | "lib": ["es2018"], 6 | "declaration": true, 7 | "strict": true, 8 | "noImplicitAny": true, 9 | "strictNullChecks": true, 10 | "noImplicitThis": true, 11 | "alwaysStrict": true, 12 | "noUnusedLocals": false, 13 | "noUnusedParameters": false, 14 | "noImplicitReturns": true, 15 | "noFallthroughCasesInSwitch": false, 16 | "inlineSourceMap": true, 17 | "inlineSources": true, 18 | "experimentalDecorators": true, 19 | "strictPropertyInitialization": false, 20 | "typeRoots": ["./node_modules/@types"] 21 | }, 22 | "exclude": ["cdk.out"] 23 | } 24 | -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "target": "ES2018", 4 | "module": "commonjs", 5 | "lib": ["es2018"], 6 | "declaration": true, 7 | "strict": true, 8 | "noImplicitAny": true, 9 | "strictNullChecks": true, 10 | "noImplicitThis": true, 11 | "alwaysStrict": true, 12 | "noUnusedLocals": false, 13 | "noUnusedParameters": false, 14 | "noImplicitReturns": true, 15 | "noFallthroughCasesInSwitch": false, 16 | "inlineSourceMap": true, 17 | "inlineSources": true, 18 | "experimentalDecorators": true, 19 | "strictPropertyInitialization": false, 20 | "typeRoots": ["./node_modules/@types"] 21 | }, 22 | "exclude": ["cdk.out"] 23 | } 24 | --------------------------------------------------------------------------------