├── .gitignore ├── Makefile ├── README.md ├── forwarder.go ├── forwarder_test.go ├── go.mod ├── go.sum ├── helper └── helper.go ├── magefile.go └── template.yml /.gitignore: -------------------------------------------------------------------------------- 1 | build 2 | config 3 | vendor 4 | test.json 5 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | TEMPLATE_FILE=template.yml 2 | OUTPUT_FILE=sam.yml 3 | FUNCTIONS=build/main 4 | 5 | build/helper: helper/*.go 6 | go build -o build/helper ./helper/ 7 | 8 | build/main: ./*.go 9 | env GOARCH=amd64 GOOS=linux go build -o build/main . 10 | 11 | clean: 12 | rm $(FUNCTIONS) 13 | 14 | test: 15 | go test -v ./lib/ 16 | 17 | sam.yml: $(TEMPLATE_FILE) $(FUNCTIONS) build/helper 18 | aws cloudformation package \ 19 | --region $(shell ./build/helper get Region) \ 20 | --template-file $(TEMPLATE_FILE) \ 21 | --s3-bucket $(shell ./build/helper get CodeS3Bucket) \ 22 | --s3-prefix $(shell ./build/helper get CodeS3Prefix) \ 23 | --output-template-file $(OUTPUT_FILE) 24 | 25 | deploy: $(OUTPUT_FILE) build/helper 26 | aws cloudformation deploy \ 27 | --region $(shell ./build/helper get Region) \ 28 | --template-file $(OUTPUT_FILE) \ 29 | --stack-name $(shell ./build/helper get StackName) \ 30 | --capabilities CAPABILITY_IAM $(shell ./build/helper mkparam) 31 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # aws-falcon-data-forwarder 2 | 3 | ## What is this 4 | 5 | This lambda function receives SQS message(s) from Data Replicator of CrowdStrike Falcon and transfer log files to your own S3 bucket. This service is deployed as AWS CloudFormation (CFn) stack with SAM technology. 6 | 7 | ## Architecture 8 | 9 | ![aws-falcon-data-forwarder-arch](https://user-images.githubusercontent.com/605953/43566627-0bc5ce66-966a-11e8-8e04-3c7a24b123b7.png) 10 | 11 | ## Prerequisite 12 | 13 | - Tools 14 | - go >= 1.11 15 | - aws-cli https://github.com/aws/aws-cli 16 | - Your AWS resources 17 | - AWS Credential for CLI (like `~/.aws/credentials` ) 18 | - S3 bucket for log data (e.g. `my-log-bucket` ) 19 | - S3 bucket for lambda function code (e.g. `my-function-code` ) 20 | - Secrets of Secrets Manager to store AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY for data replicator. 21 | - IAM role for Lambda function (e.g. `arn:aws:iam::1234567890:role/LambdaFalconDataForwarder`) 22 | - s3::PutObject for `my-log-bucket` 23 | - secretsmanager:GetSecretValue 24 | 25 | Make sure that you need CrowdStrike Falcon and Data Replicator service. 26 | 27 | ## Setup 28 | 29 | ### Setting up AWS Secrets Manager 30 | 31 | You need to put AWS API Key (AWS_ACCESS_KEY_ID) and Secret (AWS_SECRET_ACCESS_KEY) provided by CrowdStrike Falcon as secrets of Secrets Manager. Assuming AWS_ACCESS_KEY_ID is `ABCDEFG` and AWS_ACCESS_KEY_ID is `STUVWXYZ`. You can set up the secret by [AWS web console](https://ap-northeast-1.console.aws.amazon.com/secretsmanager). 32 | 33 | You need to create 2 items in the secret. 34 | 35 | - `falcon_aws_key`: set AWS_ACCESS_KEY_ID provided by CrowdStrike Falcon 36 | - `falcon_aws_secret`: set AWS_SECRET_ACCESS_KEY provided by CrowdStrike Falcon 37 | 38 | ### Configure 39 | 40 | Prepare a configuration file. (e.g. `myconfig.json` ) Please see a following sample. 41 | 42 | ```json 43 | { 44 | "StackName": "falcon-data-forwarder-staging", 45 | "Region": "ap-northeast-1", 46 | "CodeS3Bucket": "my-function-code", 47 | "CodeS3Prefix": "functions", 48 | 49 | "RoleArn": "arn:aws:iam::1234567890:role/LambdaFalconDataForwarder", 50 | "S3Bucket": "my-log-bucket", 51 | "S3Prefix": "logs/", 52 | "S3Region": "ap-northeast-1", 53 | "SqsURL": "https://us-west-1.queue.amazonaws.com/xxxxxxxxxxxxxx/some-queue-name", 54 | "SecretArn": "arn:aws:secretsmanager:ap-northeast-1:1234567890:secret:your-secret-name-4UqOs6" 55 | } 56 | ``` 57 | 58 | - Management 59 | - `StackName`: CloudFormation(CFn) stack name 60 | - `Region`: AWS region where you want to deploy the stack 61 | - `CodeS3Bucket`: S3 bucket name to save binary for lambda function 62 | - `CodeS3Prefix`: Prefix of S3 Key to save binary for lambda function 63 | - Parameters 64 | - `RoleArn`: IAM Role ARN for Lambda function 65 | - `S3Bucket`: S3 Bucket name to save log data 66 | - `S3Prefix`: Prefix of S3 Key to save log data 67 | - `S3Regio`: AWS region of `S3Bucket` 68 | - `SqsURL`: SQS URL provided by CrowdStrike Falcon 69 | - `SecretArn`: ARN of the secret that you store AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY 70 | 71 | ### Deploy 72 | 73 | ```bash 74 | $ env FORWARDER_CONFIG=myconfig.cfg make deploy 75 | ``` 76 | 77 | ## License 78 | 79 | MIT License 80 | 81 | -------------------------------------------------------------------------------- /forwarder.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "fmt" 7 | "os" 8 | "regexp" 9 | "strings" 10 | "time" 11 | 12 | "github.com/aws/aws-sdk-go/aws/credentials" 13 | 14 | "github.com/aws/aws-lambda-go/lambda" 15 | "github.com/aws/aws-sdk-go/aws" 16 | "github.com/aws/aws-sdk-go/aws/session" 17 | "github.com/aws/aws-sdk-go/service/s3" 18 | "github.com/aws/aws-sdk-go/service/s3/s3manager" 19 | "github.com/aws/aws-sdk-go/service/secretsmanager" 20 | "github.com/aws/aws-sdk-go/service/sqs" 21 | "github.com/pkg/errors" 22 | 23 | "github.com/sirupsen/logrus" 24 | ) 25 | 26 | var logger = logrus.New() 27 | 28 | func main() { 29 | logger.SetFormatter(&logrus.JSONFormatter{}) 30 | logger.SetLevel(logrus.InfoLevel) 31 | lambda.Start(handleRequest) 32 | } 33 | 34 | func handleRequest(ctx context.Context, event struct{}) error { 35 | args, err := BuildArgs() 36 | if err != nil { 37 | return err 38 | } 39 | 40 | return Handler(args) 41 | } 42 | 43 | type awsCredential struct { 44 | key string 45 | secret string 46 | } 47 | 48 | type S3Ptr struct { 49 | Region string 50 | Bucket string 51 | Key string 52 | credential *awsCredential 53 | } 54 | 55 | func Handler(args Args) error { 56 | forwardMessage := func(msg *FalconMessage) error { 57 | t := time.Unix(int64(msg.Timestamp/1000), 0) 58 | 59 | for _, f := range msg.Files { 60 | logger.WithField("f", f).Info("forwarding") 61 | 62 | src := S3Ptr{ 63 | Region: falconAwsRegion, 64 | Bucket: msg.Bucket, 65 | Key: f.Path, 66 | credential: &awsCredential{ 67 | key: args.FalconAwsKey, 68 | secret: args.FalconAwsSecret, 69 | }, 70 | } 71 | dst := S3Ptr{ 72 | Region: args.S3Region, 73 | Bucket: args.S3Bucket, 74 | Key: strings.Join([]string{ 75 | args.S3Prefix, 76 | t.Format("2006/01/02/15/"), 77 | f.Path, 78 | }, ""), 79 | } 80 | 81 | err := ForwardS3File(src, dst) 82 | if err != nil { 83 | return err 84 | } 85 | } 86 | return nil 87 | } 88 | 89 | err := ReceiveMessages(args.SqsURL, args.FalconAwsKey, args.FalconAwsSecret, 90 | forwardMessage) 91 | return err 92 | } 93 | 94 | var falconAwsRegion = "us-west-1" 95 | 96 | func getSecretValues(secretArn string, values interface{}) error { 97 | // sample: arn:aws:secretsmanager:ap-northeast-1:1234567890:secret:mytest 98 | arn := strings.Split(secretArn, ":") 99 | if len(arn) != 7 { 100 | return errors.New(fmt.Sprintf("Invalid SecretsManager ARN format: %s", secretArn)) 101 | } 102 | region := arn[3] 103 | 104 | ssn := session.Must(session.NewSession(&aws.Config{ 105 | Region: aws.String(region), 106 | })) 107 | mgr := secretsmanager.New(ssn) 108 | 109 | result, err := mgr.GetSecretValue(&secretsmanager.GetSecretValueInput{ 110 | SecretId: aws.String(secretArn), 111 | }) 112 | 113 | if err != nil { 114 | return errors.Wrap(err, "Fail to retrieve secret values") 115 | } 116 | 117 | err = json.Unmarshal([]byte(*result.SecretString), values) 118 | if err != nil { 119 | return errors.Wrap(err, "Fail to parse secret values as JSON") 120 | } 121 | 122 | return nil 123 | } 124 | 125 | // BuildArgs builds argument of receiver from environment variables. 126 | func BuildArgs() (Args, error) { 127 | args := Args{ 128 | S3Bucket: os.Getenv("S3_BUCKET"), 129 | S3Prefix: os.Getenv("S3_PREFIX"), 130 | S3Region: os.Getenv("S3_REGION"), 131 | SqsURL: os.Getenv("SQS_URL"), 132 | } 133 | 134 | err := getSecretValues(os.Getenv("SECRET_ARN"), &args) 135 | if err != nil { 136 | return args, errors.Wrap(err, "Fail to retrieve secret values") 137 | } 138 | 139 | return args, nil 140 | } 141 | 142 | type Args struct { 143 | S3Bucket string 144 | S3Prefix string 145 | S3Region string 146 | SqsURL string 147 | FalconAwsKey string `json:"falcon_aws_key"` 148 | FalconAwsSecret string `json:"falcon_aws_secret"` 149 | } 150 | 151 | type FalconMessage struct { 152 | CID string `json:"cid"` 153 | Timestamp uint `json:"timestamp"` 154 | FileCount int `json:"fileCount"` 155 | TotalSize int `json:"totalSize"` 156 | Bucket string `json:"bucket"` 157 | PathPrefix string `json:"pathPrefix"` 158 | Files []FalconLogFiles `json:"files"` 159 | } 160 | 161 | type FalconLogFiles struct { 162 | Path string `json:"path"` 163 | Size int `json:"size"` 164 | CheckSum string `json:"checksum"` 165 | } 166 | 167 | func sqsURLtoRegion(url string) (string, error) { 168 | urlPattern := []string{ 169 | // https://sqs.ap-northeast-1.amazonaws.com/21xxxxxxxxxxx/test-queue 170 | `https://sqs\.([a-z0-9\-]+)\.amazonaws\.com`, 171 | 172 | // https://us-west-1.queue.amazonaws.com/2xxxxxxxxxx/test-queue 173 | `https://([a-z0-9\-]+)\.queue\.amazonaws\.com`, 174 | } 175 | 176 | for _, ptn := range urlPattern { 177 | re := regexp.MustCompile(ptn) 178 | group := re.FindSubmatch([]byte(url)) 179 | if len(group) == 2 { 180 | return string(group[1]), nil 181 | } 182 | } 183 | 184 | return "", errors.New("unsupported SQS URL syntax") 185 | } 186 | 187 | // ReceiveMessages receives SQS message from Falcon side and invokes msgHandler per message. 188 | // In this method, not use channel because SQS queue deletion must be after handling messages 189 | // to keep idempotence. 190 | func ReceiveMessages(sqsURL, awsKey, awsSecret string, msgHandler func(msg *FalconMessage) error) error { 191 | 192 | sqsRegion, err := sqsURLtoRegion(sqsURL) 193 | if err != nil { 194 | return err 195 | } 196 | 197 | cfg := aws.Config{Region: aws.String(sqsRegion)} 198 | if awsKey != "" && awsSecret != "" { 199 | cfg.Credentials = credentials.NewStaticCredentials(awsKey, awsSecret, "") 200 | } else { 201 | logger.Warn("AWS Key and secret are not set, use role permission") 202 | } 203 | 204 | queue := sqs.New(session.Must(session.NewSession(&cfg))) 205 | 206 | for { 207 | result, err := queue.ReceiveMessage(&sqs.ReceiveMessageInput{ 208 | AttributeNames: []*string{ 209 | aws.String(sqs.MessageSystemAttributeNameSentTimestamp), 210 | }, 211 | MessageAttributeNames: []*string{ 212 | aws.String(sqs.QueueAttributeNameAll), 213 | }, 214 | QueueUrl: &sqsURL, 215 | MaxNumberOfMessages: aws.Int64(1), 216 | VisibilityTimeout: aws.Int64(36000), // 10 hours 217 | WaitTimeSeconds: aws.Int64(0), 218 | }) 219 | 220 | if err != nil { 221 | return errors.Wrap(err, "SQS recv error") 222 | } 223 | 224 | logger.WithField("result", result).Info("recv queue") 225 | 226 | if len(result.Messages) == 0 { 227 | break 228 | } 229 | 230 | for _, msg := range result.Messages { 231 | fmsg := FalconMessage{} 232 | err = json.Unmarshal([]byte(*msg.Body), &fmsg) 233 | if err != nil { 234 | return errors.Wrap(err, "Fail to parse Falcon SNS error") 235 | } 236 | 237 | if err = msgHandler(&fmsg); err != nil { 238 | return err 239 | } 240 | } 241 | 242 | _, err = queue.DeleteMessage(&sqs.DeleteMessageInput{ 243 | QueueUrl: &sqsURL, 244 | ReceiptHandle: result.Messages[0].ReceiptHandle, 245 | }) 246 | 247 | if err != nil { 248 | return errors.Wrap(err, "SQS queue delete error") 249 | } 250 | } 251 | 252 | return nil 253 | } 254 | 255 | func ForwardS3File(src, dst S3Ptr) error { 256 | cfg := aws.Config{Region: aws.String(src.Region)} 257 | if src.credential != nil { 258 | cfg.Credentials = credentials.NewStaticCredentials(src.credential.key, 259 | src.credential.secret, "") 260 | 261 | } else { 262 | logger.Warn("AWS Key and secret are not set, use role permission") 263 | } 264 | 265 | // Download 266 | downSrv := s3.New(session.Must(session.NewSession(&cfg))) 267 | getInput := &s3.GetObjectInput{ 268 | Bucket: aws.String(src.Bucket), 269 | Key: aws.String(src.Key), 270 | } 271 | 272 | getResult, err := downSrv.GetObject(getInput) 273 | if err != nil { 274 | return errors.Wrap(err, "Fail to download data from Falcon") 275 | } 276 | 277 | // Upload 278 | dstSsn := session.Must(session.NewSession(&aws.Config{ 279 | Region: aws.String(dst.Region), 280 | })) 281 | uploader := s3manager.NewUploader(dstSsn) 282 | _, err = uploader.Upload(&s3manager.UploadInput{ 283 | Bucket: aws.String(dst.Bucket), 284 | Key: aws.String(dst.Key), 285 | Body: getResult.Body, 286 | }) 287 | if err != nil { 288 | return errors.Wrap(err, "Fail to upload data to your bucket") 289 | } 290 | 291 | return nil 292 | } 293 | -------------------------------------------------------------------------------- /forwarder_test.go: -------------------------------------------------------------------------------- 1 | package main_test 2 | 3 | import ( 4 | "encoding/json" 5 | "io/ioutil" 6 | "log" 7 | "os" 8 | "path/filepath" 9 | "strings" 10 | "testing" 11 | 12 | uuid "github.com/satori/go.uuid" 13 | 14 | "github.com/aws/aws-sdk-go/aws" 15 | "github.com/aws/aws-sdk-go/aws/session" 16 | "github.com/aws/aws-sdk-go/service/s3" 17 | "github.com/aws/aws-sdk-go/service/s3/s3manager" 18 | "github.com/aws/aws-sdk-go/service/sqs" 19 | forwarder "github.com/m-mizutani/aws-falcon-data-forwarder" 20 | "github.com/stretchr/testify/assert" 21 | "github.com/stretchr/testify/require" 22 | ) 23 | 24 | type Config struct { 25 | S3Bucket string 26 | S3Prefix string 27 | S3Region string 28 | SqsURL string 29 | SecretArn string 30 | } 31 | 32 | func loadConfig() Config { 33 | cwd := os.Getenv("PWD") 34 | var fp *os.File 35 | var err error 36 | 37 | for cwd != "/" { 38 | cfgPath := filepath.Join(cwd, "test.json") 39 | 40 | cwd, _ = filepath.Split(strings.TrimRight(cwd, string(filepath.Separator))) 41 | 42 | fp, err = os.Open(cfgPath) 43 | if err == nil { 44 | break 45 | } 46 | } 47 | 48 | if fp == nil { 49 | log.Fatal("test.json is not found") 50 | } 51 | 52 | rawData, err := ioutil.ReadAll(fp) 53 | if err != nil { 54 | panic(err) 55 | } 56 | 57 | cfg := Config{} 58 | err = json.Unmarshal(rawData, &cfg) 59 | return cfg 60 | } 61 | 62 | func TestBuildConfig(t *testing.T) { 63 | // Mainly test to decrypt key 64 | cfg := loadConfig() 65 | os.Setenv("SECRET_ARN", cfg.SecretArn) 66 | defer os.Unsetenv("SECRET_ARN") 67 | 68 | _, err := forwarder.BuildArgs() 69 | 70 | assert.NoError(t, err) 71 | } 72 | 73 | func TestHandler(t *testing.T) { 74 | cfg := loadConfig() 75 | 76 | os.Setenv("SECRET_ARN", cfg.SecretArn) 77 | defer os.Unsetenv("SECRET_ARN") 78 | os.Setenv("SQS_URL", cfg.SqsURL) 79 | 80 | args, err := forwarder.BuildArgs() 81 | require.NoError(t, err) 82 | 83 | err = forwarder.Handler(args) 84 | require.NoError(t, err) 85 | } 86 | 87 | func TestReceiver(t *testing.T) { 88 | cfg := loadConfig() 89 | dataKey := "data/test_data.gz" 90 | 91 | sampleMessage := `{ 92 | "cid": "abcdefghijklmn0123456789", 93 | "timestamp": 1492726639137, 94 | "fileCount": 4, 95 | "totalSize": 349986220, 96 | "bucket": "` + cfg.S3Bucket + `", 97 | "pathPrefix": "` + cfg.S3Prefix + `", 98 | "files": [ 99 | { 100 | "path": "` + cfg.S3Prefix + dataKey + `", 101 | "size": 89118480, 102 | "checksum": "d0f566f37295e46f28c75f71ddce9422" 103 | } 104 | ] 105 | }` 106 | 107 | // Push test message. 108 | ssn := session.Must(session.NewSessionWithOptions(session.Options{ 109 | SharedConfigState: session.SharedConfigEnable, 110 | })) 111 | 112 | queue := sqs.New(ssn) 113 | _, err := queue.SendMessage(&sqs.SendMessageInput{ 114 | DelaySeconds: aws.Int64(0), 115 | MessageBody: aws.String(sampleMessage), 116 | QueueUrl: &cfg.SqsURL, 117 | }) 118 | 119 | require.NoError(t, err) 120 | os.Setenv("SECRET_ARN", cfg.SecretArn) 121 | defer os.Unsetenv("SECRET_ARN") 122 | 123 | args, err := forwarder.BuildArgs() 124 | require.NoError(t, err) 125 | 126 | msgCount := 0 127 | msgHandler := func(msg *forwarder.FalconMessage) error { 128 | msgCount++ 129 | assert.Equal(t, "abcdefghijklmn0123456789", msg.CID) 130 | assert.Equal(t, 1, len(msg.Files)) 131 | assert.Equal(t, cfg.S3Prefix+dataKey, msg.Files[0].Path) 132 | return nil 133 | } 134 | 135 | err = forwarder.ReceiveMessages(cfg.SqsURL, args.AwsKey, args.AwsSecret, msgHandler) 136 | require.NoError(t, err) 137 | assert.Equal(t, 1, msgCount) 138 | } 139 | 140 | func TestForwarder(t *testing.T) { 141 | cfg := loadConfig() 142 | 143 | os.Setenv("SECRET_ARN", cfg.SecretArn) 144 | defer os.Unsetenv("SECRET_ARN") 145 | 146 | ssn := session.Must(session.NewSessionWithOptions(session.Options{ 147 | SharedConfigState: session.SharedConfigEnable, 148 | })) 149 | uploader := s3manager.NewUploader(ssn) 150 | 151 | uniqID := uuid.NewV4().String() 152 | 153 | srcKey := cfg.S3Prefix + uniqID + "/src/data.txt" 154 | dstKey := cfg.S3Prefix + uniqID + "/dst/data.txt" 155 | // fmt.Println(srcKey) 156 | 157 | // Upload the file to S3. 158 | _, err := uploader.Upload(&s3manager.UploadInput{ 159 | Bucket: aws.String(cfg.S3Bucket), 160 | Key: aws.String(srcKey), 161 | Body: strings.NewReader("five timeless words"), 162 | }) 163 | require.NoError(t, err) 164 | 165 | // t, _ = time.Parse(time.RFC3339, "2006-01-02T15:04:05Z") 166 | 167 | src := forwarder.S3Ptr{ 168 | Region: cfg.S3Region, 169 | Bucket: cfg.S3Bucket, 170 | Key: srcKey, 171 | } 172 | 173 | dst := forwarder.S3Ptr{ 174 | Region: cfg.S3Region, 175 | Bucket: cfg.S3Bucket, 176 | Key: dstKey, 177 | } 178 | 179 | err = forwarder.ForwardS3File(src, dst) 180 | require.NoError(t, err) 181 | 182 | buf := aws.NewWriteAtBuffer([]byte{}) 183 | downloader := s3manager.NewDownloader(ssn) 184 | n, err := downloader.Download(buf, &s3.GetObjectInput{ 185 | Bucket: aws.String(cfg.S3Bucket), 186 | Key: aws.String(dstKey), 187 | }) 188 | 189 | assert.Equal(t, int64(19), n) 190 | assert.Equal(t, "five timeless words", string(buf.Bytes())) 191 | } 192 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/m-mizutani/aws-falcon-data-forwarder 2 | 3 | require ( 4 | github.com/aws/aws-lambda-go v1.8.2 5 | github.com/aws/aws-sdk-go v1.16.32 6 | github.com/davecgh/go-spew v1.1.1 7 | github.com/go-ini/ini v1.41.0 8 | github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af 9 | github.com/magefile/mage v1.8.0 10 | github.com/pkg/errors v0.8.1 11 | github.com/pmezard/go-difflib v1.0.0 12 | github.com/satori/go.uuid v1.2.0 13 | github.com/sirupsen/logrus v1.3.0 14 | github.com/stretchr/testify v1.3.0 15 | golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67 // indirect 16 | golang.org/x/sys v0.0.0-20190209173611-3b5209105503 // indirect 17 | ) 18 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/aws/aws-lambda-go v1.2.0 h1:2f0pbAKMNNhvOkjI9BCrwoeIiduSTlYpD0iKEN1neuQ= 2 | github.com/aws/aws-lambda-go v1.2.0/go.mod h1:zUsUQhAUjYzR8AuduJPCfhBuKWUaDbQiPOG+ouzmE1A= 3 | github.com/aws/aws-lambda-go v1.8.2 h1:wC8KcAG9HyVkFjbKQ9uhp87UGZutlPn9IJPq9fYM2BQ= 4 | github.com/aws/aws-lambda-go v1.8.2/go.mod h1:zUsUQhAUjYzR8AuduJPCfhBuKWUaDbQiPOG+ouzmE1A= 5 | github.com/aws/aws-sdk-go v1.15.2 h1:RH309yOKW4FwEHbRmTsY/bVP8RlldMxBvu3u74wnTVc= 6 | github.com/aws/aws-sdk-go v1.15.2/go.mod h1:mFuSZ37Z9YOHbQEwBWztmVzqXrEkub65tZoCYDt7FT0= 7 | github.com/aws/aws-sdk-go v1.16.32 h1:/grHp+bt3OAVWkdCQv2YtXkWuu58SuTlH1U8tp25n1c= 8 | github.com/aws/aws-sdk-go v1.16.32/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= 9 | github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= 10 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 11 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 12 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 13 | github.com/go-ini/ini v1.25.4/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= 14 | github.com/go-ini/ini v1.38.1 h1:hbtfM8emWUVo9GnXSloXYyFbXxZ+tG6sbepSStoe1FY= 15 | github.com/go-ini/ini v1.38.1/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= 16 | github.com/go-ini/ini v1.41.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= 17 | github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8 h1:12VvqtR6Aowv3l/EQUlocDHW2Cp4G9WJVH7uyH8QFJE= 18 | github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= 19 | github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5imkbgOkpRUYLnmbU7UEFbjtDA2hxJ1ichM= 20 | github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= 21 | github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= 22 | github.com/magefile/mage v0.0.0-20180411170307-771ebed3d686/go.mod h1:IUDi13rsHje59lecXokTfGX0QIzO45uVPlXnJYsXepA= 23 | github.com/magefile/mage v1.8.0/go.mod h1:IUDi13rsHje59lecXokTfGX0QIzO45uVPlXnJYsXepA= 24 | github.com/pkg/errors v0.8.0 h1:WdK/asTD0HN+q6hsWO3/vpuAkAr+tw6aNJNDFFf0+qw= 25 | github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 26 | github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= 27 | github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 28 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 29 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 30 | github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww= 31 | github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= 32 | github.com/sirupsen/logrus v1.3.0 h1:hI/7Q+DtNZ2kINb6qt/lS+IyXnHQe9e90POfeewL/ME= 33 | github.com/sirupsen/logrus v1.3.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= 34 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 35 | github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 36 | github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w= 37 | github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= 38 | github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= 39 | github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= 40 | golang.org/x/crypto v0.0.0-20180904163835-0709b304e793 h1:u+LnwYTOOW7Ukr/fppxEb1Nwz0AtPflrblfvUudpo+I= 41 | golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= 42 | golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67 h1:ng3VDlRp5/DHpSWl02R4rM9I+8M2rhmsuLwAMmkLQWE= 43 | golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= 44 | golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33 h1:I6FyU15t786LL7oL/hn43zqTuEGr4PN7F4XJ1p4E3Y8= 45 | golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 46 | golang.org/x/sys v0.0.0-20190209173611-3b5209105503 h1:5SvYFrOM3W8Mexn9/oA44Ji7vhXAZQ9hiP+1Q/DMrWg= 47 | golang.org/x/sys v0.0.0-20190209173611-3b5209105503/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 48 | -------------------------------------------------------------------------------- /helper/helper.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "io/ioutil" 7 | "log" 8 | "os" 9 | "strings" 10 | 11 | "github.com/sirupsen/logrus" 12 | ) 13 | 14 | var logger = logrus.New() 15 | 16 | type parameters struct { 17 | LambdaRoleArn string 18 | AlertNotifyTopic string 19 | DlqTopicName string 20 | } 21 | 22 | func appendParam(items []string, key string) []string { 23 | if v := getValue(key); v != "" { 24 | return append(items, fmt.Sprintf("%s=%s", key, v)) 25 | } 26 | 27 | return items 28 | } 29 | 30 | func getValue(key string) string { 31 | if val := os.Getenv(key); val != "" { 32 | return val 33 | } 34 | 35 | configFile := os.Getenv("FORWARDER_CONFIG") 36 | if configFile == "" { 37 | log.Fatal("missing environment variable FORWARDER_CONFIG") 38 | } 39 | 40 | fd, err := os.Open(configFile) 41 | if err != nil { 42 | logger.Fatal("Can not open FORWARDER_CONFIG: ", configFile, err) 43 | } 44 | defer fd.Close() 45 | 46 | raw, err := ioutil.ReadAll(fd) 47 | if err != nil { 48 | logger.Fatal("Fail to read FORWARDER_CONFIG", err) 49 | } 50 | 51 | var param map[string]string 52 | err = json.Unmarshal(raw, ¶m) 53 | if err != nil { 54 | logger.Fatal("Fail to unmarshal config json", err) 55 | } 56 | 57 | if val, ok := param[key]; ok { 58 | return val 59 | } 60 | 61 | return "" 62 | } 63 | 64 | func makeParameters() { 65 | parameterNames := []string{ 66 | "RoleArn", 67 | "S3Bucket", 68 | "S3Prefix", 69 | "S3Region", 70 | "SqsURL", 71 | "SecretArn", 72 | } 73 | 74 | var items []string 75 | for _, paramName := range parameterNames { 76 | items = appendParam(items, paramName) 77 | } 78 | 79 | if len(items) > 0 { 80 | fmt.Printf("--parameter-overrides %s", strings.Join(items, " ")) 81 | } 82 | } 83 | 84 | func main() { 85 | logger.SetLevel(logrus.InfoLevel) 86 | 87 | if len(os.Args) < 2 || 3 < len(os.Args) { 88 | logger.Fatalf("Usage) %s [mkparam|get ]", os.Args[0]) 89 | } 90 | 91 | switch os.Args[1] { 92 | case "mkparam": 93 | makeParameters() 94 | case "get": 95 | fmt.Print(getValue(os.Args[2])) 96 | } 97 | } 98 | -------------------------------------------------------------------------------- /magefile.go: -------------------------------------------------------------------------------- 1 | // +build mage 2 | 3 | package main 4 | 5 | import ( 6 | "bufio" 7 | "fmt" 8 | "io/ioutil" 9 | "log" 10 | "os" 11 | "os/exec" 12 | "path/filepath" 13 | "strings" 14 | "sync" 15 | 16 | "github.com/magefile/mage/mg" // mg contains helpful utility functions, like Deps 17 | "github.com/pkg/errors" 18 | ) 19 | 20 | type buildTarget struct { 21 | name string 22 | path string 23 | } 24 | 25 | var ( 26 | buildTargets = []buildTarget{ 27 | {name: "receiver", path: "./functions/receiver"}, 28 | } 29 | ) 30 | 31 | // Building binaries 32 | func Build() error { 33 | for _, target := range buildTargets { 34 | fmt.Println("Bulding ", target.name) 35 | cmd := exec.Command("go", "build", 36 | "-o", "build/"+target.name, target.path) 37 | cmd.Env = append(os.Environ(), "GOARCH=amd64", "GOOS=linux") 38 | 39 | err := cmd.Run() 40 | if err != nil { 41 | return err 42 | } 43 | } 44 | 45 | return nil 46 | } 47 | 48 | // Run test of each handler 49 | 50 | func doTest(path string) error { 51 | cmd := exec.Command("go", "test", path, "-v") 52 | out, err := cmd.CombinedOutput() 53 | fmt.Printf(string(out)) 54 | return err 55 | } 56 | 57 | func Test() error { 58 | fmt.Println("Testing...") 59 | 60 | for _, target := range buildTargets { 61 | err := doTest("./" + target.path) 62 | if err != nil { 63 | return err 64 | } 65 | } 66 | return nil 67 | } 68 | 69 | type config struct { 70 | StackName string 71 | CodeS3Bucket string 72 | CodeS3Prefix string 73 | CodeS3Region string 74 | Parameters []string 75 | } 76 | 77 | func loadConfigFile(fpath string) (config, error) { 78 | cfg := config{} 79 | cfg.Parameters = []string{} 80 | 81 | fp, err := os.Open(fpath) 82 | if err != nil { 83 | return cfg, err 84 | } 85 | defer fp.Close() 86 | 87 | scanner := bufio.NewScanner(fp) 88 | for scanner.Scan() { 89 | line := strings.TrimSpace(scanner.Text()) 90 | if len(line) == 0 { 91 | continue 92 | } 93 | 94 | idx := strings.Index(line, "=") 95 | if idx < 0 { 96 | log.Printf("Warning, invalid format of cfg file: '%s'\n", line) 97 | continue 98 | } 99 | 100 | key := line[:idx] 101 | value := line[(idx + 1):] 102 | 103 | switch key { 104 | case "StackName": 105 | cfg.StackName = value 106 | case "CodeS3Bucket": 107 | cfg.CodeS3Bucket = value 108 | case "CodeS3Prefix": 109 | cfg.CodeS3Prefix = value 110 | case "CodeS3Region": 111 | cfg.CodeS3Region = value 112 | default: 113 | cfg.Parameters = append(cfg.Parameters, line) 114 | } 115 | } 116 | 117 | return cfg, nil 118 | } 119 | 120 | func deployCFn(paramFile string) error { 121 | 122 | cfg, err := loadConfigFile(paramFile) 123 | if err != nil { 124 | return err 125 | } 126 | 127 | templateFile := "template.yml" 128 | 129 | var tmpPath string 130 | if tf, err := ioutil.TempFile("", "slam_template_"); err != nil { 131 | log.Fatal(err) 132 | } else { 133 | tmpPath = tf.Name() 134 | tf.Close() 135 | } 136 | 137 | log.Printf("[%s] Packaging...\n", paramFile) 138 | pkgCmd := exec.Command("aws", "cloudformation", "package", 139 | "--template-file", templateFile, 140 | "--s3-bucket", cfg.CodeS3Bucket, 141 | "--s3-prefix", cfg.CodeS3Prefix, 142 | "--output-template-file", tmpPath) 143 | 144 | pkgOut, err := pkgCmd.CombinedOutput() 145 | if err != nil { 146 | log.Printf("[%s] Error: %s, %s", paramFile, string(pkgOut), err) 147 | return err 148 | } 149 | log.Printf("[%s] Generated template file: %s\n", paramFile, tmpPath) 150 | 151 | // fmt.Printf("Package > %s", string(pkgOut)) 152 | log.Printf("[%s] Deploy...\n", paramFile) 153 | args := []string{ 154 | "--region", cfg.CodeS3Region, 155 | "cloudformation", "deploy", 156 | "--template-file", tmpPath, 157 | "--stack-name", cfg.StackName, 158 | "--capabilities", "CAPABILITY_IAM", 159 | "--parameter-overrides", 160 | } 161 | args = append(args, cfg.Parameters...) 162 | deployCmd := exec.Command("aws", args...) 163 | 164 | deployOut, err := deployCmd.CombinedOutput() 165 | if err != nil { 166 | log.Println("[%s] Error: %s, %s", paramFile, string(deployOut), err) 167 | return err 168 | } 169 | 170 | log.Printf("[%s] Done!", paramFile) 171 | 172 | return nil 173 | } 174 | 175 | // Deploying CloudFormation stack 176 | func Deploy() error { 177 | mg.Deps(Build) 178 | 179 | configFile := os.Getenv("PARAM_FILE") 180 | configDir := os.Getenv("PARAM_DIR") 181 | if configFile != "" { 182 | err := deployCFn(configFile) 183 | if err != nil { 184 | return err 185 | } 186 | } else if configDir != "" { 187 | files, err := ioutil.ReadDir(configDir) 188 | if err != nil { 189 | return errors.Wrap(err, "Fail to retrieve files in PARAM_DIR") 190 | } 191 | 192 | var wg sync.WaitGroup 193 | for _, finfo := range files { 194 | fpath := filepath.Join(configDir, finfo.Name()) 195 | if !strings.HasSuffix(fpath, ".cfg") || finfo.IsDir() { 196 | continue 197 | } 198 | 199 | wg.Add(1) 200 | go func(fname string) { 201 | defer wg.Done() 202 | err := deployCFn(fname) 203 | if err != nil { 204 | log.Printf("[%s] ERROR %s", fname, err) 205 | } 206 | }(fpath) 207 | } 208 | 209 | wg.Wait() 210 | 211 | } else { 212 | return errors.New("PARAM_FILE is not available. Set PARAM_FILE as environment variable.") 213 | } 214 | 215 | return nil 216 | } 217 | 218 | // Remove all built binaries 219 | func Clean() error { 220 | for _, target := range buildTargets { 221 | err := os.RemoveAll("config/build/" + target.name) 222 | if err != nil { 223 | return err 224 | } 225 | } 226 | return nil 227 | } 228 | -------------------------------------------------------------------------------- /template.yml: -------------------------------------------------------------------------------- 1 | AWSTemplateFormatVersion: '2010-09-09' 2 | Transform: AWS::Serverless-2016-10-31 3 | Description: '' 4 | Parameters: 5 | RoleArn: 6 | Type: String 7 | S3Bucket: 8 | Type: String 9 | S3Prefix: 10 | Type: String 11 | S3Region: 12 | Type: String 13 | SqsURL: 14 | Type: String 15 | SecretArn: 16 | Type: String 17 | 18 | Resources: 19 | SqsReceiver: 20 | Type: AWS::Serverless::Function 21 | Properties: 22 | CodeUri: build 23 | Handler: main 24 | Runtime: go1.x 25 | Timeout: 300 26 | MemorySize: 1024 27 | Role: 28 | Ref: RoleArn 29 | Environment: 30 | Variables: 31 | S3_BUCKET: 32 | Ref: S3Bucket 33 | S3_PREFIX: 34 | Ref: S3Prefix 35 | S3_REGION: 36 | Ref: S3Region 37 | SQS_URL: 38 | Ref: SqsURL 39 | SECRET_ARN: 40 | Ref: SecretArn 41 | Events: 42 | EveryMinutes: 43 | Type: Schedule 44 | Properties: 45 | Schedule: 'rate(1 minute)' 46 | --------------------------------------------------------------------------------