├── .circleci └── config.yml ├── .github └── workflows │ └── notify-ci-status.yml ├── .gitignore ├── Makefile ├── README.md ├── VERSION ├── cmd └── sfncli │ ├── cloudwatchreporter.go │ ├── cloudwatchreporter_test.go │ ├── ecs.go │ ├── error_names.go │ ├── runner.go │ ├── runner_test.go │ ├── sfncli.go │ ├── sfncli_test.go │ └── test_scripts │ ├── create_file.sh │ ├── echo_workdir.sh │ ├── log_to_stderr_and_wait.sh │ ├── signal_echo.sh │ ├── stderr_stdout_exitcode.sh │ ├── stderr_stdout_exitcode_onsigterm.sh │ ├── stderr_stdout_loopforever.sh │ ├── stdout_empty_output.sh │ └── stdout_parsing.sh ├── go.mod ├── go.sum ├── golang.mk ├── launch └── sfncli.yml ├── make ├── .gitignore ├── README.md ├── example.mk └── sfncli.mk └── mocks └── mocks.go /.circleci/config.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | jobs: 3 | build: 4 | working_directory: ~/go/src/github.com/Clever/sfncli 5 | docker: 6 | - image: cimg/go:1.24 7 | environment: 8 | GOPRIVATE: github.com/Clever/* 9 | CIRCLE_ARTIFACTS: /tmp/circleci-artifacts 10 | CIRCLE_TEST_REPORTS: /tmp/circleci-test-results 11 | steps: 12 | - run: 13 | command: cd $HOME && git clone --depth 1 -v https://github.com/Clever/ci-scripts.git && cd ci-scripts && git show --oneline -s 14 | name: Clone ci-scripts 15 | - checkout 16 | - setup_remote_docker 17 | - run: 18 | command: mkdir -p $CIRCLE_ARTIFACTS $CIRCLE_TEST_REPORTS 19 | name: Set up CircleCI artifacts directories 20 | - run: 21 | command: git config --global "url.ssh://git@github.com/Clever".insteadOf "https://github.com/Clever" 22 | - run: 23 | name: Add github.com to known hosts 24 | command: mkdir -p ~/.ssh && touch ~/.ssh/known_hosts && echo 'github.com ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAq2A7hRGmdnm9tUDbO9IDSwBK6TbQa+PXYPCPy6rbTrTtw7PHkccKrpp0yVhp5HdEIcKr6pLlVDBfOLX9QUsyCOV0wzfjIJNlGEYsdlLJizHhbn2mUjvSAHQqZETYP81eFzLQNnPHt4EVVUh7VfDESU84KezmD5QlWpXLmvU31/yMf+Se8xhHTvKSCZIFImWwoG6mbUoWf9nzpIoaSjB+weqqUUmpaaasXVal72J+UX2B+2RPW3RcT0eOzQgqlJL3RKrTJvdsjE3JEAvGq3lGHSZXy28G3skua2SmVi/w4yCE6gbODqnTWlg7+wC604ydGXA8VJiS5ap43JXiUFFAaQ==' >> ~/.ssh/known_hosts 25 | - run: make install_deps 26 | - run: make build 27 | - run: make test 28 | - run: if [ "${CIRCLE_BRANCH}" == "master" ]; then make release && $HOME/ci-scripts/circleci/github-release $GH_RELEASE_TOKEN release; fi; 29 | -------------------------------------------------------------------------------- /.github/workflows/notify-ci-status.yml: -------------------------------------------------------------------------------- 1 | name: Notify CI status 2 | 3 | on: 4 | check_suite: 5 | types: [completed] 6 | status: 7 | 8 | jobs: 9 | call-workflow: 10 | if: >- 11 | (github.event.branches[0].name == github.event.repository.default_branch && 12 | (github.event.state == 'error' || github.event.state == 'failure')) || 13 | (github.event.check_suite.head_branch == github.event.repository.default_branch && 14 | github.event.check_suite.conclusion != 'success') 15 | uses: Clever/ci-scripts/.github/workflows/reusable-notify-ci-status.yml@master 16 | secrets: 17 | CIRCLE_CI_INTEGRATIONS_URL: ${{ secrets.CIRCLE_CI_INTEGRATIONS_URL }} 18 | CIRCLE_CI_INTEGRATIONS_USERNAME: ${{ secrets.CIRCLE_CI_INTEGRATIONS_USERNAME }} 19 | CIRCLE_CI_INTEGRATIONS_PASSWORD: ${{ secrets.CIRCLE_CI_INTEGRATIONS_PASSWORD }} 20 | SLACK_BOT_TOKEN: ${{ secrets.DAPPLE_BOT_TOKEN }} 21 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # osx / sshfs 2 | ._* 3 | .DS_Store 4 | 5 | # emacs / vim 6 | *~ 7 | \#*\# 8 | .\#* 9 | 10 | vendor/ 11 | bin/ 12 | release/ 13 | mocks/mock_*.go 14 | cmd/sfncli/sfncli 15 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | include golang.mk 2 | .DEFAULT_GOAL := test # override default goal set in library makefile 3 | 4 | SHELL := /bin/bash 5 | PKGS := $(shell go list ./... | grep -v /vendor | grep -v /tools) 6 | VERSION := $(shell head -n 1 VERSION) 7 | EXECUTABLE := sfncli 8 | EXECUTABLE_PKG := github.com/Clever/sfncli/cmd/sfncli 9 | 10 | .PHONY: all test $(PKGS) build install_deps release clean mocks 11 | 12 | $(eval $(call golang-version-check,1.24)) 13 | 14 | all: test build release 15 | 16 | test: mocks $(PKGS) 17 | 18 | $(PKGS): golang-test-all-deps 19 | $(call golang-test-all,$@) 20 | 21 | build: 22 | mkdir -p build 23 | go build -ldflags="-X main.Version=$(VERSION)" -o bin/$(EXECUTABLE) $(EXECUTABLE_PKG) 24 | 25 | run: build 26 | ./bin/sfncli -activityname $$_DEPLOY_ENV--echo -region us-west-2 -workername `hostname` -cmd echo 27 | 28 | release: 29 | mkdir -p release 30 | GOOS=linux GOARCH=amd64 go build -ldflags="-s -w -X main.Version=$(VERSION)" \ 31 | -o="$@/$(EXECUTABLE)-$(VERSION)-linux-amd64" $(EXECUTABLE_PKG) 32 | GOOS=darwin GOARCH=amd64 go build -ldflags="-s -w -X main.Version=$(VERSION)" \ 33 | -o="$@/$(EXECUTABLE)-$(VERSION)-darwin-amd64" $(EXECUTABLE_PKG) 34 | 35 | clean: 36 | rm -rf bin release 37 | 38 | mocks: 39 | mkdir -p bin 40 | go build -o bin/mockgen -mod=vendor ./vendor/github.com/golang/mock/mockgen 41 | rm -rf mocks/mock_*.go 42 | ./bin/mockgen -source ./vendor/github.com/aws/aws-sdk-go/service/sfn/sfniface/interface.go -destination mocks/mock_sfn.go -package mocks 43 | ./bin/mockgen -source ./vendor/github.com/aws/aws-sdk-go/service/cloudwatch/cloudwatchiface/interface.go -destination mocks/mock_cloudwatch.go -package mocks 44 | 45 | install_deps: 46 | go mod vendor 47 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # sfncli 2 | 3 | Utility to create AWS Step Function (SFN) activities out of command line programs. 4 | 5 | ## Usage 6 | 7 | ``` 8 | $ sfncli -h 9 | Usage of sfncli: 10 | -activityname string 11 | The activity name to register with AWS Step Functions. $VAR and ${VAR} env variables are expanded. 12 | -cmd string 13 | The command to run to process activity tasks. 14 | -region string 15 | The AWS region to send Step Function API calls. Defaults to AWS_REGION. 16 | -cloudwatchregion string 17 | The AWS region to send metric data. Defaults to the value of region. 18 | -version 19 | Print the version and exit. 20 | -workername string 21 | The worker name to send to AWS Step Functions when processing a task. Environment variables are expanded. The magic string MAGIC_ECS_TASK_ARN will be expanded to the ECS task ARN via the metadata service. 22 | -workdirectory string 23 | A directory path that is passed to the `cmd` using an env var `WORK_DIR`. For each activity task a new directory is created in `workdirectory` and it is cleaned up after the activity task exits. Defaults to "", does not create directory or set `WORK_DIR` 24 | ``` 25 | 26 | Example: 27 | 28 | ``` 29 | sfncli -activityname sleep-100 -region us-west-2 --cloudwatchregion us-west-1 -workername sleep-worker -cmd sleep 100 30 | ``` 31 | 32 | ## High-level logic 33 | 34 | - On startup, call [`CreateActivity`](http://docs.aws.amazon.com/step-functions/latest/apireference/API_CreateActivity.html) to register an [Activity](http://docs.aws.amazon.com/step-functions/latest/dg/concepts-activities.html) with Step Functions. 35 | - Begin polling [`GetActivityTask`](http://docs.aws.amazon.com/step-functions/latest/apireference/API_GetActivityTask.html) for tasks. 36 | - Get a task. Take the JSON input for the task and 37 | - if it's a JSON object, use this as the last arg to the `cmd` passed to `sfncli`. 38 | - if it's anything else (e.g. JSON array), an error is thrown. 39 | - if `_EXECUTION_NAME` is missing from the payload, an error is thrown 40 | - the `_EXECUTION_NAME` payload attribute value is added to the environment of the `cmd` as `_EXECUTION_NAME`. 41 | - if workdirectory is set, create a sub-directory and add it to the environment of the `cmd` as `WORK_DIR`. 42 | - Start [`SendTaskHeartbeat`](http://docs.aws.amazon.com/step-functions/latest/apireference/API_SendTaskHeartbeat.html) loop. 43 | - When the command exits: 44 | - Call [`SendTaskFailure`](http://docs.aws.amazon.com/step-functions/latest/apireference/API_SendTaskFailure.html) if it exited nonzero, was killed, or `sfncli` received SIGTERM. 45 | - Call [`SendTaskSuccess`](http://docs.aws.amazon.com/step-functions/latest/apireference/API_SendTaskSuccess.html) otherwise. 46 | Parse the last line of the `stdout` of the command as the output for the task (it [must be JSON](https://states-language.net/spec.html#data)). 47 | - If `workdirectory` was set then cleanup `WORK_DIR`/sub-directory-for-task 48 | 49 | ## Errors 50 | 51 | [Error names](https://states-language.net/spec.html#error-names) in SFN state machines are useful for debugging and setting up branching/retry logic in state machine definitions. 52 | `sfncli` will report the following error names if it encounters errors it can identify: 53 | 54 | - `sfncli.TaskInputNotJSON`: input to the task was not JSON 55 | - `sfncli.TaskFailureTaskInputMissingExecutionName`: input is missing `_EXECUTION_NAME` attribute 56 | - `sfncli.CommandNotFound`: the command passed to `sfncli` was not found 57 | - `sfncli.CommandKilled`: the command process received SIGKILL 58 | - `sfncli.CommandExitedNonzero`: the command process exited with a nonzero exit code 59 | - `sfncli.TaskOutputNotJSON`: the task output (last line of command's `stdout`) was not JSON 60 | - `sfncli.CommandTerminated`: `sfncli` or the command received SIGTERM 61 | - `sfncli.Unknown`: unexpected / unclassified errors 62 | 63 | The command should signal an error by exiting with a nonzero status code. In this case, the behavior is: 64 | 1. If the last line of *stdout* was a JSON-formatted string with an `error` field, report an error to Step Functions with that field as the name and the value of the `cause` field in the output line as the cause. 65 | 2. Otherwise, report an error with name `sfncli.CommandExitedNonzero` with the last line of *stderr* as the cause. 66 | 67 | ## Local testing 68 | 69 | Start up a test activity that runs `echo` on the work it receives. 70 | 71 | ``` 72 | go run ./cmd/sfncli -region us-west-2 -activityname test-activity -workername sfncli-test -cmd echo 73 | ``` 74 | 75 | Create a new state machine that uses this activity for one of its states (this requires you to [create a role for use with Step Functions](http://docs.aws.amazon.com/step-functions/latest/dg/procedure-create-iam-role.html)): 76 | 77 | ``` 78 | aws --region us-west-2 stepfunctions create-state-machine --name test-state-machine --role-arn arn:aws:iam::589690932525:role/raf-test-step-functions --definition '{ 79 | "Comment": "Testing out step functions", 80 | "StartAt": "foo", 81 | "Version": "1.0", 82 | "TimeoutSeconds": 60, 83 | "States": { 84 | "foo": { 85 | "Resource": "arn:aws:states:us-west-2:589690932525:activity:test-activity", 86 | "Type": "Task", 87 | "End": true 88 | } 89 | } 90 | }' 91 | ``` 92 | 93 | Note that you will need to replace the `Resource` above to reflect the correct ARN with your AWS account ID. 94 | 95 | Start an execution of the state machine (again replacing the ARN below with the correct account ID): 96 | 97 | ``` 98 | aws --region us-west-2 stepfunctions start-execution --state-machine-arn arn:aws:states:us-west-2:589690932525:stateMachine:test-state-machine --input '{"_EXECUTION_NAME":"en", "hello": "world"}' 99 | ``` 100 | 101 | You should see `echo` run with the argument `{"hello": "world"}`. 102 | -------------------------------------------------------------------------------- /VERSION: -------------------------------------------------------------------------------- 1 | v0.8.0 2 | Go 1.21 update 3 | -------------------------------------------------------------------------------- /cmd/sfncli/cloudwatchreporter.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "sync" 6 | "time" 7 | 8 | "github.com/aws/aws-sdk-go/aws" 9 | "github.com/aws/aws-sdk-go/service/cloudwatch" 10 | "github.com/aws/aws-sdk-go/service/cloudwatch/cloudwatchiface" 11 | "gopkg.in/Clever/kayvee-go.v6/logger" 12 | ) 13 | 14 | const metricNameActivityActivePercent = "ActivityActivePercent" 15 | const namespaceStatesCustom = "StatesCustom" 16 | 17 | // CloudWatchReporter reports useful metrics about the activity. 18 | type CloudWatchReporter struct { 19 | cwapi cloudwatchiface.CloudWatchAPI 20 | activityArn string 21 | 22 | // state to keep track of active percent 23 | // the mutex is here to control access by two goroutines, e.g. 24 | // 1. the goroutine for `ReportActivePercent` 25 | // 2. the goroutine that calls ActiveUntilContextDone 26 | mu sync.Mutex 27 | activeState bool 28 | activeTime time.Duration 29 | lastReportingTime time.Time 30 | lastActiveStateChange time.Time 31 | pausedState bool 32 | pausedTime time.Duration 33 | lastPausedStateChange time.Time 34 | } 35 | 36 | func NewCloudWatchReporter(cwapi cloudwatchiface.CloudWatchAPI, activityArn string) *CloudWatchReporter { 37 | now := time.Now() 38 | c := &CloudWatchReporter{ 39 | cwapi: cwapi, 40 | activityArn: activityArn, 41 | 42 | activeState: false, 43 | activeTime: time.Duration(0), 44 | lastReportingTime: now, 45 | lastActiveStateChange: now, 46 | } 47 | return c 48 | } 49 | 50 | // ReportActivePercent sets up a loop that will report active percent to cloudwatch on an interval. 51 | // It stops when the context is canceled. 52 | func (c *CloudWatchReporter) ReportActivePercent(ctx context.Context, interval time.Duration) { 53 | ticker := time.NewTicker(interval) 54 | defer ticker.Stop() 55 | for ctx.Err() == nil { 56 | select { 57 | case <-ctx.Done(): 58 | break 59 | case <-ticker.C: 60 | c.report() 61 | } 62 | } 63 | } 64 | 65 | // ActiveUntilContextDone sets active state to true, and sets it false when the context is done. 66 | func (c *CloudWatchReporter) ActiveUntilContextDone(ctx context.Context) { 67 | c.SetActiveState(true) 68 | <-ctx.Done() 69 | c.SetActiveState(false) 70 | } 71 | 72 | // SetPausedState records amount of time activity is paused from working on a task 73 | func (c *CloudWatchReporter) SetPausedState(paused bool) { 74 | c.mu.Lock() 75 | defer c.mu.Unlock() 76 | if paused == c.pausedState { 77 | return 78 | } 79 | now := time.Now() 80 | if c.pausedState { 81 | c.pausedTime += now.Sub(maxTime(c.lastReportingTime, c.lastPausedStateChange)) 82 | } 83 | c.pausedState = paused 84 | c.lastPausedStateChange = now 85 | } 86 | 87 | // SetActiveState sets whether the activity is currently working on a task or not. 88 | func (c *CloudWatchReporter) SetActiveState(active bool) { 89 | c.mu.Lock() 90 | defer c.mu.Unlock() 91 | if active == c.activeState { 92 | return 93 | } 94 | now := time.Now() 95 | // going from active to inactive, so record incremental active time 96 | if c.activeState { 97 | c.activeTime += now.Sub(maxTime(c.lastReportingTime, c.lastActiveStateChange)) 98 | } 99 | c.activeState = active 100 | c.lastActiveStateChange = now 101 | } 102 | 103 | // maxTime returns the maximum between two times 104 | func maxTime(a, b time.Time) time.Time { 105 | if a.After(b) { 106 | return a 107 | } 108 | return b 109 | } 110 | 111 | // report computes and sends the active time metric to cloudwatch, resetting state related to tracking active time. 112 | func (c *CloudWatchReporter) report() { 113 | c.mu.Lock() 114 | defer c.mu.Unlock() 115 | now := time.Now() 116 | // going from active to inactive, so record incremental active time 117 | if c.activeState { 118 | c.activeTime += now.Sub(maxTime(c.lastReportingTime, c.lastActiveStateChange)) 119 | } 120 | // record incremental paused time 121 | if c.pausedState { 122 | c.pausedTime += now.Sub(maxTime(c.lastReportingTime, c.lastPausedStateChange)) 123 | } 124 | var activePercent float64 125 | totalTime := now.Sub(c.lastReportingTime) 126 | // don't divide by 0 127 | if c.pausedTime == totalTime { 128 | activePercent = 100.0 129 | } else { 130 | activePercent = 100.0 * float64(c.activeTime) / float64(totalTime-c.pausedTime) 131 | } 132 | c.lastReportingTime = now 133 | c.activeTime = time.Duration(0) 134 | c.pausedTime = time.Duration(0) 135 | // fire and forget the metric 136 | go c.putMetricData(activePercent) 137 | } 138 | 139 | func (c *CloudWatchReporter) putMetricData(activePercent float64) { 140 | log.TraceD("put-metric-data", logger.M{"activity-arn": c.activityArn, "metric-name": metricNameActivityActivePercent, "value": activePercent}) 141 | if _, err := c.cwapi.PutMetricData(&cloudwatch.PutMetricDataInput{ 142 | MetricData: []*cloudwatch.MetricDatum{{ 143 | Dimensions: []*cloudwatch.Dimension{{ 144 | Name: aws.String("ActivityArn"), 145 | Value: aws.String(c.activityArn), 146 | }}, 147 | MetricName: aws.String(metricNameActivityActivePercent), 148 | Unit: aws.String(cloudwatch.StandardUnitPercent), 149 | Value: aws.Float64(activePercent), 150 | }}, 151 | Namespace: aws.String(namespaceStatesCustom), 152 | }); err != nil { 153 | log.ErrorD("put-metric-data", logger.M{"error": err.Error()}) 154 | } 155 | } 156 | -------------------------------------------------------------------------------- /cmd/sfncli/cloudwatchreporter_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "math" 7 | "reflect" 8 | "testing" 9 | "time" 10 | 11 | "github.com/Clever/sfncli/mocks" 12 | "github.com/aws/aws-sdk-go/aws" 13 | "github.com/aws/aws-sdk-go/service/cloudwatch" 14 | "github.com/golang/mock/gomock" 15 | ) 16 | 17 | const mockActivityArn = "mockActivityArn" 18 | 19 | func TestCloudWatchReporterReportsActiveZero(t *testing.T) { 20 | testCtx, testCtxCancel := context.WithCancel(context.Background()) 21 | defer testCtxCancel() 22 | controller := gomock.NewController(t) 23 | defer controller.Finish() 24 | mockCW := mocks.NewMockCloudWatchAPI(controller) 25 | cwr := NewCloudWatchReporter(mockCW, mockActivityArn) 26 | go cwr.ReportActivePercent(testCtx, 100*time.Millisecond) 27 | mockCW.EXPECT().PutMetricData(&cloudwatch.PutMetricDataInput{ 28 | MetricData: []*cloudwatch.MetricDatum{{ 29 | Dimensions: []*cloudwatch.Dimension{{ 30 | Name: aws.String("ActivityArn"), 31 | Value: aws.String(mockActivityArn), 32 | }}, 33 | MetricName: aws.String(metricNameActivityActivePercent), 34 | Unit: aws.String(cloudwatch.StandardUnitPercent), 35 | Value: aws.Float64(0.0), 36 | }}, 37 | Namespace: aws.String(namespaceStatesCustom), 38 | }) 39 | time.Sleep(100*time.Millisecond + 10*time.Millisecond) 40 | } 41 | 42 | func TestCloudWatchReporterReportsActiveFiftyPercent(t *testing.T) { 43 | testCtx, testCtxCancel := context.WithCancel(context.Background()) 44 | defer testCtxCancel() 45 | controller := gomock.NewController(t) 46 | defer controller.Finish() 47 | mockCW := mocks.NewMockCloudWatchAPI(controller) 48 | mockCW.EXPECT().PutMetricData(fuzzy(&cloudwatch.PutMetricDataInput{ 49 | MetricData: []*cloudwatch.MetricDatum{{ 50 | Dimensions: []*cloudwatch.Dimension{{ 51 | Name: aws.String("ActivityArn"), 52 | Value: aws.String(mockActivityArn), 53 | }}, 54 | MetricName: aws.String(metricNameActivityActivePercent), 55 | Unit: aws.String(cloudwatch.StandardUnitPercent), 56 | Value: aws.Float64(50.0), 57 | }}, 58 | Namespace: aws.String(namespaceStatesCustom), 59 | })).Times(2) 60 | cwr := NewCloudWatchReporter(mockCW, mockActivityArn) 61 | go cwr.ReportActivePercent(testCtx, 1*time.Second) 62 | go func() { 63 | // active for 500 ms in first second and second second 64 | time.Sleep(500 * time.Millisecond) 65 | cwr.SetActiveState(true) 66 | time.Sleep(1 * time.Second) 67 | cwr.SetActiveState(false) 68 | }() 69 | // check after 2 seconds, should be 50% active on both intervals 70 | time.Sleep(2*time.Second + 100*time.Millisecond) 71 | } 72 | 73 | func TestCloudWatchReporterReportsActiveHundredPercent(t *testing.T) { 74 | testCtx, testCtxCancel := context.WithCancel(context.Background()) 75 | defer testCtxCancel() 76 | controller := gomock.NewController(t) 77 | defer controller.Finish() 78 | mockCW := mocks.NewMockCloudWatchAPI(controller) 79 | mockCW.EXPECT().PutMetricData(fuzzy(&cloudwatch.PutMetricDataInput{ 80 | MetricData: []*cloudwatch.MetricDatum{{ 81 | Dimensions: []*cloudwatch.Dimension{{ 82 | Name: aws.String("ActivityArn"), 83 | Value: aws.String(mockActivityArn), 84 | }}, 85 | MetricName: aws.String(metricNameActivityActivePercent), 86 | Unit: aws.String(cloudwatch.StandardUnitPercent), 87 | Value: aws.Float64(100.0), 88 | }}, 89 | Namespace: aws.String(namespaceStatesCustom), 90 | })).Times(2) 91 | cwr := NewCloudWatchReporter(mockCW, mockActivityArn) 92 | go cwr.ReportActivePercent(testCtx, 1*time.Second) 93 | go cwr.ActiveUntilContextDone(testCtx) 94 | time.Sleep(2*time.Second + 100*time.Millisecond) 95 | } 96 | 97 | func TestCloudWatchReporterReportsActiveOneHundredPercentWhenPausedForever(t *testing.T) { 98 | testCtx, testCtxCancel := context.WithCancel(context.Background()) 99 | defer testCtxCancel() 100 | controller := gomock.NewController(t) 101 | defer controller.Finish() 102 | mockCW := mocks.NewMockCloudWatchAPI(controller) 103 | mockCW.EXPECT().PutMetricData(fuzzy(&cloudwatch.PutMetricDataInput{ 104 | MetricData: []*cloudwatch.MetricDatum{{ 105 | Dimensions: []*cloudwatch.Dimension{{ 106 | Name: aws.String("ActivityArn"), 107 | Value: aws.String(mockActivityArn), 108 | }}, 109 | MetricName: aws.String(metricNameActivityActivePercent), 110 | Unit: aws.String(cloudwatch.StandardUnitPercent), 111 | Value: aws.Float64(100.0), 112 | }}, 113 | Namespace: aws.String(namespaceStatesCustom), 114 | })).Times(2) 115 | cwr := NewCloudWatchReporter(mockCW, mockActivityArn) 116 | go cwr.ReportActivePercent(testCtx, 1*time.Second) 117 | go func() { 118 | // active for 500 ms in first second and second second 119 | time.Sleep(500 * time.Millisecond) 120 | cwr.SetActiveState(true) 121 | time.Sleep(1 * time.Second) 122 | cwr.SetActiveState(false) 123 | }() 124 | go func() { 125 | // pause for 500 ms in first second and second second 126 | time.Sleep(500 * time.Millisecond) 127 | cwr.SetPausedState(true) 128 | time.Sleep(1 * time.Second) 129 | cwr.SetPausedState(false) 130 | }() 131 | // check after 2 seconds, should be 50% active on both intervals 132 | time.Sleep(2*time.Second + 100*time.Millisecond) 133 | } 134 | 135 | func TestCloudWatchReporterReportsActiveOnehundredPercentWhenPaused(t *testing.T) { 136 | testCtx, testCtxCancel := context.WithCancel(context.Background()) 137 | defer testCtxCancel() 138 | controller := gomock.NewController(t) 139 | defer controller.Finish() 140 | mockCW := mocks.NewMockCloudWatchAPI(controller) 141 | mockCW.EXPECT().PutMetricData(fuzzy(&cloudwatch.PutMetricDataInput{ 142 | MetricData: []*cloudwatch.MetricDatum{{ 143 | Dimensions: []*cloudwatch.Dimension{{ 144 | Name: aws.String("ActivityArn"), 145 | Value: aws.String(mockActivityArn), 146 | }}, 147 | MetricName: aws.String(metricNameActivityActivePercent), 148 | Unit: aws.String(cloudwatch.StandardUnitPercent), 149 | Value: aws.Float64(100.0), 150 | }}, 151 | Namespace: aws.String(namespaceStatesCustom), 152 | })).Times(2) 153 | cwr := NewCloudWatchReporter(mockCW, mockActivityArn) 154 | go cwr.ReportActivePercent(testCtx, 1*time.Second) 155 | go func() { 156 | // active for 500 ms in first second and second second 157 | time.Sleep(500 * time.Millisecond) 158 | cwr.SetActiveState(true) 159 | time.Sleep(1 * time.Second) 160 | cwr.SetActiveState(false) 161 | }() 162 | go func() { 163 | // pause after 500 ms indefinitely 164 | time.Sleep(500 * time.Millisecond) 165 | cwr.SetPausedState(true) 166 | }() 167 | // check after 2 seconds, should be 50% active on both intervals 168 | time.Sleep(2*time.Second + 100*time.Millisecond) 169 | } 170 | 171 | func TestCloudWatchReporterReportsActiveFiftyPercentWhenPaused(t *testing.T) { 172 | testCtx, testCtxCancel := context.WithCancel(context.Background()) 173 | defer testCtxCancel() 174 | controller := gomock.NewController(t) 175 | defer controller.Finish() 176 | mockCW := mocks.NewMockCloudWatchAPI(controller) 177 | mockCW.EXPECT().PutMetricData(fuzzy(&cloudwatch.PutMetricDataInput{ 178 | MetricData: []*cloudwatch.MetricDatum{{ 179 | Dimensions: []*cloudwatch.Dimension{{ 180 | Name: aws.String("ActivityArn"), 181 | Value: aws.String(mockActivityArn), 182 | }}, 183 | MetricName: aws.String(metricNameActivityActivePercent), 184 | Unit: aws.String(cloudwatch.StandardUnitPercent), 185 | Value: aws.Float64(50.0), 186 | }}, 187 | Namespace: aws.String(namespaceStatesCustom), 188 | })).Times(2) 189 | cwr := NewCloudWatchReporter(mockCW, mockActivityArn) 190 | go cwr.ReportActivePercent(testCtx, 1*time.Second) 191 | go func() { 192 | // active for 250 ms in first second and second second 193 | time.Sleep(750 * time.Millisecond) 194 | cwr.SetActiveState(true) 195 | time.Sleep(500 * time.Millisecond) 196 | cwr.SetActiveState(false) 197 | }() 198 | go func() { 199 | // paused for 500 ms in first second and second second 200 | time.Sleep(500 * time.Millisecond) 201 | cwr.SetPausedState(true) 202 | time.Sleep(1 * time.Second) 203 | cwr.SetPausedState(false) 204 | }() 205 | // check after 2 seconds, should be 50% active on both intervals 206 | time.Sleep(2*time.Second + 100*time.Millisecond) 207 | } 208 | 209 | // fuzzyMatcher is a gomock.Matcher that does a fuzzy match on cloudwatch putmetricdata values 210 | type fuzzyMatcher struct { 211 | expected *cloudwatch.PutMetricDataInput 212 | } 213 | 214 | func fuzzy(expected *cloudwatch.PutMetricDataInput) gomock.Matcher { 215 | return fuzzyMatcher{expected} 216 | } 217 | 218 | func (f fuzzyMatcher) Matches(x interface{}) bool { 219 | got, ok := x.(*cloudwatch.PutMetricDataInput) 220 | if !ok { 221 | return false 222 | } 223 | epsilon := 2.00 // within 2 percent is fine 224 | if len(f.expected.MetricData) != len(got.MetricData) { 225 | return reflect.DeepEqual(f.expected, got) 226 | } 227 | for i, md := range f.expected.MetricData { 228 | if math.Abs(aws.Float64Value(md.Value)-aws.Float64Value(got.MetricData[i].Value)) > epsilon { 229 | return false 230 | } 231 | // so that deepequal succeeds, make values match exactly if they're within epsilon 232 | f.expected.MetricData[i].Value = got.MetricData[i].Value 233 | } 234 | return reflect.DeepEqual(f.expected, x) 235 | } 236 | 237 | func (f fuzzyMatcher) String() string { 238 | return fmt.Sprintf("is equal to %v", f.expected) 239 | } 240 | -------------------------------------------------------------------------------- /cmd/sfncli/ecs.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "encoding/json" 5 | "errors" 6 | "fmt" 7 | "io/ioutil" 8 | "net/http" 9 | "os" 10 | "strings" 11 | "time" 12 | ) 13 | 14 | const magicECSTaskARN = "MAGIC_ECS_TASK_ARN" 15 | const magicECSTaskID = "MAGIC_ECS_TASK_ID" 16 | 17 | // TODO https://clever.atlassian.net/browse/INFRANG-4174. Update URI env variable 18 | // these are env vars the AWS ECS agent sets for us depending on ECS agent version or Fargate platform version 19 | const ecsContainerMetadataUriEnvVar = "ECS_CONTAINER_METADATA_URI" 20 | const ecsContainerMetadataFileEnvVar = "ECS_CONTAINER_METADATA_FILE" 21 | 22 | // ecsContainerMetadata is a subset of fields in the container metadata file. Doc for reference: 23 | // https://docs.aws.amazon.com/AmazonECS/latest/developerguide/container-metadata.html#metadata-file-format 24 | type ecsContainerMetadata struct { 25 | TaskARN string 26 | MetadataFileStatus string 27 | } 28 | 29 | // ecsTaskMetadata is a subset of fields in the task metadata JSON response. Doc for reference: 30 | // https://docs.aws.amazon.com/AmazonECS/latest/developerguide/task-metadata-endpoint.html 31 | type ecsTaskMetadata struct { 32 | TaskARN string 33 | } 34 | 35 | // expandECSMagicStrings uses ECS Container Metadata to magically populate the TaskARN or ID required to 36 | // register with AWS Step Functions. 37 | func expandECSMagicStrings(s string) (string, error) { 38 | if !strings.Contains(s, magicECSTaskARN) && !strings.Contains(s, magicECSTaskID) { 39 | return s, nil 40 | } 41 | 42 | arn, err := lookupARN() 43 | if err != nil { 44 | return "", err 45 | } 46 | 47 | arnParts := strings.Split(arn, "/") 48 | if len(arnParts) == 0 { 49 | return "", fmt.Errorf("task ARN did not contain '/'. ARN: %s", arn) 50 | } 51 | taskID := arnParts[len(arnParts)-1] 52 | 53 | s = strings.Replace(s, magicECSTaskARN, arn, 1) 54 | return strings.Replace(s, magicECSTaskID, taskID, 1), nil 55 | } 56 | 57 | // lookupARN attempts to lookup the ARN of the running task, preferring to use the metadata endpoint, or failing that, checking the metadata file 58 | func lookupARN() (string, error) { 59 | arn, errURI := arnFromMetadataURI() 60 | if errURI == nil { 61 | return arn, nil 62 | } 63 | arn, errFile := arnFromMetadataFile() 64 | if errFile == nil { 65 | return arn, nil 66 | } 67 | return "", errors.New(errURI.Error() + errFile.Error()) 68 | } 69 | 70 | func arnFromMetadataURI() (string, error) { 71 | uri, ok := os.LookupEnv(ecsContainerMetadataUriEnvVar) 72 | if !ok { 73 | return "", fmt.Errorf("%s not set", ecsContainerMetadataUriEnvVar) 74 | } 75 | resp, err := http.Get(uri + "/task") 76 | if err != nil { 77 | return "", fmt.Errorf("getting task metadata URI: %v", err) 78 | } 79 | var metadata ecsTaskMetadata 80 | defer resp.Body.Close() 81 | body, err := ioutil.ReadAll(resp.Body) 82 | if err != nil { 83 | return "", fmt.Errorf("reading response from metadata endpoint: %v", err) 84 | } 85 | if err := json.Unmarshal(body, &metadata); err != nil { 86 | return "", fmt.Errorf("unmarshaling response from metadata endpoint: %v\nresponse was %s", err, string(body)) 87 | } 88 | return metadata.TaskARN, nil 89 | } 90 | 91 | func arnFromMetadataFile() (string, error) { 92 | filePath, ok := os.LookupEnv(ecsContainerMetadataFileEnvVar) 93 | if !ok { 94 | return "", fmt.Errorf("%s not set", ecsContainerMetadataFileEnvVar) 95 | } 96 | // wait for the file to exist 97 | ticker := time.NewTicker(1 * time.Second) 98 | defer ticker.Stop() 99 | for { 100 | <-ticker.C 101 | if _, err := os.Stat(filePath); err == nil { 102 | break 103 | } 104 | } 105 | 106 | // wait until the file data has the TaskARN 107 | var metadata ecsContainerMetadata 108 | for { 109 | b, err := ioutil.ReadFile(filePath) 110 | if err != nil { 111 | return "", err 112 | } 113 | if err := json.Unmarshal(b, &metadata); err != nil { 114 | return "", err 115 | } 116 | if metadata.TaskARN != "" || metadata.MetadataFileStatus == "READY" { 117 | break 118 | } 119 | <-ticker.C 120 | } 121 | 122 | return metadata.TaskARN, nil 123 | } 124 | -------------------------------------------------------------------------------- /cmd/sfncli/error_names.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "strings" 6 | 7 | "github.com/aws/aws-sdk-go/aws" 8 | "github.com/aws/aws-sdk-go/service/sfn" 9 | "gopkg.in/Clever/kayvee-go.v6/logger" 10 | ) 11 | 12 | // States language has the concept of "Error Names"--unique strings that correspond 13 | // to specific error conditions under which a state can fail: 14 | // https://states-language.net/spec.html#error-names 15 | // SFNCLI has its own set of error names that it will report when failing a task. 16 | // These errors are described in this file. 17 | 18 | // TaskFailureError is the error reported when failing an activity task. 19 | type TaskFailureError interface { 20 | ErrorName() string 21 | ErrorCause() string 22 | 23 | error 24 | } 25 | 26 | // sendTaskFailure handles sending AWS `SendTaskFailure`. 27 | func (t TaskRunner) sendTaskFailure(err TaskFailureError) error { 28 | t.logger.ErrorD("send-task-failure", logger.M{"name": err.ErrorName(), "cause": err.ErrorCause()}) 29 | 30 | // Limits from https://docs.aws.amazon.com/step-functions/latest/apireference/API_SendTaskFailure.html 31 | const maxErrorLength = 256 32 | const maxCauseLength = 32768 33 | 34 | // don't use SendTaskFailureWithContext, since the failure itself could be from the parent 35 | // context being cancelled, but we still want to report to AWS the failure of the task. 36 | _, sendErr := t.sfnapi.SendTaskFailure(&sfn.SendTaskFailureInput{ 37 | Error: aws.String(truncateString(err.ErrorName(), maxErrorLength, "[truncated]")), 38 | Cause: aws.String(truncateString(err.ErrorCause(), maxCauseLength, "[truncated]")), 39 | TaskToken: &t.taskToken, 40 | }) 41 | if sendErr != nil { 42 | t.logger.ErrorD("send-task-failure-error", logger.M{"error": sendErr.Error()}) 43 | } 44 | return err 45 | } 46 | 47 | // Returns its input truncated to maxLength, with the ability to replace the end to indicate truncation. 48 | // 49 | // For example, truncateString(s, l, "") just truncates to length l. But truncateString(s, l, "xy") will 50 | // first truncate to length l, then replace the last two characters with "xy" 51 | func truncateString(s string, maxLength int, truncationIndicatorSuffix string) string { 52 | if len(s) <= maxLength { 53 | return s 54 | } 55 | // when we cut out some number of bytes from the end, we may be cutting in the middle of a multi-byte unicode char 56 | // if so, we can use ToValidUTF8 to trim it a teeny bit further to eliminate the whole char. 57 | // (Note, this does mean invalid UTF8 inputs will see more changes than expected, but we won't worry about that) 58 | return strings.ToValidUTF8(s[:maxLength-len(truncationIndicatorSuffix)], "") + truncationIndicatorSuffix 59 | } 60 | 61 | // TaskFailureUnknown is used for any error that is unexpected or not understood completely. 62 | type TaskFailureUnknown struct { 63 | error 64 | } 65 | 66 | func (t TaskFailureUnknown) ErrorName() string { return "sfncli.Unknown" } 67 | func (t TaskFailureUnknown) ErrorCause() string { return t.Error() } 68 | 69 | // TaskFailureTaskInputNotJSON is used when the input to the task is not a JSON object. 70 | type TaskFailureTaskInputNotJSON struct { 71 | input string 72 | } 73 | 74 | func (t TaskFailureTaskInputNotJSON) ErrorName() string { return "sfncli.TaskInputNotJSON" } 75 | func (t TaskFailureTaskInputNotJSON) ErrorCause() string { 76 | return fmt.Sprintf("task input not valid JSON: '%s'", t.input) 77 | } 78 | func (t TaskFailureTaskInputNotJSON) Error() string { return t.ErrorCause() } 79 | 80 | // TaskFailureTaskInputMissingExecutionName is used when the input to the task is not a JSON object. 81 | type TaskFailureTaskInputMissingExecutionName struct { 82 | input string 83 | } 84 | 85 | func (t TaskFailureTaskInputMissingExecutionName) ErrorName() string { 86 | return "sfncli.TaskInputMissingExecutionName" 87 | } 88 | func (t TaskFailureTaskInputMissingExecutionName) ErrorCause() string { 89 | return fmt.Sprintf("task input missing _EXECUTION_NAME attribute: '%s'", t.input) 90 | } 91 | func (t TaskFailureTaskInputMissingExecutionName) Error() string { return t.ErrorCause() } 92 | 93 | // TaskFailureCommandNotFound is used when the command passed to sfncli is not found. 94 | type TaskFailureCommandNotFound struct { 95 | path string 96 | } 97 | 98 | func (t TaskFailureCommandNotFound) ErrorName() string { return "sfncli.CommandNotFound" } 99 | func (t TaskFailureCommandNotFound) ErrorCause() string { 100 | return fmt.Sprintf("command not found: '%s'", t.path) 101 | } 102 | func (t TaskFailureCommandNotFound) Error() string { return t.ErrorCause() } 103 | 104 | // TaskFailureCommandKilled happens when the command is sent a kill signal by the OS. 105 | type TaskFailureCommandKilled struct { 106 | stderr string 107 | } 108 | 109 | func (t TaskFailureCommandKilled) ErrorName() string { return "sfncli.CommandKilled" } 110 | func (t TaskFailureCommandKilled) ErrorCause() string { return t.stderr } 111 | func (t TaskFailureCommandKilled) Error() string { 112 | return fmt.Sprintf("%s: %s", t.ErrorName(), t.ErrorCause()) 113 | } 114 | 115 | // TaskFailureCommandKilled happens when the command exits with a nonzero exit code and doesn't specifiy its own error name in the output. 116 | type TaskFailureCommandExitedNonzero struct { 117 | stderr string 118 | } 119 | 120 | func (t TaskFailureCommandExitedNonzero) ErrorName() string { return "sfncli.CommandExitedNonzero" } 121 | func (t TaskFailureCommandExitedNonzero) ErrorCause() string { return t.stderr } 122 | func (t TaskFailureCommandExitedNonzero) Error() string { 123 | return fmt.Sprintf("%s: %s", t.ErrorName(), t.ErrorCause()) 124 | } 125 | 126 | // TaskFailureCustom happens when the command exits with a nonzero exit code and outputs a custom error name to stdout. 127 | type TaskFailureCustom struct { 128 | Err string `json:"error"` 129 | Cause string `json:"cause"` 130 | } 131 | 132 | func (t TaskFailureCustom) ErrorName() string { return t.Err } 133 | func (t TaskFailureCustom) ErrorCause() string { return t.Cause } 134 | func (t TaskFailureCustom) Error() string { 135 | return fmt.Sprintf("%s: %s", t.ErrorName(), t.ErrorCause()) 136 | } 137 | 138 | // TaskFailureTaskOutputNotJSON is used when the output of the task is not a JSON object. 139 | type TaskFailureTaskOutputNotJSON struct { 140 | output string 141 | } 142 | 143 | func (t TaskFailureTaskOutputNotJSON) ErrorName() string { return "sfncli.TaskOutputNotJSON" } 144 | func (t TaskFailureTaskOutputNotJSON) ErrorCause() string { 145 | return fmt.Sprintf("stdout not valid JSON: '%s'", t.output) 146 | } 147 | func (t TaskFailureTaskOutputNotJSON) Error() string { return t.ErrorCause() } 148 | 149 | // TaskFailureCommandKilled happens when sfncli receives SIGTERM. 150 | type TaskFailureCommandTerminated struct { 151 | stderr string 152 | } 153 | 154 | func (t TaskFailureCommandTerminated) ErrorName() string { return "sfncli.CommandTerminated" } 155 | func (t TaskFailureCommandTerminated) ErrorCause() string { return t.stderr } 156 | func (t TaskFailureCommandTerminated) Error() string { 157 | return fmt.Sprintf("%s: %s", t.ErrorName(), t.ErrorCause()) 158 | } 159 | -------------------------------------------------------------------------------- /cmd/sfncli/runner.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "errors" 7 | "fmt" 8 | "io" 9 | "io/ioutil" 10 | "os" 11 | "os/exec" 12 | "os/signal" 13 | "strings" 14 | "syscall" 15 | "time" 16 | 17 | "github.com/armon/circbuf" 18 | "github.com/aws/aws-sdk-go/aws" 19 | "github.com/aws/aws-sdk-go/service/sfn" 20 | "github.com/aws/aws-sdk-go/service/sfn/sfniface" 21 | "gopkg.in/Clever/kayvee-go.v6/logger" 22 | ) 23 | 24 | // stay within documented limits of SFN APIs 25 | const ( 26 | maxTaskOutputLength = 32768 27 | maxTaskFailureCauseLength = 32768 28 | ) 29 | 30 | // TaskRunner manages resources for executing a task 31 | type TaskRunner struct { 32 | sfnapi sfniface.SFNAPI 33 | taskToken string 34 | cmd string 35 | logger logger.KayveeLogger 36 | execCmd *exec.Cmd 37 | receivedSigterm bool 38 | sigtermGracePeriod time.Duration 39 | workDirectory string 40 | ctxCancel context.CancelFunc 41 | } 42 | 43 | // NewTaskRunner instantiates a new TaskRunner 44 | func NewTaskRunner(cmd string, sfnapi sfniface.SFNAPI, taskToken string, workDirectory string) TaskRunner { 45 | return TaskRunner{ 46 | sfnapi: sfnapi, 47 | taskToken: taskToken, 48 | cmd: cmd, 49 | logger: logger.New("sfncli"), 50 | workDirectory: workDirectory, 51 | // set the default grace period to something slightly lower than the default 52 | // docker stop grace period in ECS (30s) 53 | sigtermGracePeriod: 25 * time.Second, 54 | } 55 | } 56 | 57 | // Process runs the underlying command. 58 | // The command inherits the environment of the parent process. 59 | // Any signals sent to parent process will be forwarded to the command. 60 | // If the context is canceled, the command is killed. 61 | func (t *TaskRunner) Process(ctx context.Context, args []string, input string) error { 62 | if t.sfnapi == nil { // if New failed :-/ 63 | return t.sendTaskFailure(TaskFailureUnknown{errors.New("nil sfnapi")}) 64 | } 65 | 66 | var taskInput map[string]interface{} 67 | if err := json.Unmarshal([]byte(input), &taskInput); err != nil { 68 | return t.sendTaskFailure(TaskFailureTaskInputNotJSON{input: input}) 69 | } 70 | 71 | // _EXECUTION_NAME is a required payload parameter that we inject into the environment 72 | executionName, ok := taskInput["_EXECUTION_NAME"].(string) 73 | if !ok { 74 | return t.sendTaskFailure(TaskFailureTaskInputMissingExecutionName{input: input}) 75 | } 76 | t.logger.AddContext("execution_name", executionName) 77 | 78 | marshaledInput, err := json.Marshal(taskInput) 79 | if err != nil { 80 | return t.sendTaskFailure(TaskFailureUnknown{fmt.Errorf("JSON input re-marshalling failed. This should never happen. %s", err)}) 81 | } 82 | 83 | args = append(args, string(marshaledInput)) 84 | 85 | // don't use exec.CommandContext, since we want to do graceful 86 | // sigterm + (grace period) + sigkill on the context finishing 87 | // CommandContext does sigkill immediately. 88 | t.execCmd = exec.Command(t.cmd, args...) 89 | t.execCmd.Env = append(os.Environ(), "_EXECUTION_NAME="+executionName) 90 | 91 | tmpDir := "" 92 | if t.workDirectory != "" { 93 | // make a new tmpDir for every run 94 | tmpDir, err = ioutil.TempDir(t.workDirectory, "") 95 | if err != nil { 96 | return t.sendTaskFailure(TaskFailureUnknown{fmt.Errorf("failed to create tmp dir: %s", err)}) 97 | } 98 | 99 | t.execCmd.Env = append(t.execCmd.Env, fmt.Sprintf("WORK_DIR=%s", tmpDir)) 100 | defer os.RemoveAll(tmpDir) 101 | } 102 | 103 | // Write the stdout and stderr of the process to both this process' stdout and stderr 104 | // and also write to a byte buffer so that we can send the result to step functions 105 | stderrbuf, _ := circbuf.NewBuffer(maxTaskFailureCauseLength) 106 | stdoutbuf, _ := circbuf.NewBuffer(maxTaskOutputLength) 107 | t.execCmd.Stderr = io.MultiWriter(os.Stderr, stderrbuf) 108 | t.execCmd.Stdout = io.MultiWriter(os.Stdout, stdoutbuf) 109 | 110 | // forward signals to the command, handle SIGTERM 111 | go t.handleSignals(ctx) 112 | 113 | if err := t.execCmd.Run(); err != nil { 114 | stderr := strings.TrimSpace(stderrbuf.String()) // remove trailing newline 115 | customError, _ := parseCustomErrorFromStdout(stdoutbuf.String()) // ignore parsing errors 116 | if t.receivedSigterm { 117 | if customError.ErrorName() != "" { 118 | return t.sendTaskFailure(customError) 119 | } 120 | return t.sendTaskFailure(TaskFailureCommandTerminated{stderr: stderr}) 121 | } 122 | switch err := err.(type) { 123 | case *os.PathError: 124 | return t.sendTaskFailure(TaskFailureCommandNotFound{path: err.Path}) 125 | case *exec.ExitError: 126 | if customError.ErrorName() != "" { 127 | return t.sendTaskFailure(customError) 128 | } 129 | status := err.ProcessState.Sys().(syscall.WaitStatus) 130 | switch { 131 | case status.Exited() && status.ExitStatus() > 0: 132 | return t.sendTaskFailure(TaskFailureCommandExitedNonzero{stderr: stderr}) 133 | case status.Signaled() && status.Signal() == syscall.SIGKILL: 134 | return t.sendTaskFailure(TaskFailureCommandKilled{stderr: stderr}) 135 | } 136 | } 137 | return t.sendTaskFailure(TaskFailureUnknown{err}) 138 | } 139 | 140 | // AWS / states language requires JSON output 141 | taskOutput := taskOutputFromStdout(stdoutbuf.String()) 142 | var taskOutputMap map[string]interface{} 143 | if len(taskOutput) == 0 { // Treat "" output like {}. Makes worker implementions easier. 144 | taskOutputMap = map[string]interface{}{} 145 | } else if err := json.Unmarshal([]byte(taskOutput), &taskOutputMap); err != nil { 146 | return t.sendTaskFailure(TaskFailureTaskOutputNotJSON{output: taskOutput}) 147 | } 148 | // Add _EXECUTION_NAME back into the payload in case the executing worker omits the value 149 | // in the output. 150 | taskOutputMap["_EXECUTION_NAME"] = executionName 151 | 152 | finalTaskOutput, err := json.Marshal(taskOutputMap) 153 | if err != nil { 154 | return t.sendTaskFailure(TaskFailureUnknown{fmt.Errorf("JSON output re-marshalling failed. This should never happen. %s", err)}) 155 | } 156 | _, err = t.sfnapi.SendTaskSuccessWithContext(ctx, &sfn.SendTaskSuccessInput{ 157 | Output: aws.String(string(finalTaskOutput)), 158 | TaskToken: &t.taskToken, 159 | }) 160 | if err != nil { 161 | t.logger.ErrorD("send-task-success-error", logger.M{"error": err.Error()}) 162 | } 163 | 164 | return err 165 | } 166 | 167 | func (t *TaskRunner) handleSignals(ctx context.Context) { 168 | // a buffer of one should be safe here as we're basically just catching container exits 169 | sigChan := make(chan os.Signal, 1) 170 | signal.Notify(sigChan) 171 | defer signal.Stop(sigChan) 172 | for { 173 | select { 174 | case <-ctx.Done(): 175 | // if the context has ended, but the command is still running, 176 | // initiate graceful shutdown with a much shorter grace period, 177 | // since most likely this is a case of SFN timing out the 178 | // activity. This means there is likely another activity 179 | // out there beginning work on the same input. 180 | if t.execCmd.Process != nil && t.execCmd.ProcessState == nil { 181 | sigTermAndThenKill(t.execCmd.Process.Pid, 5*time.Second) 182 | } 183 | return 184 | case sigReceived := <-sigChan: 185 | if t.execCmd.Process == nil { 186 | continue 187 | } 188 | pid := t.execCmd.Process.Pid 189 | // SIGTERM is special. If it gets sent to sfncli, initiate a docker-stop like shutdown process: 190 | // - forward the SIGTERM to the command 191 | // - after a grace period send SIGKILL to the command if it's still running 192 | if sigReceived == syscall.SIGTERM { 193 | t.receivedSigterm = true 194 | sigTermAndThenKill(pid, t.sigtermGracePeriod) 195 | return 196 | } 197 | signalProcess(pid, sigReceived) 198 | } 199 | if t.receivedSigterm { 200 | return 201 | } 202 | } 203 | } 204 | 205 | func signalProcess(pid int, signal os.Signal) { 206 | proc := os.Process{Pid: pid} 207 | proc.Signal(signal) 208 | } 209 | 210 | // sigTermAndThenKill is a docker-stop like shutdown process: 211 | // - send sigterm 212 | // - after a grace period send SIGKILL if the command is still running 213 | func sigTermAndThenKill(pid int, gracePeriod time.Duration) { 214 | signalProcess(pid, os.Signal(syscall.SIGTERM)) 215 | time.Sleep(gracePeriod) 216 | signalProcess(pid, os.Signal(syscall.SIGKILL)) 217 | } 218 | 219 | func parseCustomErrorFromStdout(stdout string) (TaskFailureCustom, error) { 220 | var customError TaskFailureCustom 221 | err := json.Unmarshal([]byte(taskOutputFromStdout(stdout)), &customError) 222 | return customError, err 223 | } 224 | 225 | func taskOutputFromStdout(stdout string) string { 226 | stdout = strings.TrimSpace(stdout) // remove trailing newline 227 | stdoutLines := strings.Split(stdout, "\n") 228 | taskOutput := "" 229 | if len(stdoutLines) > 0 { 230 | taskOutput = stdoutLines[len(stdoutLines)-1] 231 | } 232 | return taskOutput 233 | } 234 | -------------------------------------------------------------------------------- /cmd/sfncli/runner_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "os" 7 | "path" 8 | "strings" 9 | "syscall" 10 | "testing" 11 | "time" 12 | 13 | "github.com/Clever/sfncli/mocks" 14 | "github.com/aws/aws-sdk-go/aws" 15 | "github.com/aws/aws-sdk-go/service/sfn" 16 | "github.com/golang/mock/gomock" 17 | "github.com/stretchr/testify/require" 18 | ) 19 | 20 | const ( 21 | mockTaskToken = "taskToken" 22 | emptyTaskInput = `{"_EXECUTION_NAME":"fake-WFM-uuid"}` 23 | testScriptsDir = "./test_scripts" 24 | ) 25 | 26 | type workdirMatcher struct { 27 | taskToken string 28 | expectedPrefix string 29 | foundWorkdir string 30 | } 31 | 32 | func (w workdirMatcher) String() string { 33 | return "test the prefix of the work_dir value" 34 | } 35 | 36 | func (w *workdirMatcher) Matches(x interface{}) bool { 37 | input, ok := x.(*sfn.SendTaskSuccessInput) 38 | if !ok { 39 | return false 40 | } 41 | 42 | // check token 43 | if *input.TaskToken != w.taskToken { 44 | return false 45 | } 46 | 47 | workdirBlob := struct { 48 | Dir string `json:"work_dir"` 49 | }{} 50 | if err := json.Unmarshal([]byte(*input.Output), &workdirBlob); err != nil { 51 | return false 52 | } 53 | 54 | w.foundWorkdir = workdirBlob.Dir 55 | return strings.HasPrefix(workdirBlob.Dir, w.expectedPrefix) 56 | } 57 | 58 | func TestTaskFailureTaskInputNotJSON(t *testing.T) { 59 | t.Parallel() 60 | testCtx, testCtxCancel := context.WithCancel(context.Background()) 61 | defer testCtxCancel() 62 | cmd := "echo" 63 | cmdArgs := []string{} 64 | taskInput := "notjson" 65 | expectedError := TaskFailureTaskInputNotJSON{input: "notjson"} 66 | 67 | controller := gomock.NewController(t) 68 | defer controller.Finish() 69 | mockSFN := mocks.NewMockSFNAPI(controller) 70 | mockSFN.EXPECT().SendTaskFailure(&sfn.SendTaskFailureInput{ 71 | Cause: aws.String(expectedError.ErrorCause()), 72 | Error: aws.String(expectedError.ErrorName()), 73 | TaskToken: aws.String(mockTaskToken), 74 | }) 75 | taskRunner := NewTaskRunner(path.Join(testScriptsDir, cmd), mockSFN, mockTaskToken, "") 76 | err := taskRunner.Process(testCtx, cmdArgs, taskInput) 77 | require.Equal(t, err, expectedError) 78 | 79 | } 80 | 81 | func TestTaskOutputEmptyStringAsJSON(t *testing.T) { 82 | t.Parallel() 83 | testCtx, testCtxCancel := context.WithCancel(context.Background()) 84 | defer testCtxCancel() 85 | cmd := "stdout_empty_output.sh" 86 | cmdArgs := []string{} 87 | taskInput := `{"_EXECUTION_NAME":"fake-WFM-uuid"}` 88 | 89 | controller := gomock.NewController(t) 90 | defer controller.Finish() 91 | mockSFN := mocks.NewMockSFNAPI(controller) 92 | mockSFN.EXPECT().SendTaskSuccessWithContext(gomock.Any(), &sfn.SendTaskSuccessInput{ 93 | TaskToken: aws.String(mockTaskToken), 94 | Output: aws.String(`{"_EXECUTION_NAME":"fake-WFM-uuid"}`), 95 | }) 96 | taskRunner := NewTaskRunner(path.Join(testScriptsDir, cmd), mockSFN, mockTaskToken, "") 97 | err := taskRunner.Process(testCtx, cmdArgs, taskInput) 98 | require.NoError(t, err) 99 | 100 | } 101 | 102 | func TestTaskFailureCommandNotFound(t *testing.T) { 103 | t.Parallel() 104 | testCtx, testCtxCancel := context.WithCancel(context.Background()) 105 | defer testCtxCancel() 106 | cmd := "doesntexist.sh" 107 | cmdArgs := []string{} 108 | expectedError := TaskFailureCommandNotFound{path: path.Join(testScriptsDir, cmd)} 109 | 110 | controller := gomock.NewController(t) 111 | defer controller.Finish() 112 | mockSFN := mocks.NewMockSFNAPI(controller) 113 | mockSFN.EXPECT().SendTaskFailure(&sfn.SendTaskFailureInput{ 114 | Cause: aws.String(expectedError.ErrorCause()), 115 | Error: aws.String(expectedError.ErrorName()), 116 | TaskToken: aws.String(mockTaskToken), 117 | }) 118 | taskRunner := NewTaskRunner(path.Join(testScriptsDir, cmd), mockSFN, mockTaskToken, "") 119 | err := taskRunner.Process(testCtx, cmdArgs, emptyTaskInput) 120 | require.Equal(t, err, expectedError) 121 | } 122 | 123 | func TestTaskFailureCommandKilled(t *testing.T) { 124 | t.Parallel() 125 | testCtx, testCtxCancel := context.WithCancel(context.Background()) 126 | defer testCtxCancel() 127 | cmd := "log_to_stderr_and_wait.sh" 128 | cmdArgs := []string{"log this to stderr"} 129 | expectedError := TaskFailureCommandKilled{stderr: cmdArgs[0]} 130 | 131 | controller := gomock.NewController(t) 132 | defer controller.Finish() 133 | mockSFN := mocks.NewMockSFNAPI(controller) 134 | mockSFN.EXPECT().SendTaskFailure(&sfn.SendTaskFailureInput{ 135 | Cause: aws.String(expectedError.ErrorCause()), 136 | Error: aws.String(expectedError.ErrorName()), 137 | TaskToken: aws.String(mockTaskToken), 138 | }) 139 | taskRunner := NewTaskRunner(path.Join(testScriptsDir, cmd), mockSFN, mockTaskToken, "") 140 | go func() { 141 | time.Sleep(2 * time.Second) 142 | taskRunner.execCmd.Process.Signal(syscall.SIGKILL) 143 | }() 144 | err := taskRunner.Process(testCtx, cmdArgs, emptyTaskInput) 145 | require.Equal(t, err, expectedError) 146 | } 147 | 148 | func TestTaskFailureCommandExitedNonzero(t *testing.T) { 149 | t.Parallel() 150 | testCtx, testCtxCancel := context.WithCancel(context.Background()) 151 | defer testCtxCancel() 152 | cmd := "stderr_stdout_exitcode.sh" 153 | cmdArgs := []string{"stderr", `{"stdout":"mustbejson"}`, "10"} 154 | expectedError := TaskFailureCommandExitedNonzero{stderr: "stderr"} 155 | 156 | controller := gomock.NewController(t) 157 | defer controller.Finish() 158 | mockSFN := mocks.NewMockSFNAPI(controller) 159 | mockSFN.EXPECT().SendTaskFailure(&sfn.SendTaskFailureInput{ 160 | Cause: aws.String(expectedError.ErrorCause()), 161 | Error: aws.String(expectedError.ErrorName()), 162 | TaskToken: aws.String(mockTaskToken), 163 | }) 164 | taskRunner := NewTaskRunner(path.Join(testScriptsDir, cmd), mockSFN, mockTaskToken, "") 165 | err := taskRunner.Process(testCtx, cmdArgs, emptyTaskInput) 166 | require.Equal(t, err, expectedError) 167 | } 168 | 169 | func TestTaskFailureCustomErrorName(t *testing.T) { 170 | t.Parallel() 171 | testCtx, testCtxCancel := context.WithCancel(context.Background()) 172 | defer testCtxCancel() 173 | cmd := "stderr_stdout_exitcode.sh" 174 | cmdArgs := []string{"stderr", `{"error": "custom.error_name", "cause": "bar"}`, "10"} 175 | expectedError := TaskFailureCustom{Err: "custom.error_name", Cause: "bar"} 176 | 177 | controller := gomock.NewController(t) 178 | defer controller.Finish() 179 | mockSFN := mocks.NewMockSFNAPI(controller) 180 | mockSFN.EXPECT().SendTaskFailure(&sfn.SendTaskFailureInput{ 181 | Cause: aws.String(expectedError.ErrorCause()), 182 | Error: aws.String(expectedError.ErrorName()), 183 | TaskToken: aws.String(mockTaskToken), 184 | }) 185 | taskRunner := NewTaskRunner(path.Join(testScriptsDir, cmd), mockSFN, mockTaskToken, "") 186 | err := taskRunner.Process(testCtx, cmdArgs, emptyTaskInput) 187 | require.Equal(t, err, expectedError) 188 | } 189 | 190 | func TestTaskFailureTaskOutputNotJSON(t *testing.T) { 191 | t.Parallel() 192 | testCtx, testCtxCancel := context.WithCancel(context.Background()) 193 | defer testCtxCancel() 194 | cmd := "stderr_stdout_exitcode.sh" 195 | cmdArgs := []string{"stderr", `stdout not JSON!`, "0"} 196 | expectedError := TaskFailureTaskOutputNotJSON{output: "stdout not JSON!"} 197 | 198 | controller := gomock.NewController(t) 199 | defer controller.Finish() 200 | mockSFN := mocks.NewMockSFNAPI(controller) 201 | mockSFN.EXPECT().SendTaskFailure(&sfn.SendTaskFailureInput{ 202 | Cause: aws.String(expectedError.ErrorCause()), 203 | Error: aws.String(expectedError.ErrorName()), 204 | TaskToken: aws.String(mockTaskToken), 205 | }) 206 | taskRunner := NewTaskRunner(path.Join(testScriptsDir, cmd), mockSFN, mockTaskToken, "") 207 | err := taskRunner.Process(testCtx, cmdArgs, emptyTaskInput) 208 | require.Equal(t, err, expectedError) 209 | } 210 | 211 | func TestTaskFailureCommandTerminated(t *testing.T) { 212 | t.Run("command handles sigterm, exits nonzero", func(t *testing.T) { 213 | testCtx, testCtxCancel := context.WithCancel(context.Background()) 214 | defer testCtxCancel() 215 | cmd := "stderr_stdout_exitcode_onsigterm.sh" 216 | cmdArgs := []string{"stderr", "", "1"} 217 | expectedError := TaskFailureCommandTerminated{stderr: "stderr"} 218 | 219 | controller := gomock.NewController(t) 220 | defer controller.Finish() 221 | mockSFN := mocks.NewMockSFNAPI(controller) 222 | mockSFN.EXPECT().SendTaskFailure(&sfn.SendTaskFailureInput{ 223 | Cause: aws.String(expectedError.ErrorCause()), 224 | Error: aws.String(expectedError.ErrorName()), 225 | TaskToken: aws.String(mockTaskToken), 226 | }) 227 | taskRunner := NewTaskRunner(path.Join(testScriptsDir, cmd), mockSFN, mockTaskToken, "") 228 | go func() { 229 | time.Sleep(1 * time.Second) 230 | process, _ := os.FindProcess(os.Getpid()) 231 | process.Signal(syscall.SIGTERM) 232 | }() 233 | err := taskRunner.Process(testCtx, cmdArgs, emptyTaskInput) 234 | require.Equal(t, err, expectedError) 235 | }) 236 | 237 | t.Run("command handles sigterm, exits nonzero with custom error code", func(t *testing.T) { 238 | testCtx, testCtxCancel := context.WithCancel(context.Background()) 239 | defer testCtxCancel() 240 | cmd := "stderr_stdout_exitcode_onsigterm.sh" 241 | cmdArgs := []string{"stderr", `{"error": "custom.error_name", "cause": "foo"}`, "1"} 242 | expectedError := TaskFailureCustom{Err: "custom.error_name", Cause: "foo"} 243 | 244 | controller := gomock.NewController(t) 245 | defer controller.Finish() 246 | mockSFN := mocks.NewMockSFNAPI(controller) 247 | mockSFN.EXPECT().SendTaskFailure(&sfn.SendTaskFailureInput{ 248 | Cause: aws.String(expectedError.ErrorCause()), 249 | Error: aws.String(expectedError.ErrorName()), 250 | TaskToken: aws.String(mockTaskToken), 251 | }) 252 | taskRunner := NewTaskRunner(path.Join(testScriptsDir, cmd), mockSFN, mockTaskToken, "") 253 | go func() { 254 | time.Sleep(1 * time.Second) 255 | process, _ := os.FindProcess(os.Getpid()) 256 | process.Signal(syscall.SIGTERM) 257 | }() 258 | err := taskRunner.Process(testCtx, cmdArgs, emptyTaskInput) 259 | require.Equal(t, err, expectedError) 260 | }) 261 | 262 | t.Run("command does not handle sigterm", func(t *testing.T) { 263 | testCtx, testCtxCancel := context.WithCancel(context.Background()) 264 | defer testCtxCancel() 265 | cmd := "stderr_stdout_loopforever.sh" 266 | cmdArgs := []string{"stderr", ""} 267 | expectedError := TaskFailureCommandTerminated{stderr: "stderr"} 268 | 269 | controller := gomock.NewController(t) 270 | defer controller.Finish() 271 | mockSFN := mocks.NewMockSFNAPI(controller) 272 | mockSFN.EXPECT().SendTaskFailure(&sfn.SendTaskFailureInput{ 273 | Cause: aws.String(expectedError.ErrorCause()), 274 | Error: aws.String(expectedError.ErrorName()), 275 | TaskToken: aws.String(mockTaskToken), 276 | }) 277 | taskRunner := NewTaskRunner(path.Join(testScriptsDir, cmd), mockSFN, mockTaskToken, "") 278 | // lower the grace period so this test doesn't take forever 279 | taskRunner.sigtermGracePeriod = 5 * time.Second 280 | go func() { 281 | time.Sleep(1 * time.Second) 282 | process, _ := os.FindProcess(os.Getpid()) 283 | process.Signal(syscall.SIGTERM) 284 | }() 285 | err := taskRunner.Process(testCtx, cmdArgs, emptyTaskInput) 286 | require.Equal(t, err, expectedError) 287 | }) 288 | } 289 | 290 | func TestTaskSuccessSignalForwarded(t *testing.T) { 291 | testCtx, testCtxCancel := context.WithCancel(context.Background()) 292 | defer testCtxCancel() 293 | cmd := "signal_echo.sh" 294 | cmdArgs := []string{} 295 | 296 | controller := gomock.NewController(t) 297 | mockSFN := mocks.NewMockSFNAPI(controller) 298 | mockSFN.EXPECT().SendTaskSuccessWithContext(gomock.Any(), &sfn.SendTaskSuccessInput{ 299 | Output: aws.String(`{"_EXECUTION_NAME":"fake-WFM-uuid","signal":"1"}`), 300 | TaskToken: aws.String(mockTaskToken), 301 | }) 302 | defer controller.Finish() 303 | taskRunner := NewTaskRunner(path.Join(testScriptsDir, cmd), mockSFN, mockTaskToken, "") 304 | go func() { 305 | time.Sleep(1 * time.Second) 306 | process, _ := os.FindProcess(os.Getpid()) 307 | process.Signal(syscall.SIGHUP) 308 | }() 309 | require.Nil(t, taskRunner.Process(testCtx, cmdArgs, emptyTaskInput)) 310 | } 311 | 312 | func TestTaskSuccessOutputIsLastLineOfStdout(t *testing.T) { 313 | testCtx, testCtxCancel := context.WithCancel(context.Background()) 314 | defer testCtxCancel() 315 | cmd := "stdout_parsing.sh" 316 | cmdArgs := []string{} 317 | 318 | controller := gomock.NewController(t) 319 | mockSFN := mocks.NewMockSFNAPI(controller) 320 | mockSFN.EXPECT().SendTaskSuccessWithContext(gomock.Any(), &sfn.SendTaskSuccessInput{ 321 | Output: aws.String(`{"_EXECUTION_NAME":"fake-WFM-uuid","task":"output"}`), 322 | TaskToken: aws.String(mockTaskToken), 323 | }) 324 | defer controller.Finish() 325 | taskRunner := NewTaskRunner(path.Join(testScriptsDir, cmd), mockSFN, mockTaskToken, "") 326 | require.Nil(t, taskRunner.Process(testCtx, cmdArgs, emptyTaskInput)) 327 | } 328 | 329 | func TestTaskWorkDirectorySetup(t *testing.T) { 330 | t.Parallel() 331 | testCtx, testCtxCancel := context.WithCancel(context.Background()) 332 | defer testCtxCancel() 333 | cmd := "echo_workdir.sh" 334 | cmdArgs := []string{} 335 | taskInput := `{"_EXECUTION_NAME":"fake-WFM-uuid"}` 336 | 337 | controller := gomock.NewController(t) 338 | defer controller.Finish() 339 | mockSFN := mocks.NewMockSFNAPI(controller) 340 | mockSFN.EXPECT().SendTaskSuccessWithContext(gomock.Any(), &workdirMatcher{ 341 | taskToken: mockTaskToken, 342 | expectedPrefix: "/tmp", 343 | }) // returns the result of WORK_DIR 344 | taskRunner := NewTaskRunner(path.Join(testScriptsDir, cmd), mockSFN, mockTaskToken, "/tmp") 345 | err := taskRunner.Process(testCtx, cmdArgs, taskInput) 346 | require.NoError(t, err) 347 | } 348 | 349 | func TestTaskWorkDirectoryUnsetByDefault(t *testing.T) { 350 | t.Parallel() 351 | testCtx, testCtxCancel := context.WithCancel(context.Background()) 352 | defer testCtxCancel() 353 | cmd := "echo_workdir.sh" 354 | cmdArgs := []string{} 355 | taskInput := `{"_EXECUTION_NAME":"fake-WFM-uuid"}` // output a env var using the key 356 | 357 | controller := gomock.NewController(t) 358 | defer controller.Finish() 359 | mockSFN := mocks.NewMockSFNAPI(controller) 360 | mockSFN.EXPECT().SendTaskSuccessWithContext(gomock.Any(), &sfn.SendTaskSuccessInput{ 361 | TaskToken: aws.String(mockTaskToken), 362 | Output: aws.String(`{"_EXECUTION_NAME":"fake-WFM-uuid","work_dir":""}`), // returns the result of WORK_DIR 363 | }) 364 | taskRunner := NewTaskRunner(path.Join(testScriptsDir, cmd), mockSFN, mockTaskToken, "") 365 | err := taskRunner.Process(testCtx, cmdArgs, taskInput) 366 | require.NoError(t, err) 367 | } 368 | 369 | func TestTaskWorkDirectoryCleaned(t *testing.T) { 370 | t.Parallel() 371 | testCtx, testCtxCancel := context.WithCancel(context.Background()) 372 | defer testCtxCancel() 373 | cmd := "create_file.sh" 374 | cmdArgs := []string{} 375 | taskInput := `{"_EXECUTION_NAME":"fake-WFM-uuid"}` 376 | 377 | controller := gomock.NewController(t) 378 | defer controller.Finish() 379 | mockSFN := mocks.NewMockSFNAPI(controller) 380 | dirMatcher := workdirMatcher{ 381 | taskToken: mockTaskToken, 382 | expectedPrefix: "/tmp/test", 383 | } 384 | mockSFN.EXPECT().SendTaskSuccessWithContext(gomock.Any(), &dirMatcher) // returns the result of WORK_DIR 385 | 386 | os.MkdirAll("/tmp/test", os.ModeDir|0777) // base path is created by cmd/sfncli/sfncli.go 387 | defer os.RemoveAll("/tmp/test") 388 | taskRunner := NewTaskRunner(path.Join(testScriptsDir, cmd), mockSFN, mockTaskToken, "/tmp/test") 389 | err := taskRunner.Process(testCtx, cmdArgs, taskInput) 390 | require.NoError(t, err) 391 | if _, err := os.Stat(dirMatcher.foundWorkdir); os.IsExist(err) { 392 | require.Fail(t, "directory /tmp/test not deleted") 393 | } 394 | } 395 | -------------------------------------------------------------------------------- /cmd/sfncli/sfncli.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "flag" 6 | "fmt" 7 | "io/ioutil" 8 | "os" 9 | "os/signal" 10 | "syscall" 11 | "time" 12 | 13 | "github.com/aws/aws-sdk-go/aws" 14 | "github.com/aws/aws-sdk-go/aws/awserr" 15 | "github.com/aws/aws-sdk-go/aws/request" 16 | "github.com/aws/aws-sdk-go/aws/session" 17 | "github.com/aws/aws-sdk-go/service/cloudwatch" 18 | "github.com/aws/aws-sdk-go/service/sfn" 19 | "github.com/aws/aws-sdk-go/service/sfn/sfniface" 20 | "golang.org/x/time/rate" 21 | "gopkg.in/Clever/kayvee-go.v6/logger" 22 | ) 23 | 24 | var log = logger.New("sfncli") 25 | 26 | // Version denotes the version of sfncli. A value is injected at compilation via ldflags 27 | var Version string 28 | 29 | func main() { 30 | activityName := flag.String("activityname", "", "The activity name to register with AWS Step Functions. $VAR and ${VAR} env variables are expanded.") 31 | workerName := flag.String("workername", "", "The worker name to send to AWS Step Functions when processing a task. Environment variables are expanded. The magic string MAGIC_ECS_TASK_ARN will be expanded to the ECS task ARN via the metadata service.") 32 | cmd := flag.String("cmd", "", "The command to run to process activity tasks.") 33 | region := flag.String("region", "", "The AWS region to send Step Function API calls. Defaults to AWS_REGION.") 34 | cloudWatchRegion := flag.String("cloudwatchregion", "", "The AWS region to report metrics. Defaults to the value of the region flag.") 35 | workDirectory := flag.String("workdirectory", "", "Create the specified directory pass the path using the environment variable WORK_DIR to the cmd processing a task. Default is to not create the path.") 36 | printVersion := flag.Bool("version", false, "Print the version and exit.") 37 | 38 | flag.Parse() 39 | 40 | if *printVersion { 41 | fmt.Println(Version) 42 | os.Exit(0) 43 | } 44 | 45 | if *activityName == "" { 46 | fmt.Println("activityname is required") 47 | os.Exit(1) 48 | } 49 | *activityName = os.ExpandEnv(*activityName) 50 | 51 | if *workerName == "" { 52 | fmt.Println("workername is required") 53 | os.Exit(1) 54 | } 55 | *workerName = os.ExpandEnv(*workerName) 56 | if newWorkerName, err := expandECSMagicStrings(*workerName); err != nil { 57 | fmt.Printf("error expanding %s: %s", magicECSTaskARN, err) 58 | os.Exit(1) 59 | } else { 60 | *workerName = newWorkerName 61 | } 62 | 63 | if *cmd == "" { 64 | fmt.Println("cmd is required") 65 | os.Exit(1) 66 | } 67 | *cmd = os.ExpandEnv(*cmd) // Allow environment variable substition in the cmd flag. 68 | 69 | if *region == "" { 70 | *region = os.Getenv("AWS_REGION") 71 | if *region == "" { 72 | fmt.Println("region or AWS_REGION is required") 73 | os.Exit(1) 74 | } 75 | } 76 | if *cloudWatchRegion == "" { 77 | *cloudWatchRegion = *region 78 | } else { 79 | *cloudWatchRegion = os.ExpandEnv(*cloudWatchRegion) 80 | } 81 | 82 | if *workDirectory != "" { 83 | if err := validateWorkDirectory(*workDirectory); err != nil { 84 | fmt.Println(err) 85 | os.Exit(1) 86 | } 87 | } 88 | 89 | mainCtx, mainCtxCancel := context.WithCancel(context.Background()) 90 | c := make(chan os.Signal, 1) 91 | signal.Notify(c, os.Interrupt, os.Signal(syscall.SIGTERM)) 92 | go func() { 93 | for range c { 94 | // sig is a ^C, handle it 95 | mainCtxCancel() 96 | } 97 | }() 98 | 99 | // register the activity with AWS (it might already exist, which is ok) 100 | activityTags := tagsFromEnv() 101 | sfnapi := sfn.New(session.New(), aws.NewConfig().WithRegion(*region)) 102 | createOutput, err := sfnapi.CreateActivityWithContext(mainCtx, &sfn.CreateActivityInput{ 103 | Name: activityName, 104 | Tags: activityTags, 105 | }) 106 | if err != nil { 107 | fmt.Printf("error creating activity: %s\n", err) 108 | os.Exit(1) 109 | } 110 | 111 | // if the activity already exists, tags won't be applied, so explicitly 112 | // set tags here 113 | if _, err := sfnapi.TagResourceWithContext(mainCtx, &sfn.TagResourceInput{ 114 | ResourceArn: createOutput.ActivityArn, 115 | Tags: activityTags, 116 | }); err != nil { 117 | fmt.Printf("error tagging activity: %s\n", err) 118 | os.Exit(1) 119 | } 120 | 121 | log.InfoD("startup", logger.M{ 122 | "activity": *createOutput.ActivityArn, 123 | "worker-name": *workerName, 124 | "work-directory": *workDirectory, 125 | }) 126 | 127 | // set up cloudwatch metric reporting 128 | cwapi := cloudwatch.New(session.New(), aws.NewConfig().WithRegion(*cloudWatchRegion)) 129 | cw := NewCloudWatchReporter(cwapi, *createOutput.ActivityArn) 130 | go cw.ReportActivePercent(mainCtx, 60*time.Second) 131 | cw.SetActiveState(true) 132 | 133 | // allow one GetActivityTask per second, max 1 at a time 134 | limiter := rate.NewLimiter(rate.Every(1*time.Second), 1) 135 | 136 | // run getactivitytask and get some work 137 | // getactivitytask claims to initiate a polling loop, but it seems to return every few minutes with 138 | // a nil error and empty output. So wrap it in a polling loop of our own 139 | for mainCtx.Err() == nil { 140 | select { 141 | case <-mainCtx.Done(): 142 | log.Info("getactivitytask-stop") 143 | default: 144 | cw.SetActiveState(false) 145 | // setting paused here so the time spent waiting for the limiter is not counted as time 146 | // the task is inactive in the activePercent calculation 147 | cw.SetPausedState(true) 148 | if err := limiter.Wait(mainCtx); err != nil { 149 | // must unpause here because no longer waiting for limiter 150 | cw.SetPausedState(false) 151 | continue 152 | } 153 | // must unpaused here because no longer waiting for limiter 154 | cw.SetPausedState(false) 155 | 156 | log.TraceD("getactivitytask-start", logger.M{ 157 | "activity-arn": *createOutput.ActivityArn, "worker-name": *workerName, 158 | }) 159 | getATOutput, err := sfnapi.GetActivityTaskWithContext(mainCtx, &sfn.GetActivityTaskInput{ 160 | ActivityArn: createOutput.ActivityArn, 161 | WorkerName: workerName, 162 | }) 163 | if err == context.Canceled || awsErr(err, request.CanceledErrorCode) { 164 | log.Warn("getactivitytask-cancel") 165 | continue 166 | } 167 | if err != nil { 168 | log.ErrorD("getactivitytask-error", logger.M{"error": err.Error()}) 169 | continue 170 | } 171 | if getATOutput.TaskToken == nil { // No jobs to do 172 | log.Debug("getactivitytask-skip") 173 | continue 174 | } 175 | 176 | cw.SetActiveState(true) 177 | input := *getATOutput.Input 178 | token := *getATOutput.TaskToken 179 | log.TraceD("getactivitytask", logger.M{"input": input, "token": token}) 180 | 181 | // Create a context for this task. We'll cancel this context on errors. 182 | // Anything spawned on behalf of the task should use this context. 183 | taskCtx, taskCtxCancel := context.WithCancel(mainCtx) 184 | 185 | // Begin sending heartbeats 186 | go func() { 187 | if err := taskHeartbeatLoop(taskCtx, sfnapi, token); err != nil { 188 | log.ErrorD("heartbeat-error", logger.M{"error": err.Error()}) 189 | // taskHeartBeatLoop only returns errors when they should be treated as critical 190 | // e.g., if the task timed out 191 | // shut down the command in these cases 192 | taskCtxCancel() 193 | return 194 | } 195 | log.TraceD("heartbeat-end", logger.M{"token": token}) 196 | }() 197 | 198 | // Run the command. Treat unprocessed args (flag.Args()) as additional args to 199 | // send to the command on every invocation of the command 200 | taskRunner := NewTaskRunner(*cmd, sfnapi, token, *workDirectory) 201 | err = taskRunner.Process(taskCtx, flag.Args(), input) 202 | if err != nil { 203 | log.ErrorD("task-process-error", logger.M{"error": err.Error()}) 204 | taskCtxCancel() 205 | continue 206 | } 207 | 208 | // success! 209 | taskCtxCancel() 210 | } 211 | } 212 | } 213 | 214 | // tagsFromEnv computes tags for the activity from environment variables. 215 | func tagsFromEnv() []*sfn.Tag { 216 | tags := []*sfn.Tag{} 217 | if env := os.Getenv("_DEPLOY_ENV"); env != "" { 218 | tags = append(tags, &sfn.Tag{Key: aws.String("environment"), Value: aws.String(env)}) 219 | } 220 | if app := os.Getenv("_APP_NAME"); app != "" { 221 | tags = append(tags, &sfn.Tag{Key: aws.String("application"), Value: aws.String(app)}) 222 | } 223 | if pod := os.Getenv("_POD_ID"); pod != "" { 224 | tags = append(tags, &sfn.Tag{Key: aws.String("pod"), Value: aws.String(pod)}) 225 | } 226 | if shortname := os.Getenv("_POD_SHORTNAME"); shortname != "" { 227 | tags = append(tags, &sfn.Tag{Key: aws.String("pod-shortname"), Value: aws.String(shortname)}) 228 | } 229 | if region := os.Getenv("_POD_REGION"); region != "" { 230 | tags = append(tags, &sfn.Tag{Key: aws.String("pod-region"), Value: aws.String(region)}) 231 | } 232 | if account := os.Getenv("_POD_ACCOUNT"); account != "" { 233 | tags = append(tags, &sfn.Tag{Key: aws.String("pod-account"), Value: aws.String(account)}) 234 | } 235 | if team := os.Getenv("_TEAM_OWNER"); team != "" { 236 | tags = append(tags, &sfn.Tag{Key: aws.String("team"), Value: aws.String(team)}) 237 | } 238 | 239 | return tags 240 | } 241 | 242 | // validateWorkDirectory ensures the directory exists and is writable 243 | func validateWorkDirectory(dirname string) error { 244 | dirInfo, err := os.Stat(dirname) 245 | 246 | // does not exist; create dir 247 | if os.IsNotExist(err) { 248 | fmt.Printf("creating dirname %s\n", dirname) 249 | if err := os.MkdirAll(dirname, os.ModeTemporary|0700); err != nil { 250 | return fmt.Errorf("workDirectory create error: %s", err) 251 | } 252 | 253 | return nil 254 | } 255 | 256 | // dir exists; ensure permissions and mode 257 | if !dirInfo.IsDir() { 258 | return fmt.Errorf("workDirectory is not a directory") 259 | } 260 | if _, err := ioutil.TempFile(dirname, ""); err != nil { 261 | return fmt.Errorf("workDirectory write error: %s", err) 262 | } 263 | 264 | return nil 265 | } 266 | 267 | func taskHeartbeatLoop(ctx context.Context, sfnapi sfniface.SFNAPI, token string) error { 268 | if err := sendTaskHeartbeat(ctx, sfnapi, token); err != nil { 269 | return err 270 | } 271 | heartbeat := time.NewTicker(20 * time.Second) 272 | defer heartbeat.Stop() 273 | for { 274 | select { 275 | case <-ctx.Done(): 276 | return nil 277 | case <-heartbeat.C: 278 | if err := sendTaskHeartbeat(ctx, sfnapi, token); err != nil { 279 | return err 280 | } 281 | } 282 | } 283 | } 284 | 285 | func sendTaskHeartbeat(ctx context.Context, sfnapi sfniface.SFNAPI, token string) error { 286 | if _, err := sfnapi.SendTaskHeartbeatWithContext(ctx, &sfn.SendTaskHeartbeatInput{ 287 | TaskToken: aws.String(token), 288 | }); err != nil { 289 | if awsErr(err, sfn.ErrCodeInvalidToken, sfn.ErrCodeTaskDoesNotExist, sfn.ErrCodeTaskTimedOut) { 290 | return err 291 | } 292 | if err == context.Canceled || awsErr(err, request.CanceledErrorCode) { 293 | // context was canceled while sending heartbeat 294 | return nil 295 | } 296 | log.ErrorD("heartbeat-error-unknown", logger.M{"error": err.Error()}) // should investigate unknown/unclassified errors 297 | } 298 | log.Trace("heartbeat-sent") 299 | return nil 300 | } 301 | 302 | func awsErr(err error, codes ...string) bool { 303 | if err == nil { 304 | return false 305 | } 306 | if aerr, ok := err.(awserr.Error); ok { 307 | for _, code := range codes { 308 | if aerr.Code() == code { 309 | return true 310 | } 311 | } 312 | } 313 | return false 314 | } 315 | -------------------------------------------------------------------------------- /cmd/sfncli/sfncli_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "io/ioutil" 5 | "os" 6 | "testing" 7 | 8 | "github.com/stretchr/testify/assert" 9 | ) 10 | 11 | func TestValidateWorkDirectory(t *testing.T) { 12 | t.Run("creates directory if not exist", func(t *testing.T) { 13 | dirname := "/tmp/hello-there" 14 | defer os.RemoveAll(dirname) 15 | _, err := os.Stat(dirname) 16 | assert.True(t, os.IsNotExist(err)) 17 | 18 | err = validateWorkDirectory(dirname) 19 | assert.NoError(t, err) 20 | 21 | _, err = os.Stat(dirname) 22 | assert.True(t, !os.IsNotExist(err)) 23 | }) 24 | 25 | t.Run("fails if not a directory", func(t *testing.T) { 26 | f, err := ioutil.TempFile("/tmp", "filename") 27 | defer os.Remove(f.Name()) 28 | 29 | err = validateWorkDirectory(f.Name()) 30 | assert.Error(t, err) 31 | }) 32 | } 33 | -------------------------------------------------------------------------------- /cmd/sfncli/test_scripts/create_file.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | echo "{\"work_dir\":\"$WORK_DIR\"}" > $WORK_DIR/hello 4 | cat $WORK_DIR/hello 5 | -------------------------------------------------------------------------------- /cmd/sfncli/test_scripts/echo_workdir.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | echo "{\"work_dir\": \"$WORK_DIR\"}" 3 | -------------------------------------------------------------------------------- /cmd/sfncli/test_scripts/log_to_stderr_and_wait.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | echo $1 1>&2 4 | 5 | while true; do 6 | sleep 1 7 | done 8 | -------------------------------------------------------------------------------- /cmd/sfncli/test_scripts/signal_echo.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | exitCode=0 4 | if [ "$#" -eq 2 ]; then 5 | exitCode=$1 6 | fi 7 | 8 | trap_with_arg() { 9 | func="$1" ; shift 10 | for sig ; do 11 | trap "$func $sig" "$sig" 12 | done 13 | } 14 | 15 | func_trap() { 16 | echo "{\"signal\": \"$1\"}" 17 | exit $exitCode 18 | } 19 | 20 | trap_with_arg func_trap 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 21 | 22 | while true; do 23 | sleep 1 24 | done 25 | -------------------------------------------------------------------------------- /cmd/sfncli/test_scripts/stderr_stdout_exitcode.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # write something to stderr 4 | echo $1 1>&2 5 | 6 | # write stuff to stdout 7 | echo $2 8 | 9 | # exit 10 | exit $3 11 | -------------------------------------------------------------------------------- /cmd/sfncli/test_scripts/stderr_stdout_exitcode_onsigterm.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | stderr=$1 4 | stdout=$2 5 | exitcode=$3 6 | function on_sigterm { 7 | # write something to stderr 8 | echo $stderr 1>&2 9 | 10 | # write stuff to stdout 11 | echo $stdout 12 | 13 | # exit 14 | exit $exitcode 15 | } 16 | 17 | trap on_sigterm SIGTERM 18 | 19 | while true; do 20 | sleep 1 21 | done 22 | -------------------------------------------------------------------------------- /cmd/sfncli/test_scripts/stderr_stdout_loopforever.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # write something to stderr 4 | echo $1 1>&2 5 | 6 | # write stuff to stdout 7 | echo $2 8 | 9 | function on_sigterm { 10 | while true; do 11 | sleep 1 12 | done 13 | } 14 | 15 | trap on_sigterm SIGTERM 16 | 17 | while true; do 18 | sleep 1 19 | done 20 | -------------------------------------------------------------------------------- /cmd/sfncli/test_scripts/stdout_empty_output.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | -------------------------------------------------------------------------------- /cmd/sfncli/test_scripts/stdout_parsing.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | echo "some" 4 | echo "other" 5 | echo "stuff" 6 | echo "even some stderr" 2>&1 7 | echo "and now for the main event:" 8 | echo "{\"task\": \"output\"}" 9 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/Clever/sfncli 2 | 3 | go 1.24 4 | 5 | require ( 6 | github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e 7 | github.com/aws/aws-sdk-go v1.23.13 8 | github.com/golang/mock v1.6.0 9 | github.com/stretchr/testify v1.7.0 10 | golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 11 | gopkg.in/Clever/kayvee-go.v6 v6.24.0 12 | ) 13 | 14 | require ( 15 | github.com/davecgh/go-spew v1.1.1 // indirect 16 | github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af // indirect 17 | github.com/pmezard/go-difflib v1.0.0 // indirect 18 | github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f // indirect 19 | github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect 20 | github.com/xeipuuv/gojsonschema v1.1.1-0.20190423132807-354ad34c2300 // indirect 21 | golang.org/x/mod v0.4.2 // indirect 22 | golang.org/x/sys v0.0.0-20210510120138-977fb7262007 // indirect 23 | golang.org/x/tools v0.1.1 // indirect 24 | golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect 25 | gopkg.in/yaml.v2 v2.2.3-0.20190319135612-7b8349ac747c // indirect 26 | gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect 27 | ) 28 | 29 | tool github.com/golang/mock/mockgen 30 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e h1:QEF07wC0T1rKkctt1RINW/+RMTVmiwxETico2l3gxJA= 2 | github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= 3 | github.com/aws/aws-sdk-go v1.23.13 h1:l/NG+mgQFRGG3dsFzEj0jw9JIs/zYdtU6MXhY1WIDmM= 4 | github.com/aws/aws-sdk-go v1.23.13/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= 5 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 6 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 7 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 8 | github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc= 9 | github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= 10 | github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5imkbgOkpRUYLnmbU7UEFbjtDA2hxJ1ichM= 11 | github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= 12 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 13 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 14 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 15 | github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= 16 | github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 17 | github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f h1:J9EGpcZtP0E/raorCMxlFGSTBrsSlaDGf3jU/qvAE2c= 18 | github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= 19 | github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHovont7NscjpAxXsDA8S8BMYve8Y5+7cuRE7R0= 20 | github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= 21 | github.com/xeipuuv/gojsonschema v1.1.1-0.20190423132807-354ad34c2300 h1:0A2vkqfcfPZBau3ry2qbqAPSQWr7mjp1fx6aJ+9JLSg= 22 | github.com/xeipuuv/gojsonschema v1.1.1-0.20190423132807-354ad34c2300/go.mod h1:5yf86TLmAcydyeJq5YvxkGPE2fm/u4myDekKRoLuqhs= 23 | github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= 24 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 25 | golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= 26 | golang.org/x/mod v0.4.2 h1:Gz96sIWK3OalVv/I/qNygP42zyoKp3xptRVCWRFEBvo= 27 | golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= 28 | golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 29 | golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 30 | golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4 h1:4nGaVu0QrbjT/AK2PRLuQfQuh6DJve+pELhqTdAj3x0= 31 | golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= 32 | golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 33 | golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 34 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 35 | golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 36 | golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 37 | golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 38 | golang.org/x/sys v0.0.0-20210510120138-977fb7262007 h1:gG67DSER+11cZvqIMb8S8bt0vZtiN6xWYARwirrOSfE= 39 | golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 40 | golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= 41 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 42 | golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k= 43 | golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= 44 | golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 h1:SvFZT6jyqRaOeXpc5h/JSfZenJ2O330aBsf7JfSUXmQ= 45 | golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= 46 | golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 47 | golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 48 | golang.org/x/tools v0.1.1 h1:wGiQel/hW0NnEkJUk8lbzkX2gFJU6PFxf1v5OlCfuOs= 49 | golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= 50 | golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 51 | golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 52 | golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= 53 | golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 54 | gopkg.in/Clever/kayvee-go.v6 v6.24.0 h1:xOpO9c3by6CqnbWpdhzwsK+mEpNk7HKceHpVvoWFudU= 55 | gopkg.in/Clever/kayvee-go.v6 v6.24.0/go.mod h1:G0m6nBZj7Kdz+w2hiIaawmhXl5zp7E/K0ashol3Kb2A= 56 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= 57 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 58 | gopkg.in/yaml.v2 v2.2.3-0.20190319135612-7b8349ac747c h1:a0DBLLqou3UKyfyt5hQMjstKiWSqXStLNc3ABfSh1YI= 59 | gopkg.in/yaml.v2 v2.2.3-0.20190319135612-7b8349ac747c/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 60 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 61 | gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= 62 | gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 63 | -------------------------------------------------------------------------------- /golang.mk: -------------------------------------------------------------------------------- 1 | # This is the default Clever Golang Makefile. 2 | # It is stored in the dev-handbook repo, github.com/Clever/dev-handbook 3 | # Please do not alter this file directly. 4 | GOLANG_MK_VERSION := 1.3.1 5 | 6 | SHELL := /bin/bash 7 | SYSTEM := $(shell uname -a | cut -d" " -f1 | tr '[:upper:]' '[:lower:]') 8 | .PHONY: golang-test-deps golang-ensure-curl-installed 9 | 10 | # set timezone to UTC for golang to match circle and deploys 11 | export TZ=UTC 12 | 13 | # go build flags for use across all commands which accept them 14 | export GOFLAGS := -mod=vendor $(GOFLAGS) 15 | 16 | # if the gopath includes several directories, use only the first 17 | GOPATH=$(shell echo $$GOPATH | cut -d: -f1) 18 | 19 | # This block checks and confirms that the proper Go toolchain version is installed. 20 | # It uses ^ matching in the semver sense -- you can be ahead by a minor 21 | # version, but not a major version (patch is ignored). 22 | # arg1: golang version 23 | define golang-version-check 24 | _ := $(if \ 25 | $(shell \ 26 | expr >/dev/null \ 27 | `go version | cut -d" " -f3 | cut -c3- | cut -d. -f2 | sed -E 's/beta[0-9]+//'` \ 28 | \>= `echo $(1) | cut -d. -f2` \ 29 | \& \ 30 | `go version | cut -d" " -f3 | cut -c3- | cut -d. -f1` \ 31 | = `echo $(1) | cut -d. -f1` \ 32 | && echo 1), \ 33 | @echo "", \ 34 | $(error must be running Go version ^$(1) - you are running $(shell go version | cut -d" " -f3 | cut -c3-))) 35 | endef 36 | 37 | # FGT is a utility that exits with 1 whenever any stderr/stdout output is recieved. 38 | # We pin its version since its a simple tool that does its job as-is; 39 | # so we're defended against it breaking or changing in the future. 40 | FGT := $(GOPATH)/bin/fgt 41 | $(FGT): 42 | go install -mod=readonly github.com/GeertJohan/fgt@262f7b11eec07dc7b147c44641236f3212fee89d 43 | 44 | golang-ensure-curl-installed: 45 | @command -v curl >/dev/null 2>&1 || { echo >&2 "curl not installed. Please install curl."; exit 1; } 46 | 47 | # Golint is a tool for linting Golang code for common errors. 48 | # We pin its version because an update could add a new lint check which would make 49 | # previously passing tests start failing without changing our code. 50 | # this package is deprecated and frozen 51 | # Infra recommendation is to eventually move to https://github.com/golangci/golangci-lint so don't fail on linting error for now 52 | GOLINT := $(GOPATH)/bin/golint 53 | $(GOLINT): 54 | go install -mod=readonly golang.org/x/lint/golint@738671d3881b9731cc63024d5d88cf28db875626 55 | 56 | # golang-fmt-deps requires the FGT tool for checking output 57 | golang-fmt-deps: $(FGT) 58 | 59 | # golang-fmt checks that all golang files in the pkg are formatted correctly. 60 | # arg1: pkg path 61 | define golang-fmt 62 | @echo "FORMATTING $(1)..." 63 | @PKG_PATH=$$(go list -f '{{.Dir}}' $(1)); $(FGT) gofmt -l=true $${PKG_PATH}/*.go 64 | endef 65 | 66 | # golang-lint-deps requires the golint tool for golang linting. 67 | golang-lint-deps: $(GOLINT) 68 | 69 | # golang-lint calls golint on all golang files in the pkg. 70 | # arg1: pkg path 71 | define golang-lint 72 | @echo "LINTING $(1)..." 73 | @PKG_PATH=$$(go list -f '{{.Dir}}' $(1)); find $${PKG_PATH}/*.go -type f | grep -v gen_ | xargs $(GOLINT) 74 | endef 75 | 76 | # golang-lint-deps-strict requires the golint tool for golang linting. 77 | golang-lint-deps-strict: $(GOLINT) $(FGT) 78 | 79 | # golang-test-deps is here for consistency 80 | golang-test-deps: 81 | 82 | # golang-test uses the Go toolchain to run all tests in the pkg. 83 | # arg1: pkg path 84 | define golang-test 85 | @echo "TESTING $(1)..." 86 | @go test -v $(1) 87 | endef 88 | 89 | # golang-test-strict-deps is here for consistency 90 | golang-test-strict-deps: 91 | 92 | # golang-test-strict uses the Go toolchain to run all tests in the pkg with the race flag 93 | # arg1: pkg path 94 | define golang-test-strict 95 | @echo "TESTING $(1)..." 96 | @go test -v -race $(1) 97 | endef 98 | 99 | # golang-test-strict-cover-deps is here for consistency 100 | golang-test-strict-cover-deps: 101 | 102 | # golang-test-strict-cover uses the Go toolchain to run all tests in the pkg with the race and cover flag. 103 | # appends coverage results to coverage.txt 104 | # arg1: pkg path 105 | define golang-test-strict-cover 106 | @echo "TESTING $(1)..." 107 | @go test -v -race -cover -coverprofile=profile.tmp -covermode=atomic $(1) 108 | @if [ -f profile.tmp ]; then \ 109 | cat profile.tmp | tail -n +2 >> coverage.txt; \ 110 | rm profile.tmp; \ 111 | fi; 112 | endef 113 | 114 | # golang-vet-deps is here for consistency 115 | golang-vet-deps: 116 | 117 | # golang-vet uses the Go toolchain to vet all the pkg for common mistakes. 118 | # arg1: pkg path 119 | define golang-vet 120 | @echo "VETTING $(1)..." 121 | @go vet $(1) 122 | endef 123 | 124 | # golang-test-all-deps installs all dependencies needed for different test cases. 125 | golang-test-all-deps: golang-fmt-deps golang-lint-deps golang-test-deps golang-vet-deps 126 | 127 | # golang-test-all calls fmt, lint, vet and test on the specified pkg. 128 | # arg1: pkg path 129 | define golang-test-all 130 | $(call golang-fmt,$(1)) 131 | $(call golang-lint,$(1)) 132 | $(call golang-vet,$(1)) 133 | $(call golang-test,$(1)) 134 | endef 135 | 136 | # golang-test-all-strict-deps: installs all dependencies needed for different test cases. 137 | golang-test-all-strict-deps: golang-fmt-deps golang-lint-deps-strict golang-test-strict-deps golang-vet-deps 138 | 139 | # golang-test-all-strict calls fmt, lint, vet and test on the specified pkg with strict 140 | # requirements that no errors are thrown while linting. 141 | # arg1: pkg path 142 | define golang-test-all-strict 143 | $(call golang-fmt,$(1)) 144 | $(call golang-lint,$(1)) 145 | $(call golang-vet,$(1)) 146 | $(call golang-test-strict,$(1)) 147 | endef 148 | 149 | # golang-test-all-strict-cover-deps: installs all dependencies needed for different test cases. 150 | golang-test-all-strict-cover-deps: golang-fmt-deps golang-lint-deps-strict golang-test-strict-cover-deps golang-vet-deps 151 | 152 | # golang-test-all-strict-cover calls fmt, lint, vet and test on the specified pkg with strict and cover 153 | # requirements that no errors are thrown while linting. 154 | # arg1: pkg path 155 | define golang-test-all-strict-cover 156 | $(call golang-fmt,$(1)) 157 | $(call golang-lint,$(1)) 158 | $(call golang-vet,$(1)) 159 | $(call golang-test-strict-cover,$(1)) 160 | endef 161 | 162 | # golang-build: builds a golang binary 163 | # arg1: pkg path 164 | # arg2: executable name 165 | define golang-build 166 | @echo "BUILDING $(2)..." 167 | @CGO_ENABLED=0 go build -o bin/$(2) $(1); 168 | endef 169 | 170 | # golang-debug-build: builds a golang binary with debugging capabilities 171 | # arg1: pkg path 172 | # arg2: executable name 173 | define golang-debug-build 174 | @echo "BUILDING $(2) FOR DEBUG..." 175 | @CGO_ENABLED=0 go build -gcflags="all=-N -l" -o bin/$(2) $(1); 176 | endef 177 | 178 | # golang-cgo-build: builds a golang binary with CGO 179 | # arg1: pkg path 180 | # arg2: executable name 181 | define golang-cgo-build 182 | @echo "BUILDING $(2) WITH CGO ..." 183 | @CGO_ENABLED=1 go build -installsuffix cgo -o bin/$(2) $(1); 184 | endef 185 | 186 | # golang-setup-coverage: set up the coverage file 187 | golang-setup-coverage: 188 | @echo "mode: atomic" > coverage.txt 189 | 190 | # golang-update-makefile downloads latest version of golang.mk 191 | golang-update-makefile: 192 | @wget https://raw.githubusercontent.com/Clever/dev-handbook/master/make/golang-v1.mk -O /tmp/golang.mk 2>/dev/null 193 | @if ! grep -q $(GOLANG_MK_VERSION) /tmp/golang.mk; then cp /tmp/golang.mk golang.mk && echo "golang.mk updated"; else echo "golang.mk is up-to-date"; fi 194 | -------------------------------------------------------------------------------- /launch/sfncli.yml: -------------------------------------------------------------------------------- 1 | # bare minimum to support `ark start -l` 2 | {} 3 | -------------------------------------------------------------------------------- /make/.gitignore: -------------------------------------------------------------------------------- 1 | bin/ 2 | -------------------------------------------------------------------------------- /make/README.md: -------------------------------------------------------------------------------- 1 | # make 2 | 3 | `sfncli.mk` is a helpful makefile to include with projects that use `sfncli`. 4 | 5 | ## Usage 6 | 7 | See `example.mk`, or run `make -f example.mk build`. 8 | -------------------------------------------------------------------------------- /make/example.mk: -------------------------------------------------------------------------------- 1 | include sfncli.mk 2 | .DEFAULT_GOAL := build 3 | 4 | SFNCLI_VERSION := latest 5 | 6 | build: bin/sfncli 7 | 8 | run: build 9 | bin/sfncli --help 10 | 11 | clean: 12 | rm -rf bin 13 | -------------------------------------------------------------------------------- /make/sfncli.mk: -------------------------------------------------------------------------------- 1 | # This is the default sfncli Makefile. 2 | # Please do not alter this file directly. 3 | SFNCLI_MK_VERSION := 0.1.5 4 | SHELL := /bin/bash 5 | SYSTEM := $(shell uname -a | cut -d" " -f1 | tr '[:upper:]' '[:lower:]') 6 | SFNCLI_INSTALLED := $(shell [[ -e "bin/sfncli" ]] && bin/sfncli --version) 7 | # AUTH_HEADER is used to help avoid github ratelimiting 8 | AUTH_HEADER = $(shell [[ ! -z "${GITHUB_API_TOKEN}" ]] && echo "Authorization: token $(GITHUB_API_TOKEN)") 9 | SFNCLI_LATEST = $(shell \ 10 | curl --retry 5 -f -s --header "$(AUTH_HEADER)" \ 11 | https://api.github.com/repos/Clever/sfncli/releases/latest | \ 12 | grep tag_name | \ 13 | cut -d\" -f4) 14 | 15 | .PHONY: bin/sfncli sfncli-update-makefile ensure-sfncli-version-set ensure-curl-installed 16 | 17 | ensure-sfncli-version-set: 18 | @ if [[ "$(SFNCLI_VERSION)" = "" ]]; then \ 19 | echo "SFNCLI_VERSION not set in Makefile - Suggest setting 'SFNCLI_VERSION := latest'"; \ 20 | exit 1; \ 21 | fi 22 | 23 | ensure-curl-installed: 24 | @command -v curl >/dev/null 2>&1 || { echo >&2 "curl not installed. Please install curl."; exit 1; } 25 | 26 | bin/sfncli: ensure-sfncli-version-set ensure-curl-installed 27 | @mkdir -p bin 28 | $(eval SFNCLI_VERSION := $(if $(filter latest,$(SFNCLI_VERSION)),$(SFNCLI_LATEST),$(SFNCLI_VERSION))) 29 | @echo "Checking for sfncli updates..." 30 | @# AUTH_HEADER not added to curl command below because it doesn't play well with redirects 31 | @if [[ "$(SFNCLI_VERSION)" == "$(SFNCLI_INSTALLED)" ]]; then \ 32 | { [[ -z "$(SFNCLI_INSTALLED)" ]] && \ 33 | { echo "❌ Error: Failed to download sfncli. Try setting GITHUB_API_TOKEN"; exit 1; } || \ 34 | { echo "Using latest sfncli version $(SFNCLI_VERSION)"; } \ 35 | } \ 36 | else \ 37 | echo "Updating sfncli..."; \ 38 | curl --retry 5 --fail --max-time 30 -o bin/sfncli -sL https://github.com/Clever/sfncli/releases/download/$(SFNCLI_VERSION)/sfncli-$(SFNCLI_VERSION)-$(SYSTEM)-amd64 && \ 39 | chmod +x bin/sfncli && \ 40 | echo "Successfully updated sfncli to $(SFNCLI_VERSION)" || \ 41 | { [[ -z "$(SFNCLI_INSTALLED)" ]] && \ 42 | { echo "❌ Error: Failed to update sfncli"; exit 1; } || \ 43 | { echo "⚠️ Warning: Failed to update sfncli using pre-existing version"; } \ 44 | } \ 45 | ;fi 46 | 47 | sfncli-update-makefile: ensure-curl-installed 48 | @curl -o /tmp/sfncli.mk -sL https://raw.githubusercontent.com/Clever/sfncli/master/make/sfncli.mk 49 | @if ! grep -q $(SFNCLI_MK_VERSION) /tmp/sfncli.mk; then cp /tmp/sfncli.mk sfncli.mk && echo "sfncli.mk updated"; else echo "sfncli.mk is up-to-date"; fi 50 | -------------------------------------------------------------------------------- /mocks/mocks.go: -------------------------------------------------------------------------------- 1 | // Package mocks holds the generated mocks for testing. 2 | package mocks 3 | --------------------------------------------------------------------------------