├── .github └── workflows │ ├── go.yml │ └── release.yml ├── .gitignore ├── LICENSE ├── Makefile ├── Mk └── go-release.mk ├── README.md ├── awsutil ├── aws.go ├── aws_others.go └── aws_windows.go ├── cmd ├── daemon-demo │ ├── .gitignore │ ├── rotate.sh │ ├── start.sh │ └── stop.sh ├── sqs-echo │ └── sqs-echo.go ├── sqs-notify2 │ └── main.go └── sqs-send │ └── sqs-send.go ├── config.go ├── daemon.go ├── daemon_windows.go ├── doc ├── runas-service.md └── v1.md ├── go.mod ├── go.sum ├── jobs.go ├── jobs_redis.go ├── jobs_test.go ├── main.go ├── redis-example.json ├── sqsnotify └── sqsnotify.go ├── sqsnotify2 ├── cache.go ├── cache_redis.go ├── cache_test.go ├── config.go ├── sqsnotify.go ├── stage │ └── stage.go ├── version.go └── wrap_sqs.go └── worker.go /.github/workflows/go.yml: -------------------------------------------------------------------------------- 1 | name: Go 2 | 3 | on: [push] 4 | 5 | env: 6 | GO_VERSION: '>=1.21.0' 7 | 8 | jobs: 9 | 10 | build: 11 | name: Build 12 | runs-on: ${{ matrix.os }} 13 | 14 | strategy: 15 | matrix: 16 | os: [ ubuntu-latest, macos-latest, windows-latest ] 17 | steps: 18 | 19 | - uses: actions/checkout@v4 20 | 21 | - uses: actions/setup-go@v5 22 | with: 23 | go-version: ${{ env.GO_VERSION }} 24 | 25 | - run: go test ./... 26 | 27 | - run: go build 28 | 29 | - name: "Build: sqs-notify2" 30 | run: go build ./cmd/sqs-notify2 31 | 32 | - name: "Build: sqs-echo" 33 | run: go build ./cmd/sqs-echo 34 | 35 | - name: "Build: sqs-send" 36 | run: go build ./cmd/sqs-send 37 | 38 | # based on: github.com/koron-go/_skeleton/.github/workflows/go.yml 39 | -------------------------------------------------------------------------------- /.github/workflows/release.yml: -------------------------------------------------------------------------------- 1 | name: Release 2 | 3 | on: [push] 4 | 5 | env: 6 | GO_VERSION: '>=1.21.0' 7 | NAME: 'sqs-notify2' 8 | 9 | jobs: 10 | 11 | build: 12 | 13 | name: Build 14 | runs-on: ${{ matrix.os }} 15 | 16 | strategy: 17 | matrix: 18 | os: [ ubuntu-latest, macos-latest, windows-latest ] 19 | arch: [ amd64, arm64 ] 20 | exclude: 21 | - os: windows-latest 22 | arch: arm64 23 | 24 | steps: 25 | 26 | - uses: actions/checkout@v4 27 | - uses: actions/setup-go@v5 28 | with: 29 | go-version: ${{ env.GO_VERSION }} 30 | 31 | - name: Setup env 32 | id: setup 33 | shell: bash 34 | run: | 35 | if [[ ${GITHUB_REF} =~ ^refs/tags/v[0-9]+\.[0-9]+ ]] ; then 36 | export VERSION=${GITHUB_REF_NAME} 37 | else 38 | export VERSION=SNAPSHOT 39 | fi 40 | echo "VERSION=${VERSION}" >> $GITHUB_ENV 41 | case ${{ matrix.os }} in 42 | ubuntu-*) 43 | if [ "${{ matrix.arch }}" = "arm64" ] ; then 44 | sudo apt-get update 45 | sudo apt-get -y install gcc-12-aarch64-linux-gnu 46 | echo "CC=aarch64-linux-gnu-gcc-12" >> $GITHUB_ENV 47 | fi 48 | export GOOS=linux 49 | export PKGEXT=.tar.gz 50 | ;; 51 | macos-*) 52 | export GOOS=darwin 53 | export PKGEXT=.zip 54 | ;; 55 | windows-*) 56 | choco install zip 57 | export GOOS=windows 58 | export PKGEXT=.zip 59 | ;; 60 | esac 61 | export GOARCH=${{ matrix.arch }} 62 | echo "GOOS=${GOOS}" >> $GITHUB_ENV 63 | echo "GOARCH=${GOARCH}" >> $GITHUB_ENV 64 | echo "CGO_ENABLED=1" >> $GITHUB_ENV 65 | echo "PKGNAME=${NAME}_${VERSION}_${GOOS}_${GOARCH}" >> $GITHUB_ENV 66 | echo "PKGEXT=${PKGEXT}" >> $GITHUB_ENV 67 | 68 | - name: Build 69 | shell: bash 70 | run: | 71 | go build ./cmd/sqs-notify2 72 | go build ./cmd/sqs-echo 73 | go build ./cmd/sqs-send 74 | 75 | - name: Archive 76 | shell: bash 77 | run: | 78 | rm -rf _build/${PKGNAME} 79 | mkdir -p _build/${PKGNAME} 80 | cp -p LICENSE _build/${PKGNAME} 81 | cp -p README.md _build/${PKGNAME} 82 | 83 | cp -p sqs-notify2 _build/${PKGNAME} 84 | cp -p sqs-echo _build/${PKGNAME} 85 | cp -p sqs-send _build/${PKGNAME} 86 | 87 | case "${PKGEXT}" in 88 | ".tar.gz") 89 | tar cf _build/${PKGNAME}${PKGEXT} -C _build ${PKGNAME} 90 | ;; 91 | ".zip") 92 | (cd _build && zip -r9q ${PKGNAME}${PKGEXT} ${PKGNAME}) 93 | ;; 94 | esac 95 | ls -laFR _build 96 | 97 | - name: Artifact upload 98 | uses: actions/upload-artifact@v4 99 | with: 100 | name: ${{ env.GOOS }}_${{ env.GOARCH }} 101 | path: _build/${{ env.PKGNAME }}${{ env.PKGEXT }} 102 | 103 | create-release: 104 | name: Create release 105 | runs-on: ubuntu-latest 106 | if: startsWith(github.ref, 'refs/tags/') 107 | needs: 108 | - build 109 | steps: 110 | - uses: actions/download-artifact@v4 111 | with: { name: darwin_amd64 } 112 | - uses: actions/download-artifact@v4 113 | with: { name: darwin_arm64 } 114 | - uses: actions/download-artifact@v4 115 | with: { name: linux_amd64 } 116 | - uses: actions/download-artifact@v4 117 | with: { name: linux_arm64 } 118 | - uses: actions/download-artifact@v4 119 | with: { name: windows_amd64 } 120 | - run: ls -lafR 121 | - name: Release 122 | uses: softprops/action-gh-release@v2 123 | with: 124 | draft: true 125 | prerelease: ${{ contains(github.ref_name, '-alpha.') || contains(github.ref_name, '-beta.') }} 126 | files: | 127 | *.tar.gz 128 | *.zip 129 | fail_on_unmatched_files: true 130 | generate_release_notes: true 131 | append_body: true 132 | 133 | # based on: github.com/koron-go/_skeleton/.github/workflows/release.yml 134 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.exe 2 | /sqs-notify 3 | /sqs-notify2 4 | tags 5 | tmp/ 6 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2013 MURAOKA Taro 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in 13 | all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | THE SOFTWARE. 22 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | PROJECT = sqs-notify2 2 | PROJECT_DIR = ./cmd/sqs-notify2 3 | VERSION = v2.2.1 4 | RELEASE_TARGETS = release-windows-amd64 release-windows-386 release-linux-amd64 release-linux-386 5 | 6 | 7 | default: test 8 | 9 | test: 10 | go test ./... 11 | 12 | test-full: 13 | go test -v -race ./... 14 | 15 | lint: 16 | -go vet ./... 17 | @echo "" 18 | -golint ./... 19 | 20 | cyclo: 21 | -gocyclo -top 10 -avg . 22 | 23 | report: 24 | @echo "misspell" 25 | @find . -name "*.go" | xargs misspell 26 | @echo "" 27 | -errcheck ./... 28 | @echo "" 29 | -gocyclo -over 14 -avg . 30 | @echo "" 31 | -go vet ./... 32 | @echo "" 33 | -golint ./... 34 | 35 | tags: 36 | gotags -f tags -R . 37 | 38 | -include Mk/*.mk 39 | 40 | .PHONY: test test-full lint cyclo report tags 41 | -------------------------------------------------------------------------------- /Mk/go-release.mk: -------------------------------------------------------------------------------- 1 | PROJECT ?= NONAME 2 | PROJECT_DIR ?= . 3 | VERSION ?= v0 4 | REVISION ?= $(shell git rev-parse --short --verify HEAD) 5 | RELEASE_TARGETS ?= \ 6 | release-windows-amd64 \ 7 | release-windows-386 \ 8 | release-linux-amd64 \ 9 | release-linux-386 \ 10 | release-darwin-amd64 11 | 12 | RELEASE_GOVERSION=$(shell go version) 13 | RELEASE_OS=$(word 1,$(subst /, ,$(lastword $(RELEASE_GOVERSION)))) 14 | RELEASE_ARCH=$(word 2,$(subst /, ,$(lastword $(RELEASE_GOVERSION)))) 15 | 16 | RELEASE_NAME=$(PROJECT)-$(VERSION)-$(RELEASE_OS)-$(RELEASE_ARCH) 17 | RELEASE_DIR=$(PROJECT)-$(RELEASE_OS)-$(RELEASE_ARCH) 18 | 19 | 20 | release: release-build 21 | rm -rf tmp/$(RELEASE_DIR) 22 | mkdir -p tmp/$(RELEASE_DIR) 23 | cp $(PROJECT)$(SUFFIX_EXE) tmp/$(RELEASE_DIR)/ 24 | tar czf tmp/$(RELEASE_NAME).tar.gz -C tmp $(RELEASE_DIR) 25 | go clean 26 | 27 | release-build: 28 | go clean 29 | GOOS=$(RELEASE_OS) GOARCH=$(RELEASE_ARCH) go build -ldflags='-X main.version=$(VERSION) -X main.revision=$(REVISION)' $(PROJECT_DIR) 30 | 31 | release-all: $(RELEASE_TARGETS) 32 | 33 | release-windows-amd64: 34 | @$(MAKE) release RELEASE_OS=windows RELEASE_ARCH=amd64 SUFFIX_EXE=.exe 35 | 36 | release-windows-386: 37 | @$(MAKE) release RELEASE_OS=windows RELEASE_ARCH=386 SUFFIX_EXE=.exe 38 | 39 | release-linux-amd64: 40 | @$(MAKE) release RELEASE_OS=linux RELEASE_ARCH=amd64 41 | 42 | release-linux-386: 43 | @$(MAKE) release RELEASE_OS=linux RELEASE_ARCH=386 44 | 45 | release-darwin-amd64: 46 | @$(MAKE) release RELEASE_OS=darwin RELEASE_ARCH=amd64 47 | 48 | release-debug: 49 | @echo $(RELEASE_TARGETS) 50 | 51 | .PHONY: release release-build release-all release-windows-amd64 release-windows-386 release-linux-amd64 release-linux-386 release-darwin-amd64 52 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # SQS notify 2 | 3 | [![PkgGoDev](https://pkg.go.dev/badge/github.com/koron/sqs-notify)](https://pkg.go.dev/github.com/koron/sqs-notify) 4 | [![Actions/Go](https://github.com/koron/sqs-notify/workflows/Go/badge.svg)](https://github.com/koron/sqs-notify/actions?query=workflow%3AGo) 5 | [![Go Report Card](https://goreportcard.com/badge/github.com/koron/sqs-notify)](https://goreportcard.com/report/github.com/koron/sqs-notify) 6 | 7 | Listen a SQS queue, execute a command when received. A message body is passed 8 | as STDIN to the command. 9 | 10 | For old version (v1), check [doc/v1.md](./doc/v1.md). 11 | 12 | ## Installation 13 | 14 | Install and upgrade. 15 | 16 | ```console 17 | $ go install github.com/koron/sqs-notify/cmd/sqs-notify2@latest 18 | ``` 19 | 20 | ## Environment variables 21 | 22 | * `AWS_SHARED_CREDENTIALS_FILE` - Shared credentials file path can be set to 23 | instruct the SDK to use an alternate file for the shared credentials. If 24 | not set the file will be loaded from $HOME/.aws/credentials on Linux/Unix 25 | based systems, and %USERPROFILE%\.aws\credentials on Windows. 26 | 27 | ## Options 28 | 29 | From online help. 30 | 31 | ``` 32 | -cache string 33 | cache name or connection URL 34 | * memory://?capacity=1000 35 | * redis://[{USER}:{PASS}@]{HOST}/[{DBNUM}]?[{OPTIONS}] 36 | 37 | DBNUM: redis DB number (default 0) 38 | OPTIONS: 39 | * lifetime : lifetime of cachetime (ex. "10s", "2m", "3h") 40 | * prefix : prefix of keys 41 | 42 | Example to connect the redis on localhost: "redis://:6379" 43 | -createqueue 44 | create queue if not exists 45 | -endpoint string 46 | Endpoint of SQS 47 | -logfile string 48 | log file path 49 | -max-retries int 50 | max retries for AWS 51 | -multiplier value 52 | pooling the SQS in multiple runner (default 1) 53 | -pidfile string 54 | PID file path (require -logfile) 55 | -profile string 56 | AWS profile name 57 | -queue value 58 | SQS queue name 59 | -region string 60 | AWS region (default "us-east-1") 61 | -remove-policy value 62 | policy to remove messages from SQS 63 | * succeed : after execution, succeeded (default) 64 | * ignore_failure : after execution, ignore its result 65 | * before_execution : before execution (default succeed) 66 | -timeout duration 67 | timeout for command execution (default 0 - no timeout) 68 | -version 69 | show version 70 | -wait-time-seconds int 71 | wait time in seconds for next polling. (default -1, disabled, use queue default) (default -1) 72 | -workers int 73 | num of workers (default 16) 74 | ``` 75 | 76 | ## Guide 77 | 78 | Basic usage: 79 | 80 | sqs-notify2 [-region {region}] -queue {queue name} {command and args} 81 | 82 | 1. Prepare AWS auth information. 83 | 1. Use `~/.aws/credentials` (recomended). 84 | 85 | sqs-notify2 supports `~/.aws/credentials` profiles. 86 | `-profile` option can choose the profile used to. Example: 87 | 88 | ```ini 89 | [my_sqs] 90 | aws_access_key_id=foo 91 | aws_secret_access_key=bar 92 | ``` 93 | 94 | ```console 95 | $ sqs-notify2 -profile my_sqs ... 96 | ``` 97 | 98 | 2. Use two environment variables. 99 | * `AWS_ACCESS_KEY_ID` 100 | * `AWS_SECRET_ACCESS_KEY` 101 | 102 | 2. Run sqs-notify2 103 | 104 | ```console 105 | $ sqs-notify2 -queue my_queue cat 106 | ``` 107 | 108 | This example just copy messages to STDOUT. If you want to access the queue 109 | via ap-northeast-1 region, use below command. 110 | 111 | ```console 112 | $ sqs-notify2 -region ap-northeast-1 -queue my_queue cat 113 | ``` 114 | 115 | ### Name of regions 116 | 117 | * `us-east-1` (default) 118 | * `us-west-1` 119 | * `us-west-2` 120 | * `eu-west-1` 121 | * `ap-southeast-1` 122 | * `ap-southeast-2` 123 | * `ap-northeast-1` 124 | * `sp-east-1` 125 | 126 | ### Logging 127 | 128 | When `-logfile {FILE PATH}` is given, all messages which received are logged 129 | into the file. If FILE PATH is `-`, it output all logs to STDOUT not file. 130 | 131 | Using `-pidfile {FILE PATH}` with `-logfile`, sqs-notify2 writes own PID to the 132 | file. You can send SIGHUP to that PID to rotate log. 133 | 134 | ## Miscellaneous 135 | 136 | ### LF at EOF 137 | 138 | When message doesn't have LF at EOF (end of file/message), the last line can't 139 | be handled by `read` shell command or so. This is limitation of `read` 140 | command, not sqs-notify2. Therefore this kind of scripts don't work correctly 141 | for messages without LF at EOF: 142 | 143 | ```sh 144 | #!/bin/sh 145 | while read line 146 | do 147 | echo "received: $line" 148 | done 149 | ``` 150 | 151 | To work around this problem, use `xargs` like this. 152 | 153 | ```sh 154 | #!/bin/sh 155 | xargs -0 echo | ( 156 | while read line 157 | do 158 | echo "received: $line" 159 | done 160 | ) 161 | ``` 162 | 163 | ### /dev/stdin 164 | 165 | You can use `/dev/stdin` pseudo file, if your system support it, like this: 166 | 167 | ```sh 168 | #!/bin/sh 169 | 170 | data=`cat /dev/stdin` 171 | 172 | # do something for data. 173 | ``` 174 | 175 | ### sqs-echo 176 | 177 | sqs-echo is useful for debugging received SQS message with sqs-notify2. It just 178 | shows date, time, byte num and contents of received messages. Example output 179 | is below: 180 | 181 | ``` 182 | 2015/05/07 12:43:08 (7) "foo\nbar" 183 | 2015/05/07 12:43:12 (3) "qux" 184 | ``` 185 | 186 | You can install sqs-echo with below command. 187 | 188 | ``` 189 | $ go install github.com/koron/sqs-notify/cmd/sqs-echo@latest 190 | ``` 191 | 192 | ## LICENSE 193 | 194 | MIT License. See [LICENSE](./LICENSE) for details. 195 | -------------------------------------------------------------------------------- /awsutil/aws.go: -------------------------------------------------------------------------------- 1 | package awsutil 2 | 3 | import ( 4 | "errors" 5 | "os" 6 | "path/filepath" 7 | 8 | "github.com/goamz/goamz/aws" 9 | "github.com/vaughan0/go-ini" 10 | ) 11 | 12 | func loadCredFile() (ini.File, error) { 13 | credFile := os.Getenv("AWS_CREDENTIAL_FILE") 14 | if credFile == "" { 15 | home, err := getHomeDir() 16 | if err != nil { 17 | return nil, err 18 | } 19 | credFile = filepath.Join(home, ".aws", "credentials") 20 | } 21 | return ini.LoadFile(credFile) 22 | } 23 | 24 | // GetAuth returns aws.Auth from credentials or environment variables. 25 | func GetAuth(name string) (aws.Auth, error) { 26 | f, err := loadCredFile() 27 | if err != nil { 28 | if os.IsExist(err) { 29 | return aws.Auth{}, err 30 | } 31 | return aws.EnvAuth() 32 | } 33 | 34 | var prof ini.Section 35 | var ok bool 36 | 37 | if name != "" { 38 | prof, ok = f[name] 39 | } 40 | if !ok { 41 | prof, ok = f["default"] 42 | } 43 | if !ok { 44 | return aws.Auth{}, errors.New("cannot find section") 45 | } 46 | 47 | // Parse auth info from a ini's section. 48 | a := aws.Auth{ 49 | AccessKey: prof["aws_access_key_id"], 50 | SecretKey: prof["aws_secret_access_key"], 51 | } 52 | if a.AccessKey == "" { 53 | return aws.Auth{}, errors.New("empty aws_access_key_id in credentials") 54 | } 55 | if a.SecretKey == "" { 56 | return aws.Auth{}, errors.New("empty aws_secret_access_key in credentials") 57 | } 58 | return a, nil 59 | } 60 | -------------------------------------------------------------------------------- /awsutil/aws_others.go: -------------------------------------------------------------------------------- 1 | // +build !windows 2 | 3 | package awsutil 4 | 5 | import ( 6 | "errors" 7 | "os" 8 | ) 9 | 10 | func getHomeDir() (string, error) { 11 | home := os.Getenv("HOME") 12 | if home != "" { 13 | return home, nil 14 | } 15 | return "", errors.New("could not get HOME") 16 | } 17 | -------------------------------------------------------------------------------- /awsutil/aws_windows.go: -------------------------------------------------------------------------------- 1 | // +build windows 2 | 3 | package awsutil 4 | 5 | import ( 6 | "errors" 7 | "os" 8 | ) 9 | 10 | func getHomeDir() (string, error) { 11 | home := os.Getenv("HOME") 12 | if home != "" { 13 | return home, nil 14 | } 15 | home = os.Getenv("HOMEDRIVE") + os.Getenv("HOMEPATH") 16 | if home != "" { 17 | return home, nil 18 | } 19 | home = os.Getenv("USERPROFILE") 20 | if home != "" { 21 | return home, nil 22 | } 23 | return "", errors.New("could not get HOME") 24 | } 25 | -------------------------------------------------------------------------------- /cmd/daemon-demo/.gitignore: -------------------------------------------------------------------------------- 1 | /demo.log 2 | /demo.*.log 3 | /demo.pid 4 | -------------------------------------------------------------------------------- /cmd/daemon-demo/rotate.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # 3 | # USAGE: rotate.sh 4 | 5 | test -f demo.9.log && rm -f demo.9.log 6 | test -f demo.8.log && mv demo.8.log demo.9.log 7 | test -f demo.7.log && mv demo.7.log demo.8.log 8 | test -f demo.6.log && mv demo.6.log demo.7.log 9 | test -f demo.5.log && mv demo.5.log demo.6.log 10 | test -f demo.4.log && mv demo.4.log demo.5.log 11 | test -f demo.3.log && mv demo.3.log demo.4.log 12 | test -f demo.2.log && mv demo.2.log demo.3.log 13 | test -f demo.1.log && mv demo.1.log demo.2.log 14 | test -f demo.log && mv demo.log demo.1.log 15 | 16 | kill -HUP `cat demo.pid` 17 | -------------------------------------------------------------------------------- /cmd/daemon-demo/start.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # 3 | # USAGE: start.sh {region} {queue_name} 4 | 5 | region=$1 ; shift 6 | queue=$1 ; shift 7 | 8 | dir=`pwd` 9 | 10 | sqs-notify -region ${region} -daemon \ 11 | -logfile "${dir}/demo.log" \ 12 | -pidfile "${dir}/demo.pid" \ 13 | ${queue} sqs-echo 14 | -------------------------------------------------------------------------------- /cmd/daemon-demo/stop.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # 3 | # USAGE: stop.sh 4 | 5 | kill `cat demo.pid` 6 | 7 | rm -f demo.pid 8 | -------------------------------------------------------------------------------- /cmd/sqs-echo/sqs-echo.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "flag" 5 | "io/ioutil" 6 | "log" 7 | "os" 8 | "time" 9 | ) 10 | 11 | func main() { 12 | sleep := flag.Int("sleep", 0, "sleep seconds after output (default: 0)") 13 | flag.Parse() 14 | b, err := ioutil.ReadAll(os.Stdin) 15 | if err != nil { 16 | panic(err) 17 | } 18 | log.Printf("(%d) %q", len(b), b) 19 | time.Sleep(time.Duration(*sleep) * time.Second) 20 | } 21 | -------------------------------------------------------------------------------- /cmd/sqs-notify2/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "flag" 7 | "fmt" 8 | "log" 9 | "os" 10 | "os/signal" 11 | "sync" 12 | 13 | "github.com/aws/aws-sdk-go/aws/awserr" 14 | valid "github.com/koron/go-valid" 15 | "github.com/koron/hupwriter" 16 | "github.com/koron/sqs-notify/sqsnotify2" 17 | ) 18 | 19 | const ( 20 | rpSucceed = "succeed" 21 | rpIgnoreFailure = "ignore_failure" 22 | rpBeforeExecution = "before_execution" 23 | ) 24 | 25 | func toRP(s string) sqsnotify2.RemovePolicy { 26 | switch s { 27 | default: 28 | fallthrough 29 | case rpSucceed: 30 | return sqsnotify2.Succeed 31 | case rpIgnoreFailure: 32 | return sqsnotify2.IgnoreFailure 33 | case rpBeforeExecution: 34 | return sqsnotify2.BeforeExecution 35 | } 36 | } 37 | 38 | func main2() error { 39 | var ( 40 | cfg = sqsnotify2.NewConfig() 41 | version bool 42 | logfile string 43 | pidfile string 44 | 45 | waitTimeSec int64 46 | removePolicy string 47 | multiplier int 48 | ) 49 | 50 | flag.StringVar(&cfg.Profile, "profile", "", "AWS profile name") 51 | flag.StringVar(&cfg.Region, "region", "us-east-1", "AWS region") 52 | flag.StringVar(&cfg.Endpoint, "endpoint", "", "Endpoint of SQS") 53 | flag.Var(valid.String(&cfg.QueueName, "").MustSet(), "queue", "SQS queue name") 54 | flag.BoolVar(&cfg.CreateQueue, "createqueue", false, "create queue if not exists") 55 | flag.IntVar(&cfg.MaxRetries, "max-retries", cfg.MaxRetries, "max retries for AWS") 56 | flag.Int64Var(&waitTimeSec, "wait-time-seconds", -1, `wait time in seconds for next polling. (default -1, disabled, use queue default)`) 57 | 58 | flag.StringVar(&cfg.CacheName, "cache", cfg.CacheName, 59 | `cache name or connection URL 60 | * memory://?capacity=1000 61 | * redis://[{USER}:{PASS}@]{HOST}/[{DBNUM}]?[{OPTIONS}] 62 | 63 | DBNUM: redis DB number (default 0) 64 | OPTIONS: 65 | * lifetime : lifetime of cachetime (ex. "10s", "2m", "3h") 66 | * prefix : prefix of keys 67 | 68 | Example to connect the redis on localhost: "redis://:6379"`) 69 | 70 | flag.IntVar(&cfg.Workers, "workers", cfg.Workers, "num of workers") 71 | flag.Var(valid.Int(&multiplier, 1).Min(1), "multiplier", `pooling the SQS in multiple runner`) 72 | flag.DurationVar(&cfg.Timeout, "timeout", 0, "timeout for command execution (default 0 - no timeout)") 73 | flag.Var(valid.String(&removePolicy, rpSucceed). 74 | OneOf(rpSucceed, rpIgnoreFailure, rpBeforeExecution), "remove-policy", 75 | `policy to remove messages from SQS 76 | * succeed : after execution, succeeded (default) 77 | * ignore_failure : after execution, ignore its result 78 | * before_execution : before execution`) 79 | flag.BoolVar(&version, "version", false, "show version") 80 | flag.StringVar(&logfile, "logfile", "", "log file path") 81 | flag.StringVar(&pidfile, "pidfile", "", "PID file path (require -logfile)") 82 | if err := valid.Parse(flag.CommandLine, os.Args[1:]); err != nil { 83 | return err 84 | } 85 | 86 | if version { 87 | fmt.Println("sqs-notify2 version:", sqsnotify2.Version) 88 | os.Exit(1) 89 | } 90 | 91 | if flag.NArg() < 1 { 92 | return errors.New("need a notification command") 93 | } 94 | args := flag.Args() 95 | cfg.RemovePolicy = toRP(removePolicy) 96 | cfg.CmdName = args[0] 97 | cfg.CmdArgs = args[1:] 98 | if waitTimeSec >= 0 { 99 | cfg.WaitTime = &waitTimeSec 100 | } 101 | 102 | if cfg.Workers < 1 { 103 | return errors.New("\"-worker\" should be greater than 0") 104 | } 105 | if cfg.Workers > 10 { 106 | log.Print("\"WARN: -worker 10+\" doesn't have any effects, check \"-multiplier\"") 107 | } 108 | 109 | // Setup logger. 110 | // FIXME: test logging features. 111 | if pidfile != "" && logfile == "" { 112 | return errors.New("pidfile option requires logfile option") 113 | } 114 | if logfile != "" { 115 | if logfile == "-" { 116 | cfg.Logger = log.New(os.Stdout, "", log.LstdFlags) 117 | } else { 118 | w, err := hupwriter.New(logfile, pidfile) 119 | if err != nil { 120 | return err 121 | } 122 | cfg.Logger = log.New(w, "", 0) 123 | } 124 | } 125 | 126 | ctx, cancel := context.WithCancel(context.Background()) 127 | sig := make(chan os.Signal, 1) 128 | go func() { 129 | for { 130 | s := <-sig 131 | if s == os.Interrupt { 132 | cancel() 133 | signal.Stop(sig) 134 | close(sig) 135 | return 136 | } 137 | } 138 | }() 139 | signal.Notify(sig, os.Interrupt) 140 | 141 | cache, err := sqsnotify2.NewCache(ctx, cfg.CacheName) 142 | if err != nil { 143 | return err 144 | } 145 | defer cache.Close() 146 | 147 | var mu sync.Mutex 148 | var errs []error 149 | 150 | var sg sync.WaitGroup 151 | sg.Add(multiplier) 152 | for i := 0; i < multiplier; i++ { 153 | go func(id int) { 154 | defer sg.Done() 155 | err := sqsnotify2.New(cfg).Run(ctx, cache) 156 | if err == nil || isCancel(err) { 157 | return 158 | } 159 | mu.Lock() 160 | errs = append(errs, err) 161 | mu.Unlock() 162 | log.Printf("inner process #%d is terminated by error: %s", id, err) 163 | }(i) 164 | } 165 | sg.Wait() 166 | 167 | if len(errs) > 0 { 168 | return errs[0] 169 | } 170 | 171 | return nil 172 | } 173 | 174 | func isCancel(err error) bool { 175 | if err2, ok := err.(awserr.Error); ok { 176 | err = err2.OrigErr() 177 | } 178 | if err == context.Canceled { 179 | return true 180 | } 181 | return false 182 | } 183 | 184 | func main() { 185 | err := main2() 186 | if err != nil { 187 | log.Fatal(err) 188 | } 189 | } 190 | -------------------------------------------------------------------------------- /cmd/sqs-send/sqs-send.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "flag" 6 | "fmt" 7 | "log" 8 | "os" 9 | "strconv" 10 | 11 | "github.com/aws/aws-sdk-go/aws" 12 | "github.com/aws/aws-sdk-go/aws/awserr" 13 | "github.com/aws/aws-sdk-go/aws/session" 14 | "github.com/aws/aws-sdk-go/service/sqs" 15 | ) 16 | 17 | const maxSend = 10 18 | 19 | var ( 20 | endpoint string 21 | region string 22 | qname string 23 | 24 | msgnum int 25 | prefix string 26 | ) 27 | 28 | func min(a, b int) int { 29 | if a < b { 30 | return a 31 | } 32 | return b 33 | } 34 | 35 | func main() { 36 | flag.StringVar(&endpoint, "endpoint", "", "endpoint of SQS") 37 | flag.StringVar(®ion, "r", "us-east-1", "AWS region for SQS") 38 | flag.StringVar(&qname, "q", "", "queue name to send") 39 | flag.IntVar(&msgnum, "n", 1, "number of message to send") 40 | flag.StringVar(&prefix, "p", "", "prefix for messages") 41 | flag.Parse() 42 | if qname == "" { 43 | flag.Usage() 44 | fmt.Fprintf(os.Stderr, "need to specify queue name") 45 | os.Exit(1) 46 | } 47 | err := sendMessages(context.Background()) 48 | if err != nil { 49 | log.Printf("fail to send: %s", err) 50 | } 51 | } 52 | 53 | func ensureQueue(ctx context.Context, q *sqs.SQS, qn string) (*string, error) { 54 | rGet, err := q.GetQueueUrlWithContext(ctx, &sqs.GetQueueUrlInput{ 55 | QueueName: aws.String(qn), 56 | }) 57 | if err == nil { 58 | return rGet.QueueUrl, nil 59 | } 60 | if !isQueueDoesNotExist(err) { 61 | return nil, err 62 | } 63 | rCreate, err := q.CreateQueueWithContext(ctx, &sqs.CreateQueueInput{ 64 | QueueName: aws.String(qn), 65 | }) 66 | if err != nil { 67 | return nil, err 68 | } 69 | return rCreate.QueueUrl, nil 70 | } 71 | 72 | func isQueueDoesNotExist(err0 error) bool { 73 | err, ok := err0.(awserr.Error) 74 | if !ok { 75 | return false 76 | } 77 | return err.Code() == sqs.ErrCodeQueueDoesNotExist 78 | } 79 | 80 | func sendQueue(ctx context.Context, q *sqs.SQS, qurl *string, msgs []string) error { 81 | entries := make([]*sqs.SendMessageBatchRequestEntry, 0, len(msgs)) 82 | for i, m := range msgs { 83 | entries = append(entries, &sqs.SendMessageBatchRequestEntry{ 84 | Id: aws.String(strconv.Itoa(i)), 85 | MessageBody: aws.String(m), 86 | }) 87 | } 88 | _, err := q.SendMessageBatchWithContext(ctx, &sqs.SendMessageBatchInput{ 89 | Entries: entries, 90 | QueueUrl: qurl, 91 | }) 92 | if err != nil { 93 | return err 94 | } 95 | return nil 96 | } 97 | 98 | func newSQS() (*sqs.SQS, error) { 99 | cfg := aws.NewConfig() 100 | if endpoint != "" { 101 | cfg.WithEndpoint(endpoint) 102 | } 103 | if region != "" { 104 | cfg.WithRegion(region) 105 | } 106 | ses, err := session.NewSession(cfg) 107 | if err != nil { 108 | return nil, err 109 | } 110 | return sqs.New(ses), nil 111 | } 112 | 113 | func sendMessages(ctx context.Context) error { 114 | q, err := newSQS() 115 | if err != nil { 116 | return err 117 | } 118 | 119 | qurl, err := ensureQueue(ctx, q, qname) 120 | if err != nil { 121 | return err 122 | } 123 | 124 | msgs := make([]string, 0, maxSend) 125 | for i := 0; i < msgnum; { 126 | msgs = msgs[:0] 127 | n := min(msgnum-i, maxSend) 128 | for j := i; j < i+n; j++ { 129 | msgs = append(msgs, fmt.Sprintf("%s%d", prefix, j+1)) 130 | } 131 | err := sendQueue(ctx, q, qurl, msgs) 132 | if err != nil { 133 | return err 134 | } 135 | for _, m := range msgs { 136 | log.Printf("sent %q", m) 137 | } 138 | i += n 139 | } 140 | return nil 141 | } 142 | -------------------------------------------------------------------------------- /config.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "encoding/json" 5 | "errors" 6 | "flag" 7 | "io/ioutil" 8 | "log" 9 | "os" 10 | "strings" 11 | 12 | "github.com/goamz/goamz/aws" 13 | "github.com/koron/hupwriter" 14 | "github.com/koron/sqs-notify/awsutil" 15 | "github.com/koron/sqs-notify/sqsnotify" 16 | ) 17 | 18 | const ( 19 | modeAtMostOnce = "at-most-once" 20 | ) 21 | 22 | type config struct { 23 | daemon bool 24 | region string 25 | worker int 26 | nowait bool 27 | ignoreFailure bool 28 | messageCount int 29 | retryMax int 30 | msgcache int 31 | redis string 32 | logfile string 33 | pidfile string 34 | queue string 35 | cmd string 36 | args []string 37 | 38 | l *log.Logger 39 | } 40 | 41 | func getConfig() (*config, error) { 42 | var ( 43 | version bool 44 | daemon bool 45 | region string 46 | worker int 47 | nowait bool 48 | ignoreFailure bool 49 | messageCount int 50 | retryMax int 51 | msgcache int 52 | redis string 53 | logfile string 54 | pidfile string 55 | mode string 56 | ) 57 | 58 | flag.BoolVar(&version, "version", false, "show version") 59 | flag.BoolVar(&daemon, "daemon", false, "run as a daemon") 60 | flag.StringVar(®ion, "region", "us-east-1", "AWS Region for queue") 61 | flag.IntVar(&worker, "worker", 4, "Num of workers") 62 | flag.BoolVar(&nowait, "nowait", false, "Don't wait end of command") 63 | flag.BoolVar(&ignoreFailure, "ignorefailure", false, "Don't care command failures") 64 | flag.IntVar(&messageCount, "messagecount", 10, "retrieve multiple messages at once") 65 | flag.IntVar(&retryMax, "retrymax", 4, "Num of retry count") 66 | flag.IntVar(&msgcache, "msgcache", 0, "Num of last messages in cache") 67 | flag.StringVar(&redis, "redis", "", "Use redis as messages cache") 68 | flag.StringVar(&logfile, "logfile", "", "Log file path") 69 | flag.StringVar(&pidfile, "pidfile", "", "PID file path (require -logfile)") 70 | flag.StringVar(&mode, "mode", "", "pre-defined set of options for specific usecases") 71 | flag.Usage = usage 72 | flag.Parse() 73 | 74 | if version { 75 | showVersion() 76 | } 77 | 78 | // Parse arguments. 79 | args := flag.Args() 80 | if len(args) < 2 { 81 | usage() 82 | } 83 | 84 | // Check consistencies of options 85 | if len(pidfile) > 0 && len(logfile) == 0 { 86 | return nil, errors.New("`-pidfile' requires `-logfile' option") 87 | } 88 | 89 | if messageCount <= 0 { 90 | return nil, errors.New("`-messagecount` should be > 0") 91 | } else if messageCount > 10 { 92 | return nil, errors.New("`-messagecount` should be <= 10") 93 | } 94 | 95 | // Apply modes. 96 | switch strings.ToLower(mode) { 97 | case modeAtMostOnce: 98 | if nowait { 99 | return nil, errors.New("`-nowait' conflicts with at-most-once") 100 | } 101 | if msgcache == 0 && redis == "" { 102 | return nil, errors.New("`-msgcache' or `-redis' is required for at-most-once") 103 | } 104 | nowait = false 105 | ignoreFailure = true 106 | } 107 | 108 | return &config{ 109 | daemon: daemon, 110 | region: region, 111 | worker: worker, 112 | nowait: nowait, 113 | ignoreFailure: ignoreFailure, 114 | messageCount: messageCount, 115 | retryMax: retryMax, 116 | msgcache: msgcache, 117 | redis: redis, 118 | logfile: logfile, 119 | pidfile: pidfile, 120 | queue: args[0], 121 | cmd: args[1], 122 | args: args[2:], 123 | }, nil 124 | } 125 | 126 | func (c *config) toApp() (*app, error) { 127 | // Retrieve an AWS auth. 128 | auth, err := awsutil.GetAuth("sqs-notify") 129 | if err != nil { 130 | return nil, err 131 | } 132 | 133 | // Determine a region. 134 | region, ok := aws.Regions[c.region] 135 | if !ok { 136 | return nil, errors.New("unknown region:" + c.region) 137 | } 138 | 139 | logger, err := c.logger() 140 | if err != nil { 141 | return nil, err 142 | } 143 | 144 | sqsnotify.Logger = logger 145 | notify := sqsnotify.New(auth, region, c.queue) 146 | // TODO: should another option be used? (github#19) 147 | notify.FailMax = c.retryMax 148 | 149 | jobs, err := c.newJobs() 150 | if err != nil { 151 | return nil, err 152 | } 153 | 154 | return &app{ 155 | logger: logger, 156 | auth: auth, 157 | region: region, 158 | worker: c.worker, 159 | nowait: c.nowait, 160 | ignoreFailure: c.ignoreFailure, 161 | messageCount: c.messageCount, 162 | retryMax: c.retryMax, 163 | jobs: jobs, 164 | notify: notify, 165 | cmd: c.cmd, 166 | args: c.args, 167 | }, nil 168 | } 169 | 170 | func (c *config) logger() (*log.Logger, error) { 171 | if c.l != nil { 172 | return c.l, nil 173 | } 174 | if len(c.logfile) > 0 { 175 | if c.logfile == "-" { 176 | c.l = log.New(os.Stdout, "", log.LstdFlags) 177 | } else { 178 | w, err := hupwriter.New(c.logfile, c.pidfile) 179 | if err != nil { 180 | return nil, err 181 | } 182 | c.l = log.New(w, "", log.LstdFlags) 183 | } 184 | } 185 | if c.l == nil { 186 | c.l = log.New(ioutil.Discard, "", 0) 187 | } 188 | return c.l, nil 189 | } 190 | 191 | func (c *config) newJobs() (jobs, error) { 192 | if c.redis != "" { 193 | rj, err := loadRedisJobs(c.redis) 194 | if err != nil { 195 | return nil, err 196 | } 197 | rj.logger, err = c.logger() 198 | if err != nil { 199 | return nil, err 200 | } 201 | return rj, nil 202 | } 203 | return newJobs(c.msgcache) 204 | } 205 | 206 | func loadRedisJobs(path string) (*redisJobsManager, error) { 207 | b, err := ioutil.ReadFile(path) 208 | if err != nil { 209 | return nil, err 210 | } 211 | var opt redisJobsOptions 212 | if err := json.Unmarshal(b, &opt); err != nil { 213 | return nil, err 214 | } 215 | return newRedisJobs(opt) 216 | } 217 | -------------------------------------------------------------------------------- /daemon.go: -------------------------------------------------------------------------------- 1 | // +build !windows 2 | 3 | package main 4 | 5 | import ( 6 | "github.com/VividCortex/godaemon" 7 | ) 8 | 9 | func makeDaemon() { 10 | godaemon.MakeDaemon(&godaemon.DaemonAttr{}) 11 | } 12 | -------------------------------------------------------------------------------- /daemon_windows.go: -------------------------------------------------------------------------------- 1 | // +build windows 2 | 3 | package main 4 | 5 | import "log" 6 | 7 | func makeDaemon() { 8 | log.Fatalln("sqs-notify:", "windows doesn't support daemon") 9 | } 10 | -------------------------------------------------------------------------------- /doc/runas-service.md: -------------------------------------------------------------------------------- 1 | # Run sqs-notify2 as Service 2 | 3 | This document describes "How make sqs-notify2 work as service" on Ubuntu. 4 | Some paths or commands in this document should be modified for other Linux 5 | distributions. 6 | 7 | ## common preparations 8 | 9 | Build and install sqs-notify2: 10 | 11 | ```console 12 | $ mkdir -p /opt/sqsnotify 13 | $ go build -v ./cmd/sqs-notify2 14 | $ sudo cp ./sqs-notify2 /opt/sqsnotify 15 | ``` 16 | 17 | (OPTION) Install message processing command: 18 | 19 | ```console 20 | $ go build ./cmd/sqs-echo 21 | $ sudo cp ./sqs-echo /opt/sqsnotify 22 | ``` 23 | 24 | Create `/opt/sqsnotify/credentilas` for AWS credentials: 25 | 26 | ``` 27 | [default] 28 | aws_access_key_id=XXXXXXXXXXXXXXXXXXXX 29 | aws_secret_access_key=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX 30 | ``` 31 | 32 | Create `/opt/sqsnotify/env` file to use above credentials file. 33 | 34 | ``` 35 | AWS_SHARED_CREDENTIALS_FILE=/opt/sqsnotify/credentials 36 | ``` 37 | 38 | ## using systemd 39 | 40 | Then create a service file `/etc/systemd/system/sqsnotify.service` to define a 41 | service unit. 42 | 43 | ``` 44 | [Unit] 45 | Description = sqs-notify daemon 46 | 47 | [Service] 48 | ExecStart = /opt/sqsnotify/sqs-notify2 -region {YOUR_REGION} -queue {YOUR_QUEUE} [OPTIONS] {YOUR_COMMAND} [YOUR_COMMAND_OPTIONS] 49 | Restart = always 50 | EnvironmentFile = /opt/sqsnotify/env 51 | Type = simple 52 | 53 | [Install] 54 | WantedBy = multi-user.target 55 | ``` 56 | 57 | Enable (install) sqsnotify service unit: 58 | 59 | ```console 60 | sudo systemctl enable sqsnotify 61 | ``` 62 | 63 | Start the service: 64 | 65 | ```console 66 | sudo systemctl start sqsnotify 67 | ``` 68 | 69 | Check staus of the service: 70 | 71 | ```console 72 | $ sudo systemctl status sqsnotify 73 | ``` 74 | 75 | See log (journal) of the service: 76 | 77 | ```console 78 | $ sudo journalctl -u sqsnotify 79 | ``` 80 | 81 | Stop the service: 82 | 83 | ```console 84 | $ sudo systemctl stop sqsnotify 85 | ``` 86 | 87 | Restart the service: 88 | 89 | ```console 90 | $ sudo systemctl restart sqsnotify 91 | ``` 92 | 93 | ## using supervisord 94 | 95 | Install supervisor. 96 | 97 | ```console 98 | $ sudo apt install supervisor 99 | ``` 100 | 101 | Create a service file `/etc/supervisor/conf.d/sqsnotify.conf` to define the 102 | sqs-notify service. 103 | 104 | ``` 105 | [program:sqsnotify] 106 | command = /opt/sqsnotify/sqs-notify2 -region {YOUR_REGION} -queue {YOUR_QUEUE} [OPTIONS] {YOUR_COMMAND} [YOUR_COMMAND_OPTIONS] 107 | directory = /opt/sqsnotify 108 | process_name = sqsnotify 109 | numprocs = 1 110 | autostart = true 111 | autorestart = true 112 | environment = AWS_SHARED_CREDENTIALS_FILE="/opt/sqsnotify/credentials" 113 | ``` 114 | 115 | Restart (or start) supervisor service to start sqsnotify. 116 | 117 | ```console 118 | $ sudo systemctl restart supervisor 119 | ``` 120 | 121 | Check staus of sqsnotify: 122 | 123 | ```console 124 | $ sudo supervisorctl status sqsnotify 125 | ``` 126 | 127 | Logs from sqsnotify are stored in `/var/log/supervisor/sqsnotify-std*.log`. 128 | 129 | See for more details of supervisord. 130 | -------------------------------------------------------------------------------- /doc/v1.md: -------------------------------------------------------------------------------- 1 | # SQS notify V1 2 | 3 | This document describes old version. 4 | Check [this doc](../README.md) for newer version. 5 | 6 | ## Installation 7 | 8 | ### Pre built binaries 9 | 10 | Please check below URL for pre built binaries: 11 | 12 | 13 | 14 | ### Build by yourself 15 | 16 | ``` 17 | $ go get github.com/koron/sqs-notify 18 | ``` 19 | 20 | or update 21 | 22 | ``` 23 | $ go get -u github.com/koron/sqs-notify 24 | ``` 25 | 26 | Require git command for dependencies. 27 | 28 | ### for developer 29 | 30 | ``` 31 | $ go get github.com/goamz/goamz 32 | ``` 33 | 34 | 35 | ## Usage 36 | 37 | From online help. 38 | 39 | ``` 40 | Usage: sqs-notify [OPTIONS] {queue name} {command and args...} 41 | 42 | OPTIONS: 43 | -daemon 44 | run as a daemon 45 | -ignorefailure 46 | Don't care command failures 47 | -logfile string 48 | Log file path 49 | -messagecount int 50 | retrieve multiple messages at once (default 10) 51 | -mode string 52 | pre-defined set of options for specific usecases 53 | -msgcache int 54 | Num of last messages in cache 55 | -nowait 56 | Don't wait end of command 57 | -pidfile string 58 | PID file path (require -logfile) 59 | -redis string 60 | Use redis as messages cache 61 | -region string 62 | AWS Region for queue (default "us-east-1") 63 | -retrymax int 64 | Num of retry count (default 4) 65 | -version 66 | show version 67 | -worker int 68 | Num of workers (default 4) 69 | 70 | Source: https://github.com/koron/sqs-notify 71 | ``` 72 | 73 | ### Guide 74 | 75 | Basic usage: 76 | 77 | sqs-notify [-region {region}] {queue name} {command and args} 78 | 79 | 1. Prepare AWS auth information. 80 | 1. Use `~/.aws/credentials` (recomended). 81 | 82 | sqs-notify supports `~/.aws/credentials` file, and use info from 83 | `sqs-notify` or `default` section in this order. Example: 84 | 85 | ```ini 86 | [sqs-notify] 87 | aws_access_key_id=foo 88 | aws_secret_access_key=bar 89 | ``` 90 | 91 | 2. Use two environment variables. 92 | * `AWS_ACCESS_KEY_ID` 93 | * `AWS_SECRET_ACCESS_KEY` 94 | 2. Run sqs-notify 95 | 96 | ``` 97 | $ sqs-notify your-queue cat 98 | ``` 99 | 100 | This example just copy messages to STDOUT. If you want to access the queue 101 | via ap-northeast-1 region, use below command. 102 | 103 | ``` 104 | $ sqs-notify -region ap-northeast-1 your-queue cat 105 | ``` 106 | 107 | ### Name of regions 108 | 109 | * `us-east-1` (default) 110 | * `us-west-1` 111 | * `us-west-2` 112 | * `eu-west-1` 113 | * `ap-southeast-1` 114 | * `ap-southeast-2` 115 | * `ap-northeast-1` 116 | * `sp-east-1` 117 | 118 | ### Logging 119 | 120 | When `-logfile {FILE PATH}` is given, all messages which received are logged 121 | into the file. If FILE PATH is `-`, it output all logs to STDOUT not file. 122 | 123 | Using `-pidfile {FILE PATH}` with `-logfile`, sqs-notify writes own PID to the 124 | file. You can send SIGHUP to that PID to rotate log. 125 | 126 | Log example: 127 | 128 | ``` 129 | 2014/05/28 00:13:29 EXECUTED queue:kaoriya body:"abc\n" cmd:cat status:0 130 | 2014/05/28 00:14:27 EXECUTED queue:kaoriya body:"def\n" cmd:cat status:0 131 | 2014/05/28 00:14:54 EXECUTED queue:kaoriya body:"foo\nbar\n" cmd:cat status:0 132 | ``` 133 | 134 | Each items in one log are separated by tab char (`\t`). 135 | 136 | 137 | ## Miscellaneous 138 | 139 | ### LF at EOF 140 | 141 | When message doesn't have LF at EOF (end of file/message), the last line can't 142 | be handled by `read` shell command or so. This is limitation of `read` 143 | command, not sqs-notify. Therefore this kind of scripts don't work correctly 144 | for messages without LF at EOF: 145 | 146 | ```sh 147 | #!/bin/sh 148 | while read line 149 | do 150 | echo "received: $line" 151 | done 152 | ``` 153 | 154 | To work around this problem, use `xargs` like this. 155 | 156 | ```sh 157 | #!/bin/sh 158 | xargs -0 echo | ( 159 | while read line 160 | do 161 | echo "received: $line" 162 | done 163 | ) 164 | ``` 165 | 166 | ### /dev/stdin 167 | 168 | You can use `/dev/stdin` pseudo file, if your system support it, like this: 169 | 170 | ```sh 171 | #!/bin/sh 172 | 173 | data=`cat /dev/stdin` 174 | 175 | # do something for data. 176 | ``` 177 | 178 | ### sqs-echo 179 | 180 | sqs-echo is useful for debugging received SQS message with sqs-notify. It just 181 | shows date, time, byte num and contents of received messages. Example output 182 | is below: 183 | 184 | ``` 185 | 2015/05/07 12:43:08 (7) "foo\nbar" 186 | 2015/05/07 12:43:12 (3) "qux" 187 | ``` 188 | 189 | You can install sqs-echo with below command. 190 | 191 | ``` 192 | $ go install github.com/koron/sqs-notify/cmd/sqs-echo 193 | ``` 194 | 195 | ### Suppress for duplicated message 196 | 197 | To suppress executions of the command for duplicated (seen recently) messages, 198 | please use `-msgcache {num}` option. `{num}` means cache size for messages. 199 | 200 | Below command shows how to enable suppression with cache size 10. 201 | 202 | $ sqs-notify -msgcache 10 -logfile - my-queue sqs-echo 203 | 204 | You'll get log like this when sqs-notify detect duplicated message. 205 | 206 | ``` 207 | 2015/05/07 12:13:08 EXECUTED queue:my-queue body:"foo\n" cmd:cat status:0 208 | 2015/05/07 12:43:11 SKIPPED queue:my-queue body:"foo\n" 209 | ``` 210 | 211 | sqs-notify stores hashs (MD5) of message to detect duplication. 212 | 213 | #### Use redis to suppress duplicated messages 214 | 215 | sqs-notify では `-redis` オプションに設定ファイル(JSON)を指定すると、メッセージ 216 | ハッシュの保存先をRedisに変更できます。 217 | 218 | 実行例: 219 | 220 | $ sqs-notify -redis redis-example.json -logfile - my-queue sqs-echo 221 | 222 | JSON内で使用できる設定項目は以下のとおりです。 223 | 224 | Name |Mandatory? |Description 225 | -----------|------------|------------ 226 | addr |YES |"host:port" address 227 | keyPrefix |YES |prefix of keys 228 | expiration |YES |time to expiration for keys (acceptable [format](http://golang.org/pkg/time/#ParseDuration)) 229 | network |NO |"tcp" or "udp" (default "tcp") 230 | password |NO |password to connect redis 231 | maxRetries |NO |default: no retry 232 | 233 | [redis-example.json](./redis-example.json) には最小限の設定が記載されています。 234 | 235 | ### at-most-once mode 236 | 237 | **at-most-once** mode provides at-most-once command execution. 238 | 239 | $ sqs-notify -mode at-most-once -msgcache 10 my-queue sqs-echo 240 | 241 | This mode implies `-ignorefailure` option, excludes `-nowait` option, and 242 | requires one of `-msgcache` or `-redis` option. 243 | 244 | ## LICENSE 245 | 246 | MIT License. See [LICENSE](../LICENSE) for details. 247 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/koron/sqs-notify 2 | 3 | go 1.21.11 4 | 5 | require ( 6 | github.com/VividCortex/godaemon v1.0.0 7 | github.com/aws/aws-sdk-go v1.54.10 8 | github.com/go-redis/redis v6.15.9+incompatible 9 | github.com/goamz/goamz v0.0.0-20180131231218-8b901b531db8 10 | github.com/koron/go-valid v1.0.0 11 | github.com/koron/hupwriter v1.0.0 12 | github.com/vaughan0/go-ini v0.0.0-20130923145212-a98ad7ee00ec 13 | golang.org/x/sync v0.7.0 14 | gopkg.in/redis.v3 v3.6.4 15 | ) 16 | 17 | require ( 18 | github.com/garyburd/redigo v1.6.4 // indirect 19 | github.com/jmespath/go-jmespath v0.4.0 // indirect 20 | github.com/kr/pretty v0.3.1 // indirect 21 | github.com/onsi/ginkgo v1.16.5 // indirect 22 | github.com/onsi/gomega v1.27.1 // indirect 23 | gopkg.in/bsm/ratelimit.v1 v1.0.0-20160220154919-db14e161995a // indirect 24 | gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect 25 | ) 26 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/VividCortex/godaemon v1.0.0 h1:aHYrScWvgaSOdAoYCdObWXLm+e1rldP9Pwb1ZvuZkQw= 2 | github.com/VividCortex/godaemon v1.0.0/go.mod h1:hBWe/72KbGt/lb95E+Sh9ersdYbB57Dt6CG66S1YPno= 3 | github.com/aws/aws-sdk-go v1.54.10 h1:dvkMlAttUsyacKj2L4poIQBLzOSWL2JG2ty+yWrqets= 4 | github.com/aws/aws-sdk-go v1.54.10/go.mod h1:eRwEWoyTWFMVYVQzKMNHWP5/RV4xIUGMQfXQHfHkpNU= 5 | github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= 6 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 7 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 8 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 9 | github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= 10 | github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= 11 | github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= 12 | github.com/garyburd/redigo v1.6.4 h1:LFu2R3+ZOPgSMWMOL+saa/zXRjw0ID2G8FepO53BGlg= 13 | github.com/garyburd/redigo v1.6.4/go.mod h1:rTb6epsqigu3kYKBnaF028A7Tf/Aw5s0cqA47doKKqw= 14 | github.com/go-redis/redis v6.15.9+incompatible h1:K0pv1D7EQUjfyoMql+r/jZqCLizCGKFlFgcHWWmHQjg= 15 | github.com/go-redis/redis v6.15.9+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA= 16 | github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= 17 | github.com/goamz/goamz v0.0.0-20180131231218-8b901b531db8 h1:G1U0vew/vA/1/hBmf1XNeyIzJJbPFVv+kb+HPl6rj6c= 18 | github.com/goamz/goamz v0.0.0-20180131231218-8b901b531db8/go.mod h1:/Ya1YZsqLQp17bDgHdyE9/XBR1uIH1HKasTvLxcoM/A= 19 | github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 20 | github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= 21 | github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= 22 | github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= 23 | github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= 24 | github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= 25 | github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= 26 | github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= 27 | github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= 28 | github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 29 | github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= 30 | github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= 31 | github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= 32 | github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= 33 | github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= 34 | github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= 35 | github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= 36 | github.com/koron/go-valid v1.0.0 h1:0i+eszV1r1v+fx01xrK6lMte7oUaU1P5rMQ/4HSPQyI= 37 | github.com/koron/go-valid v1.0.0/go.mod h1:IlYKifUShurgElmG2Os/BtspVyoxZnY9T3hEQq0plk4= 38 | github.com/koron/hupwriter v1.0.0 h1:KPVYndiH2IqyrcXMwJVjJOMfvywwEzXkTv5dqr3udSU= 39 | github.com/koron/hupwriter v1.0.0/go.mod h1:u+hx32qhFPd8zzaUNS2XVl5SipRtRhZn3LWfQoEDP6Y= 40 | github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= 41 | github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= 42 | github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= 43 | github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= 44 | github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= 45 | github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= 46 | github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= 47 | github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= 48 | github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= 49 | github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= 50 | github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= 51 | github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= 52 | github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= 53 | github.com/onsi/gomega v1.27.1 h1:rfztXRbg6nv/5f+Raen9RcGoSecHIFgBBLQK3Wdj754= 54 | github.com/onsi/gomega v1.27.1/go.mod h1:aHX5xOykVYzWOV4WqQy0sy8BQptgukenXpCXfadcIAw= 55 | github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= 56 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 57 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 58 | github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= 59 | github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= 60 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 61 | github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= 62 | github.com/vaughan0/go-ini v0.0.0-20130923145212-a98ad7ee00ec h1:DGmKwyZwEB8dI7tbLt/I/gQuP559o/0FrAkHKlQM/Ks= 63 | github.com/vaughan0/go-ini v0.0.0-20130923145212-a98ad7ee00ec/go.mod h1:owBmyHYMLkxyrugmfwE/DLJyW8Ro9mkphwuVErQ0iUw= 64 | github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= 65 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 66 | golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= 67 | golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= 68 | golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= 69 | golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 70 | golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 71 | golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 72 | golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= 73 | golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= 74 | golang.org/x/net v0.7.0 h1:rJrUqqhjsgNp7KqAIc25s9pZnjU7TUcSY7HcVZjdn1g= 75 | golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= 76 | golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 77 | golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 78 | golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 79 | golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= 80 | golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= 81 | golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 82 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 83 | golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 84 | golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 85 | golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 86 | golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 87 | golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 88 | golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 89 | golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 90 | golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU= 91 | golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 92 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 93 | golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= 94 | golang.org/x/text v0.7.0 h1:4BRB4x83lYWy72KwLD/qYDuTu7q9PjSagHvijDw7cLo= 95 | golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= 96 | golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 97 | golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 98 | golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= 99 | golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 100 | golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 101 | golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 102 | golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 103 | google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= 104 | google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= 105 | google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= 106 | google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= 107 | google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= 108 | google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= 109 | gopkg.in/bsm/ratelimit.v1 v1.0.0-20160220154919-db14e161995a h1:stTHdEoWg1pQ8riaP5ROrjS6zy6wewH/Q2iwnLCQUXY= 110 | gopkg.in/bsm/ratelimit.v1 v1.0.0-20160220154919-db14e161995a/go.mod h1:KF9sEfUPAXdG8Oev9e99iLGnl2uJMjc5B+4y3O7x610= 111 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 112 | gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= 113 | gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 114 | gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= 115 | gopkg.in/redis.v3 v3.6.4 h1:u7XgPH1rWwsdZnR+azldXC6x9qDU2luydOIeU/l52fE= 116 | gopkg.in/redis.v3 v3.6.4/go.mod h1:6XeGv/CrsUFDU9aVbUdNykN7k1zVmoeg83KC9RbQfiU= 117 | gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= 118 | gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= 119 | gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 120 | gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 121 | gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 122 | gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU= 123 | gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 124 | gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= 125 | gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 126 | -------------------------------------------------------------------------------- /jobs.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "container/list" 5 | "sync" 6 | ) 7 | 8 | type jobState int 9 | 10 | const ( 11 | jobStarted jobState = iota + 1 12 | jobRunning 13 | jobCompleted 14 | ) 15 | 16 | func (s jobState) String() string { 17 | switch s { 18 | case jobStarted: 19 | return "started" 20 | case jobRunning: 21 | return "running" 22 | case jobCompleted: 23 | return "completed" 24 | default: 25 | return "" 26 | } 27 | } 28 | 29 | func parseJobState(s string) (jobState, bool) { 30 | switch s { 31 | case "started": 32 | return jobStarted, true 33 | case "running": 34 | return jobRunning, true 35 | case "completed": 36 | return jobCompleted, true 37 | default: 38 | return 0, false 39 | } 40 | } 41 | 42 | type jobs interface { 43 | StartTry(id string) (jobState, error) 44 | Fail(id string) 45 | Complete(id string) 46 | Close() 47 | } 48 | 49 | func newJobs(capacity int) (jobs, error) { 50 | return &jobManager{ 51 | capacity: capacity, 52 | table: make(map[string]*jobItem), 53 | keys: list.New(), 54 | }, nil 55 | } 56 | 57 | type jobItem struct { 58 | el *list.Element 59 | state jobState 60 | } 61 | 62 | type jobManager struct { 63 | capacity int 64 | lock sync.Mutex 65 | table map[string]*jobItem 66 | keys *list.List 67 | } 68 | 69 | func (m *jobManager) StartTry(id string) (jobState, error) { 70 | if m.capacity <= 0 { 71 | return jobStarted, nil 72 | } 73 | // get lock for table. 74 | m.lock.Lock() 75 | defer m.lock.Unlock() 76 | // search from cached key. 77 | s, ok := m.table[id] 78 | if ok { 79 | return s.state, nil 80 | } 81 | // remove old entries, if over capacity. 82 | for m.keys.Len() >= m.capacity { 83 | f := m.keys.Front() 84 | delete(m.table, f.Value.(string)) 85 | m.keys.Remove(f) 86 | } 87 | // add a key. 88 | el := m.keys.PushBack(id) 89 | m.table[id] = &jobItem{ 90 | el: el, 91 | state: jobRunning, 92 | } 93 | return jobStarted, nil 94 | } 95 | 96 | func (m *jobManager) Fail(id string) { 97 | if m.capacity <= 0 { 98 | return 99 | } 100 | // get lock for table. 101 | m.lock.Lock() 102 | defer m.lock.Unlock() 103 | // search cached key. 104 | s, ok := m.table[id] 105 | if !ok { 106 | return 107 | } 108 | // remove a key. 109 | delete(m.table, id) 110 | m.keys.Remove(s.el) 111 | } 112 | 113 | func (m *jobManager) Complete(id string) { 114 | if m.capacity <= 0 { 115 | return 116 | } 117 | // get lock for table. 118 | m.lock.Lock() 119 | defer m.lock.Unlock() 120 | // search cached key. 121 | s, ok := m.table[id] 122 | if !ok { 123 | return 124 | } 125 | // remove a key. 126 | s.state = jobCompleted 127 | } 128 | 129 | func (m *jobManager) Close() { 130 | // nothing to do. 131 | } 132 | -------------------------------------------------------------------------------- /jobs_redis.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "log" 7 | "sync" 8 | "time" 9 | 10 | "gopkg.in/redis.v3" 11 | ) 12 | 13 | var ( 14 | errNilCmd = errors.New("redis client returned nil response") 15 | ) 16 | 17 | type redisJobsOptions struct { 18 | redis.Options 19 | KeyPrefix string 20 | Expiration string 21 | } 22 | 23 | type redisJobsManager struct { 24 | logger *log.Logger 25 | lock sync.Mutex 26 | client *redis.Client 27 | keyPrefix string 28 | expiration time.Duration 29 | } 30 | 31 | func newRedisJobs(opt redisJobsOptions) (*redisJobsManager, error) { 32 | expiration, err := time.ParseDuration(opt.Expiration) 33 | if err != nil { 34 | return nil, err 35 | } 36 | c := redis.NewClient(&opt.Options) 37 | if c == nil { 38 | return nil, fmt.Errorf("redis.NewClient failed: %#v", opt.Options) 39 | } 40 | // check connection. 41 | if err := c.Ping().Err(); err != nil { 42 | _ = c.Close() 43 | return nil, err 44 | } 45 | return &redisJobsManager{ 46 | client: c, 47 | keyPrefix: opt.KeyPrefix, 48 | expiration: expiration, 49 | }, nil 50 | } 51 | 52 | func (m *redisJobsManager) StartTry(id string) (jobState, error) { 53 | m.lock.Lock() 54 | defer m.lock.Unlock() 55 | s, ok, err := m.get(id) 56 | if err != nil { 57 | return 0, err 58 | } 59 | if ok { 60 | return s, nil 61 | } 62 | if _, err := m.insert(id, jobRunning); err != nil { 63 | return 0, err 64 | } 65 | return jobStarted, nil 66 | } 67 | 68 | func (m *redisJobsManager) Fail(id string) { 69 | m.lock.Lock() 70 | defer m.lock.Unlock() 71 | // all errors are logged in m.remove(). 72 | _, _ = m.remove(id) 73 | } 74 | 75 | func (m *redisJobsManager) Complete(id string) { 76 | m.lock.Lock() 77 | defer m.lock.Unlock() 78 | // all errors are logged in m.update(). 79 | _, _ = m.update(id, jobCompleted) 80 | } 81 | 82 | func (m *redisJobsManager) Close() { 83 | m.lock.Lock() 84 | defer m.lock.Unlock() 85 | if m.client != nil { 86 | _ = m.client.Close() 87 | m.client = nil 88 | } 89 | } 90 | 91 | func (m *redisJobsManager) logNilCmd(cmd string) { 92 | m.logErr(fmt.Errorf("command %s returns nil", cmd)) 93 | } 94 | 95 | func (m *redisJobsManager) logCmdErr(cmd string, err error) { 96 | m.logErr(fmt.Errorf("command %s returns error: %v", cmd, err)) 97 | } 98 | 99 | func (m *redisJobsManager) logErr(err error) { 100 | if m.logger == nil { 101 | return 102 | } 103 | m.logger.Printf("REDIS: %s", err) 104 | } 105 | 106 | func (m *redisJobsManager) key(id string) string { 107 | return m.keyPrefix + id 108 | } 109 | 110 | func (m *redisJobsManager) insert(id string, s jobState) (bool, error) { 111 | c := m.client.SetNX(m.key(id), s.String(), m.expiration) 112 | if c == nil { 113 | m.logNilCmd("SetNX") 114 | return false, errNilCmd 115 | } 116 | b, err := c.Result() 117 | if err != nil { 118 | if err != redis.Nil { 119 | m.logCmdErr("SetNX", err) 120 | } else { 121 | err = nil 122 | } 123 | return false, err 124 | } 125 | return b, nil 126 | } 127 | 128 | func (m *redisJobsManager) update(id string, s jobState) (bool, error) { 129 | c := m.client.SetXX(m.key(id), s.String(), m.expiration) 130 | if c == nil { 131 | m.logNilCmd("SetXX") 132 | return false, errNilCmd 133 | } 134 | b, err := c.Result() 135 | if err != nil { 136 | if err != redis.Nil { 137 | m.logCmdErr("SetXX", err) 138 | } else { 139 | err = nil 140 | } 141 | return false, err 142 | } 143 | return b, nil 144 | } 145 | 146 | func (m *redisJobsManager) remove(id string) (bool, error) { 147 | c := m.client.Del(m.key(id)) 148 | if c == nil { 149 | m.logNilCmd("Del") 150 | return false, errNilCmd 151 | } 152 | n, err := c.Result() 153 | if err != nil { 154 | if err != redis.Nil { 155 | m.logCmdErr("Del", err) 156 | } else { 157 | err = nil 158 | } 159 | return false, err 160 | } 161 | if n != 1 { 162 | return false, nil 163 | } 164 | return true, nil 165 | } 166 | 167 | func (m *redisJobsManager) get(id string) (jobState, bool, error) { 168 | c := m.client.Get(m.key(id)) 169 | if c == nil { 170 | m.logNilCmd("Get") 171 | return 0, false, errNilCmd 172 | } 173 | v, err := c.Result() 174 | if err != nil { 175 | if err != redis.Nil { 176 | m.logCmdErr("Get", err) 177 | } else { 178 | err = nil 179 | } 180 | return 0, false, err 181 | } 182 | s, ok := parseJobState(v) 183 | if !ok { 184 | err := fmt.Errorf("uknown state: %#v", v) 185 | m.logErr(err) 186 | return 0, false, err 187 | } 188 | return s, true, nil 189 | } 190 | -------------------------------------------------------------------------------- /jobs_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import "testing" 4 | 5 | func jobAssertStart(t *testing.T, j jobs, id string, n int) { 6 | s, err := j.StartTry(id) 7 | if err != nil { 8 | t.Fatalf("failed to start job: %s", err) 9 | } 10 | if s != jobStarted { 11 | t.Errorf(`couldn't start job:%s(%d) with %#v`, id, n, s) 12 | } 13 | } 14 | 15 | func jobAssertNotStart(t *testing.T, j jobs, id string, n int) { 16 | s, err := j.StartTry(id) 17 | if err != nil { 18 | t.Fatalf("failed to start job: %s", err) 19 | } 20 | if s == jobStarted { 21 | t.Errorf(`job:%s(%d) must not be started`, id, n) 22 | } 23 | } 24 | 25 | func TestJobs(t *testing.T) { 26 | j, err := newJobs(5) 27 | if err != nil { 28 | t.Fatalf("newJobs must not return error: %s", err) 29 | } 30 | if j == nil { 31 | t.Fatal("newJobs must not return nil") 32 | } 33 | 34 | // basic 35 | jobAssertStart(t, j, "foo", 1) 36 | jobAssertNotStart(t, j, "foo", 2) 37 | 38 | // add more jobs 39 | jobAssertStart(t, j, "bar", 1) 40 | jobAssertStart(t, j, "baz", 1) 41 | jobAssertNotStart(t, j, "bar", 2) 42 | jobAssertNotStart(t, j, "baz", 2) 43 | jobAssertNotStart(t, j, "foo", 3) 44 | 45 | // capacity 46 | jobAssertStart(t, j, "qux", 1) 47 | jobAssertNotStart(t, j, "foo", 4) 48 | jobAssertStart(t, j, "quux", 1) 49 | jobAssertStart(t, j, "corge", 1) 50 | jobAssertStart(t, j, "foo", 5) 51 | jobAssertNotStart(t, j, "baz", 3) 52 | jobAssertNotStart(t, j, "qux", 2) 53 | jobAssertNotStart(t, j, "quux", 2) 54 | jobAssertNotStart(t, j, "corge", 2) 55 | jobAssertNotStart(t, j, "foo", 6) 56 | 57 | // fail 58 | j.Fail("baz") 59 | jobAssertStart(t, j, "baz", 4) 60 | 61 | // complete 62 | j.Complete("corge") 63 | if s, _ := j.StartTry("corge"); s != jobCompleted { 64 | t.Errorf(`"corge" must not started with %d, but %d`, jobCompleted, s) 65 | } 66 | if s, _ := j.StartTry("foo"); s != jobRunning { 67 | t.Errorf(`"corge" must not started with %d, but %d`, jobRunning, s) 68 | } 69 | } 70 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "errors" 5 | "flag" 6 | "fmt" 7 | "io" 8 | "log" 9 | "math/rand" 10 | "os" 11 | "os/exec" 12 | "os/signal" 13 | "strconv" 14 | "time" 15 | 16 | "github.com/goamz/goamz/aws" 17 | "github.com/goamz/goamz/sqs" 18 | "github.com/koron/sqs-notify/sqsnotify" 19 | ) 20 | 21 | const progname = "sqs-notify" 22 | 23 | var ( 24 | version = "1.5.6" 25 | revision = "" 26 | ) 27 | 28 | type app struct { 29 | logger *log.Logger 30 | auth aws.Auth 31 | region aws.Region 32 | worker int 33 | nowait bool 34 | ignoreFailure bool 35 | messageCount int 36 | retryMax int 37 | jobs jobs 38 | notify *sqsnotify.SQSNotify 39 | cmd string 40 | args []string 41 | 42 | w *workers 43 | } 44 | 45 | func showVersion() { 46 | v := version 47 | if revision != "" { 48 | v = fmt.Sprintf("%s (rev:%s)", version, revision) 49 | } 50 | fmt.Printf("%s version %s\n", progname, v) 51 | os.Exit(1) 52 | } 53 | 54 | func usage() { 55 | fmt.Printf(`Usage: %s [OPTIONS] {queue name} {command and args...} 56 | 57 | OPTIONS: 58 | `, progname) 59 | flag.PrintDefaults() 60 | fmt.Println("\nSource: https://github.com/koron/sqs-notify") 61 | os.Exit(1) 62 | } 63 | 64 | func retryDuration(c int) time.Duration { 65 | limit := (1 << uint(c)) - 1 66 | if limit > 50 { 67 | limit = 50 68 | } 69 | v := rand.Intn(limit) 70 | return time.Duration(v*200) * time.Millisecond 71 | } 72 | 73 | func (a *app) log(v ...interface{}) { 74 | if a.logger == nil { 75 | return 76 | } 77 | a.logger.Print(v...) 78 | } 79 | 80 | func (a *app) logf(s string, args ...interface{}) { 81 | if a.logger == nil { 82 | return 83 | } 84 | a.logger.Printf(s, args...) 85 | } 86 | 87 | func (a *app) logOk(m string, r workerResult) { 88 | if a.logger == nil { 89 | return 90 | } 91 | // Log as OK. 92 | a.logger.Printf("\tEXECUTED\tqueue:%s\tbody:%#v\tcmd:%s\tstatus:%d", 93 | a.notify.Name(), m, a.cmd, r.Code) 94 | } 95 | 96 | func (a *app) logSkip(m string) { 97 | if a.logger == nil { 98 | return 99 | } 100 | // Log as SKIP. 101 | a.logger.Printf("\tSKIPPED\tqueue:%s\tbody:%#v\t", a.notify.Name(), m) 102 | } 103 | 104 | func (a *app) logNg(m string, err error) { 105 | if a.logger == nil { 106 | return 107 | } 108 | a.logger.Printf("\tNOT_EXECUTED\tqueue:%s\tbody:%#v\terror:%s", 109 | a.notify.Name(), m, err) 110 | } 111 | 112 | func (a *app) logAbort(err error) { 113 | s := a.errorSQS(err) 114 | a.log("abort:", s) 115 | log.Println("sqs-notify (abort):", s) 116 | } 117 | 118 | func (a *app) logRetry(err error) { 119 | s := a.errorSQS(err) 120 | a.log("retry:", s) 121 | log.Println("sqs-notify (retry):", s) 122 | } 123 | 124 | func (a *app) errorSQS(err error) string { 125 | switch err := err.(type) { 126 | case *sqs.Error: 127 | return fmt.Sprintf("%s (Code:%s, RequestId:%s)", 128 | err.Message, err.Code, err.RequestId) 129 | default: 130 | return err.Error() 131 | } 132 | } 133 | 134 | func (a *app) deleteSQSMessage(m *sqsnotify.SQSMessage) { 135 | a.notify.ReserveDelete(m) 136 | } 137 | 138 | func (a *app) messageID(m *sqsnotify.SQSMessage) string { 139 | return m.ID() 140 | } 141 | 142 | func (a *app) run() (err error) { 143 | // Open a queue. 144 | sqsnotify.MessageCount = a.messageCount 145 | err = a.notify.Open() 146 | if err != nil { 147 | return 148 | } 149 | 150 | // Listen queue. 151 | c, err := a.notify.Listen() 152 | if err != nil { 153 | return 154 | } 155 | defer a.notify.Stop() 156 | 157 | // accept CTRL+C to terminate. 158 | sig := make(chan os.Signal, 1) 159 | go func() { 160 | for { 161 | s := <-sig 162 | if s == os.Interrupt { 163 | break 164 | } 165 | } 166 | signal.Stop(sig) 167 | close(sig) 168 | close(c) 169 | }() 170 | signal.Notify(sig, os.Interrupt) 171 | 172 | a.w = newWorkers(a.worker) 173 | defer a.waitWorkers() 174 | 175 | // Receive *sqsnotify.SQSMessage via channel. 176 | retryCount := 0 177 | for m := range c { 178 | if m.Error != nil { 179 | if retryCount >= a.retryMax { 180 | a.logAbort(m.Error) 181 | return errors.New("over retry: " + strconv.Itoa(retryCount)) 182 | } 183 | a.logRetry(m.Error) 184 | retryCount++ 185 | // sleep before retry. 186 | time.Sleep(retryDuration(retryCount)) 187 | continue 188 | } else { 189 | retryCount = 0 190 | } 191 | 192 | body := *m.Body() 193 | jid := a.messageID(m) 194 | st, err := a.jobs.StartTry(jid) 195 | if err != nil { 196 | return fmt.Errorf("failed to register/start a job: %s", err) 197 | } 198 | switch st { 199 | case jobRunning: 200 | a.logSkip(body) 201 | if a.nowait { 202 | a.deleteSQSMessage(m) 203 | } 204 | continue 205 | case jobCompleted: 206 | a.logSkip(body) 207 | a.deleteSQSMessage(m) 208 | continue 209 | } 210 | 211 | // Create and setup a exec.Cmd. 212 | if err := a.execCmd(m, jid, body); err != nil { 213 | a.logNg(body, err) 214 | a.jobs.Fail(jid) 215 | } 216 | } 217 | 218 | return 219 | } 220 | 221 | func (a *app) waitWorkers() { 222 | a.w.Wait() 223 | } 224 | 225 | func (a *app) execCmd(m *sqsnotify.SQSMessage, jid, body string) error { 226 | // Create and setup a exec.Cmd. 227 | cmd := exec.Command(a.cmd, a.args...) 228 | stdin, err := cmd.StdinPipe() 229 | if err != nil { 230 | return err 231 | } 232 | stdout, err := cmd.StdoutPipe() 233 | if err != nil { 234 | return err 235 | } 236 | stderr, err := cmd.StderrPipe() 237 | if err != nil { 238 | return err 239 | } 240 | 241 | if a.nowait { 242 | a.deleteSQSMessage(m) 243 | a.w.Run(workerJob{cmd, func(r workerResult) { 244 | a.logOk(body, r) 245 | if r.Success() || a.ignoreFailure { 246 | a.jobs.Complete(jid) 247 | } else { 248 | a.jobs.Fail(jid) 249 | } 250 | }}) 251 | } else { 252 | a.w.Run(workerJob{cmd, func(r workerResult) { 253 | a.logOk(body, r) 254 | if r.Success() || a.ignoreFailure { 255 | a.jobs.Complete(jid) 256 | a.deleteSQSMessage(m) 257 | } else { 258 | a.jobs.Fail(jid) 259 | } 260 | }}) 261 | } 262 | 263 | go io.Copy(os.Stdout, stdout) 264 | go io.Copy(os.Stderr, stderr) 265 | go func() { 266 | _, err := stdin.Write([]byte(body)) 267 | if err != nil { 268 | a.logf("\tWARN: failed to write body\tID:%s\tBODY:%s", 269 | m.Message.MessageId, body) 270 | } 271 | _ = stdin.Close() 272 | }() 273 | 274 | return nil 275 | } 276 | 277 | func main() { 278 | c, err := getConfig() 279 | if err != nil { 280 | log.Fatalln("sqs-notify:", err) 281 | } 282 | 283 | if c.daemon { 284 | makeDaemon() 285 | } 286 | 287 | a, err := c.toApp() 288 | if err != nil { 289 | log.Fatalln("sqs-notify:", err) 290 | } 291 | if a.jobs != nil { 292 | defer a.jobs.Close() 293 | } 294 | 295 | err = a.run() 296 | if err != nil { 297 | log.Fatalln("sqs-notify:", err) 298 | } 299 | } 300 | -------------------------------------------------------------------------------- /redis-example.json: -------------------------------------------------------------------------------- 1 | { 2 | "addr": "127.0.0.1:6379", 3 | "keyPrefix": "mysqsnotify-", 4 | "expiration": "1h" 5 | } 6 | -------------------------------------------------------------------------------- /sqsnotify/sqsnotify.go: -------------------------------------------------------------------------------- 1 | package sqsnotify 2 | 3 | import ( 4 | "container/list" 5 | "fmt" 6 | "io/ioutil" 7 | "log" 8 | "sync" 9 | 10 | "github.com/goamz/goamz/aws" 11 | "github.com/goamz/goamz/sqs" 12 | ) 13 | 14 | // MessageCount specifies message amount to get at once. 15 | var MessageCount = 1 16 | 17 | // Logger provides log for sqsnotify. 18 | var Logger = log.New(ioutil.Discard, "", log.LstdFlags) 19 | 20 | const maxDelete = 10 21 | 22 | func min(a, b int) int { 23 | if a < b { 24 | return a 25 | } 26 | return b 27 | } 28 | 29 | // SQSNotify provides SQS message stream. 30 | type SQSNotify struct { 31 | auth aws.Auth 32 | region aws.Region 33 | name string 34 | 35 | queue *sqs.Queue 36 | 37 | running bool 38 | 39 | // for delete queue 40 | dql sync.Mutex 41 | dqID *list.List 42 | dqMsg map[string]sqs.Message 43 | 44 | // FailMax is limit for continuous errors. 45 | FailMax int 46 | 47 | failCnt int 48 | } 49 | 50 | // New creates and returns a SQSNotify instance. 51 | func New(auth aws.Auth, region aws.Region, name string) *SQSNotify { 52 | return &SQSNotify{ 53 | auth: auth, 54 | region: region, 55 | name: name, 56 | queue: nil, 57 | running: false, 58 | } 59 | } 60 | 61 | // Open prepare internal resources. 62 | func (n *SQSNotify) Open() (err error) { 63 | awsSQS := sqs.New(n.auth, n.region) 64 | n.queue, err = awsSQS.GetQueue(n.name) 65 | if err != nil { 66 | return err 67 | } 68 | n.dqID = list.New() 69 | n.dqMsg = make(map[string]sqs.Message) 70 | return nil 71 | } 72 | 73 | // Listen starts the stream. 74 | func (n *SQSNotify) Listen() (chan *SQSMessage, error) { 75 | ch := make(chan *SQSMessage, 1) 76 | go func() { 77 | n.running = true 78 | loop: 79 | for n.running { 80 | if err := n.flushDeleteQueue(); err != nil { 81 | ch <- newErrorMessage(err) 82 | } 83 | resp, err := n.queue.ReceiveMessage(MessageCount) 84 | if err != nil { 85 | ch <- newErrorMessage(err) 86 | continue 87 | } 88 | for _, m := range n.unique(resp.Messages) { 89 | ch <- newMessage(m, n.queue) 90 | if !n.running { 91 | break loop 92 | } 93 | } 94 | } 95 | close(ch) 96 | }() 97 | return ch, nil 98 | } 99 | 100 | func (n *SQSNotify) unique(list []sqs.Message) []sqs.Message { 101 | uniq := make([]sqs.Message, 0, len(list)) 102 | index := make(map[string]int) 103 | for _, m := range list { 104 | k := m.MessageId 105 | n, ok := index[k] 106 | if ok { 107 | uniq[n] = m 108 | continue 109 | } 110 | index[k] = len(uniq) 111 | uniq = append(uniq, m) 112 | } 113 | return uniq 114 | } 115 | 116 | func (n *SQSNotify) addDeleteQueue(m *SQSMessage) { 117 | if m.IsEmpty() || m.deleted { 118 | return 119 | } 120 | id := m.Message.MessageId 121 | n.dql.Lock() 122 | defer n.dql.Unlock() 123 | if _, ok := n.dqMsg[id]; ok { 124 | // update ReceiptHandle (github#19) 125 | n.dqMsg[id] = m.Message 126 | return 127 | } 128 | n.dqMsg[id] = m.Message 129 | n.dqID.PushBack(id) 130 | m.deleted = true 131 | } 132 | 133 | // ReserveDelete reserves to delete message. 134 | func (n *SQSNotify) ReserveDelete(m *SQSMessage) { 135 | n.addDeleteQueue(m) 136 | // flush to delete ASAP when the queue beyond max delete messages. 137 | if n.dqID.Len() >= maxDelete { 138 | _ = n.flushDeleteQueue() 139 | } 140 | } 141 | 142 | type deleteFault struct { 143 | ID string 144 | Code string 145 | } 146 | 147 | func (n *SQSNotify) logDeleteMessageBatchError(resp *sqs.DeleteMessageBatchResponse, err error) { 148 | var faults []deleteFault 149 | for _, r := range resp.DeleteMessageBatchResult { 150 | if !r.SenderFault { 151 | continue 152 | } 153 | faults = append(faults, deleteFault{ID: r.Id, Code: r.Code}) 154 | } 155 | if len(faults) > 0 { 156 | Logger.Printf("\tDELETE_FAULTS\t%+v", faults) 157 | } 158 | if err != nil { 159 | Logger.Printf("\tDELETE_ERROR\terror:%s", err) 160 | } 161 | } 162 | 163 | func (n *SQSNotify) flushDeleteQueue() error { 164 | err := n.flushDeleteQueue0() 165 | if err == nil { 166 | n.failCnt = 0 167 | return nil 168 | } 169 | if n.FailMax > 0 { 170 | n.failCnt++ 171 | if n.failCnt >= n.failCnt { 172 | // TODO: better failure propagation. (github#19) 173 | panic(fmt.Sprintf("delete fails last %d times", n.failCnt)) 174 | } 175 | } 176 | return err 177 | } 178 | 179 | func (n *SQSNotify) flushDeleteQueue0() error { 180 | n.dql.Lock() 181 | defer n.dql.Unlock() 182 | for n.dqID.Len() > 0 { 183 | msgs := make([]sqs.Message, 0, maxDelete) 184 | el := n.dqID.Front() 185 | for el != nil && len(msgs) < maxDelete { 186 | id := el.Value.(string) 187 | el = el.Next() 188 | msgs = append(msgs, n.dqMsg[id]) 189 | } 190 | resp, err := n.queue.DeleteMessageBatch(msgs) 191 | n.logDeleteMessageBatchError(resp, err) 192 | if err != nil { 193 | return err 194 | } 195 | for _, m := range msgs { 196 | delete(n.dqMsg, m.MessageId) 197 | n.dqID.Remove(n.dqID.Front()) 198 | } 199 | } 200 | return nil 201 | } 202 | 203 | // Name returns queue name. 204 | func (n *SQSNotify) Name() string { 205 | return n.name 206 | } 207 | 208 | // Stop terminates listen loop. 209 | func (n *SQSNotify) Stop() { 210 | n.running = false 211 | _ = n.flushDeleteQueue0() 212 | } 213 | 214 | // SQSMessage represent a SQS message. 215 | type SQSMessage struct { 216 | Error error 217 | Message sqs.Message 218 | 219 | deleted bool 220 | queue *sqs.Queue 221 | } 222 | 223 | // IsEmpty checks MessageId is empty or not. 224 | func (m SQSMessage) IsEmpty() bool { 225 | return m.Message.MessageId == "" 226 | } 227 | 228 | func newErrorMessage(err error) *SQSMessage { 229 | return &SQSMessage{ 230 | Error: err, 231 | deleted: true, 232 | queue: nil, 233 | } 234 | } 235 | 236 | func newMessage(m sqs.Message, q *sqs.Queue) *SQSMessage { 237 | return &SQSMessage{ 238 | Message: m, 239 | deleted: false, 240 | queue: q, 241 | } 242 | } 243 | 244 | // ID returns MessageId of the message. 245 | func (m *SQSMessage) ID() string { 246 | return m.Message.MessageId 247 | } 248 | 249 | // Body returns body of message. 250 | func (m *SQSMessage) Body() *string { 251 | if m.IsEmpty() { 252 | return nil 253 | } 254 | return &m.Message.Body 255 | } 256 | 257 | // Delete requests to delete message to SQS. 258 | func (m *SQSMessage) Delete() (err error) { 259 | if m.deleted { 260 | return nil 261 | } 262 | _, err = m.queue.DeleteMessage(&m.Message) 263 | if err == nil { 264 | m.deleted = true 265 | } 266 | return 267 | } 268 | -------------------------------------------------------------------------------- /sqsnotify2/cache.go: -------------------------------------------------------------------------------- 1 | package sqsnotify2 2 | 3 | import ( 4 | "container/list" 5 | "context" 6 | "errors" 7 | "fmt" 8 | "net/url" 9 | "strconv" 10 | "sync" 11 | 12 | "github.com/koron/sqs-notify/sqsnotify2/stage" 13 | ) 14 | 15 | const minCapacity = maxMsg 16 | 17 | var ( 18 | errCacheFound = errors.New("cache found") 19 | errCacheNotFound = errors.New("cache not found") 20 | ) 21 | 22 | // Cache defines cache storage interface. 23 | type Cache interface { 24 | Insert(id string, stg stage.Stage) error 25 | Update(id string, stg stage.Stage) error 26 | Delete(id string) error 27 | Close() error 28 | } 29 | 30 | type memoryCache struct { 31 | c int 32 | l sync.Mutex 33 | m map[string]mcEntry 34 | k *list.List 35 | } 36 | 37 | type mcEntry struct { 38 | el *list.Element 39 | stg stage.Stage 40 | } 41 | 42 | func newMemoryCache(capacity int) *memoryCache { 43 | return &memoryCache{ 44 | c: capacity, 45 | m: make(map[string]mcEntry), 46 | k: list.New(), 47 | } 48 | } 49 | 50 | func (mc *memoryCache) Insert(id string, stg stage.Stage) error { 51 | mc.l.Lock() 52 | defer mc.l.Unlock() 53 | 54 | if stg == stage.None { 55 | return nil 56 | } 57 | if mc.c < minCapacity { 58 | return nil 59 | } 60 | _, ok := mc.m[id] 61 | if ok { 62 | return errCacheFound 63 | } 64 | // remove old entries. 65 | for mc.k.Len() >= mc.c { 66 | f := mc.k.Front() 67 | delete(mc.m, f.Value.(string)) 68 | mc.k.Remove(f) 69 | } 70 | // add an entry. 71 | el := mc.k.PushBack(id) 72 | mc.m[id] = mcEntry{el: el, stg: stg} 73 | return nil 74 | } 75 | 76 | func (mc *memoryCache) Update(id string, stg stage.Stage) error { 77 | mc.l.Lock() 78 | defer mc.l.Unlock() 79 | 80 | if stg == stage.None { 81 | return nil 82 | } 83 | if mc.c < minCapacity { 84 | return nil 85 | } 86 | v, ok := mc.m[id] 87 | if !ok { 88 | return errCacheNotFound 89 | } 90 | v.stg = stg 91 | return nil 92 | } 93 | 94 | func (mc *memoryCache) Delete(id string) error { 95 | mc.l.Lock() 96 | defer mc.l.Unlock() 97 | 98 | if mc.c < minCapacity { 99 | return nil 100 | } 101 | v, ok := mc.m[id] 102 | if !ok { 103 | return nil 104 | } 105 | delete(mc.m, id) 106 | mc.k.Remove(v.el) 107 | return nil 108 | } 109 | 110 | func (mc *memoryCache) Close() error { 111 | return nil 112 | } 113 | 114 | // NewCache creates a cache implementation. 115 | func NewCache(ctx context.Context, name string) (Cache, error) { 116 | u, err := url.Parse(name) 117 | if err != nil { 118 | return nil, err 119 | } 120 | switch u.Scheme { 121 | case "", "memory": 122 | q := u.Query() 123 | var capacity int 124 | s := q.Get("capacity") 125 | if s != "" { 126 | capacity, err = strconv.Atoi(s) 127 | if err != nil { 128 | return nil, err 129 | } 130 | } 131 | return newMemoryCache(capacity), nil 132 | 133 | case "redis": 134 | return newRedisCache(ctx, u) 135 | } 136 | return nil, fmt.Errorf("not supported cache: %s", name) 137 | } 138 | -------------------------------------------------------------------------------- /sqsnotify2/cache_redis.go: -------------------------------------------------------------------------------- 1 | package sqsnotify2 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "net/url" 7 | "strconv" 8 | "time" 9 | 10 | "github.com/go-redis/redis" 11 | "github.com/koron/sqs-notify/sqsnotify2/stage" 12 | ) 13 | 14 | type redisCache struct { 15 | c *redis.Client 16 | 17 | prefix string 18 | lifetime time.Duration 19 | } 20 | 21 | func newRedisCache(ctx context.Context, u *url.URL) (*redisCache, error) { 22 | if u.Scheme != "redis" { 23 | return nil, fmt.Errorf("unexpected scheme: %s", u.Scheme) 24 | } 25 | var ( 26 | err error 27 | opt = &redis.Options{Addr: u.Host} 28 | rc = &redisCache{} 29 | ) 30 | if u.User != nil { 31 | opt.Password, _ = u.User.Password() 32 | } 33 | if len(u.Path) >= 2 { 34 | opt.DB, err = strconv.Atoi(u.Path[1:]) 35 | if err != nil { 36 | return nil, fmt.Errorf("failed to parse path as DB num: %s", err) 37 | } 38 | } 39 | v := u.Query() 40 | if s := v.Get("lifetime"); s != "" { 41 | rc.lifetime, err = time.ParseDuration(s) 42 | if err != nil { 43 | return nil, fmt.Errorf("failed to parse lifetime: %s", err) 44 | } 45 | } 46 | if s := v.Get("prefix"); s != "" { 47 | rc.prefix = s 48 | } 49 | c := redis.NewClient(opt).WithContext(ctx) 50 | if _, err := c.Ping().Result(); err != nil { 51 | if err != nil { 52 | c.Close() 53 | return nil, fmt.Errorf("failed to ping: %s", err) 54 | } 55 | } 56 | rc.c = c 57 | return rc, nil 58 | } 59 | 60 | func (rc *redisCache) key(id string) string { 61 | return rc.prefix + id 62 | } 63 | 64 | func (rc *redisCache) Insert(id string, stg stage.Stage) error { 65 | if stg == stage.None { 66 | return nil 67 | } 68 | b, err := rc.c.SetNX(rc.key(id), stg, rc.lifetime).Result() 69 | if err != nil { 70 | return err 71 | } 72 | if !b { 73 | return errCacheFound 74 | } 75 | return nil 76 | } 77 | 78 | func (rc *redisCache) Update(id string, stg stage.Stage) error { 79 | if stg == stage.None { 80 | return nil 81 | } 82 | b, err := rc.c.SetXX(rc.key(id), stg, rc.lifetime).Result() 83 | if err != nil { 84 | return err 85 | } 86 | if !b { 87 | return errCacheNotFound 88 | } 89 | return nil 90 | } 91 | 92 | func (rc *redisCache) Delete(id string) error { 93 | _, err := rc.c.Del(rc.key(id)).Result() 94 | if err != nil { 95 | return err 96 | } 97 | return nil 98 | } 99 | 100 | func (rc *redisCache) Close() error { 101 | if rc.c != nil { 102 | err := rc.c.Close() 103 | rc.c = nil 104 | return err 105 | } 106 | return nil 107 | } 108 | -------------------------------------------------------------------------------- /sqsnotify2/cache_test.go: -------------------------------------------------------------------------------- 1 | package sqsnotify2 2 | 3 | import ( 4 | "context" 5 | "net/url" 6 | "os" 7 | "testing" 8 | "time" 9 | 10 | "github.com/koron/sqs-notify/sqsnotify2/stage" 11 | ) 12 | 13 | func testCache(t *testing.T, c Cache) { 14 | id1 := "1234" 15 | id2 := "abcd" 16 | 17 | err := c.Insert(id1, stage.Recv) 18 | if err != nil { 19 | t.Fatalf("failed to insert: %v", err) 20 | } 21 | err = c.Insert(id1, stage.Recv) 22 | if err != errCacheFound { 23 | t.Fatalf("unexpected insertion: %v", err) 24 | } 25 | 26 | err = c.Update(id1, stage.Exec) 27 | if err != nil { 28 | t.Fatalf("failed to update: %v", err) 29 | } 30 | err = c.Update(id2, stage.Exec) 31 | if err != errCacheNotFound { 32 | t.Fatalf("unexpected update: %v", err) 33 | } 34 | 35 | err = c.Delete(id1) 36 | if err != nil { 37 | t.Fatalf("failed to delete: %v", err) 38 | } 39 | err = c.Delete(id1) 40 | if err != nil { 41 | t.Fatalf("failed to delete none: %v", err) 42 | } 43 | } 44 | 45 | func TestRedisCache(t *testing.T) { 46 | s := os.Getenv("REDIS_URL") 47 | if s == "" { 48 | t.Skip("skipping test because REDIS_URL isn't given") 49 | return 50 | } 51 | u, err := url.Parse(s) 52 | if err != nil { 53 | t.Fatalf("failed to parse REDIS_URL: %v", err) 54 | } 55 | rc, err := newRedisCache(context.Background(), u) 56 | if err != nil { 57 | t.Fatalf("failed to create redisCache: %v", err) 58 | } 59 | defer rc.Close() 60 | rc.prefix = t.Name() 61 | rc.lifetime = 10 * time.Second 62 | 63 | testCache(t, rc) 64 | } 65 | 66 | func TestMemoryCache(t *testing.T) { 67 | mc := newMemoryCache(minCapacity) 68 | testCache(t, mc) 69 | } 70 | -------------------------------------------------------------------------------- /sqsnotify2/config.go: -------------------------------------------------------------------------------- 1 | package sqsnotify2 2 | 3 | import ( 4 | "log" 5 | "runtime" 6 | "time" 7 | ) 8 | 9 | // RemovePolicy is a policy to remove SQS message. 10 | type RemovePolicy int 11 | 12 | const ( 13 | // Succeed means "remove a message after notification succeeded" 14 | Succeed RemovePolicy = 0 15 | // IgnoreFailure means "remove a message after notification always" 16 | IgnoreFailure = 1 17 | // BeforeExecution means "remove a message before notification" 18 | BeforeExecution = 2 19 | ) 20 | 21 | // Config configures sqsnotify2 service 22 | type Config struct { 23 | Profile string 24 | Region string 25 | Endpoint string 26 | QueueName string 27 | CreateQueue bool 28 | MaxRetries int 29 | WaitTime *int64 30 | 31 | CacheName string 32 | 33 | Workers int 34 | Timeout time.Duration 35 | RemovePolicy RemovePolicy 36 | CmdName string 37 | CmdArgs []string 38 | 39 | Logger *log.Logger 40 | } 41 | 42 | // NewConfig creates a new Config object. 43 | func NewConfig() *Config { 44 | return &Config{ 45 | Region: "us-east-1", 46 | Workers: runtime.NumCPU(), 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /sqsnotify2/sqsnotify.go: -------------------------------------------------------------------------------- 1 | package sqsnotify2 2 | 3 | import ( 4 | "context" 5 | "io" 6 | "io/ioutil" 7 | "log" 8 | "os" 9 | "os/exec" 10 | "sync" 11 | 12 | "github.com/aws/aws-sdk-go/aws" 13 | "github.com/aws/aws-sdk-go/aws/session" 14 | "github.com/aws/aws-sdk-go/service/sqs" 15 | "github.com/aws/aws-sdk-go/service/sqs/sqsiface" 16 | "github.com/koron/sqs-notify/sqsnotify2/stage" 17 | "golang.org/x/sync/semaphore" 18 | ) 19 | 20 | const maxMsg = 10 21 | 22 | var discardLog = log.New(ioutil.Discard, "", 0) 23 | 24 | // SQSNotify provides SQS consumer and job manager. 25 | type SQSNotify struct { 26 | Config 27 | 28 | l sync.Mutex 29 | results []*result 30 | cache Cache 31 | } 32 | 33 | // New creates a SQSNotify object with configuration. 34 | func New(cfg *Config) *SQSNotify { 35 | if cfg == nil { 36 | cfg = NewConfig() 37 | } 38 | return &SQSNotify{ 39 | Config: *cfg, 40 | } 41 | } 42 | 43 | func (sn *SQSNotify) log() *log.Logger { 44 | if sn.Config.Logger == nil { 45 | return discardLog 46 | } 47 | return sn.Config.Logger 48 | } 49 | 50 | func (sn *SQSNotify) logResult(r *result) { 51 | if r.err == nil { 52 | sn.log().Printf("\tEXECUTED\tbody:%#v", *r.msg.Body) 53 | return 54 | } 55 | sn.log().Printf("\tNOT_EXECUTED\tstage:%[2]s error:%[1]s", r.err, r.stg) 56 | } 57 | 58 | // Run runs SQS notification service. 59 | // ctx is not supported yet. 60 | func (sn *SQSNotify) Run(ctx context.Context, cache Cache) error { 61 | svc, err := sn.newSQS() 62 | if err != nil { 63 | return err 64 | } 65 | sn.cache = cache 66 | 67 | return sn.run(ctx, svc) 68 | } 69 | 70 | func (sn *SQSNotify) newSQS() (*sqs.SQS, error) { 71 | s, err := session.NewSessionWithOptions(session.Options{ 72 | Profile: sn.Profile, 73 | }) 74 | if err != nil { 75 | return nil, err 76 | } 77 | cfg := aws.NewConfig() 78 | if sn.Region != "" { 79 | cfg.WithRegion(sn.Region) 80 | } 81 | if sn.MaxRetries > 0 { 82 | cfg.WithMaxRetries(sn.MaxRetries) 83 | } 84 | if sn.Endpoint != "" { 85 | cfg.WithEndpoint(sn.Endpoint) 86 | } 87 | return sqs.New(s, cfg), nil 88 | } 89 | 90 | func (sn *SQSNotify) run(ctx context.Context, api sqsiface.SQSAPI) error { 91 | qu, err := getQueueURL(api, sn.QueueName, sn.CreateQueue) 92 | if err != nil { 93 | return err 94 | } 95 | var round = 0 96 | for { 97 | // receive messages. 98 | msgs, err := sn.receiveQ(ctx, api, qu, maxMsg) 99 | if err != nil { 100 | return err 101 | } 102 | if len(msgs) == 0 { 103 | //sn.log().Printf("round %d polling timed out, proceed next", round) 104 | round++ 105 | continue 106 | } 107 | 108 | // remove messsages first when RemovePolicy == BeforeExecution 109 | if sn.RemovePolicy == BeforeExecution { 110 | entries := make([]*sqs.DeleteMessageBatchRequestEntry, 0, len(msgs)) 111 | for _, m := range msgs { 112 | entries = append(entries, &sqs.DeleteMessageBatchRequestEntry{ 113 | Id: m.MessageId, 114 | ReceiptHandle: m.ReceiptHandle, 115 | }) 116 | } 117 | err := sn.deleteQ(ctx, api, qu, entries) 118 | if err != nil { 119 | return err 120 | } 121 | } 122 | 123 | // run as commands 124 | sem := sn.newWeighted() 125 | var wg sync.WaitGroup 126 | for i, m := range msgs { 127 | res := &result{round: round, index: i, msg: m} 128 | err := sn.cacheInsert(res, stage.Recv) 129 | if err != nil { 130 | sn.addResult(res.withErr(err)) 131 | continue 132 | } 133 | wg.Add(1) 134 | go func(r, n int, m *sqs.Message, res *result) { 135 | defer wg.Done() 136 | res.stg = stage.Lock 137 | err := sem.Acquire(ctx, 1) 138 | if err != nil { 139 | sn.addResult(res.withErr(err)) 140 | return 141 | } 142 | defer sem.Release(1) 143 | res.stg = stage.Exec 144 | err = sn.cacheUpdate(res, stage.Exec) 145 | if err != nil { 146 | sn.addResult(res.withErr(err)) 147 | return 148 | } 149 | err = sn.execCmd(ctx, m) 150 | if err != nil { 151 | sn.addResult(res.withErr(err)) 152 | return 153 | } 154 | res.stg = stage.Done 155 | err = sn.cacheUpdate(res, stage.Done) 156 | if err != nil { 157 | sn.addResult(res.withErr(err)) 158 | return 159 | } 160 | sn.addResult(&result{round: r, index: n, msg: m}) 161 | }(round, i, m, res) 162 | } 163 | wg.Wait() 164 | 165 | // delete messages 166 | err = sn.deleteQ(ctx, api, qu, sn.deleteEntries()) 167 | if err != nil { 168 | return err 169 | } 170 | sn.clearResults() 171 | round++ 172 | } 173 | } 174 | 175 | func (sn *SQSNotify) deleteEntries() []*sqs.DeleteMessageBatchRequestEntry { 176 | var entries []*sqs.DeleteMessageBatchRequestEntry 177 | for _, r := range sn.results { 178 | if !sn.shouldRemoveAfter(r) { 179 | continue 180 | } 181 | entries = append(entries, &sqs.DeleteMessageBatchRequestEntry{ 182 | Id: r.msg.MessageId, 183 | ReceiptHandle: r.msg.ReceiptHandle, 184 | }) 185 | } 186 | return entries 187 | } 188 | 189 | func (sn *SQSNotify) cacheInsert(r *result, stg stage.Stage) error { 190 | r.stg = stg 191 | err := sn.cache.Insert(*r.msg.MessageId, stg) 192 | if err != nil { 193 | return err 194 | } 195 | return nil 196 | } 197 | 198 | func (sn *SQSNotify) cacheUpdate(r *result, stg stage.Stage) error { 199 | r.stg = stg 200 | err := sn.cache.Update(*r.msg.MessageId, stg) 201 | if err != nil { 202 | // FIXME: consider errCacheNotFound 203 | return err 204 | } 205 | return nil 206 | } 207 | 208 | func (sn *SQSNotify) shouldRemoveAfter(r *result) bool { 209 | switch sn.RemovePolicy { 210 | default: 211 | fallthrough 212 | case Succeed: 213 | return r.err == nil 214 | case IgnoreFailure: 215 | if r.stg == stage.Exec { 216 | sn.log().Printf("command failed but message is deleted: id=%s err=%s", *r.msg.MessageId, r.err) 217 | return true 218 | } 219 | return r.err == nil 220 | case BeforeExecution: 221 | return false 222 | } 223 | } 224 | 225 | // execCmd executes a command for a message, and returns its exit code. 226 | func (sn *SQSNotify) execCmd(ctx context.Context, m *sqs.Message) error { 227 | if sn.Timeout != 0 { 228 | var cancel context.CancelFunc 229 | ctx, cancel = context.WithTimeout(ctx, sn.Timeout) 230 | defer cancel() 231 | } 232 | cmd := exec.CommandContext(ctx, sn.CmdName, sn.CmdArgs...) 233 | 234 | stdout, err := cmd.StdoutPipe() 235 | if err != nil { 236 | return err 237 | } 238 | go io.Copy(os.Stdout, stdout) 239 | stderr, err := cmd.StderrPipe() 240 | if err != nil { 241 | return err 242 | } 243 | go io.Copy(os.Stderr, stderr) 244 | 245 | stdin, err := cmd.StdinPipe() 246 | if err != nil { 247 | return err 248 | } 249 | go func() { 250 | defer stdin.Close() 251 | _, err := io.WriteString(stdin, *m.Body) 252 | if err != nil { 253 | sn.handleCopyMessageFailure(err, m) 254 | } 255 | }() 256 | 257 | err = cmd.Run() 258 | if err != nil { 259 | return err 260 | } 261 | return nil 262 | } 263 | 264 | func (sn *SQSNotify) receiveQ(ctx context.Context, api sqsiface.SQSAPI, queueURL *string, max int64) ([]*sqs.Message, error) { 265 | msgs, err := receiveMessages(ctx, api, queueURL, maxMsg, sn.WaitTime) 266 | if err != nil { 267 | return nil, err 268 | } 269 | return msgs, nil 270 | } 271 | 272 | func (sn *SQSNotify) deleteQ(ctx context.Context, api sqsiface.SQSAPI, queueURL *string, entries []*sqs.DeleteMessageBatchRequestEntry) error { 273 | if len(entries) == 0 { 274 | return nil 275 | } 276 | err := deleteMessages(ctx, api, queueURL, entries) 277 | if err != nil { 278 | if f, ok := err.(*deleteFailure); ok { 279 | // TODO: retry or skip failed entries. 280 | // 1. "not exists" be skipped (ignored) 281 | // 2. others are retried or logged 282 | _ = f 283 | } 284 | return err 285 | } 286 | return nil 287 | } 288 | 289 | func (sn *SQSNotify) handleCopyMessageFailure(err error, m *sqs.Message) { 290 | sn.log().Printf("failed to pass message body: id=%s err=%s", *m.MessageId, err) 291 | } 292 | 293 | func (sn *SQSNotify) newWeighted() *semaphore.Weighted { 294 | n := sn.Workers 295 | if n < 0 || n > maxMsg { 296 | n = 4 297 | } 298 | return semaphore.NewWeighted(int64(n)) 299 | } 300 | 301 | func (sn *SQSNotify) clearResults() { 302 | sn.l.Lock() 303 | sn.results = sn.results[:0] 304 | sn.l.Unlock() 305 | } 306 | 307 | func (sn *SQSNotify) addResult(r *result) { 308 | sn.logResult(r) 309 | sn.l.Lock() 310 | sn.results = append(sn.results, r) 311 | sn.l.Unlock() 312 | } 313 | 314 | type result struct { 315 | round int 316 | index int 317 | msg *sqs.Message 318 | stg stage.Stage 319 | err error 320 | } 321 | 322 | func (r *result) withErr(err error) *result { 323 | r.err = err 324 | return r 325 | } 326 | -------------------------------------------------------------------------------- /sqsnotify2/stage/stage.go: -------------------------------------------------------------------------------- 1 | package stage 2 | 3 | // Stage represents execution stages. 4 | type Stage int 5 | 6 | const ( 7 | // None is zero value of Stage 8 | None Stage = iota 9 | // Recv means "received message, but not executed" 10 | Recv 11 | // Lock means "waiting execution in queue" 12 | Lock 13 | // Exec means "executing" 14 | Exec 15 | // Done means "done execution" 16 | Done 17 | ) 18 | 19 | // MarshalBinary marshal Stage into []byte. used by go-redis/redis. 20 | func (stg Stage) MarshalBinary() ([]byte, error) { 21 | return []byte{byte(stg)}, nil 22 | } 23 | 24 | func (stg Stage) String() string { 25 | switch stg { 26 | case None: 27 | return "None" 28 | case Recv: 29 | return "Recv" 30 | case Lock: 31 | return "Lock" 32 | case Exec: 33 | return "Exec" 34 | case Done: 35 | return "Done" 36 | default: 37 | return "Unknown" 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /sqsnotify2/version.go: -------------------------------------------------------------------------------- 1 | package sqsnotify2 2 | 3 | // Version is a string for sqs-notify2's version 4 | const Version = "v2.2.2" 5 | -------------------------------------------------------------------------------- /sqsnotify2/wrap_sqs.go: -------------------------------------------------------------------------------- 1 | package sqsnotify2 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | 7 | "github.com/aws/aws-sdk-go/aws" 8 | "github.com/aws/aws-sdk-go/aws/awserr" 9 | "github.com/aws/aws-sdk-go/service/sqs" 10 | "github.com/aws/aws-sdk-go/service/sqs/sqsiface" 11 | ) 12 | 13 | func getQueueURL(api sqsiface.SQSAPI, queueName string, create bool) (*string, error) { 14 | rGet, err := api.GetQueueUrl(&sqs.GetQueueUrlInput{ 15 | QueueName: aws.String(queueName), 16 | }) 17 | if err == nil { 18 | return rGet.QueueUrl, nil 19 | } 20 | if !create || !isQueueDoesNotExist(err) { 21 | return nil, err 22 | } 23 | 24 | rCreate, err := api.CreateQueue(&sqs.CreateQueueInput{ 25 | QueueName: aws.String(queueName), 26 | }) 27 | if err != nil { 28 | return nil, err 29 | } 30 | return rCreate.QueueUrl, nil 31 | } 32 | 33 | func isQueueDoesNotExist(err0 error) bool { 34 | err, ok := err0.(awserr.Error) 35 | if !ok { 36 | return false 37 | } 38 | return err.Code() == sqs.ErrCodeQueueDoesNotExist 39 | } 40 | 41 | func receiveMessages(ctx context.Context, api sqsiface.SQSAPI, queueURL *string, max int64, waitTime *int64) ([]*sqs.Message, error) { 42 | out, err := api.ReceiveMessageWithContext(ctx, &sqs.ReceiveMessageInput{ 43 | QueueUrl: queueURL, 44 | MaxNumberOfMessages: &max, 45 | WaitTimeSeconds: waitTime, 46 | }) 47 | if err != nil { 48 | return nil, err 49 | } 50 | return out.Messages, nil 51 | } 52 | 53 | type deleteFailure struct { 54 | failed []*sqs.BatchResultErrorEntry 55 | } 56 | 57 | func (f *deleteFailure) Error() string { 58 | return fmt.Sprintf("failed to delete %d messages", len(f.failed)) 59 | } 60 | 61 | func deleteMessages(ctx context.Context, api sqsiface.SQSAPI, queueURL *string, entries []*sqs.DeleteMessageBatchRequestEntry) error { 62 | out, err := api.DeleteMessageBatchWithContext(ctx, &sqs.DeleteMessageBatchInput{ 63 | QueueUrl: queueURL, 64 | Entries: entries, 65 | }) 66 | if err != nil { 67 | return err 68 | } 69 | if len(out.Failed) > 0 { 70 | return &deleteFailure{failed: out.Failed} 71 | } 72 | return nil 73 | } 74 | -------------------------------------------------------------------------------- /worker.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "os" 5 | "os/exec" 6 | "sync" 7 | "syscall" 8 | ) 9 | 10 | type workerResult struct { 11 | Error error 12 | Code int 13 | ProcessState *os.ProcessState 14 | } 15 | 16 | func (r *workerResult) Success() bool { 17 | return r.ProcessState != nil && r.ProcessState.Success() 18 | } 19 | 20 | type workerJob struct { 21 | Cmd *exec.Cmd 22 | Finish func(workerResult) 23 | } 24 | 25 | type workers struct { 26 | num int 27 | jobs chan workerJob 28 | wait *sync.WaitGroup 29 | cmds []*exec.Cmd 30 | } 31 | 32 | func newWorkers(num int) *workers { 33 | jobs := make(chan workerJob, 1) 34 | w := &workers{ 35 | num: num, 36 | jobs: jobs, 37 | wait: &sync.WaitGroup{}, 38 | cmds: make([]*exec.Cmd, num), 39 | } 40 | 41 | for i := 0; i < num; i++ { 42 | go w.startWorker(i, jobs) 43 | } 44 | return w 45 | } 46 | 47 | func (w *workers) startWorker(num int, jobs chan workerJob) { 48 | for j := range jobs { 49 | w.cmds[num] = j.Cmd 50 | err := j.Cmd.Run() 51 | res := workerResult{ 52 | Code: getStatusCode(err), 53 | Error: err, 54 | ProcessState: j.Cmd.ProcessState, 55 | } 56 | if j.Finish != nil { 57 | j.Finish(res) 58 | } 59 | w.wait.Done() 60 | w.cmds[num] = nil 61 | } 62 | } 63 | 64 | func (w *workers) Run(job workerJob) { 65 | w.wait.Add(1) 66 | w.jobs <- job 67 | } 68 | 69 | func (w *workers) Wait() { 70 | w.wait.Wait() 71 | } 72 | 73 | func (w *workers) Kill() { 74 | for _, c := range w.cmds { 75 | if c == nil || c.Process == nil { 76 | continue 77 | } 78 | c.Process.Kill() 79 | } 80 | } 81 | 82 | // Get status code. It works for Windows and UNIX. 83 | func getStatusCode(err error) int { 84 | if err != nil { 85 | if errexit, ok := err.(*exec.ExitError); ok { 86 | if status, ok := errexit.Sys().(syscall.WaitStatus); ok { 87 | return status.ExitStatus() 88 | } 89 | } 90 | } 91 | return 0 92 | } 93 | --------------------------------------------------------------------------------