├── .gitignore ├── LICENSE.md ├── Makefile ├── README.md ├── go.mod ├── go.sum └── main.go /.gitignore: -------------------------------------------------------------------------------- 1 | # Compiled Object files, Static and Dynamic libs (Shared Objects) 2 | *.o 3 | *.a 4 | *.so 5 | 6 | # Folders 7 | _obj 8 | _test 9 | 10 | # Architecture specific extensions/prefixes 11 | *.[568vq] 12 | [568vq].out 13 | 14 | *.cgo1.go 15 | *.cgo2.c 16 | _cgo_defun.c 17 | _cgo_gotypes.go 18 | _cgo_export.* 19 | 20 | _testmain.go 21 | 22 | *.exe 23 | *.test 24 | *.prof 25 | dist 26 | -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2016-2018 Scott Barr 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | # command to build and run on the local OS. 2 | GO_BUILD = go build 3 | 4 | # command to compiling the distributable. Specify GOOS and GOARCH for 5 | # the target OS. 6 | GO_DIST = CGO_ENABLED=0 GOOS=linux GOARCH=amd64 $(GO_BUILD) -a -tags netgo -ldflags '-w' 7 | 8 | APP = sqsmv 9 | TMP_BUILD = tmp/$(APP) 10 | DIST_DIR = dist 11 | 12 | TAG ?= `git describe --tags | sed -e 's/^v//'` 13 | 14 | .PHONY: dist 15 | 16 | all: clean tools lint goimports vet dist 17 | 18 | deps: 19 | go get -t ./... 20 | 21 | prepare: 22 | mkdir -p build dist 23 | 24 | tools: 25 | go get golang.org/x/tools/cmd/goimports 26 | go get golang.org/x/lint/golint 27 | 28 | lint: golint vet goimports vet 29 | 30 | vet: 31 | go vet 32 | 33 | golint: 34 | ret=0 && test -z "$$(golint . | tee /dev/stderr)" || ret=1 ; exit $$ret 35 | 36 | goimports: 37 | ret=0 && test -z "$$(goimports -l . | tee /dev/stderr)" || ret=1 ; exit $$ret 38 | 39 | dist: prepare dist-linux dist-darwin dist-windows 40 | 41 | dist-linux: 42 | GOOS=linux GOARCH=amd64 go build -o $(TMP_BUILD) 43 | tar -C tmp -zcvf $(DIST_DIR)/$(APP)-$(TAG)-linux.gz $(APP) 44 | rm $(TMP_BUILD) 45 | 46 | dist-darwin: 47 | GOOS=darwin GOARCH=amd64 go build -o $(TMP_BUILD) 48 | tar -C tmp -zcvf $(DIST_DIR)/$(APP)-$(TAG)-darwin.gz $(APP) 49 | rm $(TMP_BUILD) 50 | 51 | dist-windows: 52 | GOOS=windows GOARCH=amd64 go build -o $(TMP_BUILD) 53 | tar -C tmp -zcvf $(DIST_DIR)/$(APP)-$(TAG)-windows.gz $(APP) 54 | rm $(TMP_BUILD) 55 | 56 | clean: 57 | rm -rf build dist 58 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # sqsmv 2 | 3 | Move all messages from one SQS queue to another. 4 | 5 | ## Installation 6 | 7 | ### Source 8 | 9 | ``` 10 | git clone git@github.com:vercel/sqsmv.git 11 | cd sqmv 12 | go build 13 | ``` 14 | 15 | Note: **DO NOT** use `go get`. It will install an old version 16 | 17 | ## Configuration 18 | 19 | The `AWS_SECRET_ACCESS_KEY`, `AWS_ACCESS_KEY_ID`, and `AWS_REGION` 20 | environment variables must be set. 21 | 22 | ## Usage 23 | 24 | Supply source and destination URL endpoints. 25 | 26 | sqsmv -src https://region.queue.amazonaws.com/123/queue-a -dest https://region.queue.amazonaws.com/123/queue-b 27 | 28 | 29 | You can also limit the number of messages you want to move (⚠️ note that the limit is approximate - in the worse case you will get 10 more messages than the limit you entered): 30 | 31 | sqsmv -src https://region.queue.amazonaws.com/123/queue-a -dest https://region.queue.amazonaws.com/123/queue-b -maxMsgCount 20 32 | 33 | ## License 34 | 35 | The MIT License (MIT) 36 | 37 | Copyright (c) 2016-2018 Scott Barr 38 | 39 | See [LICENSE.md](LICENSE.md) 40 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/vercel/sqsmv 2 | 3 | go 1.15 4 | 5 | require github.com/aws/aws-sdk-go v1.36.8 6 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/aws/aws-sdk-go v1.36.8 h1:3nvY3Ax2RC6PN1i0OKppxjq3doHWqiYtvenLQ/oZ5jI= 2 | github.com/aws/aws-sdk-go v1.36.8/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= 3 | github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= 4 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 5 | github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= 6 | github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= 7 | github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= 8 | github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= 9 | github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 10 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 11 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 12 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 13 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 14 | golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= 15 | golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 16 | golang.org/x/net v0.0.0-20201110031124-69a78807bb2b h1:uwuIcX0g4Yl1NC5XAz37xsr2lTtcqevgzYNVt49waME= 17 | golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= 18 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 19 | golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 20 | golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 21 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 22 | golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k= 23 | golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= 24 | golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 25 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= 26 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 27 | gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= 28 | gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 29 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "flag" 5 | "log" 6 | "os" 7 | "regexp" 8 | "sync" 9 | 10 | "github.com/aws/aws-sdk-go/aws" 11 | "github.com/aws/aws-sdk-go/aws/session" 12 | "github.com/aws/aws-sdk-go/service/sqs" 13 | ) 14 | 15 | func main() { 16 | src := flag.String("src", "", "source queue") 17 | dest := flag.String("dest", "", "destination queue") 18 | // TODO make the 1 million be the max safe int in go (how do you do that?) 19 | maxMessageCount := flag.Int("maxMsgCount", 1000000, "max number of messages to be moved") 20 | flag.Parse() 21 | 22 | if *src == "" || *dest == "" { 23 | flag.Usage() 24 | os.Exit(1) 25 | } 26 | 27 | re := regexp.MustCompile(`[a-z]{2}-(?:north|south|east|west)[^-]*-\d`) 28 | srcRegion := re.FindString(*src) 29 | destRegion := re.FindString(*dest) 30 | 31 | log.Printf("source queue : %v (%v)", *src, srcRegion) 32 | log.Printf("destination queue : %v (%v)", *dest, destRegion) 33 | 34 | // enable automatic use of AWS_PROFILE like awscli and other tools do. 35 | opts := session.Options{ 36 | SharedConfigState: session.SharedConfigEnable, 37 | } 38 | 39 | session, err := session.NewSessionWithOptions(opts) 40 | if err != nil { 41 | panic(err) 42 | } 43 | 44 | srcClient := sqs.New(session, aws.NewConfig().WithRegion(srcRegion)) 45 | destClient := sqs.New(session, aws.NewConfig().WithRegion(destRegion)) 46 | 47 | maxMessages := int64(10) 48 | waitTime := int64(0) 49 | messageAttributeNames := aws.StringSlice([]string{"All"}) 50 | 51 | rmin := &sqs.ReceiveMessageInput{ 52 | QueueUrl: src, 53 | MaxNumberOfMessages: &maxMessages, 54 | WaitTimeSeconds: &waitTime, 55 | MessageAttributeNames: messageAttributeNames, 56 | } 57 | 58 | lastMessageCount := int(1) 59 | totalMessageCount := int(0) 60 | // loop as long as there are messages on the queue 61 | for { 62 | resp, err := srcClient.ReceiveMessage(rmin) 63 | 64 | if err != nil { 65 | panic(err) 66 | } 67 | 68 | if lastMessageCount == 0 && len(resp.Messages) == 0 { 69 | // no messages returned twice now, the queue is probably empty 70 | log.Printf("done") 71 | return 72 | } 73 | 74 | lastMessageCount = len(resp.Messages) 75 | totalMessageCount += lastMessageCount 76 | log.Printf("received %v messages...", len(resp.Messages)) 77 | 78 | var wg sync.WaitGroup 79 | wg.Add(len(resp.Messages)) 80 | 81 | for _, m := range resp.Messages { 82 | go func(m *sqs.Message) { 83 | defer wg.Done() 84 | 85 | // write the message to the destination queue 86 | smi := sqs.SendMessageInput{ 87 | MessageAttributes: m.MessageAttributes, 88 | MessageBody: m.Body, 89 | QueueUrl: dest, 90 | } 91 | 92 | _, err := destClient.SendMessage(&smi) 93 | 94 | if err != nil { 95 | log.Printf("ERROR sending message to destination %v", err) 96 | return 97 | } 98 | 99 | // message was sent, dequeue from source queue 100 | dmi := &sqs.DeleteMessageInput{ 101 | QueueUrl: src, 102 | ReceiptHandle: m.ReceiptHandle, 103 | } 104 | 105 | if _, err := srcClient.DeleteMessage(dmi); err != nil { 106 | log.Printf("ERROR dequeueing message ID %v : %v", 107 | *m.ReceiptHandle, 108 | err) 109 | } 110 | }(m) 111 | } 112 | 113 | // wait for all jobs from this batch... 114 | wg.Wait() 115 | 116 | if (totalMessageCount >= *maxMessageCount) { 117 | log.Printf("done, already moved more than max") 118 | return 119 | } 120 | } 121 | } 122 | --------------------------------------------------------------------------------