├── COPYRIGHT ├── LICENSE ├── Makefile ├── README ├── SUPPORT ├── TODO ├── consts.go ├── dialer.go ├── ec2 ├── Makefile ├── consts.go ├── instance.go ├── service.go └── util │ ├── Makefile │ └── ec2_main.go ├── elb ├── Makefile ├── consts.go ├── response.go ├── service.go └── util │ ├── Makefile │ └── elb_main.go ├── escape.go ├── flags ├── Makefile └── init.go ├── http_dialer.go ├── s3 ├── Makefile ├── bucket_opers.go ├── consts.go ├── service_opers.go └── util │ ├── Makefile │ └── s3_main.go ├── samples └── s3_proxy │ ├── Makefile │ ├── README │ ├── config.go │ ├── config.json │ └── main.go ├── sdb ├── Makefile ├── attribute.go ├── consts.go ├── domain.go ├── response.go ├── service.go └── util │ ├── Makefile │ └── sdb_main.go ├── signer.go ├── sqs ├── Makefile ├── consts.go ├── error.go ├── queue.go ├── service.go └── util │ ├── Makefile │ └── sqs_main.go ├── timeformats.go └── util ├── Makefile ├── common ├── Makefile └── common.go └── main.go /COPYRIGHT: -------------------------------------------------------------------------------- 1 | All module code is Copyright(c) 2010-2011, Abneptis LLC, All rights reserved. 2 | 3 | All Go core functionality is Copyright (c) 2009-2011 The Go Authors, 4 | All rights reserved. 5 | 6 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Released under the Go language license (copied below) 2 | 3 | // Redistribution and use in source and binary forms, with or without 4 | // modification, are permitted provided that the following conditions are 5 | // met: 6 | // 7 | // * Redistributions of source code must retain the above copyright 8 | // notice, this list of conditions and the following disclaimer. 9 | // * Redistributions in binary form must reproduce the above 10 | // copyright notice, this list of conditions and the following disclaimer 11 | // in the documentation and/or other materials provided with the 12 | // distribution. 13 | // * Neither the name of Google Inc. nor the names of its 14 | // contributors may be used to endorse or promote products derived from 15 | // this software without specific prior written permission. 16 | // 17 | // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 18 | // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 19 | // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 20 | // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 21 | // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 22 | // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 23 | // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 24 | // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 25 | // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 | // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27 | // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | // 29 | // Subject to the terms and conditions of this License, Google hereby 30 | // grants to You a perpetual, worldwide, non-exclusive, no-charge, 31 | // royalty-free, irrevocable (except as stated in this section) patent 32 | // license to make, have made, use, offer to sell, sell, import, and 33 | // otherwise transfer this implementation of Go, where such license 34 | // applies only to those patent claims licensable by Google that are 35 | // necessarily infringed by use of this implementation of Go. If You 36 | // institute patent litigation against any entity (including a 37 | // cross-claim or counterclaim in a lawsuit) alleging that this 38 | // implementation of Go or a Contribution incorporated within this 39 | // implementation of Go constitutes direct or contributory patent 40 | // infringement, then any patent licenses granted to You under this 41 | // License for this implementation of Go shall terminate as of the date 42 | // such litigation is filed. 43 | 44 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | include $(GOROOT)/src/Make.inc 2 | 3 | TARG=aws 4 | GOFILES=\ 5 | consts.go\ 6 | dialer.go\ 7 | http_dialer.go\ 8 | signer.go\ 9 | escape.go\ 10 | timeformats.go\ 11 | 12 | include $(GOROOT)/src/Make.pkg 13 | 14 | module.%: % 15 | gomake -C $* 16 | 17 | module_install.%: module.% 18 | gomake -C $* install 19 | 20 | module_clean.%: 21 | gomake -C $* clean 22 | 23 | modules: module.sqs module.s3 module.sdb module.ec2 module.elb 24 | modules_install: module_install.sqs module_install.s3 module_install.sdb module_install.ec2 module_install.elb 25 | modules_clean: module_clean.sqs module_clean.s3 module_clean.sdb module_clean.elb module_clean.ec2 26 | -------------------------------------------------------------------------------- /README: -------------------------------------------------------------------------------- 1 | BREAKAGE WARNING! 2 | ================= 3 | The API is going under some substantial cleanups; If you are building from sources prior to Jun 20, 2011, 4 | you will likely find near-zero one-one compatibility. 5 | - Generally, the new API's are applied to SQS, S3 and SDB (in progress) 6 | - Check godoc for the new structures & shapes. 7 | - The tools have been aggregated into a single built tool (in progress) 8 | 9 | Some key functions are not yet implemented (ACL's, subresources), 10 | and most of it is not yet thoroughly tested outside the direct 11 | paths required for the command line tool. 12 | 13 | For quick usage examples, see {elb,s3,simpledb,sqs}/util/*.go. The tools are 14 | fully capable of basic administration of ELB, S3, SQS, and SimpleDB resources, but 15 | are intended primarily for testing and API usage examples. 16 | 17 | This package should build/goinstall on its own, but to make use of the sub-modules, 18 | you will need to run 'make modules modules_install'. (If anyone can submit a patch 19 | to allow goinstall to do the entire project, it will be gladly accepted!) 20 | 21 | Bugs/Notes 22 | ========== 23 | - All the core modules maintain a persistant connection to the service endpoint 24 | - S3 doesn't currently follow redirects for puts 25 | - NO api's automatically retry, it is left to the caller to retry if so desired. 26 | - Errors are not consistant across all modules/calls, work is ongoing to fix this. 27 | - There are a number of issues related to Go's XML parsing that have made 28 | full implementation potentially buggy. 29 | - Most basic commands/operations are available, but we do not currently support 30 | setting ACL's or IAM (directly). 31 | 32 | 33 | Alternatives (let me know if I miss any) 34 | ======================================== 35 | goamz - https://wiki.ubuntu.com/goamz - seems to have much more complete 36 | ec2 support, and s3 37 | -------------------------------------------------------------------------------- /SUPPORT: -------------------------------------------------------------------------------- 1 | Abneptis Go Module Support 2 | =========================== 3 | 4 | This module is still in pre-alpha, and so no user-lists have been created. 5 | 6 | Please don't hesitate to file an issue/pull request against any of our projects 7 | 8 | If you have an interest in such a list, please contact support@abneptis.com 9 | to let us know what module(s) you're interested in, and we'll get a list 10 | created. 11 | 12 | Abneptis Go Module Engineering 13 | ============================== 14 | 15 | While this module is released free to you for any purpose (subject to the 16 | LICENSE file), Abneptis LLC is available for custom and contractual work. 17 | 18 | Please contact sales@abneptis.com for details. 19 | -------------------------------------------------------------------------------- /TODO: -------------------------------------------------------------------------------- 1 | SQS: 2 | - (med) add tests 3 | - (med) add SSL support 4 | - (low) add ACL support 5 | 6 | S3: 7 | - (med) add tests 8 | - (low) ACL support 9 | 10 | Fps/Cobranded: 11 | - all 12 | 13 | Fps: 14 | - all 15 | 16 | Ec2: 17 | - all 18 | 19 | All: 20 | - Refactor some of the general auth/message structures as other types are 21 | needed. 22 | -------------------------------------------------------------------------------- /consts.go: -------------------------------------------------------------------------------- 1 | package aws 2 | 3 | import ( 4 | "os" 5 | "http" 6 | ) 7 | 8 | const ( 9 | DEFAULT_SIGNATURE_VERSION = "2" 10 | DEFAULT_SIGNATURE_METHOD = "HmacSHA256" 11 | ) 12 | 13 | 14 | var ErrorNotFound os.Error = os.NewError("Not found") 15 | var ErrorUnexpectedResponse os.Error = os.NewError("Unexpected response code") 16 | var ErrorConflicts os.Error = os.NewError("Conflicts with another resources") 17 | var ErrorForbidden os.Error = os.NewError("Access denied") 18 | 19 | func CodeToError(i int) (err os.Error) { 20 | switch i { 21 | case http.StatusOK: 22 | case http.StatusNotFound: 23 | err = ErrorNotFound 24 | case http.StatusConflict: 25 | err = ErrorConflicts 26 | case http.StatusForbidden: 27 | err = ErrorForbidden 28 | default: 29 | err = ErrorUnexpectedResponse 30 | } 31 | return 32 | } 33 | -------------------------------------------------------------------------------- /dialer.go: -------------------------------------------------------------------------------- 1 | package aws 2 | 3 | import ( 4 | "net" 5 | "os" 6 | "sync" 7 | // "log" 8 | ) 9 | 10 | // Dev notes: lower-case (private) functions assume the lock is held, 11 | // upper-case functions should use a defer lock.Unlock to ensure 12 | // underlying dialer/socket panics will not leave locks hanging. 13 | 14 | var ErrUnderlyingNotconnected = os.NewError("Underlying socket is not connected") 15 | 16 | // A Dialer is usually a closuer that 17 | // is pre-configured to the callers tastes. 18 | // 19 | // (see URLDialer for an example/default generator) 20 | type Dialer func() (net.Conn, os.Error) 21 | 22 | // A Reusable conn is a syncronized structure around a 23 | // Dialer / net.Conn pair. All net.Conn calls are wrapped 24 | // around the underlying structure. Errors are bubbled 25 | // up, and trigger closure of the underlying socket (to 26 | // be reopened on the next call) 27 | type ReusableConn struct { 28 | lock *sync.Mutex 29 | dialer Dialer 30 | conn net.Conn 31 | readTimeout int64 32 | writeTimeout int64 33 | } 34 | 35 | const ( 36 | _UNSET_TIMEOUT int64 = -1 37 | ) 38 | 39 | // Create a new reusable connection with a sepcific dialer. 40 | func NewReusableConnection(d Dialer) (c *ReusableConn) { 41 | return &ReusableConn{ 42 | dialer: d, 43 | conn: nil, 44 | lock: &sync.Mutex{}, 45 | readTimeout: _UNSET_TIMEOUT, 46 | writeTimeout: _UNSET_TIMEOUT, 47 | } 48 | } 49 | 50 | // Dial is idempotent, and safe to call; 51 | func (self *ReusableConn) Dial() (err os.Error) { 52 | // log.Printf("Public Dial() called") 53 | self.lock.Lock() 54 | defer self.lock.Unlock() 55 | return self.dial() 56 | } 57 | 58 | // Dial will redial if conn is nil, and set 59 | // timeouts if they've been set by the caller. 60 | // 61 | // It simply returns nil if the socket appears already connected 62 | func (self *ReusableConn) dial() (err os.Error) { 63 | // log.Printf("Private dial() called (%v)", self.conn) 64 | if self.conn == nil { 65 | self.conn, err = self.dialer() 66 | if err == nil && self.readTimeout != _UNSET_TIMEOUT { 67 | err = self.setReadTimeout(self.readTimeout) 68 | } 69 | if err == nil && self.writeTimeout != _UNSET_TIMEOUT { 70 | err = self.setWriteTimeout(self.writeTimeout) 71 | } 72 | } 73 | // log.Printf("Private dial() complete (%v)", self.conn) 74 | return 75 | } 76 | 77 | func (self *ReusableConn) close() (err os.Error) { 78 | if self.conn != nil { 79 | err = self.conn.Close() 80 | self.conn = nil 81 | } 82 | return 83 | } 84 | 85 | // Unlike close on a traditional socket, no error 86 | // is raised if you close a closed (nil) connection. 87 | func (self *ReusableConn) Close() (err os.Error) { 88 | self.lock.Lock() 89 | defer self.lock.Unlock() 90 | return self.close() 91 | } 92 | 93 | // TODO: What's an appropriate responsde when we're not connected? 94 | // ATM, we return whatever the other side says, or the nil net.Addr. 95 | func (self *ReusableConn) RemoteAddr() (a net.Addr) { 96 | self.lock.Lock() 97 | defer self.lock.Unlock() 98 | if self.conn != nil { 99 | a = self.conn.RemoteAddr() 100 | } 101 | return 102 | } 103 | 104 | // See RemoteAddr for notes. 105 | func (self *ReusableConn) LocalAddr() (a net.Addr) { 106 | self.lock.Lock() 107 | defer self.lock.Unlock() 108 | if self.conn != nil { 109 | a = self.conn.RemoteAddr() 110 | } 111 | return 112 | } 113 | 114 | func (self *ReusableConn) read(in []byte) (n int, err os.Error) { 115 | err = self.dial() 116 | if err == nil { 117 | n, err = self.conn.Read(in) 118 | if err != nil { 119 | self.close() 120 | } 121 | } 122 | return 123 | } 124 | 125 | func (self *ReusableConn) write(in []byte) (n int, err os.Error) { 126 | err = self.dial() 127 | if err == nil { 128 | n, err = self.conn.Write(in) 129 | if err != nil { 130 | self.close() 131 | } 132 | } 133 | return 134 | } 135 | 136 | 137 | // Read from the underlying connection, triggering a dial if needed. 138 | // NB: For the expected case (HTTP), this shouldn't happen before the 139 | // first Write. 140 | func (self *ReusableConn) Read(in []byte) (n int, err os.Error) { 141 | self.lock.Lock() 142 | defer self.lock.Unlock() 143 | return self.read(in) 144 | } 145 | 146 | // Write to the underlying connection, triggering a dial if needed. 147 | func (self *ReusableConn) Write(out []byte) (n int, err os.Error) { 148 | self.lock.Lock() 149 | defer self.lock.Unlock() 150 | return self.write(out) 151 | } 152 | 153 | 154 | func (self *ReusableConn) setReadTimeout(t int64) (err os.Error) { 155 | err = self.dial() 156 | if err == nil { 157 | err = self.conn.SetReadTimeout(t) 158 | if err == nil { 159 | self.readTimeout = t 160 | } 161 | } 162 | return 163 | } 164 | 165 | func (self *ReusableConn) setWriteTimeout(t int64) (err os.Error) { 166 | err = self.dial() 167 | if err == nil { 168 | err = self.conn.SetWriteTimeout(t) 169 | if err == nil { 170 | self.writeTimeout = t 171 | } 172 | } 173 | return 174 | } 175 | 176 | 177 | // Sets the read timeout on the underlying socket, as well 178 | // as an internal flag for any future re-opened connections. 179 | func (self *ReusableConn) SetReadTimeout(t int64) (err os.Error) { 180 | self.lock.Lock() 181 | defer self.lock.Unlock() 182 | return self.setReadTimeout(t) 183 | } 184 | 185 | // Sets the write timeout on the underlying socket, as well 186 | // as an internal flag for any future re-opened connections. 187 | func (self *ReusableConn) SetWriteTimeout(t int64) (err os.Error) { 188 | self.lock.Lock() 189 | defer self.lock.Unlock() 190 | return self.setWriteTimeout(t) 191 | } 192 | 193 | // Conveinience function for Set(read|write)timeout 194 | func (self *ReusableConn) SetTimeout(t int64) (err os.Error) { 195 | err = self.SetReadTimeout(t) 196 | if err == nil { 197 | err = self.SetWriteTimeout(t) 198 | } 199 | return 200 | } 201 | -------------------------------------------------------------------------------- /ec2/Makefile: -------------------------------------------------------------------------------- 1 | include $(GOROOT)/src/Make.inc 2 | 3 | TARG=aws/ec2 4 | GOFILES=\ 5 | consts.go\ 6 | instance.go\ 7 | service.go\ 8 | 9 | DEPS=\ 10 | ../ 11 | #../\ 12 | 13 | CLEANFILES+=\ 14 | 15 | include $(GOROOT)/src/Make.pkg 16 | 17 | -------------------------------------------------------------------------------- /ec2/consts.go: -------------------------------------------------------------------------------- 1 | package ec2 2 | 3 | const ( 4 | DEFAULT_VERSION = "2011-05-15" 5 | ) 6 | -------------------------------------------------------------------------------- /ec2/instance.go: -------------------------------------------------------------------------------- 1 | package ec2 2 | 3 | 4 | import ( 5 | "xml" 6 | ) 7 | /* 8 | The XML package doesn't do deeply-nested types with reflection well, 9 | so much of the functionality provided is simply hacks 10 | around the very-deep responses given by EC2. 11 | */ 12 | 13 | type describeInstancesResponse struct { 14 | RequestId string "requestId" 15 | Reservations []ReservationSet "reservationSet>item" 16 | } 17 | 18 | type GroupSet struct { 19 | GroupId string "groupId" 20 | GroupName string "groupName" 21 | } 22 | 23 | type ReservationSet struct { 24 | ReservationId string 25 | OwnerId string 26 | Groups []GroupSet "groupSet>item" 27 | Instances []Instance "instancesSet>item" 28 | } 29 | 30 | 31 | // At the level of depth we're at, XML's 32 | // not happy with us 33 | type Instance struct { 34 | XMLName xml.Name 35 | InstanceId string 36 | ImageId string 37 | PrivateDNSName string 38 | DNSName string 39 | PrivateIPAddress string 40 | IPAddress string 41 | AvailabilityZone string "placement>availabilityZone" 42 | MonitoringState string "monitoring>state" 43 | InstanceType string 44 | RootDeviceName string 45 | RootDeviceType string 46 | KernelId string 47 | } 48 | 49 | /* Example EC2 Output (2011-06-21) 50 | 51 | 52 | 9e7685a5-dc96-4ee8-b587-0d4c3b42aed6 53 | 54 | 55 | r-8982afe5 56 | 930374178234 57 | 58 | 59 | sg-b0dc65d9 60 | default 61 | 62 | 63 | 64 | 65 | i-21672e4f 66 | ami-e00df089 67 | 68 | 16 69 | running 70 | 71 | domU-12-31-39-0B-0E-8E.compute-1.internal 72 | ec2-184-72-81-29.compute-1.amazonaws.com 73 | 74 | oddgeneration 75 | 0 76 | 77 | t1.micro 78 | 2011-06-18T17:19:07.000Z 79 | 80 | us-east-1a 81 | 82 | default 83 | 84 | aki-4e7d9527 85 | 86 | disabled 87 | 88 | 10.214.17.120 89 | 184.72.81.29 90 | 91 | 92 | sg-b0dc65d9 93 | default 94 | 95 | 96 | x86_64 97 | ebs 98 | /dev/sda1 99 | 100 | 101 | /dev/sda 102 | 103 | vol-fbf98690 104 | attached 105 | 2011-06-18T17:19:30.000Z 106 | true 107 | 108 | 109 | 110 | paravirtual 111 | 112 | 113 | 114 | Name 115 | build0 116 | 117 | 118 | xen 119 | 120 | 121 | 058890971305 122 | 123 | 124 | r-5fc2eb33 125 | 930374178234 126 | 127 | 128 | sg-b0dc65d9 129 | default 130 | 131 | 132 | 133 | 134 | i-8782d6e9 135 | ami-e00df089 136 | 137 | 16 138 | running 139 | 140 | ip-10-122-46-4.ec2.internal 141 | ec2-50-17-140-146.compute-1.amazonaws.com 142 | 143 | oddgeneration 144 | 0 145 | 146 | t1.micro 147 | 2011-06-20T05:53:35.000Z 148 | 149 | us-east-1c 150 | 151 | default 152 | 153 | aki-4e7d9527 154 | 155 | disabled 156 | 157 | 10.122.46.4 158 | 50.17.140.146 159 | 160 | 161 | sg-b0dc65d9 162 | default 163 | 164 | 165 | x86_64 166 | ebs 167 | /dev/sda1 168 | 169 | 170 | /dev/sda 171 | 172 | vol-dbbfc7b0 173 | attached 174 | 2011-06-20T05:53:54.000Z 175 | true 176 | 177 | 178 | 179 | spot 180 | sir-fa163214 181 | paravirtual 182 | 183 | xen 184 | 185 | 186 | 854251627541 187 | 188 | 189 | 190 | */ 191 | -------------------------------------------------------------------------------- /ec2/service.go: -------------------------------------------------------------------------------- 1 | package ec2 2 | 3 | import ( 4 | "aws" 5 | ) 6 | 7 | import ( 8 | "http" 9 | "log" 10 | "os" 11 | "xml" 12 | ) 13 | 14 | type Service struct { 15 | conn *aws.Conn 16 | URL *http.URL 17 | } 18 | 19 | func NewService(url *http.URL) (s *Service) { 20 | return &Service{ 21 | URL: url, 22 | conn: aws.NewConn(aws.URLDialer(url, nil)), 23 | } 24 | } 25 | 26 | func (self *Service) DescribeInstances(id *aws.Signer, filter http.Values, ic chan Instance) (err os.Error) { 27 | if filter == nil { 28 | filter = http.Values{} 29 | } 30 | filter.Set("Action", "DescribeInstances") 31 | req := aws.NewRequest(self.URL, "GET", nil, filter) 32 | err = id.SignRequestV2(req, aws.Canonicalize, DEFAULT_VERSION, 15) 33 | if err != nil { 34 | return 35 | } 36 | resp, err := self.conn.Request(req) 37 | if err == nil { 38 | defer resp.Body.Close() 39 | xresp := describeInstancesResponse{} 40 | err := xml.Unmarshal(resp.Body, &xresp) 41 | if err == nil { 42 | log.Printf("XRESP == %+v", xresp) 43 | } else { 44 | log.Printf("XERR == %+v", err) 45 | } 46 | ob, _ := http.DumpResponse(resp, true) 47 | os.Stdout.Write(ob) 48 | } 49 | 50 | return 51 | } 52 | 53 | // Closes the underlying connection 54 | func (self *Service) Close() (err os.Error) { 55 | return self.conn.Close() 56 | } 57 | -------------------------------------------------------------------------------- /ec2/util/Makefile: -------------------------------------------------------------------------------- 1 | include $(GOROOT)/src/Make.inc 2 | 3 | TARG=aws/ec2/ec2_util 4 | GOFILES=ec2_main.go\ 5 | 6 | DEPS=\ 7 | ../\ 8 | ../../\ 9 | ../../flags/\ 10 | ../../util/common/\ 11 | 12 | CLEANFILES+=\ 13 | 14 | include $(GOROOT)/src/Make.pkg 15 | 16 | -------------------------------------------------------------------------------- /ec2/util/ec2_main.go: -------------------------------------------------------------------------------- 1 | package ec2_util 2 | 3 | import ( 4 | . "aws/util/common" 5 | . "aws/flags" 6 | "aws/ec2" 7 | "aws" 8 | ) 9 | 10 | import ( 11 | "http" 12 | "os" 13 | "fmt" 14 | "flag" 15 | ) 16 | 17 | // Safety warning 18 | // These are globals to allow the code to be more readable, 19 | // since the tool is "single-tasked" it has no threading issues. 20 | // 21 | // You are of course encouraged to take a more thread-safe approach 22 | // if you intend to use multiple threads. 23 | 24 | var flag_endpoint_url string = "" 25 | 26 | // Convenience method to clean up calls. 27 | func DefaultEC2Service() (id *aws.Signer, s *ec2.Service, err os.Error) { 28 | id, err = DefaultSigner() 29 | if err == nil { 30 | url, err := http.ParseURL(flag_endpoint_url) 31 | if err == nil { 32 | s = ec2.NewService(url) 33 | } 34 | } 35 | return 36 | } 37 | 38 | func init() { 39 | AddModule("ec2", func() { 40 | flag.StringVar(&flag_endpoint_url, "ec2-endpoint", "https://ec2.amazonaws.com/", "Endpoint to use for EC2 calls") 41 | }) 42 | Modules["ec2"].Calls["instances"] = func(args []string) (err os.Error) { 43 | id, s, err := DefaultEC2Service() 44 | if err != nil { 45 | return 46 | } 47 | c := make(chan ec2.Instance) 48 | go func() { 49 | for i := range c { 50 | fmt.Printf("%+v\n", i) 51 | } 52 | }() 53 | err = s.DescribeInstances(id, nil, c) 54 | return 55 | } 56 | } 57 | 58 | func Nil() {} 59 | -------------------------------------------------------------------------------- /elb/Makefile: -------------------------------------------------------------------------------- 1 | include $(GOROOT)/src/Make.inc 2 | 3 | TARG=aws/elb 4 | GOFILES=\ 5 | consts.go\ 6 | response.go\ 7 | service.go\ 8 | 9 | DEPS=../ 10 | 11 | CLEANFILES+=\ 12 | tools/*.[568vq]\ 13 | tools/simpledb 14 | 15 | include $(GOROOT)/src/Make.pkg 16 | 17 | 18 | tools: tools/simpledb 19 | 20 | tools/simpledb.$(O): tools/simpledb.go 21 | $(GC) -o $@ $^ 22 | 23 | tools/simpledb: tools/simpledb.$(O) 24 | $(LD) -o $@ $^ 25 | -------------------------------------------------------------------------------- /elb/consts.go: -------------------------------------------------------------------------------- 1 | package elb 2 | 3 | const DEFAULT_VERSION = "2011-04-05" 4 | -------------------------------------------------------------------------------- /elb/response.go: -------------------------------------------------------------------------------- 1 | package elb 2 | 3 | type LoadBalancerDescription struct { 4 | CanonicalHostedZoneName string 5 | CanonicalHostedZoneNameID string 6 | CreatedTime string 7 | LoadBalancerName string 8 | SourceSecurityGroupOwnerAlias string "sourcesecuritygroup>owneralias" 9 | SourceSecurityGroupGroupName string "sourcesecuritygroup>groupname" 10 | DNSName string 11 | Listeners []Listener "listenerdescriptions>member>listener" 12 | AvailabilityZones []string "AvailabilityZones>member" 13 | } 14 | 15 | type LoadBalancerQueryResult struct { 16 | LoadBalancerDescription []LoadBalancerDescription "DescribeLoadBalancersResult>LoadBalancerDescriptions>member" 17 | RequestId string "requestmetadata>requestid" 18 | ErrorCode string "error>errorcode" 19 | } 20 | -------------------------------------------------------------------------------- /elb/service.go: -------------------------------------------------------------------------------- 1 | package elb 2 | 3 | import ( 4 | "aws" 5 | ) 6 | 7 | import ( 8 | "crypto" 9 | "http" 10 | . "fmt" 11 | "xml" 12 | "os" 13 | "strconv" 14 | ) 15 | 16 | const ( 17 | DEFAULT_HASH = crypto.SHA256 18 | MAX_MESSAGE_SIZE = 64 * 1024 19 | ) 20 | 21 | type Service struct { 22 | URL *http.URL 23 | conn *aws.Conn 24 | } 25 | 26 | func NewService(url *http.URL) *Service { 27 | return &Service{ 28 | URL: url, 29 | conn: aws.NewConn(aws.URLDialer(url, nil)), 30 | } 31 | } 32 | 33 | type Listener struct { 34 | InstancePort int 35 | LoadBalancerPort int 36 | Protocol string 37 | SSLCertificateID string 38 | } 39 | 40 | func (self Listener) SetValues(v http.Values, i int) { 41 | v.Set(Sprintf("Listeners.members.%d.LoadBalancerPort", i), strconv.Itoa(self.LoadBalancerPort)) 42 | v.Set(Sprintf("Listeners.members.%d.InstancePort", i), strconv.Itoa(self.InstancePort)) 43 | v.Set(Sprintf("Listeners.members.%d.Protocol", i), self.Protocol) 44 | } 45 | 46 | func (self *Service) CreateLoadBalancer(id *aws.Signer, name string, zones []string, listeners []Listener) (err os.Error) { 47 | parms := http.Values{} 48 | parms.Set("Action", "CreateLoadBalancer") 49 | parms.Set("LoadBalancerName", name) 50 | for zi := range zones { 51 | parms.Set(Sprintf("AvailabilityZones.members.%d", zi+1), zones[zi]) 52 | } 53 | for li := range listeners { 54 | listeners[li].SetValues(parms, li+1) 55 | } 56 | req := aws.NewRequest(self.URL, "GET", nil, parms) 57 | err = id.SignRequestV2(req, aws.Canonicalize, DEFAULT_VERSION, 0) 58 | if err != nil { 59 | return 60 | } 61 | resp, err := self.conn.Request(req) 62 | if err == nil { 63 | defer resp.Body.Close() 64 | ob, _ := http.DumpResponse(resp, true) 65 | os.Stdout.Write(ob) 66 | } 67 | return 68 | 69 | } 70 | 71 | func (self *Service) DescribeLoadBalancers(id *aws.Signer) (lbs []LoadBalancerDescription, err os.Error) { 72 | parms := http.Values{} 73 | parms.Set("Action", "DescribeLoadBalancers") 74 | req := aws.NewRequest(self.URL, "GET", nil, parms) 75 | err = id.SignRequestV2(req, aws.Canonicalize, DEFAULT_VERSION, 0) 76 | if err != nil { 77 | return 78 | } 79 | resp, err := self.conn.Request(req) 80 | if err == nil { 81 | qr := LoadBalancerQueryResult{} 82 | defer resp.Body.Close() 83 | err = xml.Unmarshal(resp.Body, &qr) 84 | if err == nil { 85 | lbs = qr.LoadBalancerDescription 86 | } 87 | } 88 | 89 | return 90 | } 91 | 92 | 93 | // Users note: amazon will only return an error if the request is bad, 94 | // thus an error will not be raised when deleting a non-existent LB. 95 | func (self *Service) DeleteLoadBalancer(id *aws.Signer, name string) (err os.Error) { 96 | parms := http.Values{} 97 | parms.Set("Action", "DeleteLoadBalancer") 98 | parms.Set("LoadBalancerName", name) 99 | req := aws.NewRequest(self.URL, "GET", nil, parms) 100 | err = id.SignRequestV2(req, aws.Canonicalize, DEFAULT_VERSION, 0) 101 | if err != nil { 102 | return 103 | } 104 | resp, err := self.conn.Request(req) 105 | if err != nil { 106 | return 107 | } 108 | defer resp.Body.Close() 109 | if resp.StatusCode != 200 { 110 | err = aws.CodeToError(resp.StatusCode) 111 | } 112 | qr := LoadBalancerQueryResult{} 113 | err = xml.Unmarshal(resp.Body, &qr) 114 | if err == nil { 115 | if qr.ErrorCode != "" { 116 | err = os.NewError(qr.ErrorCode) 117 | } 118 | } 119 | return 120 | } 121 | 122 | // Closes the underlying connection 123 | func (self *Service) Close() (err os.Error) { 124 | return self.conn.Close() 125 | } 126 | -------------------------------------------------------------------------------- /elb/util/Makefile: -------------------------------------------------------------------------------- 1 | include $(GOROOT)/src/Make.inc 2 | 3 | TARG=aws/elb/elb_util 4 | GOFILES=elb_main.go\ 5 | 6 | DEPS=\ 7 | ../\ 8 | ../../\ 9 | ../../flags/\ 10 | ../../util/common/\ 11 | 12 | CLEANFILES+=\ 13 | 14 | include $(GOROOT)/src/Make.pkg 15 | 16 | -------------------------------------------------------------------------------- /elb/util/elb_main.go: -------------------------------------------------------------------------------- 1 | package elb_util 2 | 3 | import ( 4 | "aws/elb" 5 | . "aws/flags" 6 | . "aws/util/common" 7 | "aws" 8 | ) 9 | 10 | import ( 11 | "flag" 12 | "fmt" 13 | "http" 14 | "os" 15 | "strconv" 16 | "strings" 17 | ) 18 | 19 | // Safety warning 20 | // These are globals to allow the code to be more readable, 21 | // since the tool is "single-tasked" it has no threading issues. 22 | // 23 | // You are of course encouraged to take a more thread-safe approach 24 | // if you intend to use multiple threads. 25 | 26 | var flag_endpoint_url string = "" 27 | var signer *aws.Signer 28 | var service *elb.Service 29 | 30 | // Convenience method to clean up calls. 31 | func DefaultELBService() (id *aws.Signer, s *elb.Service, err os.Error) { 32 | id, err = DefaultSigner() 33 | if err == nil { 34 | url, err := http.ParseURL(flag_endpoint_url) 35 | if err == nil { 36 | s = elb.NewService(url) 37 | } 38 | } 39 | return 40 | } 41 | 42 | 43 | func init() { 44 | AddModule("elb", func() { 45 | flag.StringVar(&flag_endpoint_url, "elb-endpoint", "https://elasticloadbalancing.amazonaws.com/", "Endpoint to use for S3 calls") 46 | }) 47 | Modules["elb"].Setup = func() (err os.Error) { 48 | signer, service, err = DefaultELBService() 49 | return 50 | } 51 | Modules["elb"].Calls["destroy"] = func(args []string) (err os.Error) { 52 | if len(args) != 1 { 53 | err = os.NewError("Usage: destroy lbname") 54 | } 55 | err = service.DeleteLoadBalancer(signer, args[0]) 56 | return 57 | } 58 | Modules["elb"].Calls["list"] = func(args []string) (err os.Error) { 59 | if len(args) != 0 { 60 | err = os.NewError("Usage: list") 61 | } 62 | lbd, err := service.DescribeLoadBalancers(signer) 63 | if err == nil { 64 | for lb := range lbd { 65 | fmt.Printf("%s\t%v\t%s\n", lbd[lb].LoadBalancerName, lbd[lb].AvailabilityZones, lbd[lb].DNSName) 66 | } 67 | } 68 | return 69 | } 70 | 71 | Modules["elb"].Calls["create"] = func(args []string) (err os.Error) { 72 | if len(args) != 3 { 73 | return os.NewError("Usage: create name zone[,zone2] lbport,iport,proto") 74 | } 75 | name := args[0] 76 | zones := strings.Split(args[1], ",", -1) 77 | triple := strings.Split(args[2], ",", 3) 78 | if len(triple) != 3 { 79 | return os.NewError("Invalid lbport/iport/proto triple") 80 | } 81 | l := elb.Listener{} 82 | l.InstancePort, err = strconv.Atoi(triple[0]) 83 | if err != nil { 84 | return 85 | } 86 | l.LoadBalancerPort, err = strconv.Atoi(triple[1]) 87 | if err != nil { 88 | return 89 | } 90 | l.Protocol = triple[2] 91 | err = service.CreateLoadBalancer(signer, name, zones, []elb.Listener{l}) 92 | 93 | return 94 | } 95 | 96 | } 97 | 98 | func Nil() {} 99 | -------------------------------------------------------------------------------- /escape.go: -------------------------------------------------------------------------------- 1 | package aws 2 | 3 | import ( 4 | "http" 5 | "sort" 6 | ) 7 | 8 | // (2011-06-21) - The standard go http.Values.Escape 9 | // works properly for SQS and S3, but it should be 10 | // noted that at least SDB requiers more to be escaped 11 | // than is officially standard. 12 | // 13 | // Sorted Escape also sorts the keys before joining them (needed 14 | // for canonicalization). 15 | func SortedEscape(v http.Values) (out string) { 16 | keys := []string{} 17 | for k, _ := range v { 18 | keys = append(keys, k) 19 | } 20 | sort.SortStrings(keys) 21 | for k := range keys { 22 | if k > 0 { 23 | out += "&" 24 | } 25 | // out += http.URLEscape(keys[k]) + "=" + http.URLEscape(v.Get(keys[k])) 26 | out += escape(keys[k]) + "=" + escape(v.Get(keys[k])) 27 | } 28 | return 29 | } 30 | 31 | func escapeTest(b byte) (out bool) { 32 | switch b { 33 | case 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 34 | 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 35 | 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', 36 | 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', 37 | '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '-', '.', '_': 38 | out = false 39 | default: 40 | out = true 41 | } 42 | return 43 | } 44 | 45 | func escape(in string) (out string) { 46 | return _escape(in, escapeTest, _percentUpper) 47 | } 48 | 49 | // Copied from urltools to avoid the need to import it 50 | 51 | // Escape the string as follows: if etf(b) returns true, use the value generated 52 | // by ef(b), else use b. 53 | func _escape(in string, etf func(byte) bool, ef func(byte) string) (out string) { 54 | for i := range in { 55 | if etf(in[i]) { 56 | out += ef(in[i]) 57 | } else { 58 | out += string(in[i]) 59 | } 60 | } 61 | return 62 | } 63 | 64 | // Returns the hex encoded (upper-case) value of byte. 65 | func _hexEncodeUpper(b byte) (out string) { 66 | hb := []byte{'0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'A', 'B', 'C', 'D', 'E', 'F'} 67 | out = string([]byte{hb[(b&0xf0)>>4], hb[(b & 0x0f)]}) 68 | return 69 | } 70 | 71 | // Returns the hex encoded (upper-case) value of byte with a % prepended. 72 | func _percentUpper(b byte) (out string) { 73 | return "%" + _hexEncodeUpper(b) 74 | } 75 | -------------------------------------------------------------------------------- /flags/Makefile: -------------------------------------------------------------------------------- 1 | include $(GOROOT)/src/Make.inc 2 | 3 | TARG=aws/flags 4 | GOFILES=init.go\ 5 | 6 | DEPS=\ 7 | ../\ 8 | 9 | CLEANFILES+=\ 10 | 11 | include $(GOROOT)/src/Make.pkg 12 | 13 | -------------------------------------------------------------------------------- /flags/init.go: -------------------------------------------------------------------------------- 1 | package flags 2 | // The flags module is primarily a convenience module for those 3 | // who want to request the AWS identity on the command line 4 | // It should be noted that this approach is not particularly secure 5 | // against a malicious root, but very little is. 6 | 7 | import ( 8 | "aws" 9 | ) 10 | 11 | import ( 12 | "flag" 13 | "os" 14 | ) 15 | 16 | 17 | var accessKey *string = flag.String("aws-access-key", 18 | os.Getenv("AWS_ACCESS_KEY_ID"), "AWS Access Key") 19 | 20 | var secretKey *string = flag.String("aws-secret-key", 21 | os.Getenv("AWS_SECRET_ACCESS_KEY"), "AWS Secret Key") 22 | 23 | // Returns the aws.Signer associated with the aws-*-key flags. 24 | func DefaultSigner() (signer *aws.Signer, err os.Error) { 25 | if accessKey == nil || secretKey == nil { 26 | flag.Parse() 27 | } 28 | if *accessKey == "" || *secretKey == "" { 29 | err = os.NewError("No default access key provided") 30 | } else { 31 | signer = aws.NewSigner(*accessKey, *secretKey) 32 | } 33 | return 34 | } 35 | 36 | // An empty function to allow easy package usage (for side-effect only imports) 37 | func UseFlags() {} 38 | -------------------------------------------------------------------------------- /http_dialer.go: -------------------------------------------------------------------------------- 1 | package aws 2 | 3 | import ( 4 | "crypto/tls" 5 | "http" 6 | "net" 7 | "os" 8 | ) 9 | 10 | // The conn structure represents a 'semi detached' http-client 11 | // It handles redialing & reconnecting on connection errors. 12 | type Conn struct { 13 | uc *ReusableConn 14 | c *http.ClientConn 15 | } 16 | 17 | // Creates a new connection with the specified dialer function. 18 | func NewConn(d Dialer) *Conn { 19 | return &Conn{ 20 | uc: NewReusableConnection(d), 21 | c: nil, 22 | } 23 | } 24 | 25 | func (self *Conn) dial() (err os.Error) { 26 | if self.c == nil { 27 | // Get the underlying connection (or redial) 28 | err = self.uc.Dial() 29 | if err == nil { 30 | self.c = http.NewClientConn(self.uc, nil) 31 | } 32 | } 33 | return 34 | } 35 | 36 | 37 | // Closes the underlying connection 38 | // 39 | // NB: if you re-use the connection after 40 | // this, it will be redialed. 41 | func (self *Conn) Close() (err os.Error) { 42 | if self.c != nil { 43 | self.c.Close() 44 | self.c = nil 45 | } 46 | if self.uc != nil { 47 | err = self.uc.Close() 48 | } 49 | return 50 | } 51 | 52 | // Write a request and read the response; 53 | // This function will also fix-up req.Form for 'GET's 54 | func (self *Conn) Request(req *http.Request) (resp *http.Response, err os.Error) { 55 | err = self.dial() 56 | if err == nil { 57 | if req.Form != nil && req.Method == "GET" { 58 | if req.URL.RawQuery != "" { 59 | req.URL.RawQuery += "&" 60 | } 61 | req.URL.RawQuery += req.Form.Encode() 62 | req.Form = nil 63 | } 64 | err = self.c.Write(req) 65 | if err == nil { 66 | resp, err = self.c.Read(req) 67 | } 68 | } 69 | if err != nil { 70 | if err == http.ErrPersistEOF { 71 | err = nil 72 | } 73 | self.Close() 74 | } 75 | return 76 | } 77 | 78 | 79 | // A generic Dialer to handle both TLS and non TLS http connections. 80 | func URLDialer(u *http.URL, conf *tls.Config) (f func() (c net.Conn, err os.Error)) { 81 | host, port, _ := net.SplitHostPort(u.Host) 82 | if port == "" { 83 | if u.Scheme == "http" { 84 | port = "80" 85 | } 86 | if u.Scheme == "https" { 87 | port = "443" 88 | } 89 | } 90 | if host == "" { 91 | host = u.Host 92 | } 93 | useTLS := (u.Scheme == "https") 94 | 95 | f = func() (c net.Conn, err os.Error) { 96 | if useTLS { 97 | return tls.Dial("tcp", host+":"+port, conf) 98 | } 99 | return net.Dial("tcp", host+":"+port) 100 | } 101 | return 102 | } 103 | 104 | // Constructs a basic http.Request based off of a fully-qualified URL 105 | func NewRequest(url *http.URL, method string, hdrs http.Header, params http.Values) (req *http.Request ){ 106 | req = &http.Request{ 107 | Method: method, 108 | URL: &http.URL{ 109 | Path: url.Path, 110 | RawQuery: url.RawQuery, 111 | }, 112 | Host: url.Host, 113 | Header: hdrs, 114 | Form: params, 115 | } 116 | if req.URL.RawQuery != "" { req.URL.RawQuery += "&" } 117 | req.URL.RawQuery += params.Encode() 118 | return 119 | } 120 | -------------------------------------------------------------------------------- /s3/Makefile: -------------------------------------------------------------------------------- 1 | include $(GOROOT)/src/Make.inc 2 | 3 | TARG=aws/s3 4 | 5 | DEPS=\ 6 | ../ 7 | 8 | GOFILES=\ 9 | consts.go\ 10 | service_opers.go\ 11 | bucket_opers.go\ 12 | 13 | include $(GOROOT)/src/Make.pkg 14 | 15 | -------------------------------------------------------------------------------- /s3/bucket_opers.go: -------------------------------------------------------------------------------- 1 | package s3 2 | 3 | import ( 4 | "aws" 5 | ) 6 | 7 | import ( 8 | "bytes" 9 | "http" 10 | "io" 11 | "io/ioutil" 12 | "net" 13 | "os" 14 | "path" 15 | "xml" 16 | ) 17 | 18 | 19 | // Represents a URL and connection to an S3 bucket. 20 | type Bucket struct { 21 | Name string 22 | URL *http.URL 23 | conn *aws.Conn 24 | } 25 | 26 | // NewBucket creates a new *Bucket object from an endpoint URL. 27 | // 28 | // URL is the _endpoint_ url (nil will default to https://s3.amazonaws.com/) 29 | // Name should be the bucket name; If you pass this as an empty string, you will regret it. 30 | // conn is OPTIONAL, but allows you to re-use another aws.Conn if you'd like. 31 | // 32 | // If you omit conn, the dialer used will be based off the VHost of your bucket (if possible), 33 | // to ensure best performance (e.g., endpoint associated w/ your bucket, and any 34 | // regional lb's) 35 | func NewBucket(u *http.URL, Name string, conn *aws.Conn) (b *Bucket) { 36 | if u == nil { 37 | u = &http.URL{Scheme: "https", Host: USEAST_HOST, Path: "/"} 38 | } 39 | if conn == nil { 40 | vname := VhostName(Name, u) 41 | addrs, err := net.LookupHost(vname) 42 | if err == nil && len(addrs) > 0 { 43 | dial_url := &http.URL{ 44 | Scheme: u.Scheme, 45 | Host: vname, 46 | Path: u.Path, 47 | } 48 | conn = aws.NewConn(aws.URLDialer(dial_url, nil)) 49 | } else { 50 | conn = aws.NewConn(aws.URLDialer(u, nil)) 51 | } 52 | } 53 | b = &Bucket{ 54 | URL: u, 55 | Name: Name, 56 | conn: conn, 57 | } 58 | return 59 | } 60 | 61 | // Returns the vhost name of the bucket (bucket.s3.amazonaws.com) 62 | func VhostName(b string, ep *http.URL) string { 63 | return b + "." + ep.Host 64 | } 65 | 66 | func (self *Bucket) key_url(key string) *http.URL { 67 | return &http.URL{ 68 | Scheme: self.URL.Scheme, 69 | Host: self.URL.Host, 70 | Path: path.Join(self.URL.Path, self.Name, key), 71 | } 72 | } 73 | 74 | 75 | // Will open a local file, size it, and upload it to the named key. 76 | // This is a convenience wrapper aroudn PutFile. 77 | func (self *Bucket) PutLocalFile(id *aws.Signer, key, file string) (err os.Error) { 78 | fp, err := os.Open(file) 79 | if err == nil { 80 | defer fp.Close() 81 | err = self.PutFile(id, key, fp) 82 | } 83 | return 84 | } 85 | 86 | // Will put an open file descriptor to the named key. Size is determined 87 | // by statting the fd (so a partially read file will not work). 88 | // TODO: ACL's & content-type/headers support 89 | func (self *Bucket) PutFile(id *aws.Signer, key string, fp *os.File) (err os.Error) { 90 | var resp *http.Response 91 | if fp == nil { 92 | return os.NewError("invalid file descriptor") 93 | } 94 | fi, err := fp.Stat() 95 | if err == nil { 96 | fsize := fi.Size 97 | hdr := http.Header{} 98 | hreq := aws.NewRequest(self.key_url(key), "PUT", hdr, nil) 99 | hreq.ContentLength = fsize 100 | hreq.Body = fp 101 | err = id.SignRequestV1(hreq, aws.CanonicalizeS3, 15) 102 | if err == nil { 103 | resp, err = self.conn.Request(hreq) 104 | if err == nil { 105 | defer resp.Body.Close() 106 | err = aws.CodeToError(resp.StatusCode) 107 | } 108 | } 109 | } 110 | return 111 | } 112 | 113 | 114 | func (self *Bucket) PutKeyBytes(id *aws.Signer, key string, buff []byte, hdr http.Header) (err os.Error) { 115 | var resp *http.Response 116 | hreq := aws.NewRequest(self.key_url(key), "PUT", hdr, nil) 117 | hreq.ContentLength = int64(len(buff)) 118 | hreq.Body = ioutil.NopCloser(bytes.NewBuffer(buff)) 119 | err = id.SignRequestV1(hreq, aws.CanonicalizeS3, 15) 120 | if err == nil { 121 | resp, err = self.conn.Request(hreq) 122 | if err == nil { 123 | defer resp.Body.Close() 124 | err = aws.CodeToError(resp.StatusCode) 125 | } 126 | } 127 | return 128 | } 129 | 130 | // NB: Length is required as we do not buffer the reader 131 | // NB(2): We do NOT close your reader (hence the io.Reader), 132 | // we wrap it with a NopCloser. 133 | func (self *Bucket) PutKeyReader(id *aws.Signer, key string, r io.Reader, l int64, hdr http.Header) (err os.Error) { 134 | var resp *http.Response 135 | hreq := aws.NewRequest(self.key_url(key), "PUT", hdr, nil) 136 | hreq.ContentLength = l 137 | hreq.Body = ioutil.NopCloser(io.LimitReader(r, l)) 138 | err = id.SignRequestV1(hreq, aws.CanonicalizeS3, 15) 139 | if err == nil { 140 | resp, err = self.conn.Request(hreq) 141 | if err == nil { 142 | defer resp.Body.Close() 143 | err = aws.CodeToError(resp.StatusCode) 144 | } 145 | if err == aws.ErrorUnexpectedResponse { 146 | ob, _ := http.DumpResponse(resp, true) 147 | os.Stdout.Write(ob) 148 | } 149 | } 150 | return 151 | } 152 | 153 | // Deletes the named key from the bucket. To delete a bucket, see *Service.DeleteBucket() 154 | func (self *Bucket) Delete(id *aws.Signer, key string) (err os.Error) { 155 | var resp *http.Response 156 | if key == "" { 157 | return os.NewError("Key cannot be empty!") 158 | } 159 | hreq := aws.NewRequest(self.key_url(key), "DELETE", nil, nil) 160 | err = id.SignRequestV1(hreq, aws.CanonicalizeS3, 15) 161 | if err == nil { 162 | resp, err = self.conn.Request(hreq) 163 | } 164 | if err == nil { 165 | defer resp.Body.Close() 166 | if resp.StatusCode != http.StatusNoContent { 167 | err = aws.CodeToError(resp.StatusCode) 168 | } 169 | } 170 | 171 | return 172 | } 173 | 174 | // Opens the named key and copys it to the named io.Writer IFF the response.Status is 200. 175 | // Also returns the http headers for convenience (regardless of status code, as long as a resp is generated). 176 | func (self *Bucket) GetKey(id *aws.Signer, key string, w io.Writer) (hdr http.Header, err os.Error) { 177 | var resp *http.Response 178 | hreq := aws.NewRequest(self.key_url(key), "GET", nil, nil) 179 | err = id.SignRequestV1(hreq, aws.CanonicalizeS3, 15) 180 | if err == nil { 181 | resp, err = self.conn.Request(hreq) 182 | } 183 | if err == nil { 184 | defer resp.Body.Close() 185 | err = aws.CodeToError(resp.StatusCode) 186 | hdr = resp.Header 187 | if err == nil { 188 | _, err2 := io.Copy(w, resp.Body) 189 | if err == nil { 190 | err = err2 191 | } 192 | } 193 | } 194 | return 195 | } 196 | 197 | // Performs a HEAD request on the bucket and returns nil of the key appears 198 | // valid (returns 200). 199 | func (self *Bucket) Exists(id *aws.Signer, key string) (err os.Error) { 200 | _, err = self.HeadKey(id, key) 201 | return 202 | } 203 | 204 | // Performs a HEAD request on the bucket and returns the response object. 205 | // The body is CLOSED, and it is an error to try and read from it. 206 | func (self *Bucket) HeadKey(id *aws.Signer, key string) (resp *http.Response, err os.Error) { 207 | hreq := aws.NewRequest(self.key_url(key), "HEAD", nil, nil) 208 | err = id.SignRequestV1(hreq, aws.CanonicalizeS3, 15) 209 | if err == nil { 210 | resp, err = self.conn.Request(hreq) 211 | } 212 | if err == nil { 213 | resp.Body.Close() 214 | err = aws.CodeToError(resp.StatusCode) 215 | } 216 | return 217 | } 218 | 219 | type owner struct { 220 | ID string 221 | DisplayName string 222 | } 223 | 224 | // Walks a bucket and writes the resulting strings to the channel. 225 | // * There is currently NO (correct) way to abort a running walk. 226 | func (self *Bucket) ListKeys(id *aws.Signer, 227 | prefix, delim, marker string, out chan<- string) (err os.Error) { 228 | var done bool 229 | var resp *http.Response 230 | var last string 231 | form := http.Values{"prefix": []string{prefix}, 232 | "delimeter": []string{delim}, 233 | "marker":[]string{marker}} 234 | for err == nil && !done { 235 | result := listBucketResult{} 236 | result.Prefix = prefix 237 | result.Marker = marker 238 | if last != "" {form.Set("marker", last) } 239 | 240 | hreq := aws.NewRequest(self.key_url("/"), "GET", nil, form) 241 | err = id.SignRequestV1(hreq, aws.CanonicalizeS3, 15) 242 | 243 | if err == nil { 244 | resp, err = self.conn.Request(hreq) 245 | } 246 | if err == nil { 247 | err = aws.CodeToError(resp.StatusCode) 248 | if err == nil { 249 | err = xml.Unmarshal(resp.Body, &result) 250 | if err == nil { 251 | for i := range result.Contents { 252 | out <- result.Contents[i].Key 253 | } 254 | if len(result.Contents) > 0 { 255 | last = result.Contents[len(result.Contents) - 1].Key 256 | } 257 | done = !result.IsTruncated 258 | } 259 | } 260 | resp.Body.Close() 261 | } 262 | } 263 | close(out) 264 | return 265 | } 266 | 267 | type listBucketResult struct { 268 | Name string 269 | Prefix string 270 | Marker string 271 | MaxKeys int 272 | IsTruncated bool 273 | Contents []keyItem 274 | } 275 | 276 | type keyItem struct { 277 | Key string 278 | LastModified string // TODO: date 279 | ETag string 280 | Size int64 281 | Owner owner 282 | StorageClass string 283 | } 284 | 285 | // Closes the underlying connection 286 | func (self *Bucket) Close() (err os.Error) { 287 | return self.conn.Close() 288 | } 289 | -------------------------------------------------------------------------------- /s3/consts.go: -------------------------------------------------------------------------------- 1 | package s3 2 | 3 | import ( 4 | "crypto" 5 | ) 6 | 7 | 8 | // Do S3 endpoints actually play any role? 9 | const ( 10 | USWEST_HOST = "us-west-1.s3.amazonaws.com" 11 | USEAST_HOST = "s3.amazonaws.com" 12 | APSOUTHEAST_HOST = "ap-southeast-1.s3.amazonaws.com" 13 | EUWEST_HOST = "eu-west-1.s3.amazonaws.com" 14 | ) 15 | 16 | const ( 17 | DEFAULT_HASH = crypto.SHA1 18 | ) 19 | -------------------------------------------------------------------------------- /s3/service_opers.go: -------------------------------------------------------------------------------- 1 | package s3 2 | 3 | import ( 4 | "aws" 5 | ) 6 | 7 | import ( 8 | "http" 9 | "os" 10 | "path" 11 | "xml" 12 | ) 13 | 14 | type Service struct { 15 | URL *http.URL 16 | conn *aws.Conn 17 | } 18 | 19 | // Initilalize a new Service object with a specific 20 | // S3 endpoint. If URL is omitted, it defaults to the 21 | // us-east endpoint over HTTPS (https://s3.amazonaws.com/) 22 | func NewService(url *http.URL) (s *Service) { 23 | s = &Service{ 24 | URL: url, 25 | } 26 | if s.URL == nil { 27 | s.URL, _ = http.ParseURL("https://" + USEAST_HOST + "/") 28 | } 29 | s.conn = aws.NewConn(aws.URLDialer(s.URL, nil)) 30 | return 31 | } 32 | 33 | // Returns a new *Bucket with the same URL data as the Service connection. 34 | // You MUST have already created the bucket in order to make use of the 35 | // Bucket object. 36 | // 37 | // See CreateBucket to create a new bucket. 38 | func (self *Service) Bucket(name string) *Bucket { 39 | // We deliberately do NOT re-use our conn here, in order to take advantage 40 | // of vhosted bucket DNS perks. 41 | return NewBucket(self.URL, name, nil) 42 | } 43 | 44 | func s3Path(bucket, key string) string { 45 | return path.Join("/", bucket, key) 46 | } 47 | 48 | func (self *Service) bucket_url(bucket string) *http.URL { 49 | return &http.URL{ 50 | Host: self.URL.Host, 51 | Path: path.Join(self.URL.Path, s3Path(bucket, "")), 52 | Scheme: self.URL.Scheme, 53 | } 54 | } 55 | 56 | // Deletes the named bucket from the S3 service. 57 | func (self *Service) DeleteBucket(id *aws.Signer, name string) (err os.Error) { 58 | var resp *http.Response 59 | hreq := aws.NewRequest(self.bucket_url(name), "DELETE", nil, nil) 60 | err = id.SignRequestV1(hreq, aws.CanonicalizeS3, 15) 61 | 62 | if err == nil { 63 | resp, err = self.conn.Request(hreq) 64 | } 65 | 66 | if err == nil { 67 | defer resp.Body.Close() 68 | if resp.StatusCode != http.StatusNoContent { 69 | err = aws.CodeToError(resp.StatusCode) 70 | } 71 | } 72 | 73 | return 74 | } 75 | 76 | // Creates a new bucket 77 | // TODO: Will (probably) create the bucket in US-east no matter 78 | // what underlying endpoint you've chosen. 79 | func (self *Service) CreateBucket(id *aws.Signer, name string) (err os.Error) { 80 | var resp *http.Response 81 | hreq := aws.NewRequest(self.bucket_url(name), "PUT", nil, nil) 82 | err = id.SignRequestV1(hreq, aws.CanonicalizeS3, 15) 83 | 84 | if err == nil { 85 | resp, err = self.conn.Request(hreq) 86 | } 87 | if err == nil { 88 | defer resp.Body.Close() 89 | err = aws.CodeToError(resp.StatusCode) 90 | } 91 | return 92 | } 93 | 94 | 95 | // Returns a list of bucket names known by the endpoint. Depending on the 96 | // endpoint used, your list may be global or regional in nature. 97 | func (self *Service) ListBuckets(id *aws.Signer) (out []string, err os.Error) { 98 | var resp *http.Response 99 | hreq := aws.NewRequest(self.bucket_url(""), "GET", nil, nil) 100 | err = id.SignRequestV1(hreq, aws.CanonicalizeS3, 15) 101 | if err == nil { 102 | resp, err = self.conn.Request(hreq) 103 | } 104 | if err == nil { 105 | err = aws.CodeToError(resp.StatusCode) 106 | } 107 | if err == nil { 108 | defer resp.Body.Close() 109 | result := listAllMyBucketsResult{} 110 | err = xml.Unmarshal(resp.Body, &result) 111 | if err == nil { 112 | out = result.Buckets 113 | } 114 | } 115 | return 116 | } 117 | 118 | type listAllMyBucketsResult struct { 119 | Owner owner 120 | Buckets []string "Buckets>Bucket>Name" 121 | } 122 | 123 | // Closes the underlying connection 124 | func (self *Service) Close() (err os.Error) { 125 | return self.conn.Close() 126 | } 127 | -------------------------------------------------------------------------------- /s3/util/Makefile: -------------------------------------------------------------------------------- 1 | include $(GOROOT)/src/Make.inc 2 | 3 | TARG=aws/s3/s3_util 4 | GOFILES=s3_main.go\ 5 | 6 | DEPS=\ 7 | ../\ 8 | ../../\ 9 | ../../flags/\ 10 | ../../util/common/\ 11 | 12 | CLEANFILES+=\ 13 | 14 | include $(GOROOT)/src/Make.pkg 15 | 16 | -------------------------------------------------------------------------------- /s3/util/s3_main.go: -------------------------------------------------------------------------------- 1 | package s3_util 2 | 3 | import ( 4 | "aws/s3" 5 | . "aws/flags" 6 | . "aws/util/common" 7 | "aws" 8 | ) 9 | 10 | import ( 11 | "flag" 12 | "fmt" 13 | "http" 14 | "os" 15 | "path" 16 | ) 17 | 18 | // Safety warning 19 | // These are globals to allow the code to be more readable, 20 | // since the tool is "single-tasked" it has no threading issues. 21 | // 22 | // You are of course encouraged to take a more thread-safe approach 23 | // if you intend to use multiple threads. 24 | 25 | var flag_endpoint_url string = "" 26 | var signer *aws.Signer 27 | var service *s3.Service 28 | 29 | // Convenience method to clean up calls. 30 | func DefaultS3Service() (id *aws.Signer, s *s3.Service, err os.Error) { 31 | id, err = DefaultSigner() 32 | if err == nil { 33 | url, err := http.ParseURL(flag_endpoint_url) 34 | if err == nil { 35 | s = s3.NewService(url) 36 | } 37 | } 38 | return 39 | } 40 | 41 | 42 | func init() { 43 | AddModule("s3", func() { 44 | flag.StringVar(&flag_endpoint_url, "s3-endpoint", "https://s3.amazonaws.com/", "Endpoint to use for S3 calls") 45 | }) 46 | 47 | Modules["s3"].Setup = func() (err os.Error) { 48 | signer, service, err = DefaultS3Service() 49 | return 50 | } 51 | 52 | // awstool.s3.ls 53 | Modules["s3"].Calls["ls"] = func(args []string) (err os.Error) { 54 | if len(args) != 1 { 55 | return os.NewError("USAGE: ls BUCKET") 56 | } 57 | keys := make(chan string) 58 | go func() { 59 | for i := range keys { 60 | fmt.Printf("%s\n", i) 61 | } 62 | }() 63 | err = service.Bucket(args[0]).ListKeys(signer, "", "", "", keys) 64 | if err != nil { 65 | close(keys) 66 | } 67 | return 68 | } 69 | 70 | // awstool.s3.buckets 71 | Modules["s3"].Calls["buckets"] = func(args []string) (err os.Error) { 72 | if len(args) != 0 { 73 | return os.NewError("USAGE: buckets") 74 | } 75 | lb, err := service.ListBuckets(signer) 76 | if err == nil { 77 | for b := range lb { 78 | fmt.Println(lb[b]) 79 | } 80 | } 81 | return 82 | } 83 | 84 | Modules["s3"].Calls["cat"] = func(args []string) (err os.Error) { 85 | if len(args) != 2 { 86 | return os.NewError("Usage: get BUCKET KEY") 87 | } 88 | if err == nil { 89 | _, err = service.Bucket(args[0]).GetKey(signer, args[1], os.Stdout) 90 | } 91 | return 92 | } 93 | Modules["s3"].Calls["exists"] = func(args []string) (err os.Error) { 94 | if len(args) == 2 { 95 | fmt.Printf("Usage: exists BUCKET KEY\n") 96 | os.Exit(1) 97 | } 98 | err = service.Bucket(args[0]).Exists(signer, args[1]) 99 | return 100 | } 101 | Modules["s3"].Calls["drop"] = func(args []string) (err os.Error) { 102 | if len(args) != 1 { 103 | return os.NewError("Usage: drop BUCKET") 104 | } 105 | err = service.DeleteBucket(signer, args[0]) 106 | return 107 | } 108 | Modules["s3"].Calls["create"] = func(args []string) (err os.Error) { 109 | if len(args) != 1 { 110 | return os.NewError("Usage: create BUCKET") 111 | } 112 | err = service.CreateBucket(signer, args[0]) 113 | return 114 | } 115 | Modules["s3"].Calls["rm"] = func(args []string) (err os.Error) { 116 | if len(args) < 2 { 117 | return os.NewError("Usage: rm BUCKET KEY [KEY2...]") 118 | } 119 | bucket := args[0] 120 | args = args[1:] 121 | for i := range args { 122 | err = service.Bucket(bucket).Delete(signer, args[i]) 123 | if err != nil { 124 | break 125 | } 126 | } 127 | return 128 | } 129 | Modules["s3"].Calls["put"] = func(args []string) (err os.Error) { 130 | if len(args) < 3 { 131 | return os.NewError("Usage: put BUCKET PREFIX FILE [FILE2...]") 132 | } 133 | bucket := args[0] 134 | prefix := args[1] 135 | keys := args[2:] 136 | for i := range keys { 137 | err = service.Bucket(bucket).PutLocalFile(signer, path.Join(prefix, path.Base(keys[i])), keys[i]) 138 | if err != nil { 139 | break 140 | } 141 | } 142 | return 143 | } 144 | 145 | } 146 | 147 | func Nil() {} 148 | -------------------------------------------------------------------------------- /samples/s3_proxy/Makefile: -------------------------------------------------------------------------------- 1 | include $(GOROOT)/src/Make.inc 2 | 3 | TARG=s3_proxy 4 | GOFILES=\ 5 | config.go\ 6 | main.go\ 7 | 8 | include $(GOROOT)/src/Make.cmd 9 | -------------------------------------------------------------------------------- /samples/s3_proxy/README: -------------------------------------------------------------------------------- 1 | S3_proxy 2 | ======== 3 | 4 | An example application of the GoAWS tools, and an occasionally useful tool in your S3 toolbox. 5 | 6 | S3_proxy simply maps specific hostnames to s3 buckets (possibly with a prefix or dedicated AWS identity), 7 | and returns the result directly to the user. 8 | 9 | It is NOT a security proxy and makes very little effort to ensure the requests are proper, and 10 | as such should probably be used only as a starting point for a more customized S3 proxy. 11 | 12 | Potential use cases: 13 | ==================== 14 | 15 | Apt-get from S3 16 | --------------- 17 | 18 | # apt.conf 19 | deb http://localhost/debian localdist main contrib non-free 20 | # config.js 21 | { "localhost" : { "Bucket":"myAptbucket", "Prefix":"/OS/debian/apt/" } } 22 | 23 | Would rewrite all incoming GET's for http://localhost/... to be proxied 24 | from s3://myAptBucket/OS/debian/apt/... 25 | 26 | 27 | Legacy/closed source compatibility 28 | ---------------------------------- 29 | Add "free" s3 logic to any existing application that supports an http proxy 30 | or supports the http_proxy environment variable: 31 | 32 | # config.js 33 | { "logs.mydomain.com" : { "Bucket":"logs.mydomain.com" }} 34 | $ http_proxy="http://localhost:8080" curl http://logs.mydomain.com/some-key 35 | .... 36 | 37 | Serving a partial site from S3 with a content-prefix. 38 | ----------------------------------------------------- 39 | # nginx conf stub 40 | location /s3/ { 41 | proxy_pass http://localhost:8080/ 42 | } 43 | 44 | # config.js 45 | { "localhost" : { "Bucket":"static.bucket.tld", "Prefix":"/static_content/" }} 46 | 47 | (Please read the comments and notes throughout before trying this in production, 48 | as there are a number of places you could improve performance if you needed it, 49 | and transferring very large files will require a very large amount of memory 50 | as the example buffers the entire response from S3 into memory prior to writing 51 | it to the client.) 52 | 53 | Use notes 54 | ========= 55 | - Currently it strips off the port of the incoming request before mapping, so you cannot specify things like "localhost:8080". 56 | - Only GET is supported 57 | - Requesting '/' in S3 is a ListBucket request. 58 | We don't act to block such attempts in this example. 59 | - It binds to localhost:8080 by default, and can be changed on the command line. 60 | - You may use either the AWS_PUBLIC_KEY_ID and AWS_SECRET_KEY environment 61 | variables which are used for all domains that have no per-domain credentials 62 | in the config ( via the "AccessKey" and "SecretKey" fields). 63 | 64 | Have fun! 65 | -------------------------------------------------------------------------------- /samples/s3_proxy/config.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | . "aws/flags" 5 | "aws" 6 | "aws/s3" 7 | ) 8 | 9 | import ( 10 | "json" 11 | "http" 12 | "os" 13 | ) 14 | 15 | type conf map[string]*proxyConf 16 | 17 | type proxyConf struct { 18 | Bucket *s3.Bucket 19 | Prefix string 20 | Identity *aws.Signer 21 | } 22 | 23 | func (self *proxyConf) UnmarshalJSON(in []byte) (err os.Error) { 24 | confmap := map[string]string{} 25 | err = json.Unmarshal(in, &confmap) 26 | if err == nil { 27 | if _, ok := confmap["Bucket"]; !ok { 28 | return os.NewError("Bucket is required") 29 | } 30 | self.Prefix = confmap["Prefix"] 31 | if self.Prefix == "" { 32 | self.Prefix = "/" 33 | } 34 | self.Bucket = s3.NewBucket(&http.URL{ 35 | Scheme: "http", 36 | Host: "s3.amazonaws.com", 37 | Path: self.Prefix, 38 | }, confmap["Bucket"], nil) 39 | 40 | if confmap["AccessKey"] != "" && confmap["SecretKey"] != "" { 41 | self.Identity = aws.NewSigner(confmap["AccessKey"], confmap["SecretKey"]) 42 | } else { 43 | self.Identity, err = DefaultSigner() 44 | } 45 | } 46 | return 47 | } 48 | -------------------------------------------------------------------------------- /samples/s3_proxy/config.json: -------------------------------------------------------------------------------- 1 | { "localhost" : { "Bucket":"apt.abneptis.com", "Prefix":"" }, 2 | "logs.mydomain.com" : { "Bucket":"logs.mydomain.com" } 3 | } 4 | -------------------------------------------------------------------------------- /samples/s3_proxy/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | . "log" 5 | "flag" 6 | "http" 7 | "net" 8 | "path" 9 | "json" 10 | "os" 11 | "strconv" 12 | ) 13 | 14 | type Service struct { 15 | conf conf 16 | } 17 | 18 | func (self Service) ServeHTTP(rw http.ResponseWriter, req *http.Request) { 19 | host, _, err := net.SplitHostPort(req.Host) 20 | if err != nil { 21 | host = req.Host 22 | } 23 | if req.Method != "GET" { 24 | http.Error(rw, "Only GET is supported", http.StatusMethodNotAllowed) 25 | return 26 | } 27 | if req.URL.RawQuery != "" { 28 | http.Error(rw, "Sorry, we do not permit query-string parameters", http.StatusBadRequest) 29 | return 30 | } 31 | if domain, ok := self.conf[host]; ok { 32 | Printf("%s %s %s", req.RemoteAddr, host, req.URL.Path) 33 | key_path := path.Join(domain.Prefix, req.URL.Path) 34 | resp, err := domain.Bucket.HeadKey(domain.Identity, key_path) 35 | if err != nil { 36 | resp, err = domain.Bucket.HeadKey(domain.Identity, key_path) 37 | } 38 | // Error or no, we copy in the headers if we got a valid response 39 | if resp != nil { 40 | for k, v := range resp.Header { 41 | rw.Header()[k] = v 42 | } 43 | // We're only copying the body in for 200's. 44 | if resp.ContentLength > 0 && err == nil { 45 | rw.Header().Set("Content-Length", strconv.Itoa64(resp.ContentLength)) 46 | } 47 | } 48 | outcode := http.StatusInternalServerError 49 | if resp != nil { 50 | outcode = resp.StatusCode 51 | } 52 | rw.WriteHeader(outcode) 53 | if err == nil { 54 | _, err = domain.Bucket.GetKey(domain.Identity, key_path, rw) 55 | } 56 | Printf("%d: %s %s %s %d %v", outcode, req.RemoteAddr, host, req.URL.Path, resp.ContentLength, err) 57 | return 58 | } else { 59 | Printf("%s %s %s - Host Unknown", req.RemoteAddr, host, req.URL.Path) 60 | http.Error(rw, "Invalid host", http.StatusForbidden) 61 | } 62 | } 63 | 64 | var flag_bind_addr *string = flag.String("listen", "127.0.0.1:8080", "Address/port to listen to") 65 | 66 | func main() { 67 | conf := conf{} 68 | flag.Parse() 69 | fp, err := os.Open("config.json") 70 | if err != nil { 71 | Fatalf("Couldn't open config file: %v", err) 72 | } 73 | err = json.NewDecoder(fp).Decode(&conf) 74 | if err != nil { 75 | Fatalf("Couldn't parse config file: %v", err) 76 | } 77 | err = http.ListenAndServe(*flag_bind_addr, Service{conf: conf}) 78 | if err != nil { 79 | Fatalf("Couldn't setup listener: %v", err) 80 | } 81 | return 82 | } 83 | -------------------------------------------------------------------------------- /sdb/Makefile: -------------------------------------------------------------------------------- 1 | include $(GOROOT)/src/Make.inc 2 | 3 | TARG=aws/sdb 4 | GOFILES=\ 5 | service.go\ 6 | consts.go\ 7 | response.go\ 8 | domain.go\ 9 | attribute.go\ 10 | # handler.go\ 11 | # response.go\ 12 | 13 | DEPS=../ 14 | 15 | CLEANFILES+=\ 16 | 17 | include $(GOROOT)/src/Make.pkg 18 | 19 | 20 | -------------------------------------------------------------------------------- /sdb/attribute.go: -------------------------------------------------------------------------------- 1 | package sdb 2 | 3 | import ( 4 | "http" 5 | "strconv" 6 | ) 7 | 8 | type Item struct { 9 | Name string 10 | Attribute []Attribute 11 | } 12 | 13 | type Attribute struct { 14 | Name string 15 | Value string 16 | Exists *bool 17 | Replace *bool 18 | } 19 | 20 | type AttributeList []Attribute 21 | 22 | type AttrListType string 23 | 24 | const ( 25 | ATTRIBUTE_LIST AttrListType = "Attribute." 26 | EXPECTED_LIST AttrListType = "Expected." 27 | ) 28 | 29 | func (self AttributeList) Values(afix AttrListType) (v http.Values) { 30 | v = http.Values{} 31 | for i := range self { 32 | prefix := string(afix) + strconv.Itoa(i+1) + "." 33 | v.Set(prefix+"Name", self[i].Name) 34 | if self[i].Value != "" { 35 | v.Set(prefix+"Value", self[i].Value) 36 | } 37 | if self[i].Replace != nil { 38 | if *self[i].Replace { 39 | v.Set(prefix+"Replace", "true") 40 | } else { 41 | v.Set(prefix+"Replace", "false") 42 | } 43 | } 44 | if self[i].Exists != nil { 45 | if *self[i].Exists { 46 | v.Set(prefix+"Exists", "true") 47 | } else { 48 | v.Set(prefix+"Exists", "false") 49 | } 50 | } 51 | } 52 | return 53 | } 54 | 55 | 56 | // miscelanious helper functions. 57 | func AttrMissing(name string) Attribute { 58 | f := false 59 | return Attribute{Name: name, Exists: &f} 60 | } 61 | 62 | func AttrExists(name string) Attribute { 63 | t := true 64 | return Attribute{Name: name, Exists: &t} 65 | } 66 | 67 | func AttrEquals(name string, value string) Attribute { 68 | t := true 69 | return Attribute{Name: name, Value: value, Exists: &t} 70 | } 71 | -------------------------------------------------------------------------------- /sdb/consts.go: -------------------------------------------------------------------------------- 1 | package sdb 2 | 3 | const DEFAULT_API_VERSION = "2009-04-15" 4 | -------------------------------------------------------------------------------- /sdb/domain.go: -------------------------------------------------------------------------------- 1 | package sdb 2 | 3 | import ( 4 | "aws" 5 | ) 6 | 7 | import ( 8 | "fmt" 9 | "http" 10 | "os" 11 | "xml" 12 | ) 13 | 14 | type Domain struct { 15 | URL *http.URL 16 | conn *aws.Conn 17 | Name string 18 | } 19 | 20 | // if domain != "" { params["DomainName"] = domain } 21 | // params["Action"] = action 22 | 23 | 24 | func (self *Domain) DeleteAttribute(s *aws.Signer, item string, attrs, expected AttributeList) (err os.Error) { 25 | var resp *http.Response 26 | 27 | vl := attrs.Values(ATTRIBUTE_LIST) 28 | 29 | for k, v := range expected.Values(EXPECTED_LIST) { 30 | vl[k] = v 31 | } 32 | 33 | vl.Set("Action", "DeleteAttribute") 34 | vl.Set("DomainName", self.Name) 35 | vl.Set("ItemName", item) 36 | 37 | req := aws.NewRequest(self.URL, "GET", nil, vl) 38 | err = s.SignRequestV2(req, aws.Canonicalize, DEFAULT_API_VERSION, 0) 39 | if err == nil { 40 | resp, err = self.conn.Request(req) 41 | } 42 | if err == nil { 43 | defer resp.Body.Close() 44 | err = aws.CodeToError(resp.StatusCode) 45 | } 46 | return 47 | } 48 | 49 | func (self *Domain) GetAttribute(s *aws.Signer, item string, attrs AttributeList, consist bool) (a []Attribute, err os.Error) { 50 | var resp *http.Response 51 | 52 | vl := attrs.Values(ATTRIBUTE_LIST) 53 | 54 | vl.Set("Action", "GetAttributes") 55 | vl.Set("DomainName", self.Name) 56 | vl.Set("ItemName", item) 57 | 58 | if consist { 59 | vl.Set("ConsistentRead", "true") 60 | } 61 | 62 | req := aws.NewRequest(self.URL, "GET", nil, vl) 63 | err = s.SignRequestV2(req, aws.Canonicalize, DEFAULT_API_VERSION, 0) 64 | if err == nil { 65 | resp, err = self.conn.Request(req) 66 | } 67 | if err == nil { 68 | defer resp.Body.Close() 69 | err = aws.CodeToError(resp.StatusCode) 70 | } 71 | if err == nil { 72 | var response getattributesresponse 73 | ob, _ := http.DumpResponse(resp, true) 74 | os.Stdout.Write(ob) 75 | err = xml.Unmarshal(resp.Body, &response) 76 | if err == nil { 77 | a = response.Attributes 78 | } 79 | } 80 | return 81 | } 82 | 83 | func (self *Domain) Select(id *aws.Signer, what, where string, consist bool, items chan<- Item) (err os.Error) { 84 | var resp *http.Response 85 | 86 | vl := http.Values{} 87 | 88 | vl.Set("Action", "Select") 89 | if where != "" { 90 | where = " where " + where 91 | } 92 | vl.Set("SelectExpression", fmt.Sprintf("select %s from %s%s", what, self.Name, where)) 93 | 94 | if consist { 95 | vl.Set("ConsistentRead", "true") 96 | } 97 | done := false 98 | nextToken := "" 99 | for err == nil && !done { 100 | vl.Del("NextToken") 101 | if nextToken != "" { 102 | vl.Set("NextToken", nextToken) 103 | } 104 | req := aws.NewRequest(self.URL, "GET", nil, vl) 105 | err = id.SignRequestV2(req, aws.Canonicalize, DEFAULT_API_VERSION, 0) 106 | if err == nil { 107 | resp, err = self.conn.Request(req) 108 | } 109 | if err == nil { 110 | ob, _ := http.DumpResponse(resp, true) 111 | os.Stdout.Write(ob) 112 | xresp := selectresponse{} 113 | err = xml.Unmarshal(resp.Body, &xresp) 114 | if err == nil { 115 | fmt.Printf("XML == %+v", xresp) 116 | for i := range xresp.Items { 117 | items <- xresp.Items[i] 118 | } 119 | nextToken = xresp.NextToken 120 | done = (nextToken == "") 121 | } 122 | resp.Body.Close() 123 | } 124 | } 125 | return 126 | } 127 | 128 | // Closes the underlying connection 129 | func (self *Domain) Close() (err os.Error) { 130 | return self.conn.Close() 131 | } 132 | -------------------------------------------------------------------------------- /sdb/response.go: -------------------------------------------------------------------------------- 1 | package sdb 2 | 3 | 4 | type listdomainsresponse struct { 5 | Domains []string "ListDomainsResult>DomainName" 6 | NextToken string "ListDomainsResult>NextToken" 7 | RequestId string "ResponseMetadata>RequestId" 8 | BoxUsage string "ResponseMetadata>BoxUsage" 9 | ErrorMessage string "Errors>Error>Message" 10 | ErrorCode string "Errors>Error>Code" 11 | } 12 | 13 | 14 | type getattributesresponse struct { 15 | Attributes []Attribute "GetAttributesResult>Attribute" 16 | ErrorMessage string "Errors>Error>Message" 17 | ErrorCode string "Errors>Error>Code" 18 | } 19 | 20 | type selectresponse struct { 21 | Items []Item "SelectResult>Item" 22 | ErrorMessage string "Errors>Error>Message" 23 | ErrorCode string "Errors>Error>Code" 24 | NextToken string 25 | } 26 | -------------------------------------------------------------------------------- /sdb/service.go: -------------------------------------------------------------------------------- 1 | package sdb 2 | 3 | import ( 4 | "aws" 5 | ) 6 | 7 | import ( 8 | "http" 9 | "os" 10 | "xml" 11 | ) 12 | 13 | type Service struct { 14 | URL *http.URL 15 | conn *aws.Conn 16 | } 17 | 18 | func NewService(url *http.URL) *Service { 19 | return &Service{ 20 | URL: url, 21 | conn: aws.NewConn(aws.URLDialer(url, nil)), 22 | } 23 | } 24 | 25 | func (self *Service) Domain(name string) *Domain { 26 | return &Domain{ 27 | URL: &http.URL{ 28 | Scheme: self.URL.Scheme, 29 | Host: self.URL.Host, 30 | Path: self.URL.Path, 31 | }, 32 | conn: self.conn, 33 | Name: name, 34 | } 35 | } 36 | 37 | func (self *Service) CreateDomain(id *aws.Signer, name string) (err os.Error) { 38 | var resp *http.Response 39 | parms := http.Values{} 40 | parms.Set("DomainName", name) 41 | parms.Set("Action", "CreateDomain") 42 | req := aws.NewRequest(self.URL, "GET", nil, parms) 43 | err = id.SignRequestV2(req, aws.Canonicalize, DEFAULT_API_VERSION, 0) 44 | if err == nil { 45 | resp, err = self.conn.Request(req) 46 | } 47 | if err == nil { 48 | if resp.StatusCode != http.StatusOK { 49 | err = os.NewError("Unexpected response") 50 | } 51 | } 52 | return 53 | } 54 | 55 | func (self *Service) DestroyDomain(id *aws.Signer, name string) (err os.Error) { 56 | var resp *http.Response 57 | parms := http.Values{} 58 | parms.Set("DomainName", name) 59 | parms.Set("Action", "DeleteDomain") 60 | req := aws.NewRequest(self.URL, "GET", nil, parms) 61 | 62 | err = id.SignRequestV2(req, aws.Canonicalize, DEFAULT_API_VERSION, 0) 63 | if err == nil { 64 | resp, err = self.conn.Request(req) 65 | } 66 | 67 | if err == nil { 68 | resp, err = self.conn.Request(req) 69 | } 70 | if err == nil { 71 | if resp.StatusCode != http.StatusOK { 72 | err = os.NewError("Unexpected response") 73 | } 74 | } 75 | return 76 | } 77 | 78 | func (self *Service) ListDomains(id *aws.Signer) (out []string, err os.Error) { 79 | var resp *http.Response 80 | parms := http.Values{} 81 | parms.Set("Action", "ListDomains") 82 | parms.Set("MaxNumberOfDomains", "100") 83 | var done bool 84 | nextToken := "" 85 | for err == nil && !done { 86 | xmlresp := listdomainsresponse{} 87 | if nextToken != "" { 88 | parms.Set("NextToken", nextToken) 89 | } else { 90 | parms.Del("NextToken") 91 | } 92 | req := aws.NewRequest(self.URL, "GET", nil, parms) 93 | 94 | err = id.SignRequestV2(req, aws.Canonicalize, DEFAULT_API_VERSION, 0) 95 | if err == nil { 96 | resp, err = self.conn.Request(req) 97 | } 98 | 99 | if err == nil { 100 | resp, err = self.conn.Request(req) 101 | } 102 | if err == nil { 103 | defer resp.Body.Close() 104 | if resp.StatusCode != http.StatusOK { 105 | err = os.NewError("Unexpected response") 106 | ob, _ := http.DumpResponse(resp, true) 107 | os.Stdout.Write(ob) 108 | } 109 | if err == nil { 110 | err = xml.Unmarshal(resp.Body, &xmlresp) 111 | if err == nil { 112 | if xmlresp.ErrorCode != "" { 113 | err = os.NewError(xmlresp.ErrorCode) 114 | } 115 | if err == nil { 116 | for d := range xmlresp.Domains { 117 | out = append(out, xmlresp.Domains[d]) 118 | } 119 | } 120 | nextToken = xmlresp.NextToken 121 | } 122 | } 123 | } 124 | done = (nextToken == "") 125 | } 126 | return 127 | } 128 | 129 | // Closes the underlying connection 130 | func (self *Service) Close() (err os.Error) { 131 | return self.conn.Close() 132 | } 133 | -------------------------------------------------------------------------------- /sdb/util/Makefile: -------------------------------------------------------------------------------- 1 | include $(GOROOT)/src/Make.inc 2 | 3 | TARG=aws/sdb/sdb_util 4 | GOFILES=sdb_main.go\ 5 | 6 | DEPS=\ 7 | ../\ 8 | ../../\ 9 | ../../flags/\ 10 | ../../util/common/\ 11 | 12 | CLEANFILES+=\ 13 | 14 | include $(GOROOT)/src/Make.pkg 15 | 16 | -------------------------------------------------------------------------------- /sdb/util/sdb_main.go: -------------------------------------------------------------------------------- 1 | package sdb_util 2 | 3 | import ( 4 | . "aws/flags" 5 | . "aws/util/common" 6 | "aws" 7 | "aws/sdb" 8 | ) 9 | 10 | import ( 11 | "flag" 12 | "fmt" 13 | "http" 14 | "os" 15 | ) 16 | 17 | var flag_endpoint_url string 18 | var id *aws.Signer 19 | var service *sdb.Service 20 | 21 | func DefaultSDBService() (id *aws.Signer, s *sdb.Service, err os.Error) { 22 | id, err = DefaultSigner() 23 | if err == nil { 24 | url, err := http.ParseURL(flag_endpoint_url) 25 | if err == nil { 26 | s = sdb.NewService(url) 27 | } 28 | } 29 | return 30 | } 31 | 32 | func init() { 33 | AddModule("sdb", func() { 34 | flag.StringVar(&flag_endpoint_url, "sdb-endpoint", "https://sdb.amazonaws.com/", "Endpoint to use for S3 calls") 35 | }) 36 | 37 | Modules["sdb"].Setup = func() (err os.Error) { 38 | id, service, err = DefaultSDBService() 39 | return 40 | } 41 | Modules["sdb"].Calls["rm"] = func(args []string) (err os.Error) { 42 | if len(args) < 2 { 43 | return os.NewError("Usage: rm domain_name item [...]") 44 | } 45 | d := service.Domain(args[0]) 46 | args = args[1:] 47 | for i := range args { 48 | err = d.DeleteAttribute(id, args[i], nil, nil) 49 | if err != nil { 50 | return 51 | } 52 | } 53 | return 54 | } 55 | 56 | Modules["sdb"].Calls["get"] = func(args []string) (err os.Error) { 57 | if len(args) < 2 { 58 | return os.NewError("Usage: get domain_name item [...]") 59 | } 60 | d := service.Domain(args[0]) 61 | args = args[1:] 62 | for i := range args { 63 | var attrs []sdb.Attribute 64 | attrs, err = d.GetAttribute(id, args[i], nil, false) 65 | if err != nil { 66 | return 67 | } 68 | fmt.Printf("Item: %+v", attrs) 69 | } 70 | return 71 | } 72 | Modules["sdb"].Calls["create"] = func(args []string) (err os.Error) { 73 | if len(args) != 1 { 74 | return os.NewError("Usage: create domain_name") 75 | } 76 | err = service.CreateDomain(id, args[0]) 77 | return 78 | } 79 | Modules["sdb"].Calls["drop"] = func(args []string) (err os.Error) { 80 | if len(args) != 0 { 81 | return os.NewError("Usage: drop domain_name") 82 | } 83 | err = service.DestroyDomain(id, args[0]) 84 | return 85 | } 86 | Modules["sdb"].Calls["select"] = func(args []string) (err os.Error) { 87 | if len(args) < 2 || len(args) > 3 { 88 | return os.NewError("Usage: service.lect ('*'|col,col2,...) domain_name [extended expression]") 89 | } 90 | colstr := args[0] 91 | d := service.Domain(args[1]) 92 | expr := "" 93 | if len(args) == 3 { 94 | expr = args[2] 95 | } 96 | c := make(chan sdb.Item) 97 | go func() { 98 | for i := range c { 99 | fmt.Printf("%s\n", i.Name) 100 | for ai := range i.Attribute { 101 | fmt.Printf("\t%s\t%s\n", i.Attribute[ai].Name, i.Attribute[ai].Value) 102 | } 103 | } 104 | }() 105 | err = d.Select(id, colstr, expr, true, c) 106 | close(c) 107 | 108 | return 109 | } 110 | Modules["sdb"].Calls["domains"] = func(args []string) (err os.Error) { 111 | doms, err := service.ListDomains(id) 112 | for i := range doms { 113 | fmt.Printf("%s\n", doms[i]) 114 | } 115 | return 116 | } 117 | } 118 | 119 | func Nil() {} 120 | -------------------------------------------------------------------------------- /signer.go: -------------------------------------------------------------------------------- 1 | package aws 2 | 3 | import ( 4 | "bytes" 5 | "crypto" 6 | "crypto/hmac" 7 | "encoding/base64" 8 | "hash" 9 | "http" 10 | "os" 11 | "strings" 12 | "strconv" 13 | "time" 14 | ) 15 | 16 | // A signer simply holds the access & secret access keys 17 | // necessary for aws, and proivides helper functions 18 | // to assist in generating an appropriate signature. 19 | type Signer struct { 20 | AccessKey string 21 | secretAccessKey []byte 22 | } 23 | 24 | func NewSigner(akid, sak string) *Signer { 25 | return &Signer{akid, bytes.NewBufferString(sak).Bytes()} 26 | } 27 | 28 | // the core function of the Signer, generates the raw hmac of he bytes. 29 | func (self *Signer) SignBytes(h crypto.Hash, buff []byte) (sig []byte, err os.Error) { 30 | hh := hmac.New(func() hash.Hash { 31 | return h.New() 32 | }, self.secretAccessKey) 33 | _, err = hh.Write(buff) 34 | if err == nil { 35 | sig = hh.Sum() 36 | } 37 | return 38 | } 39 | 40 | // Same as SignBytes, but with strings. 41 | func (self *Signer) SignString(h crypto.Hash, s string) (os string, err os.Error) { 42 | ob, err := self.SignBytes(h, bytes.NewBufferString(s).Bytes()) 43 | if err == nil { 44 | os = string(ob) 45 | } 46 | return 47 | } 48 | 49 | // SignBytes, but will base64 encode based on the specified encoder. 50 | func (self *Signer) SignEncoded(h crypto.Hash, s string, e *base64.Encoding) (out []byte, err os.Error) { 51 | ob, err := self.SignBytes(h, bytes.NewBufferString(s).Bytes()) 52 | if err == nil { 53 | out = make([]byte, e.EncodedLen(len(ob))) 54 | e.Encode(out, ob) 55 | } 56 | return 57 | } 58 | 59 | // The V2 denotes amazon signing version 2, not version 2 of this particular function... 60 | // V2 is used by all services but S3; 61 | // Note, some services vary in their exact implementation of escaping w/r/t signatures, 62 | // so it is recommended you use this function. 63 | // 64 | // Final note: if exp is set to 0, a Timestamp will be used, otherwise an expiration. 65 | func (self *Signer) SignRequestV2(req *http.Request, canon func(*http.Request)(string, os.Error), api_ver string, exp int64) (err os.Error) { 66 | // log.Printf("Signing request...") 67 | 68 | qstring, err := http.ParseQuery(req.URL.RawQuery) 69 | if err != nil { return } 70 | qstring["SignatureVersion"] = []string{DEFAULT_SIGNATURE_VERSION} 71 | if _, ok := qstring["SignatureMethod"]; !ok || len(qstring["SignatureMethod"]) == 0 { 72 | qstring["SignatureMethod"] = []string{DEFAULT_SIGNATURE_METHOD} 73 | } 74 | qstring["Version"] = []string{api_ver} 75 | 76 | if exp > 0 { 77 | qstring["Expires"] = []string{strconv.Itoa64(time.Seconds()+exp)} 78 | } else { 79 | qstring["Timestamp"] = []string{time.UTC().Format(ISO8601TimestampFormat)} 80 | } 81 | qstring["Signature"] = nil, false 82 | qstring["AWSAccessKeyId"] = []string{ self.AccessKey} 83 | 84 | var sig []byte 85 | req.URL.RawQuery = http.Values(qstring).Encode() 86 | can, err := canon(req) 87 | if err != nil { return } 88 | //log.Printf("String-to-sign: '%s'", can) 89 | 90 | switch qstring["SignatureMethod"][0] { 91 | case "HmacSHA256": 92 | sig, err = self.SignEncoded(crypto.SHA256, can, base64.StdEncoding) 93 | case "HmacSHA1": 94 | sig, err = self.SignEncoded(crypto.SHA1, can, base64.StdEncoding) 95 | default: 96 | err = os.NewError("Unknown SignatureMethod:" + req.Form.Get("SignatureMethod")) 97 | } 98 | 99 | if err == nil { 100 | req.URL.RawQuery += "&" + http.Values{"Signature": []string{string(sig)}}.Encode() 101 | } 102 | 103 | return 104 | } 105 | 106 | // Used exclusively by S3 to the best of my knowledge... 107 | func (self *Signer) SignRequestV1(req *http.Request, canon func(*http.Request) (string, os.Error), exp int64) (err os.Error) { 108 | qstring, err := http.ParseQuery(req.URL.RawQuery) 109 | 110 | if err != nil { return } 111 | 112 | if exp > 0 { 113 | qstring["Expires"] = []string{strconv.Itoa64(time.Seconds()+exp)} 114 | } else { 115 | qstring["Timestamp"] = []string{time.UTC().Format(ISO8601TimestampFormat)} 116 | } 117 | qstring["Signature"] = nil, false 118 | qstring["AWSAccessKeyId"] = []string{ self.AccessKey} 119 | 120 | 121 | req.URL.RawQuery = http.Values(qstring).Encode() 122 | 123 | 124 | can, err := canon(req) 125 | if err != nil { return } 126 | 127 | var sig []byte 128 | sig, err = self.SignEncoded(crypto.SHA1, can, base64.StdEncoding) 129 | 130 | if err == nil { 131 | req.URL.RawQuery += "&" + http.Values{"Signature": []string{string(sig)}}.Encode() 132 | } 133 | 134 | return 135 | } 136 | 137 | // Generates the canonical string-to-sign for (most) AWS services. 138 | // You shouldn't need to use this directly. 139 | func Canonicalize(req *http.Request) (out string, err os.Error) { 140 | fv, err := http.ParseQuery(req.URL.RawQuery) 141 | if err == nil { 142 | out = strings.Join([]string{req.Method, req.Host, req.URL.Path, SortedEscape(fv)}, "\n") 143 | } 144 | return 145 | } 146 | 147 | // Generates the canonical string-to-sign for S3 services. 148 | // You shouldn't need to use this directly unless you're pre-signing URL's. 149 | func CanonicalizeS3(req *http.Request) (out string, err os.Error ){ 150 | fv, err := http.ParseQuery(req.URL.RawQuery) 151 | if err == nil || len(fv["Expires"]) != 1 { 152 | out = strings.Join([]string{req.Method, req.Header.Get("Content-Md5"), req.Header.Get("Content-Type"), fv["Expires"][0], req.URL.Path}, "\n") 153 | } 154 | return 155 | } 156 | -------------------------------------------------------------------------------- /sqs/Makefile: -------------------------------------------------------------------------------- 1 | include $(GOROOT)/src/Make.inc 2 | 3 | TARG=aws/sqs 4 | GOFILES=\ 5 | \ 6 | queue.go\ 7 | service.go\ 8 | consts.go\ 9 | 10 | DEPS=\ 11 | ../\ 12 | 13 | include $(GOROOT)/src/Make.pkg 14 | 15 | -------------------------------------------------------------------------------- /sqs/consts.go: -------------------------------------------------------------------------------- 1 | package sqs 2 | 3 | 4 | const ( 5 | DEFAULT_VERSION = "2009-02-01" 6 | ) 7 | -------------------------------------------------------------------------------- /sqs/error.go: -------------------------------------------------------------------------------- 1 | package sqs 2 | 3 | type SQSError struct { 4 | Type string 5 | Code string 6 | Message string 7 | // 8 | RequestId string 9 | } 10 | -------------------------------------------------------------------------------- /sqs/queue.go: -------------------------------------------------------------------------------- 1 | package sqs 2 | 3 | import ( 4 | "aws" 5 | ) 6 | 7 | import ( 8 | "http" 9 | "os" 10 | "strconv" 11 | "xml" 12 | ) 13 | 14 | type Queue struct { 15 | URL *http.URL 16 | conn *aws.Conn 17 | } 18 | 19 | func NewQueue(url *http.URL) *Queue { 20 | return &Queue{ 21 | URL: url, 22 | conn: aws.NewConn(aws.URLDialer(url, nil)), 23 | } 24 | } 25 | 26 | func (self *Queue) DeleteQueue(id *aws.Signer) (err os.Error) { 27 | var resp *http.Response 28 | parms := http.Values{} 29 | parms.Set("Action", "DeleteQueue") 30 | 31 | req := aws.NewRequest(self.URL, "GET", nil, parms) 32 | err = id.SignRequestV2(req, aws.Canonicalize, DEFAULT_VERSION, 15) 33 | if err == nil { 34 | resp, err = self.conn.Request(req) 35 | if err == nil { 36 | defer resp.Body.Close() 37 | if resp.StatusCode != http.StatusOK { 38 | err = os.NewError("Unexpected response") 39 | } 40 | } 41 | } 42 | 43 | return 44 | } 45 | 46 | func (self *Queue) Push(id *aws.Signer, body []byte) (err os.Error) { 47 | var resp *http.Response 48 | parms := http.Values{} 49 | parms.Set("Action", "SendMessage") 50 | parms.Set("MessageBody", string(body)) 51 | req := aws.NewRequest(self.URL, "GET", nil, parms) 52 | err = id.SignRequestV2(req, aws.Canonicalize, DEFAULT_VERSION, 15) 53 | if err == nil { 54 | resp, err = self.conn.Request(req) 55 | if err == nil { 56 | defer resp.Body.Close() 57 | if resp.StatusCode != http.StatusOK { 58 | err = os.NewError("Unexpected response") 59 | } 60 | } 61 | } 62 | return 63 | } 64 | 65 | // Note: 0 is a valid timeout!! 66 | func (self *Queue) Peek(id *aws.Signer, vt int) (body []byte, msgid string, err os.Error) { 67 | var resp *http.Response 68 | parms := http.Values{} 69 | parms.Set("Action", "ReceiveMessage") 70 | if vt >= 0 { 71 | parms.Set("VisibilityTimeout", strconv.Itoa(vt)) 72 | } 73 | req := aws.NewRequest(self.URL, "GET", nil, parms) 74 | err = id.SignRequestV2(req, aws.Canonicalize, DEFAULT_VERSION, 15) 75 | if err == nil { 76 | resp, err = self.conn.Request(req) 77 | if err == nil { 78 | defer resp.Body.Close() 79 | } 80 | if err == nil && resp.StatusCode != http.StatusOK { 81 | err = os.NewError("Unexpected response") 82 | } 83 | if err == nil { 84 | msg := message{} 85 | err = xml.Unmarshal(resp.Body, &msg) 86 | if err == nil { 87 | body, msgid = msg.Body, msg.ReceiptHandle 88 | } 89 | } 90 | } 91 | return 92 | } 93 | 94 | // Note: 0 is a valid timeout!! 95 | func (self *Queue) Delete(id *aws.Signer, mid string) (err os.Error) { 96 | var resp *http.Response 97 | parms := http.Values{} 98 | parms.Set("Action", "DeleteMessage") 99 | parms.Set("ReceiptHandle", mid) 100 | req := aws.NewRequest(self.URL, "GET", nil, parms) 101 | err = id.SignRequestV2(req, aws.Canonicalize, DEFAULT_VERSION, 15) 102 | if err == nil { 103 | resp, err = self.conn.Request(req) 104 | if err == nil { 105 | defer resp.Body.Close() 106 | } 107 | if resp.StatusCode != http.StatusOK { 108 | err = os.NewError("Unexpected response") 109 | } 110 | } 111 | return 112 | } 113 | 114 | type message struct { 115 | MessageId string "ReceiveMessageResult>Message>MessageId" 116 | ReceiptHandle string "ReceiveMessageResult>Message>ReceiptHandle" 117 | MD5OfBody string "ReceiveMessageResult>Message>MD5OfBody" 118 | Body []byte "ReceiveMessageResult>Message>Body" 119 | } 120 | 121 | // Closes the underlying connection 122 | func (self *Queue) Close() (err os.Error) { 123 | return self.conn.Close() 124 | } 125 | -------------------------------------------------------------------------------- /sqs/service.go: -------------------------------------------------------------------------------- 1 | package sqs 2 | 3 | import ( 4 | "aws" 5 | ) 6 | 7 | import ( 8 | "crypto" 9 | "http" 10 | "os" 11 | "strconv" 12 | "xml" 13 | ) 14 | 15 | const ( 16 | DEFAULT_HASH = crypto.SHA256 17 | MAX_MESSAGE_SIZE = 64 * 1024 18 | ) 19 | 20 | type Service struct { 21 | URL *http.URL 22 | conn *aws.Conn 23 | } 24 | 25 | func NewService(url *http.URL) *Service { 26 | return &Service{ 27 | URL: url, 28 | conn: aws.NewConn(aws.URLDialer(url, nil)), 29 | } 30 | } 31 | 32 | func (self *Service) ListQueues(id *aws.Signer, prefix string) (mq []string, err os.Error) { 33 | var resp *http.Response 34 | parms := http.Values{} 35 | parms.Set("Action", "ListQueues") 36 | if prefix != "" { 37 | parms.Set("QueueNamePrefix", prefix) 38 | } 39 | 40 | req := aws.NewRequest(self.URL, "GET", nil, parms) 41 | err = id.SignRequestV2(req, aws.Canonicalize, DEFAULT_VERSION, 15) 42 | resp, err = self.conn.Request(req) 43 | if err == nil { 44 | defer resp.Body.Close() 45 | xresp := listQueuesResponse{} 46 | if resp.StatusCode == http.StatusOK { 47 | err = xml.Unmarshal(resp.Body, &xresp) 48 | } else { 49 | err = os.NewError("Unexpected response code") 50 | } 51 | if err == nil { 52 | mq = xresp.QueueURL 53 | } 54 | 55 | } 56 | return 57 | } 58 | 59 | // Create a queue, returning the Queue object. 60 | func (self *Service) CreateQueue(id *aws.Signer, name string, dvtimeout int) (mq *Queue, err os.Error) { 61 | var resp *http.Response 62 | parms := http.Values{} 63 | parms.Set("Action", "CreateQueue") 64 | parms.Set("QueueName", name) 65 | parms.Set("DefaultVisibilityTimeout", strconv.Itoa(dvtimeout)) 66 | 67 | req := aws.NewRequest(self.URL, "GET", nil, parms) 68 | err = id.SignRequestV2(req, aws.Canonicalize, DEFAULT_VERSION, 15) 69 | if err == nil { 70 | resp, err = self.conn.Request(req) 71 | if err == nil { 72 | defer resp.Body.Close() 73 | if resp.StatusCode == http.StatusOK { 74 | xmlresp := createQueueResponse{} 75 | err = xml.Unmarshal(resp.Body, &xmlresp) 76 | if err == nil { 77 | var qrl *http.URL 78 | qrl, err = http.ParseURL(xmlresp.QueueURL) 79 | if err == nil { 80 | mq = NewQueue(qrl) 81 | } 82 | } 83 | } else { 84 | err = os.NewError("Unexpected response") 85 | } 86 | } 87 | } 88 | 89 | return 90 | } 91 | 92 | type createQueueResponse struct { 93 | QueueURL string "CreateQueueResult>QueueUrl" 94 | RequestId string "ResponseMetadata>RequestId" 95 | } 96 | 97 | type listQueuesResponse struct { 98 | QueueURL []string "ListQueuesResult>QueueUrl" 99 | RequestId string "ResponseMetadata>RequestId" 100 | } 101 | 102 | // Closes the underlying connection 103 | func (self *Service) Close() (err os.Error) { 104 | return self.conn.Close() 105 | } 106 | -------------------------------------------------------------------------------- /sqs/util/Makefile: -------------------------------------------------------------------------------- 1 | include $(GOROOT)/src/Make.inc 2 | 3 | TARG=aws/sqs/sqs_util 4 | GOFILES=sqs_main.go\ 5 | 6 | DEPS=\ 7 | ../\ 8 | ../../\ 9 | ../../flags/\ 10 | ../../util/common/\ 11 | 12 | CLEANFILES+=\ 13 | 14 | include $(GOROOT)/src/Make.pkg 15 | 16 | -------------------------------------------------------------------------------- /sqs/util/sqs_main.go: -------------------------------------------------------------------------------- 1 | package sqs_util 2 | 3 | import ( 4 | . "aws/flags" 5 | . "aws/util/common" 6 | "aws" 7 | "aws/sqs" 8 | ) 9 | 10 | import ( 11 | "flag" 12 | "fmt" 13 | "http" 14 | "io" 15 | "os" 16 | ) 17 | 18 | var flag_endpoint_url string 19 | var flag_default_timeout int 20 | var flag_pop_timeout int 21 | 22 | var signer *aws.Signer 23 | var s *sqs.Service 24 | 25 | func DefaultSQSService() (id *aws.Signer, s *sqs.Service, err os.Error) { 26 | id, err = DefaultSigner() 27 | if err == nil { 28 | url, err := http.ParseURL(flag_endpoint_url) 29 | if err == nil { 30 | s = sqs.NewService(url) 31 | } 32 | } 33 | return 34 | } 35 | 36 | 37 | func init() { 38 | AddModule("sqs", func() { 39 | flag.StringVar(&flag_endpoint_url, "sqs-endpoint", "https://queue.amazonaws.com/", "Endpoint to use") 40 | flag.IntVar(&flag_default_timeout, "sqs-queue-timeout", 90, "Queue timeout (create/delete)") 41 | flag.IntVar(&flag_pop_timeout, "sqs-message-timeout", 90, "Queue timeout (pop/peek)") 42 | }) 43 | Modules["sqs"].Setup = func() (err os.Error) { 44 | signer, s, err = DefaultSQSService() 45 | return 46 | } 47 | Modules["sqs"].Calls["create"] = func(args []string) (err os.Error) { 48 | if len(args) != 1 { 49 | return os.NewError("Usage: create QUEUE") 50 | } 51 | Q, err := s.CreateQueue(signer, args[0], flag_default_timeout) 52 | if err == nil { 53 | fmt.Printf("%s\n", Q.URL) 54 | } 55 | return 56 | } 57 | 58 | Modules["sqs"].Calls["list"] = func(args []string) (err os.Error) { 59 | if len(args) != 0 { 60 | return os.NewError("Usage: list") 61 | } 62 | qs, err := s.ListQueues(signer, "") 63 | if err == nil { 64 | for i := range qs { 65 | fmt.Printf("%s\n", qs[i]) 66 | } 67 | } 68 | return 69 | } 70 | 71 | Modules["sqs"].Calls["drop"] = func(args []string) (err os.Error) { 72 | if len(args) != 1 { 73 | return os.NewError("Usage: drop queue") 74 | } 75 | Q, err := s.CreateQueue(signer, args[0], flag_default_timeout) 76 | if err == nil { 77 | err = Q.DeleteQueue(signer) 78 | } 79 | return 80 | } 81 | 82 | Modules["sqs"].Calls["push"] = func(args []string) (err os.Error) { 83 | if len(args) != 1 { 84 | return os.NewError("Usage: push queuename") 85 | } 86 | Q, err := s.CreateQueue(signer, args[0], flag_default_timeout) 87 | if err == nil { 88 | var n int 89 | lr := io.LimitReader(os.Stdin, sqs.MAX_MESSAGE_SIZE) 90 | buff := make([]byte, sqs.MAX_MESSAGE_SIZE) 91 | n, err = io.ReadFull(lr, buff) 92 | if err == nil || err == io.ErrUnexpectedEOF { 93 | buff = buff[0:n] 94 | err = Q.Push(signer, buff) 95 | } 96 | } 97 | return 98 | } 99 | Modules["sqs"].Calls["rm"] = func(args []string) (err os.Error) { 100 | if len(args) != 2 { 101 | return os.NewError("Usage: rm queuename receipthandle") 102 | } 103 | Q, err := s.CreateQueue(signer, args[0], flag_default_timeout) 104 | if err == nil { 105 | err = Q.Delete(signer, args[1]) 106 | } 107 | return 108 | } 109 | Modules["sqs"].Calls["peek"] = func(args []string) (err os.Error) { 110 | if len(args) != 1 { 111 | return os.NewError("Usage: peek queuename") 112 | } 113 | Q, err := s.CreateQueue(signer, args[0], flag_default_timeout) 114 | var body []byte 115 | var id string 116 | if err == nil { 117 | body, id, err = Q.Peek(signer, flag_pop_timeout) 118 | } 119 | if err == nil { 120 | fmt.Printf("# MessageId %s\n", id) 121 | os.Stdout.Write(body) 122 | } 123 | return 124 | } 125 | } 126 | 127 | func Nil() {} 128 | -------------------------------------------------------------------------------- /timeformats.go: -------------------------------------------------------------------------------- 1 | package aws 2 | 3 | /* 4 | Copyright (c) 2010, Abneptis LLC. 5 | See COPYRIGHT and LICENSE for details. 6 | */ 7 | 8 | var SQSTimestampFormat = "2006-01-02T15:04:05MST" 9 | var ISO8601TimestampFormat = "2006-01-02T15:04:05Z" 10 | 11 | -------------------------------------------------------------------------------- /util/Makefile: -------------------------------------------------------------------------------- 1 | include $(GOROOT)/src/Make.inc 2 | 3 | TARG=awstool 4 | GOFILES=main.go\ 5 | 6 | DEPS=\ 7 | ../s3/util/\ 8 | ../sqs/util/\ 9 | ../sdb/util/\ 10 | ../ec2/util/\ 11 | ../elb/util/\ 12 | ../flags/\ 13 | ./common/\ 14 | 15 | CLEANFILES+=\ 16 | 17 | include $(GOROOT)/src/Make.cmd 18 | 19 | -------------------------------------------------------------------------------- /util/common/Makefile: -------------------------------------------------------------------------------- 1 | include $(GOROOT)/src/Make.inc 2 | 3 | TARG=aws/util/common 4 | GOFILES=common.go\ 5 | 6 | DEPS=\ 7 | 8 | CLEANFILES+=\ 9 | 10 | include $(GOROOT)/src/Make.pkg 11 | 12 | -------------------------------------------------------------------------------- /util/common/common.go: -------------------------------------------------------------------------------- 1 | package common 2 | 3 | import ( 4 | "os" 5 | ) 6 | 7 | type FunctionModule struct { 8 | FlagFunc func() 9 | Setup func() os.Error 10 | Calls map[string]UserCall 11 | } 12 | 13 | func (self FunctionModule) Names() (out []string) { 14 | for k, _ := range self.Calls { 15 | out = append(out, k) 16 | } 17 | return 18 | } 19 | 20 | func NewFunctionModule(f func()) *FunctionModule { 21 | return &FunctionModule{ 22 | FlagFunc: f, 23 | Calls: make(map[string]UserCall), 24 | } 25 | } 26 | 27 | var Modules map[string]*FunctionModule 28 | 29 | // NOT THREAD SAFE! 30 | // DO NOT USE OUTSIDE OF init()! 31 | func AddModule(mod string, ffunc func()) { 32 | Modules[mod] = NewFunctionModule(ffunc) 33 | } 34 | 35 | // Common functionality to ease sub-modules 36 | 37 | type UserCall func([]string) os.Error 38 | 39 | 40 | func init() { 41 | Modules = make(map[string]*FunctionModule) 42 | } 43 | -------------------------------------------------------------------------------- /util/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | . "aws/util/common" // AWS ID Flags 5 | . "aws/flags" // AWS ID Flags 6 | "aws/ec2/ec2_util" 7 | "aws/elb/elb_util" 8 | "aws/s3/s3_util" 9 | "aws/sqs/sqs_util" 10 | "aws/sdb/sdb_util" 11 | ) 12 | 13 | import ( 14 | "flag" 15 | "os" 16 | "fmt" 17 | ) 18 | 19 | func keys(in map[string]interface{}) (out []string) { 20 | for k, _ := range in { 21 | out = append(out, k) 22 | } 23 | return 24 | } 25 | 26 | func main() { 27 | flag.Parse() 28 | if flag.NArg() < 2 { 29 | fmt.Printf("USAGE: aws [ec2|elb|s3|sdb|sqs] subcommand ...\n") 30 | os.Exit(1) 31 | } 32 | module := flag.Arg(0) 33 | cmd := flag.Arg(1) 34 | os.Args = os.Args[2:] 35 | var err os.Error 36 | modulenames := []string{} 37 | for k, _ := range Modules { 38 | modulenames = append(modulenames, k) 39 | } 40 | if m, ok := Modules[module]; ok { 41 | m.FlagFunc() 42 | flag.Parse() 43 | if c, ok := m.Calls[cmd]; ok { 44 | err = m.Setup() 45 | if err == nil { 46 | err = c(flag.Args()) 47 | } 48 | } else { 49 | err = os.NewError(fmt.Sprintf("Invalid subcommand: %s, expected one of %v", 50 | cmd, m.Names())) 51 | } 52 | } else { 53 | err = os.NewError(fmt.Sprintf("Invalid modulle : %s, expected one of %v", 54 | flag.Arg(0), modulenames)) 55 | 56 | } 57 | if err != nil { 58 | fmt.Printf("Error: %v\n", err) 59 | os.Exit(1) 60 | } 61 | os.Exit(0) 62 | UseFlags() // we want the side effects of import... 63 | ec2_util.Nil() 64 | elb_util.Nil() 65 | s3_util.Nil() 66 | sdb_util.Nil() 67 | sqs_util.Nil() 68 | } 69 | --------------------------------------------------------------------------------