├── spec ├── spec_helper.cr ├── aws_spec.cr └── sqs │ └── message_spec.cr ├── .travis.yml ├── .gitignore ├── .editorconfig ├── shard.yml ├── src ├── aws.cr ├── sqs │ └── message.cr ├── client.cr ├── sns.cr ├── s3.cr └── sqs.cr ├── LICENSE └── README.md /spec/spec_helper.cr: -------------------------------------------------------------------------------- 1 | require "spec" 2 | -------------------------------------------------------------------------------- /spec/aws_spec.cr: -------------------------------------------------------------------------------- 1 | require "./spec_helper" 2 | 3 | describe Aws do 4 | # TODO: Write tests 5 | 6 | it "works" do 7 | false.should eq(true) 8 | end 9 | end 10 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: crystal 2 | 3 | # Uncomment the following if you'd like Travis to run specs and check code formatting 4 | # script: 5 | # - crystal spec 6 | # - crystal tool format --check 7 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /docs/ 2 | /lib/ 3 | /bin/ 4 | /.shards/ 5 | *.dwarf 6 | 7 | # Libraries don't need dependency lock 8 | # Dependencies will be locked in applications that use them 9 | /shard.lock 10 | -------------------------------------------------------------------------------- /.editorconfig: -------------------------------------------------------------------------------- 1 | root = true 2 | 3 | [*.cr] 4 | charset = utf-8 5 | end_of_line = lf 6 | insert_final_newline = true 7 | indent_style = space 8 | indent_size = 2 9 | trim_trailing_whitespace = true 10 | -------------------------------------------------------------------------------- /shard.yml: -------------------------------------------------------------------------------- 1 | name: aws 2 | version: 0.1.0 3 | 4 | authors: 5 | - Jamie Gaskins 6 | 7 | dependencies: 8 | awscr-signer: 9 | github: taylorfinnell/awscr-signer 10 | branch: master 11 | db: 12 | github: crystal-lang/crystal-db 13 | 14 | crystal: ">= 0.35.0, < 2.0.0" 15 | 16 | license: MIT 17 | -------------------------------------------------------------------------------- /src/aws.cr: -------------------------------------------------------------------------------- 1 | module AWS 2 | VERSION = "0.1.0" 3 | 4 | class_property access_key_id : String = ENV["AWS_ACCESS_KEY_ID"]? || "" 5 | class_property secret_access_key : String = ENV["AWS_SECRET_ACCESS_KEY"]? || "" 6 | class_property region : String = ENV["AWS_REGION"]? || "" 7 | 8 | class Exception < ::Exception 9 | end 10 | class InvalidCredentials < Exception 11 | end 12 | end 13 | -------------------------------------------------------------------------------- /spec/sqs/message_spec.cr: -------------------------------------------------------------------------------- 1 | require "../spec_helper" 2 | require "../../src/sqs/message" 3 | 4 | module AWS 5 | module SQS 6 | describe Message do 7 | it "parses its XML representation" do 8 | id = UUID.random 9 | xml = <<-XML 10 | 11 | #{id} 12 | receipt_handle 13 | md5_of_body 14 | body! 15 | 16 | XML 17 | 18 | message = Message.from_xml(xml) 19 | 20 | message.id.should eq id 21 | message.receipt_handle.should eq "receipt_handle" 22 | message.md5.should eq "md5_of_body" 23 | message.body.should eq "body!" 24 | end 25 | end 26 | end 27 | end 28 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2019 Jamie Gaskins 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in 13 | all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | THE SOFTWARE. 22 | -------------------------------------------------------------------------------- /src/sqs/message.cr: -------------------------------------------------------------------------------- 1 | require "xml" 2 | require "uuid" 3 | 4 | module AWS 5 | module SQS 6 | struct Message 7 | getter id, receipt_handle, md5, body 8 | 9 | def self.from_xml(xml : String) 10 | from_xml XML.parse(xml).first_element_child.not_nil! 11 | end 12 | 13 | def self.from_xml(xml : XML::Node) 14 | new( 15 | id: UUID.new(get_xml_child(xml, "MessageId")), 16 | receipt_handle: get_xml_child(xml, "ReceiptHandle"), 17 | md5: get_xml_child(xml, "MD5OfBody"), 18 | body: get_xml_child(xml, "Body"), 19 | attributes: get_attributes(xml), 20 | message_attributes: get_message_attributes(xml), 21 | ) 22 | end 23 | 24 | @id : UUID 25 | @receipt_handle : String 26 | @md5 : String 27 | @body : String 28 | @attributes : Hash(String, String) 29 | @message_attributes : Hash(String, String | Bytes | Int64 | Float64) 30 | 31 | def initialize(@id, @receipt_handle, @md5, @body, @attributes, @message_attributes) 32 | end 33 | 34 | private def self.get_attributes(xml : XML::Node) 35 | attributes = {} of String => String 36 | xml.xpath_nodes("xmlns:Attribute").each do |attribute| 37 | name_node = attribute.xpath_node("./xmlns:Name").not_nil! 38 | value_node = attribute.xpath_node("./xmlns:Value").not_nil! 39 | 40 | attributes[name_node.text] = value_node.text 41 | end 42 | 43 | attributes 44 | end 45 | 46 | private def self.get_message_attributes(xml : XML::Node) 47 | attributes = {} of String => String | Bytes | Int64 | Float64 48 | xml.xpath_nodes("xmlns:MessageAttribute").each do |attribute| 49 | name_node = attribute.xpath_node("./xmlns:Name").not_nil! 50 | value_node = attribute.xpath_node("./xmlns:Value").not_nil! 51 | value_type = value_node.xpath_node("./xmlns:DataType").not_nil! 52 | value = case value_type.text 53 | when "String" 54 | value_node.xpath_node("./xmlns:StringValue").not_nil!.text 55 | when "Number" 56 | string = value_node.xpath_node("./xmlns:StringValue").not_nil!.text 57 | if string.includes? '.' 58 | string.to_f64 59 | else 60 | string.to_i64 61 | end 62 | when "Binary" 63 | Base64.decode(value_node.xpath_node("./xmlns:BinaryValue").not_nil!.text) 64 | else # Return bytes for custom data types in case it's not UTF8 strings 65 | value_node.xpath_node("./xmlns:BinaryValue").not_nil!.text.to_slice 66 | end 67 | 68 | attributes[name_node.text] = value 69 | end 70 | 71 | attributes 72 | end 73 | 74 | private def self.get_xml_child(xml, name) : String 75 | xml.xpath_node("*[name()='#{name}']/text()").to_s 76 | end 77 | end 78 | end 79 | end 80 | -------------------------------------------------------------------------------- /src/client.cr: -------------------------------------------------------------------------------- 1 | require "http" 2 | require "awscr-signer" 3 | require "db/pool" 4 | 5 | require "./aws" 6 | 7 | # It doesn't handle `Connection: keep-alive` headers :-\ 8 | Awscr::Signer::HeaderCollection::BLACKLIST_HEADERS << "connection" 9 | 10 | module AWS 11 | abstract class Client 12 | macro service_name 13 | {{SERVICE_NAME}} 14 | end 15 | 16 | def initialize( 17 | @access_key_id = AWS.access_key_id, 18 | @secret_access_key = AWS.secret_access_key, 19 | @region = AWS.region, 20 | @endpoint = URI.parse("https://#{service_name}.#{region}.amazonaws.com"), 21 | ) 22 | @signer = Awscr::Signer::Signers::V4.new(service_name, region, access_key_id, secret_access_key) 23 | @connection_pools = Hash({String, Int32?, Bool}, DB::Pool(HTTP::Client)).new 24 | end 25 | 26 | DEFAULT_HEADERS = HTTP::Headers { 27 | "Connection" => "keep-alive", 28 | "User-Agent" => "Crystal AWS #{VERSION}", 29 | } 30 | def get(path : String, headers = HTTP::Headers.new) 31 | headers = DEFAULT_HEADERS.dup.merge!(headers) 32 | http(&.get(path, headers: headers)) 33 | end 34 | 35 | def get(path : String, headers = HTTP::Headers.new, &block : HTTP::Client::Response ->) 36 | headers = DEFAULT_HEADERS.dup.merge!(headers) 37 | http(&.get(path, headers: headers, &block)) 38 | end 39 | 40 | def post(path : String, body : String, headers = HTTP::Headers.new) 41 | headers = DEFAULT_HEADERS.dup.merge!(headers) 42 | http(&.post(path, body: body, headers: headers)) 43 | end 44 | 45 | def put(path : String, body : IO, headers = HTTP::Headers.new) 46 | headers = DEFAULT_HEADERS.dup.merge!(headers) 47 | http(&.put(path, body: body, headers: headers)) 48 | end 49 | 50 | def head(path : String, headers : HTTP::Headers) 51 | headers = DEFAULT_HEADERS.dup.merge!(headers) 52 | http(&.head(path, headers)) 53 | end 54 | 55 | def delete(path : String, headers = HTTP::Headers.new) 56 | headers = DEFAULT_HEADERS.dup.merge!(headers) 57 | http(&.delete(path, headers: headers)) 58 | end 59 | 60 | protected getter endpoint 61 | 62 | protected def http(host = endpoint.host.not_nil!, port = endpoint.port, tls = true) 63 | pool = @connection_pools.fetch({host, port, tls}) do |key| 64 | @connection_pools[key] = DB::Pool.new(DB::Pool::Options.new(initial_pool_size: 0, max_idle_pool_size: 20)) do 65 | if port 66 | http = HTTP::Client.new(host, port, tls: tls) 67 | else 68 | http = HTTP::Client.new(host, tls: tls) 69 | end 70 | http.before_request do |request| 71 | # Apparently Connection: keep-alive causes trouble with signatures. 72 | # See https://github.com/taylorfinnell/awscr-signer/issues/56#issue-801172534 73 | request.headers.delete "Authorization" 74 | request.headers.delete "X-Amz-Content-Sha256" 75 | request.headers.delete "X-Amz-Date" 76 | @signer.sign request 77 | end 78 | 79 | http 80 | end 81 | end 82 | 83 | pool.checkout { |http| yield http } 84 | end 85 | end 86 | end 87 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Crystal AWS 2 | 3 | This shard provides clients for various services on AWS. So far, clients implemented are: 4 | 5 | - S3 6 | - SQS 7 | - SNS 8 | 9 | ## Installation 10 | 11 | 1. Add the dependency to your `shard.yml`: 12 | 13 | ```yaml 14 | dependencies: 15 | aws: 16 | github: jgaskins/aws 17 | ``` 18 | 19 | 2. Run `shards install` 20 | 21 | ## Usage 22 | 23 | To use each service client, you need to require that client specifically. This avoids loading the entire suite of clients just to use a single one. So for example, to use S3, you would use `require "aws/s3"`. 24 | 25 | All AWS clients are instantiated with credentials and a service endpoint. The defaults can be set either programmatically or via environment variables. To set them programmatically, you can set them directly on the `AWS` namespace: 26 | 27 | ```crystal 28 | AWS.access_key_id = "AKIAOMGLOLWTFBBQ" 29 | AWS.secret_access_key = "this is a secret, don't tell anyone" 30 | AWS.region = "us-east-1" 31 | ``` 32 | 33 | There is no global default endpoint since those are specific to the service. If you wish to use a nonstandard endpoint (for example, to use DigitalOcean Spaces or a MinIO instance instead of S3), you must set it when instantiating the client. 34 | 35 | To set the defaults via environment variables 36 | 37 | | Property | Environment Variable | 38 | |-|-| 39 | | `access_key_id` | `AWS_ACCESS_KEY_ID` | 40 | | `secret_access_key` | `AWS_SECRET_ACCESS_KEY` | 41 | | `region` | `AWS_REGION` | 42 | 43 | Individual services and their APIs are documented below. All examples assume credentials are set globally and use default AWS endpoints for brevity. 44 | 45 | Whenever feasible, method and argument names are snake-cased versions of those of the AWS REST API for ease of translating docs to application code. For example, with SQS, the `ReceiveMessage` API takes `QueueUrl`, `MaxNumberOfMessages`, and `WaitTimeSeconds` arguments. Your application would call `sqs.receive_message(queue_url: url, max_number_of_messages: 10, wait_time_seconds: 20)`. Additional method overrides are planned to make some of these API calls easier to read. 46 | 47 | ### S3 48 | 49 | ```crystal 50 | require "aws/s3" 51 | 52 | s3 = AWS::S3::Client.new 53 | 54 | s3.list_buckets 55 | s3.list_objects(bucket_name: "my-bucket") 56 | s3.get_object(bucket_name: "my-bucket", key: "my-object") 57 | s3.head_object(bucket_name: "my-bucket", key: "my-object") 58 | s3.put_object( 59 | bucket_name: "my-bucket", 60 | key: "my-object", 61 | headers: HTTP::Headers { 62 | "Content-Type" => "image/jpeg", 63 | "Cache-Control" => "private, max-age=3600", 64 | }, 65 | body: body, # String | IO - if you provide an IO, it MUST be rewindable! 66 | ) 67 | s3.delete_object(bucket_name: "my-bucket", key: "my-object") 68 | 69 | # Pre-signed URLs for direct uploads or serving tags for user-uploaded objects 70 | s3.presigned_url("PUT", "my-bucket", "my-object", ttl: 10.minutes) 71 | ``` 72 | 73 | ### SNS 74 | 75 | ```crystal 76 | require "aws/sns" 77 | 78 | sns = AWS::SNS::Client.new 79 | 80 | topic = sns.create_topic("my-topic") 81 | 82 | # Publishing a message - subject is optional 83 | sns.publish topic_arn: topic.arn, message: "hello" 84 | sns.publish topic_arn: topic.arn, message: "hello", subject: "MySubject" 85 | 86 | # This endpoint requires an SQS client and a queue instance from that client. If 87 | # you omit the client, it will create one for you. 88 | sns.subscribe topic: topic.arn, queue: queue, sqs: sqs 89 | ``` 90 | 91 | ### SQS 92 | 93 | ```crystal 94 | require "aws/sqs" 95 | 96 | sqs = AWS::SQS::Client.new 97 | 98 | queue = sqs.create_queue(queue_name: "my-queue") 99 | 100 | sqs.send_message( 101 | queue_url: queue.url, 102 | message_body: "hi", 103 | ) 104 | msgs = sqs.receive_message( 105 | queue_url: queue.url, 106 | max_number_of_messages: 10, 107 | wait_time_seconds: 20, 108 | ) 109 | msgs.each do |msg| 110 | process msg 111 | end 112 | 113 | sqs.delete_message_batch msgs 114 | ``` 115 | 116 | ## Connection pooling 117 | 118 | This shard maintains its own connection pools, so you can assign clients directly to a constant for use throughout your application: 119 | 120 | ```crystal 121 | require "aws/s3" 122 | 123 | S3 = AWS::S3::Client.new 124 | ``` 125 | 126 | You can use this client anywhere in your application by calling the methods listed above directly on the constant. There is no worry about collisions due to concurrent access. The client manages that transparently. 127 | 128 | ## Contributing 129 | 130 | 1. Fork it () 131 | 2. Create your feature branch (`git checkout -b my-new-feature`) 132 | 3. Commit your changes (`git commit -am 'Add some feature'`) 133 | 4. Push to the branch (`git push origin my-new-feature`) 134 | 5. Create a new Pull Request 135 | 136 | ## Contributors 137 | 138 | - [Jamie Gaskins](https://github.com/jgaskins) - creator and maintainer 139 | -------------------------------------------------------------------------------- /src/sns.cr: -------------------------------------------------------------------------------- 1 | require "json" 2 | require "http" 3 | require "xml" 4 | 5 | require "./client" 6 | require "./sqs" 7 | 8 | module AWS 9 | module SNS 10 | class Client < AWS::Client 11 | SERVICE_NAME = "sns" 12 | 13 | def publish(topic_arn : String, message : String, subject : String = "") 14 | response = http(&.post( 15 | path: "/", 16 | form: { 17 | "Action" => "Publish", 18 | "TopicArn" => topic_arn, 19 | "Message" => message, 20 | "Subject" => subject, 21 | "Version" => "2010-03-31", 22 | }.select { |key, value| !value.empty? }, 23 | )) 24 | 25 | if response.success? 26 | true 27 | else 28 | raise "AWS::SNS#publish: #{XML.parse(response.body).to_xml}" 29 | end 30 | end 31 | 32 | def create_topic(name : String) 33 | http do |http| 34 | response = http.post( 35 | path: "/", 36 | form: { 37 | "Action" => "CreateTopic", 38 | "Name" => name, 39 | }, 40 | ) 41 | 42 | if response.success? 43 | Topic.from_xml response.body 44 | else 45 | raise "AWS::SNS#create_topic: #{XML.parse(response.body).to_xml}" 46 | end 47 | end 48 | end 49 | 50 | def subscribe( 51 | topic : Topic, 52 | queue : SQS::Queue, 53 | sqs = SQS::Client.new( 54 | access_key_id: access_key_id, 55 | secret_access_key: secret_access_key, 56 | region: region, 57 | endpoint: endpoint, 58 | ), 59 | ) 60 | subscribe( 61 | topic_arn: topic.arn, 62 | protocol: "sqs", 63 | endpoint: sqs.get_queue_attributes(queue.url, %w[QueueArn])["QueueArn"] 64 | ) 65 | end 66 | 67 | def subscribe( 68 | topic_arn : String, 69 | protocol : String, 70 | endpoint : String, 71 | ) 72 | http do |http| 73 | response = http.post( 74 | path: "/", 75 | form: { 76 | "Action" => "Subscribe", 77 | "TopicArn" => topic_arn, 78 | "Protocol" => protocol, 79 | "Endpoint" => endpoint, 80 | }, 81 | ) 82 | 83 | if response.success? 84 | TopicSubscription.from_xml(response.body) 85 | else 86 | raise "AWS::SNS#subscribe: #{XML.parse(response.body).to_xml}" 87 | end 88 | end 89 | end 90 | end 91 | 92 | struct Topic 93 | getter arn 94 | 95 | def self.from_xml(xml : String) 96 | from_xml XML.parse xml 97 | end 98 | 99 | def self.from_xml(xml : XML::Node) 100 | new(arn: xml.xpath_node("//xmlns:TopicArn").not_nil!.text) 101 | end 102 | 103 | def initialize(@arn : String) 104 | end 105 | end 106 | 107 | struct TopicSubscription 108 | getter arn 109 | 110 | def self.from_xml(xml : String) 111 | from_xml XML.parse xml 112 | end 113 | 114 | def self.from_xml(xml : XML::Node) 115 | new(arn: xml.xpath_node("//xmlns:SubscriptionArn").not_nil!.text) 116 | end 117 | 118 | def initialize(@arn : String) 119 | end 120 | end 121 | 122 | struct Message 123 | include JSON::Serializable 124 | 125 | @[JSON::Field(key: "Type")] 126 | getter type : String 127 | 128 | @[JSON::Field(key: "Subject")] 129 | getter subject : String 130 | 131 | @[JSON::Field(key: "MessageId")] 132 | getter message_id : String 133 | 134 | @[JSON::Field(key: "TopicArn")] 135 | getter topic_arn : String 136 | 137 | @[JSON::Field(key: "Message")] 138 | getter message : String 139 | 140 | # Because srsly calling message.message is ridiculous 141 | def body 142 | message 143 | end 144 | 145 | @[JSON::Field(key: "Timestamp", converter: ::AWS::SNS::TimestampConverter)] 146 | getter timestamp : Time 147 | 148 | @[JSON::Field(key: "SignatureVersion")] 149 | getter signature_version : String 150 | 151 | @[JSON::Field(key: "Signature")] 152 | getter signature : String 153 | 154 | @[JSON::Field(key: "SigningCertURL", converter: ::AWS::SNS::URIConverter)] 155 | getter signing_cert_url : URI 156 | 157 | @[JSON::Field(key: "UnsubscribeURL", converter: ::AWS::SNS::URIConverter)] 158 | getter unsubscribe_url : URI 159 | end 160 | 161 | module TimestampConverter 162 | FORMAT = Time::Format::ISO_8601_DATE_TIME 163 | 164 | def self.from_json(json : JSON::PullParser) : Time 165 | FORMAT.parse json.read_string 166 | end 167 | 168 | def self.to_json(value : Time, json : JSON::Builder) 169 | FORMAT.format value, json 170 | end 171 | end 172 | 173 | module URIConverter 174 | def self.from_json(json : JSON::PullParser) : URI 175 | URI.parse json.read_string 176 | end 177 | end 178 | end 179 | end 180 | -------------------------------------------------------------------------------- /src/s3.cr: -------------------------------------------------------------------------------- 1 | require "xml" 2 | 3 | require "./client" 4 | 5 | module AWS 6 | module S3 7 | class Exception < AWS::Exception 8 | end 9 | 10 | struct Bucket 11 | getter name, creation_date 12 | 13 | def self.new(xml : XML::Node) 14 | if (name = xml.xpath_node("./xmlns:Name")) && (creation_date = xml.xpath_node("./xmlns:CreationDate")) 15 | new( 16 | name: name.text, 17 | creation_date: Time::Format::ISO_8601_DATE_TIME.parse(creation_date.text), 18 | ) 19 | else 20 | raise XMLIsNotABucket.new("The following XML does not represent an AWS S3 bucket: #{xml}") 21 | end 22 | end 23 | 24 | def initialize(@name : String, @creation_date : Time) 25 | end 26 | end 27 | 28 | class Client < AWS::Client 29 | SERVICE_NAME = "s3" 30 | 31 | def list_buckets 32 | xml = get("/").body 33 | doc = XML.parse xml 34 | 35 | if buckets = doc.xpath_node("//xmlns:Buckets") 36 | buckets.children.map { |b| Bucket.new b } 37 | else 38 | raise UnexpectedResponse.new("The following XML was unexpected from the ListBuckets request: #{xml}") 39 | end 40 | end 41 | 42 | def list_objects(bucket : Bucket) 43 | list_objects bucket.name 44 | end 45 | 46 | def list_objects(bucket_name : String) 47 | xml = get("/?list-type=2", headers: HTTP::Headers{ 48 | "Host" => "#{bucket_name}.#{endpoint.host}", 49 | }).body 50 | XML.parse(xml).to_xml 51 | ListBucketResult.from_xml xml 52 | end 53 | 54 | def get_object(bucket : Bucket, key : String) 55 | get_object bucket.name, key 56 | end 57 | 58 | def get_object(bucket_name : String, key : String) : String 59 | get("/#{key}", headers: HTTP::Headers{ 60 | "Host" => "#{bucket_name}.#{endpoint.host}", 61 | }).body 62 | end 63 | 64 | def get_object(bucket_name : String, key : String, io : IO) : Nil 65 | headers = HTTP::Headers{ 66 | "Host" => "#{bucket_name}.#{endpoint.host}", 67 | } 68 | get "/#{key}", headers: headers do |response| 69 | IO.copy response.body_io, io 70 | end 71 | end 72 | 73 | def head_object(bucket : Bucket, key : String) 74 | head_object bucket.name, key 75 | end 76 | 77 | def head_object(bucket_name : String, key : String) 78 | head("/#{key}", headers: HTTP::Headers{ 79 | "Host" => "#{bucket_name}.#{endpoint.host}", 80 | }) 81 | end 82 | 83 | def presigned_url(method : String, bucket_name : String, key : String, ttl = 10.minutes, headers = HTTP::Headers.new) 84 | date = Time.utc.to_s("%Y%m%dT%H%M%SZ") 85 | algorithm = "AWS4-HMAC-SHA256" 86 | scope = "#{date[0...8]}/#{@region}/s3/aws4_request" 87 | credential = "#{@access_key_id}/#{scope}" 88 | headers = headers.dup # Don't mutate headers we received 89 | headers["Host"] = "#{bucket_name}.#{endpoint.host}" 90 | 91 | unless key.starts_with? "/" 92 | key = "/#{key}" 93 | end 94 | request = HTTP::Request.new( 95 | method: method, 96 | resource: key, 97 | headers: headers, 98 | ) 99 | 100 | canonical_headers = headers 101 | .to_a 102 | .sort_by { |(key, values)| key.downcase } 103 | signed_headers = canonical_headers 104 | .map { |(key, values)| key.downcase } 105 | .join(';') 106 | params = URI::Params{ 107 | "X-Amz-Algorithm" => algorithm, 108 | "X-Amz-Credential" => credential, 109 | "X-Amz-Date" => date, 110 | "X-Amz-Expires" => ttl.total_seconds.to_i.to_s, 111 | "X-Amz-SignedHeaders" => signed_headers, 112 | } 113 | 114 | canonical_request = String.build { |str| 115 | str << method << '\n' 116 | str << key << '\n' 117 | str << params 118 | .to_a 119 | .sort_by { |(key, value)| key } 120 | .each_with_object(URI::Params.new) { |(key, value), params| params[key] = value.gsub(/\s+/, ' ') } 121 | str << '\n' 122 | 123 | canonical_headers 124 | .each do |(key, values)| 125 | values.each do |value| 126 | str << key.downcase << ':' << value.strip << '\n' 127 | end 128 | end 129 | str << '\n' 130 | 131 | str << signed_headers << '\n' 132 | str << "UNSIGNED-PAYLOAD" 133 | } 134 | 135 | string_to_sign = <<-STRING 136 | #{algorithm} 137 | #{date} 138 | #{scope} 139 | #{(OpenSSL::Digest.new("SHA256") << canonical_request).final.hexstring} 140 | STRING 141 | 142 | date_key = OpenSSL::HMAC.digest(OpenSSL::Algorithm::SHA256, "AWS4#{@secret_access_key}", date[0...8]) 143 | region_key = OpenSSL::HMAC.digest(OpenSSL::Algorithm::SHA256, date_key, @region) 144 | service_key = OpenSSL::HMAC.digest(OpenSSL::Algorithm::SHA256, region_key, "s3") 145 | signing_key = OpenSSL::HMAC.digest(OpenSSL::Algorithm::SHA256, service_key, "aws4_request") 146 | signature = OpenSSL::HMAC.hexdigest(OpenSSL::Algorithm::SHA256, signing_key, string_to_sign) 147 | uri = URI.parse("#{endpoint.scheme}://#{bucket_name}.#{endpoint.host}#{request.resource}") 148 | 149 | params["X-Amz-Signature"] = signature 150 | 151 | uri.query = params.to_s 152 | uri 153 | end 154 | 155 | def put_object(bucket_name : String, key : String, headers my_headers : HTTP::Headers, body : IO) 156 | headers = HTTP::Headers{ 157 | "Host" => "#{bucket_name}.#{endpoint.host}", 158 | } 159 | headers.merge! my_headers 160 | 161 | response = put( 162 | "/#{key}", 163 | headers: headers, 164 | body: body 165 | ) 166 | 167 | unless response.success? 168 | raise Exception.new("S3 PutObject returned HTTP status #{response.status}: #{XML.parse(response.body).to_xml}") 169 | end 170 | 171 | response 172 | end 173 | 174 | def put_object(bucket_name : String, key : String, headers : HTTP::Headers, body : String) 175 | put_object bucket_name, 176 | key: key, 177 | headers: HTTP::Headers{"Content-Length" => body.bytesize.to_s} 178 | .tap(&.merge!(headers)), 179 | body: IO::Memory.new(body) 180 | end 181 | 182 | def delete_object(bucket_name : String, key : String) 183 | delete("/#{key}", headers: HTTP::Headers{ 184 | "Host" => "#{bucket_name}.#{endpoint.host}", 185 | }) 186 | end 187 | end 188 | 189 | struct ListBucketResult 190 | getter name, prefix, key_count, max_keys, contents 191 | getter? truncated 192 | 193 | def self.from_xml(xml : String) 194 | from_xml XML.parse(xml).root.not_nil! 195 | end 196 | 197 | def self.from_xml(xml : XML::Node) 198 | name = xml.xpath_node("./xmlns:Name") 199 | prefix = xml.xpath_node("./xmlns:Prefix") 200 | max_keys = xml.xpath_node("./xmlns:MaxKeys") 201 | key_count = xml.xpath_node("./xmlns:KeyCount") 202 | truncated = xml.xpath_node("./xmlns:IsTruncated") 203 | 204 | if name && prefix && max_keys && truncated 205 | contents = xml.xpath_nodes("./xmlns:Contents") 206 | new( 207 | name: name.text, 208 | prefix: prefix.text, 209 | max_keys: max_keys.text.to_i, 210 | key_count: key_count.try(&.text.to_i), 211 | truncated: truncated.text == "true", 212 | contents: contents.map { |c| Contents.from_xml c }, 213 | ) 214 | else 215 | raise InvalidXML.new("The following XML does not represent a ListBucketResult: #{xml}") 216 | end 217 | end 218 | 219 | def initialize( 220 | @name : String, 221 | @prefix : String, 222 | @key_count : Int32?, 223 | @max_keys : Int32, 224 | @truncated : Bool, 225 | @contents : Array(Contents) 226 | ) 227 | end 228 | 229 | struct Contents 230 | getter key, last_modified, etag, size, storage_class 231 | 232 | def self.from_xml(xml : String) 233 | from_xml XML.parse xml 234 | end 235 | 236 | def self.from_xml(xml : XML::Node) 237 | if (key = xml.xpath_node("./xmlns:Key")) && (size = xml.xpath_node("./xmlns:Size")) 238 | new( 239 | key: key.text, 240 | last_modified: Time.utc, 241 | etag: (xml.xpath_node("./xmlns:ETag").try(&.text) || "").gsub('"', ""), 242 | size: size.text.to_i64, 243 | storage_class: xml.xpath_node("./xmlns:StorageClass").try(&.text) || "", 244 | ) 245 | else 246 | raise InvalidXML.new("The following XML is not a ListBucketResult::Contents: #{xml}") 247 | end 248 | end 249 | 250 | def initialize( 251 | @key : String, 252 | @last_modified : Time, 253 | @etag : String, 254 | @size : Int64, 255 | @storage_class : String 256 | ) 257 | end 258 | end 259 | end 260 | 261 | class InvalidXML < Exception 262 | end 263 | 264 | class Exception < ::AWS::Exception 265 | end 266 | 267 | class UnknownBucket < Exception 268 | end 269 | 270 | class UnknownObject < Exception 271 | end 272 | 273 | class XMLIsNotABucket < Exception 274 | end 275 | 276 | class UnexpectedResponse < Exception 277 | end 278 | end 279 | end 280 | -------------------------------------------------------------------------------- /src/sqs.cr: -------------------------------------------------------------------------------- 1 | require "http" 2 | require "xml" 3 | 4 | require "./sqs/message" 5 | require "./client" 6 | 7 | module AWS 8 | module SQS 9 | class Client < AWS::Client 10 | SERVICE_NAME = "sqs" 11 | 12 | def receive_message( 13 | queue_url : URI, 14 | max_number_of_messages : Int = 1, 15 | wait_time_seconds : Int = 0, 16 | ) 17 | params = HTTP::Params.encode({ 18 | Action: "ReceiveMessage", 19 | MaxNumberOfMessages: max_number_of_messages.to_s, 20 | WaitTimeSeconds: wait_time_seconds.to_s, 21 | "AttributeName.1": "All", 22 | "MessageAttributeName.1": "All", 23 | }) 24 | 25 | xml = XML.parse( 26 | http(queue_url.host.not_nil!, &.get( 27 | "#{queue_url.path}?#{params}" 28 | )).body 29 | ) 30 | ReceiveMessageResult.from_xml(xml) 31 | end 32 | 33 | def send_message( 34 | queue_url : URI, 35 | message_body : String, 36 | ) 37 | http(queue_url.host.not_nil!) do |http| 38 | headers = DEFAULT_HEADERS.dup.merge!({ 39 | "Host" => queue_url.host.not_nil!, 40 | "Content-Type" => "application/x-www-form-urlencoded", 41 | }) 42 | params = HTTP::Params { 43 | "Action" => "SendMessage", 44 | "MessageBody" => message_body 45 | } 46 | response = http.post(queue_url.path, body: params.to_s, headers: headers) 47 | SendMessageResponse.from_xml response.body 48 | end 49 | end 50 | 51 | def delete_message(queue_url : URI, receipt_handle : String) 52 | http(queue_url.host.not_nil!) do |http| 53 | params = HTTP::Params.encode({ 54 | Action: "DeleteMessage", 55 | ReceiptHandle: receipt_handle, 56 | }) 57 | 58 | http 59 | .delete("#{queue_url.path}?#{params}") 60 | .body 61 | end 62 | end 63 | 64 | def delete_message_batch( 65 | queue_url : URI, 66 | delete_message_batch_request_entries messages : Enumerable(Message), 67 | ) 68 | http(queue_url.host.not_nil!) do |http| 69 | params = HTTP::Params{"Action" => "DeleteMessageBatch"} 70 | messages.each_with_index(1) do |message, index| 71 | params["DeleteMessageBatchRequestEntry.#{index}.Id"] = message.id.to_s 72 | params["DeleteMessageBatchRequestEntry.#{index}.ReceiptHandle"] = message.receipt_handle 73 | end 74 | http.delete("#{queue_url.path}?#{params}").body 75 | end 76 | end 77 | 78 | def change_message_visibility_batch(queue_url, change_message_visibility_batch_request_entries, visibility_timeout : Time::Span) 79 | change_message_visibility_batch( 80 | queue_url, 81 | change_message_visibility_batch_request_entries, 82 | visibility_timeout: visibility_timeout.total_seconds.to_i, 83 | ) 84 | end 85 | 86 | def change_message_visibility_batch( 87 | queue_url : URI, 88 | change_message_visibility_batch_request_entries messages : Enumerable(Message), 89 | visibility_timeout : Int32 | String, 90 | ) 91 | http(queue_url.host.not_nil!) do |http| 92 | params = HTTP::Params{"Action" => "DeleteMessageBatch"} 93 | messages.each_with_index(1) do |message, index| 94 | params["ChangeMessageVisibilityBatchRequestEntry.#{index}.Id"] = message.id.to_s 95 | params["ChangeMessageVisibilityBatchRequestEntry.#{index}.ReceiptHandle"] = message.receipt_handle 96 | params["ChangeMessageVisibilityBatchRequestEntry.#{index}.VisibilityTimeout"] = visibility_timeout.to_s 97 | end 98 | http.delete("#{queue_url.path}?#{params}").body 99 | end 100 | end 101 | 102 | def create_queue(queue_name name : String) 103 | http do |http| 104 | params = HTTP::Params{"Action" => "CreateQueue", "QueueName" => name} 105 | response = http.post("/?#{params}") 106 | if response.success? 107 | Queue.from_xml response.body 108 | else 109 | raise "AWS::SQS#create_queue: #{XML.parse(response.body).to_xml}" 110 | end 111 | end 112 | end 113 | 114 | def list_queues(queue_name_prefix : String? = nil) 115 | http do |http| 116 | params = HTTP::Params{"Action" => "ListQueues"} 117 | if queue_name_prefix 118 | params["QueueNamePrefix"] = queue_name_prefix 119 | end 120 | 121 | response = http.get("/?#{pp params}") 122 | 123 | if response.success? 124 | ListQueuesResult.from_xml response.body 125 | else 126 | raise "AWS::SQS#list_queues: #{XML.parse(response.body).to_xml}" 127 | end 128 | end 129 | end 130 | 131 | def get_queue_attributes(queue_url : URI, attributes : Enumerable(String)) 132 | q_attrs = Hash(String, String).new(initial_capacity: attributes.size) 133 | 134 | http do |http| 135 | params = HTTP::Params{ 136 | "Action" => "GetQueueAttributes", 137 | "QueueUrl" => queue_url.to_s 138 | } 139 | attributes.each_with_index(1) do |attribute, index| 140 | params["AttributeName.#{index}"] = attribute 141 | end 142 | response = http.get("?#{params}") 143 | 144 | if response.success? 145 | XML.parse(response.body).xpath_nodes("//xmlns:Attribute").each do |attribute| 146 | q_attrs[attribute.xpath_node("./xmlns:Name").not_nil!.text] = 147 | attribute.xpath_node("./xmlns:Value").not_nil!.text 148 | end 149 | else 150 | raise "AWS::SQS#get_queue_attributes: #{XML.parse(response.body).to_xml}" 151 | end 152 | end 153 | 154 | q_attrs 155 | end 156 | end 157 | 158 | struct ListQueuesResult 159 | getter queues : Enumerable(Queue) 160 | getter next_token : String? 161 | 162 | def self.from_xml(xml : String) 163 | from_xml XML.parse xml 164 | end 165 | 166 | def self.from_xml(xml : XML::Node) 167 | queues = xml.xpath_nodes("//xmlns:QueueUrl").map do |url| 168 | Queue.new(url.text) 169 | end 170 | 171 | new(queues: queues) 172 | end 173 | 174 | def initialize(@queues : Enumerable(Queue), @next_token = nil) 175 | end 176 | end 177 | struct Queue 178 | getter url : URI 179 | 180 | def self.from_xml(xml : String) 181 | from_xml XML.parse xml 182 | end 183 | 184 | def self.from_xml(xml : XML::Node) 185 | if xml.document? 186 | from_xml xml.root.not_nil! 187 | else 188 | new(url: xml.xpath_node("//xmlns:QueueUrl").not_nil!.text) 189 | end 190 | end 191 | 192 | def initialize(url : String) 193 | initialize URI.parse(url) 194 | end 195 | 196 | def initialize(@url : URI) 197 | end 198 | end 199 | 200 | struct SendMessageResponse 201 | def self.from_xml(xml : String) 202 | from_xml XML.parse xml 203 | end 204 | 205 | def self.from_xml(xml : XML::Node) 206 | if xml.document? 207 | from_xml xml.root.not_nil! 208 | else 209 | new( 210 | send_message_result: SendMessageResult.from_xml(xml.xpath_node("./xmlns:SendMessageResult").not_nil!), 211 | response_metadata: ResponseMetadata.from_xml(xml.xpath_node("./xmlns:ResponseMetadata").not_nil!), 212 | ) 213 | end 214 | end 215 | 216 | getter send_message_result : SendMessageResult 217 | getter response_metadata : ResponseMetadata 218 | 219 | def initialize(@send_message_result, @response_metadata) 220 | end 221 | 222 | struct SendMessageResult 223 | def self.from_xml(xml : XML::Node) 224 | new( 225 | message_id: UUID.new(xml.xpath_string("string(./xmlns:MessageId)")), 226 | md5_of_message_body: xml.xpath_string("string(./xmlns:MD5OfMessageBody)"), 227 | ) 228 | end 229 | 230 | def initialize(@message_id : UUID, md5_of_message_body : String) 231 | end 232 | end 233 | 234 | struct ResponseMetadata 235 | def self.from_xml(xml : XML::Node) 236 | new( 237 | request_id: UUID.new(xml.xpath_string("string(./xmlns:RequestId)")), 238 | ) 239 | end 240 | 241 | def initialize(@request_id : UUID) 242 | end 243 | end 244 | end 245 | 246 | struct ReceiveMessageResult 247 | getter messages 248 | 249 | def self.from_xml(xml : String) 250 | from_xml XML.parse xml 251 | end 252 | 253 | def self.from_xml(xml : XML::Node) 254 | if xml.name == "ReceiveMessageResult" 255 | new(xml.children.map { |message| Message.from_xml(message) }) 256 | elsif xml.name == "ErrorResponse" 257 | raise ERROR_MAP.fetch(xml.xpath_node("//xmlns:Code").not_nil!.text, Exception).new( 258 | message: xml.xpath_node("//xmlns:Message").not_nil!.text, 259 | ) 260 | else 261 | from_xml(xml.children.first) 262 | end 263 | end 264 | 265 | def initialize(@messages : Array(Message)) 266 | end 267 | end 268 | 269 | class Exception < AWS::Exception 270 | end 271 | class ResultParsingError < Exception 272 | end 273 | class InvalidParameterValue < Exception 274 | end 275 | 276 | ERROR_MAP = { 277 | "InvalidParameterValue" => InvalidParameterValue, 278 | } 279 | end 280 | end 281 | --------------------------------------------------------------------------------