├── .gitignore ├── README.rdoc ├── Rakefile ├── VERSION ├── ca-bundle.crt ├── config ├── locales │ └── en.yml └── s3.yml.example ├── init.rb ├── lang └── en.yml ├── lib ├── aws-sdk.rb ├── aws.rb ├── aws │ ├── LICENSE.txt │ ├── NOTICE.txt │ ├── core.rb │ ├── core │ │ ├── api_config.rb │ │ ├── async_handle.rb │ │ ├── authorize_v2.rb │ │ ├── authorize_v3.rb │ │ ├── authorize_with_session_token.rb │ │ ├── autoloader.rb │ │ ├── cacheable.rb │ │ ├── client.rb │ │ ├── client_logging.rb │ │ ├── collection.rb │ │ ├── collection │ │ │ ├── batchable.rb │ │ │ ├── limitable.rb │ │ │ └── simple.rb │ │ ├── configuration.rb │ │ ├── configured_client_methods.rb │ │ ├── configured_grammars.rb │ │ ├── configured_option_grammars.rb │ │ ├── configured_xml_grammars.rb │ │ ├── default_signer.rb │ │ ├── http │ │ │ ├── curb_handler.rb │ │ │ ├── handler.rb │ │ │ ├── httparty_handler.rb │ │ │ ├── net_http_handler.rb │ │ │ ├── request.rb │ │ │ └── response.rb │ │ ├── ignore_result_element.rb │ │ ├── indifferent_hash.rb │ │ ├── inflection.rb │ │ ├── lazy_error_classes.rb │ │ ├── meta_utils.rb │ │ ├── model.rb │ │ ├── naming.rb │ │ ├── option_grammar.rb │ │ ├── page_result.rb │ │ ├── policy.rb │ │ ├── resource.rb │ │ ├── resource_cache.rb │ │ ├── response.rb │ │ ├── response_cache.rb │ │ ├── service_interface.rb │ │ ├── uri_escape.rb │ │ └── xml_grammar.rb │ ├── errors.rb │ ├── s3.rb │ └── s3 │ │ ├── access_control_list.rb │ │ ├── acl_object.rb │ │ ├── bucket.rb │ │ ├── bucket_collection.rb │ │ ├── bucket_version_collection.rb │ │ ├── client.rb │ │ ├── client │ │ └── xml.rb │ │ ├── config.rb │ │ ├── data_options.rb │ │ ├── errors.rb │ │ ├── multipart_upload.rb │ │ ├── multipart_upload_collection.rb │ │ ├── object_collection.rb │ │ ├── object_metadata.rb │ │ ├── object_upload_collection.rb │ │ ├── object_version.rb │ │ ├── object_version_collection.rb │ │ ├── paginated_collection.rb │ │ ├── policy.rb │ │ ├── prefix_and_delimiter_collection.rb │ │ ├── prefixed_collection.rb │ │ ├── presigned_post.rb │ │ ├── request.rb │ │ ├── s3_object.rb │ │ ├── tree.rb │ │ ├── tree │ │ ├── branch_node.rb │ │ ├── child_collection.rb │ │ ├── leaf_node.rb │ │ ├── node.rb │ │ └── parent.rb │ │ ├── uploaded_part.rb │ │ └── uploaded_part_collection.rb ├── redmine_s3 │ ├── attachment_patch.rb │ ├── attachments_controller_patch.rb │ └── connection.rb └── tasks │ └── files_to_s3.rake ├── rails └── init.rb ├── redmine_s3.gemspec └── test └── test_helper.rb /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | .swp 3 | -------------------------------------------------------------------------------- /README.rdoc: -------------------------------------------------------------------------------- 1 | = S3 plugin for Redmine 2 | 3 | == Description 4 | This Redmine[http://www.redmine.org] plugin makes file attachments be stored on "Amazon S3"[http://aws.amazon.com/s3] rather than on the local filesystem. 5 | 6 | == Installation 7 | 1. Make sure Redmine is installed and cd into it's root directory 8 | 2. git clone git://github.com/tigrish/redmine_s3.git vendor/plugins/redmine_s3 9 | 3. cp vendor/plugins/redmine_s3/config/s3.yml.example config/s3.yml 10 | 4. Edit config/s3.yml with your favourite editor 11 | 5. Restart mongrel/upload to production/whatever 12 | 6. _Optional:_ Run 'rake redmine_s3:files_to_s3' to upload files in your files folder to s3 13 | 14 | == Options Overview 15 | * The bucket specified in s3.yml will be created automatically when the plugin is loaded (this is generally when the server starts). 16 | * _Deprecated_ (no longer supported, specify endpoint option instead) If you have created a CNAME entry for your bucket set the cname_bucket option to true in s3.yml and your files will be served from that domain. 17 | * After files are uploaded they are made public, unless private is set to true. 18 | * Public and private files can use HTTPS urls using the secure option 19 | * Files can use private signed urls using the private option 20 | * Private file urls can expire a set time after the links were generated using the expires option 21 | 22 | == Options Detail 23 | * access_key_id: string key (required) 24 | * secret_access_key: string key (required) 25 | * bucket: string bucket name (required) 26 | * endpoint: string endpoint instead of s3.amazonaws.com 27 | * secure: boolean true/false 28 | * private: boolean true/false 29 | * expires: integer number of seconds for private links to expire after being generated 30 | * Defaults to private: false, secure: false, default endpoint, and default expires 31 | 32 | == Reporting Bugs and Getting Help 33 | Bugs and feature requests may be filed at https://github.com/tigrish/redmine_s3/issues 34 | -------------------------------------------------------------------------------- /VERSION: -------------------------------------------------------------------------------- 1 | 0.0.3 -------------------------------------------------------------------------------- /config/locales/en.yml: -------------------------------------------------------------------------------- 1 | # English strings go here for Rails i18n 2 | en: 3 | my_label: "My label" 4 | -------------------------------------------------------------------------------- /config/s3.yml.example: -------------------------------------------------------------------------------- 1 | production: 2 | access_key_id: 3 | secret_access_key: 4 | bucket: 5 | endpoint: 6 | secure: 7 | private: 8 | expires: 9 | 10 | development: 11 | access_key_id: 12 | secret_access_key: 13 | bucket: 14 | endpoint: 15 | secure: 16 | private: 17 | expires: 18 | -------------------------------------------------------------------------------- /init.rb: -------------------------------------------------------------------------------- 1 | require 'redmine' 2 | require 'dispatcher' # Patches to the Redmine core. 3 | 4 | Dispatcher.to_prepare :redmine_s3 do 5 | require_dependency 'attachment' 6 | unless Attachment.included_modules.include? RedmineS3::AttachmentPatch 7 | Attachment.send(:include, RedmineS3::AttachmentPatch) 8 | end 9 | 10 | app_dependency = Redmine::VERSION.to_a.slice(0,3).join('.') > '0.8.4' ? 'application_controller' : 'application' 11 | require_dependency(app_dependency) 12 | require_dependency 'attachments_controller' 13 | unless AttachmentsController.included_modules.include? RedmineS3::AttachmentsControllerPatch 14 | AttachmentsController.send(:include, RedmineS3::AttachmentsControllerPatch) 15 | end 16 | 17 | RedmineS3::Connection.create_bucket 18 | end 19 | 20 | Redmine::Plugin.register :redmine_s3_attachments do 21 | name 'S3' 22 | author 'Chris Dell' 23 | description 'Use Amazon S3 as a storage engine for attachments' 24 | version '0.0.3' 25 | end 26 | -------------------------------------------------------------------------------- /lang/en.yml: -------------------------------------------------------------------------------- 1 | # English strings go here 2 | my_label: "My label" 3 | -------------------------------------------------------------------------------- /lib/aws-sdk.rb: -------------------------------------------------------------------------------- 1 | # Copyright 2011 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). You 4 | # may not use this file except in compliance with the License. A copy of 5 | # the License is located at 6 | # 7 | # http://aws.amazon.com/apache2.0/ 8 | # 9 | # or in the "license" file accompanying this file. This file is 10 | # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF 11 | # ANY KIND, either express or implied. See the License for the specific 12 | # language governing permissions and limitations under the License. 13 | 14 | require 'aws' 15 | -------------------------------------------------------------------------------- /lib/aws.rb: -------------------------------------------------------------------------------- 1 | # Copyright 2011 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). You 4 | # may not use this file except in compliance with the License. A copy of 5 | # the License is located at 6 | # 7 | # http://aws.amazon.com/apache2.0/ 8 | # 9 | # or in the "license" file accompanying this file. This file is 10 | # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF 11 | # ANY KIND, either express or implied. See the License for the specific 12 | # language governing permissions and limitations under the License. 13 | 14 | require 'aws/core' 15 | 16 | # redmine_s3 only needs s3 support 17 | 18 | #require 'aws/ec2/config' 19 | #require 'aws/elb/config' 20 | #require 'aws/iam/config' 21 | require 'aws/s3/config' 22 | #require 'aws/simple_db/config' 23 | #require 'aws/simple_email_service/config' 24 | #require 'aws/sns/config' 25 | #require 'aws/sqs/config' 26 | #require 'aws/sts/config' 27 | 28 | module AWS 29 | register_autoloads(self) do 30 | # autoload :EC2, 'ec2' 31 | # autoload :ELB, 'elb' 32 | # autoload :IAM, 'iam' 33 | autoload :S3, 's3' 34 | # autoload :SimpleDB, 'simple_db' 35 | # autoload :SimpleEmailService, 'simple_email_service' 36 | # autoload :SNS, 'sns' 37 | # autoload :SQS, 'sqs' 38 | # autoload :STS, 'sts' 39 | # autoload :Record, 'record' 40 | end 41 | end 42 | 43 | # require 'aws/rails' 44 | -------------------------------------------------------------------------------- /lib/aws/NOTICE.txt: -------------------------------------------------------------------------------- 1 | AWS SDK for Ruby 2 | Copyright 2011 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | -------------------------------------------------------------------------------- /lib/aws/core/api_config.rb: -------------------------------------------------------------------------------- 1 | # Copyright 2011 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). You 4 | # may not use this file except in compliance with the License. A copy of 5 | # the License is located at 6 | # 7 | # http://aws.amazon.com/apache2.0/ 8 | # 9 | # or in the "license" file accompanying this file. This file is 10 | # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF 11 | # ANY KIND, either express or implied. See the License for the specific 12 | # language governing permissions and limitations under the License. 13 | 14 | require 'yaml' 15 | require 'pathname' 16 | 17 | module AWS 18 | module Core 19 | 20 | # @private 21 | module ApiConfig 22 | 23 | include Naming 24 | 25 | protected 26 | def api_config 27 | 28 | config_file = $:.map do |load_path| 29 | if config_dir = Pathname.new(load_path) + 30 | "aws" + "api_config" and 31 | config_dir.directory? 32 | config_dir.children.select do |child| 33 | child.basename.to_s =~ /^#{service_name}/ 34 | end.sort.last 35 | end 36 | end.compact.sort.last 37 | 38 | YAML.load(config_file.read) 39 | 40 | end 41 | 42 | end 43 | end 44 | end 45 | -------------------------------------------------------------------------------- /lib/aws/core/async_handle.rb: -------------------------------------------------------------------------------- 1 | # Copyright 2011 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). You 4 | # may not use this file except in compliance with the License. A copy of 5 | # the License is located at 6 | # 7 | # http://aws.amazon.com/apache2.0/ 8 | # 9 | # or in the "license" file accompanying this file. This file is 10 | # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF 11 | # ANY KIND, either express or implied. See the License for the specific 12 | # language governing permissions and limitations under the License. 13 | 14 | module AWS 15 | module Core 16 | 17 | # Mixin that provides a generic callback facility for asynchronous 18 | # tasks that can either succeed or fail. 19 | # @private 20 | module AsyncHandle 21 | 22 | # Called to signal success and fire off the success and complete callbacks. 23 | def signal_success 24 | __send_signal(:success) 25 | end 26 | 27 | # Called to signal failure and fire off the failure and complete callbacks. 28 | def signal_failure 29 | __send_signal(:failure) 30 | end 31 | 32 | # Registers a callback to be called on successful completion of 33 | # the task. 34 | # 35 | # handle.on_success { puts "It worked!" } 36 | # 37 | # If this is called when the task has already completed 38 | # successfully, it will call the callback immediately. 39 | def on_success(&blk) 40 | if @_async_status == :success 41 | blk.call 42 | else 43 | (@_async_callbacks ||= []) << { :success => blk } 44 | end 45 | end 46 | 47 | # Registers a callback to be called when the task fails. 48 | # 49 | # handle.on_failure { puts "It didn't work!" } 50 | # 51 | # If this is called when the task has already failed, it will 52 | # call the callback immediately. 53 | def on_failure(&blk) 54 | if @_async_status == :failure 55 | blk.call 56 | else 57 | (@_async_callbacks ||= []) << { :failure => blk } 58 | end 59 | end 60 | 61 | # Registers a callback to be called when the task is complete, 62 | # regardless of its status. Yields the status to the block. 63 | # 64 | # handle.on_complete do |status| 65 | # puts "It #{status == :success ? 'did' : 'did not'} work!" 66 | # end 67 | # 68 | # If this is called when the task has already completed, it will 69 | # call the callback immediately. 70 | def on_complete(&blk) 71 | if !@_async_status.nil? 72 | blk.call(@_async_status) 73 | else 74 | (@_async_callbacks ||= []) << { 75 | :failure => lambda { blk.call(:failure) }, 76 | :success => lambda { blk.call(:success) } 77 | } 78 | end 79 | end 80 | 81 | private 82 | def __send_signal(kind) 83 | @_async_status = kind 84 | @_async_callbacks.map do |cb| 85 | cb[kind] 86 | end.compact.each { |blk| blk.call } if @_async_callbacks 87 | end 88 | 89 | end 90 | end 91 | end 92 | -------------------------------------------------------------------------------- /lib/aws/core/authorize_v2.rb: -------------------------------------------------------------------------------- 1 | # Copyright 2011 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). You 4 | # may not use this file except in compliance with the License. A copy of 5 | # the License is located at 6 | # 7 | # http://aws.amazon.com/apache2.0/ 8 | # 9 | # or in the "license" file accompanying this file. This file is 10 | # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF 11 | # ANY KIND, either express or implied. See the License for the specific 12 | # language governing permissions and limitations under the License. 13 | 14 | module AWS 15 | module Core 16 | 17 | # Mixed into clients that use v2 authorization. 18 | # @private 19 | module AuthorizeV2 20 | 21 | def string_to_sign 22 | parts = [http_method, 23 | host, 24 | path, 25 | params.sort.collect { |p| p.encoded }.join('&')] 26 | parts.join("\n") 27 | end 28 | 29 | def add_authorization! signer 30 | self.access_key_id = signer.access_key_id 31 | add_param('AWSAccessKeyId', access_key_id) 32 | add_param('SignatureVersion', '2') 33 | add_param('SignatureMethod', 'HmacSHA256') 34 | add_param('Signature', signer.sign(string_to_sign)) 35 | end 36 | 37 | end 38 | end 39 | end 40 | -------------------------------------------------------------------------------- /lib/aws/core/authorize_v3.rb: -------------------------------------------------------------------------------- 1 | # Copyright 2011 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). You 4 | # may not use this file except in compliance with the License. A copy of 5 | # the License is located at 6 | # 7 | # http://aws.amazon.com/apache2.0/ 8 | # 9 | # or in the "license" file accompanying this file. This file is 10 | # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF 11 | # ANY KIND, either express or implied. See the License for the specific 12 | # language governing permissions and limitations under the License. 13 | 14 | require 'time' 15 | 16 | module AWS 17 | module Core 18 | 19 | # Mixed into clients that use v3 authorization. 20 | # @private 21 | module AuthorizeV3 22 | 23 | def string_to_sign 24 | headers['date'] ||= Time.now.rfc822 25 | end 26 | 27 | def add_authorization! signer 28 | self.access_key_id = signer.access_key_id 29 | parts = [] 30 | parts << "AWS3-HTTPS AWSAccessKeyId=#{access_key_id}" 31 | parts << "Algorithm=HmacSHA256" 32 | parts << "Signature=#{signer.sign(string_to_sign)}" 33 | headers['x-amzn-authorization'] = parts.join(',') 34 | end 35 | 36 | end 37 | end 38 | end 39 | -------------------------------------------------------------------------------- /lib/aws/core/authorize_with_session_token.rb: -------------------------------------------------------------------------------- 1 | # Copyright 2011 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). You 4 | # may not use this file except in compliance with the License. A copy of 5 | # the License is located at 6 | # 7 | # http://aws.amazon.com/apache2.0/ 8 | # 9 | # or in the "license" file accompanying this file. This file is 10 | # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF 11 | # ANY KIND, either express or implied. See the License for the specific 12 | # language governing permissions and limitations under the License. 13 | 14 | module AWS 15 | module Core 16 | 17 | # @private 18 | module AuthorizeWithSessionToken 19 | 20 | def add_authorization! signer 21 | if signer.respond_to?(:session_token) and token = signer.session_token 22 | add_param("SecurityToken", token) 23 | end 24 | super 25 | end 26 | 27 | end 28 | end 29 | end 30 | -------------------------------------------------------------------------------- /lib/aws/core/autoloader.rb: -------------------------------------------------------------------------------- 1 | # Copyright 2011 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). You 4 | # may not use this file except in compliance with the License. A copy of 5 | # the License is located at 6 | # 7 | # http://aws.amazon.com/apache2.0/ 8 | # 9 | # or in the "license" file accompanying this file. This file is 10 | # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF 11 | # ANY KIND, either express or implied. See the License for the specific 12 | # language governing permissions and limitations under the License. 13 | 14 | require 'set' 15 | 16 | module AWS 17 | 18 | @@eager = false 19 | @@autoloads = {} 20 | 21 | def self.register_autoloads klass, prefix = nil, &block 22 | autoloader = Core::Autoloader.new(klass, prefix) 23 | autoloader.instance_eval(&block) 24 | autoloader.autoloads.each_pair do |const_name, file_path| 25 | require(file_path) if @@eager 26 | @@autoloads["#{klass}::#{const_name}"] = file_path 27 | end 28 | end 29 | 30 | def self.eager_autoload! 31 | unless @@eager 32 | @@eager = true 33 | @@autoloads.values.uniq.each {|file_path| require(file_path) } 34 | end 35 | end 36 | 37 | def self.autoloads 38 | @@autoloads 39 | end 40 | 41 | module Core 42 | 43 | # @private 44 | class Autoloader 45 | 46 | def initialize klass, prefix = nil 47 | @klass = klass 48 | @prefix = prefix || klass.name.gsub(/::/, '/').downcase 49 | @autoloads = {} 50 | end 51 | 52 | attr_reader :autoloads 53 | 54 | def autoload const_name, file_name 55 | path = "#{@prefix}/#{file_name}" 56 | @klass.autoload(const_name, path) 57 | @autoloads[const_name] = path 58 | end 59 | 60 | end 61 | 62 | end 63 | 64 | end 65 | -------------------------------------------------------------------------------- /lib/aws/core/cacheable.rb: -------------------------------------------------------------------------------- 1 | # Copyright 2011 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). You 4 | # may not use this file except in compliance with the License. A copy of 5 | # the License is located at 6 | # 7 | # http://aws.amazon.com/apache2.0/ 8 | # 9 | # or in the "license" file accompanying this file. This file is 10 | # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF 11 | # ANY KIND, either express or implied. See the License for the specific 12 | # language governing permissions and limitations under the License. 13 | 14 | module AWS 15 | module Core 16 | 17 | # @private 18 | module Cacheable 19 | 20 | # @private 21 | class NoData < StandardError; end 22 | 23 | def self.included base 24 | base.extend Naming unless base.respond_to?(:service_ruby_name) 25 | end 26 | 27 | # @private 28 | protected 29 | def local_cache_key 30 | raise NotImplementedError 31 | end 32 | 33 | # @private 34 | protected 35 | def cache_key 36 | @cache_key ||= begin 37 | endpoint_method = self.class.service_ruby_name + "_endpoint" 38 | config.signer.access_key_id + ":" + 39 | config.send(endpoint_method) + ":" + 40 | self.class.name + ":" + 41 | local_cache_key 42 | end 43 | end 44 | 45 | # @private 46 | public 47 | def retrieve_attribute attr, &block 48 | 49 | if cache = AWS.response_cache 50 | 51 | if cache.resource_cache.cached?(cache_key, attr.name) 52 | return cache.resource_cache.get(cache_key, attr.name) 53 | end 54 | 55 | cache.select(*attr.request_types).each do |response| 56 | if attributes = attributes_from_response(response) 57 | cache.resource_cache.store(cache_key, attributes) 58 | return attributes[attr.name] if attributes.key?(attr.name) 59 | end 60 | end 61 | 62 | end 63 | 64 | response = yield 65 | 66 | if attributes = attributes_from_response(response) 67 | if cache = AWS.response_cache 68 | cache.resource_cache.store(cache_key, attributes) 69 | end 70 | attributes[attr.name] if attributes.key?(attr.name) 71 | else 72 | raise NoData.new("no data in #{response.request_type} response") 73 | end 74 | end 75 | 76 | end 77 | end 78 | end 79 | -------------------------------------------------------------------------------- /lib/aws/core/client_logging.rb: -------------------------------------------------------------------------------- 1 | # Copyright 2011 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). You 4 | # may not use this file except in compliance with the License. A copy of 5 | # the License is located at 6 | # 7 | # http://aws.amazon.com/apache2.0/ 8 | # 9 | # or in the "license" file accompanying this file. This file is 10 | # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF 11 | # ANY KIND, either express or implied. See the License for the specific 12 | # language governing permissions and limitations under the License. 13 | 14 | require 'benchmark' 15 | 16 | module AWS 17 | module Core 18 | 19 | # @private 20 | module ClientLogging 21 | 22 | def log_client_request(name, options) 23 | response = nil 24 | time = Benchmark.measure do 25 | response = yield 26 | end 27 | 28 | if options[:async] 29 | response.on_complete do 30 | log_client_request_on_success(name, options, response, time) 31 | end 32 | else 33 | log_client_request_on_success(name, options, response, time) 34 | end 35 | response 36 | end 37 | 38 | # Summarizes long strings and adds file size information 39 | # @private 40 | def sanitize_options(options) 41 | sanitize_hash(options) 42 | end 43 | 44 | protected 45 | def log severity, message 46 | config.logger.send(severity, message + "\n") if config.logger 47 | end 48 | 49 | protected 50 | def log_client_request_on_success(method_name, options, response, time) 51 | status = response.http_response.status 52 | service = self.class.service_name 53 | 54 | pattern = "[AWS %s %s %.06f] %s(%s)" 55 | parts = [service, status, time.real, method_name, sanitize_options(options)] 56 | severity = :info 57 | 58 | if response.error 59 | pattern += " %s: %s" 60 | parts << response.error.class 61 | parts << response.error.message 62 | severity = :error 63 | end 64 | 65 | if response.cached 66 | pattern << " [CACHED]" 67 | end 68 | 69 | log(severity, pattern % parts) 70 | end 71 | 72 | protected 73 | def sanitize_value(value) 74 | case value 75 | when Hash 76 | '{' + sanitize_hash(value) + '}' 77 | when Array 78 | sanitize_array(value) 79 | when File 80 | sanitize_file(value) 81 | when String 82 | sanitize_string(value) 83 | else 84 | value.inspect 85 | end 86 | end 87 | 88 | protected 89 | def sanitize_string str 90 | inspected = str.inspect 91 | if inspected.size > config.logger_truncate_strings_at 92 | summarize_string(str) 93 | else 94 | inspected 95 | end 96 | end 97 | 98 | protected 99 | def summarize_string str 100 | # skip the openning " 101 | string_start = str.inspect[1,config.logger_truncate_strings_at] 102 | "#" 103 | end 104 | 105 | protected 106 | def sanitize_file file 107 | "#" 108 | end 109 | 110 | protected 111 | def sanitize_array array 112 | "[" + array.map { |v| sanitize_value(v) }.join(",") + "]" 113 | end 114 | 115 | protected 116 | def sanitize_hash hash 117 | hash.map do |k,v| 118 | "#{sanitize_value(k)}=>#{sanitize_value(v)}" 119 | end.sort.join(",") 120 | end 121 | 122 | end 123 | end 124 | end 125 | -------------------------------------------------------------------------------- /lib/aws/core/collection.rb: -------------------------------------------------------------------------------- 1 | # Copyright 2011 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # Licensed under the Apache License, Version 2.0 (the "License"). You 3 | # may not use this file except in compliance with the License. A copy of 4 | # the License is located at 5 | # 6 | # http://aws.amazon.com/apache2.0/ 7 | # 8 | # or in the "license" file accompanying this file. This file is 9 | # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF 10 | # ANY KIND, either express or implied. See the License for the specific 11 | # language governing permissions and limitations under the License. 12 | 13 | module AWS 14 | module Core 15 | 16 | # = Different Collection Types in AWS 17 | # 18 | # The Collection module acts as a namespace and base implementation for 19 | # the primary collection types in AWS: 20 | # 21 | # * {AWS::Core::Collection::Simple} 22 | # * {AWS::Core::Collection::Limitable} 23 | # 24 | # Each AWS service allows provides a method to enumerate resources. 25 | # 26 | # * Services that return all results in a single response use 27 | # {AWS::Core::Collection::Simple}. 28 | # 29 | # * Services that truncate large results sets *AND* allow you to provide 30 | # a perfered maximum number of results use 31 | # {AWS::Core::Collection::Limitable}. 32 | # 33 | module Collection 34 | 35 | AWS.register_autoloads(self) do 36 | autoload :Simple, 'simple' 37 | autoload :Limitable, 'limitable' 38 | end 39 | 40 | include Enumerable 41 | 42 | # Yields once for every item in this collection. 43 | # 44 | # collection.each {|item| ... } 45 | # 46 | # @note If you want fewer than all items, it is generally better 47 | # to call #{page} than {#each} with a +:limit+. 48 | # 49 | # @param [Hash] options 50 | # 51 | # @option options [Integer] :limit (nil) The maximum number of 52 | # items to enumerate from this collection. 53 | # 54 | # @option options [next_token] :next_token (nil) Next tokens 55 | # act as offsets into the collection. Next tokens vary in 56 | # format from one service to the next, (e.g. may be a number, 57 | # an opaque string, a hash of values, etc). 58 | # 59 | # {#each} and {#each_batch} return a +:next_token+ when called 60 | # with +:limit+ and there were more items matching the request. 61 | # 62 | # *NOTE* It is generally better to call {#page} if you only 63 | # want a few items with the ability to request more later. 64 | # 65 | # @return [nil_or_next_token] Returns nil if all items were enumerated. 66 | # If some items were excluded because of a +:limit+ option then 67 | # a +next_token+ is returned. Calling an enumerable method on 68 | # the same collection with the +next_token+ acts like an offset. 69 | # 70 | def each options = {}, &block 71 | each_batch(options) do |batch| 72 | batch.each(&block) 73 | end 74 | end 75 | 76 | # Yields items from this collection in batches. 77 | # 78 | # collection.each_batch do |batch| 79 | # batch.each do |item| 80 | # # ... 81 | # end 82 | # end 83 | # 84 | # == Variable Batch Sizes 85 | # 86 | # Each AWS service has its own rules on how it returns results. 87 | # Because of this batch size may very based on: 88 | # 89 | # * Service limits (e.g. S3 limits keys to 1000 per response) 90 | # 91 | # * The size of response objects (SimpleDB limits responses to 1MB) 92 | # 93 | # * Time to process the request 94 | # 95 | # Because of these variables, batch sizes may not be consistent for 96 | # a single collection. Each batch represents all of the items returned 97 | # in a single resopnse. 98 | # 99 | # @note If you require fixed size batches, see {#in_groups_of}. 100 | # 101 | # @param (see #each) 102 | # 103 | # @option (see #each) 104 | # 105 | # @return (see #each) 106 | # 107 | def each_batch options = {}, &block 108 | raise NotImplementedError 109 | end 110 | 111 | # Use this method when you want to call a method provided by 112 | # Enumerable, but you need to pass options: 113 | # 114 | # # raises an error because collect does not accept arguments 115 | # collection.collect(:limit => 10) {|i| i.name } 116 | # 117 | # # not an issue with the enum method 118 | # collection.enum(:limit => 10).collect(&:name) 119 | # 120 | # @param (see #each) 121 | # 122 | # @option (see #each) 123 | # 124 | # @return [Enumerable::Enumerator] Returns an enumerator for this 125 | # collection. 126 | # 127 | def enum options = {} 128 | Enumerator.new(self, :each, options) 129 | end 130 | alias_method :enumerator, :enum 131 | 132 | # Returns the first item from this collection. 133 | # 134 | # @return [item_or_nil] Returns the first item from this collection or 135 | # nil if the collection is empty. 136 | # 137 | def first options = {} 138 | enum(options.merge(:limit => 1)).first 139 | end 140 | 141 | # Yields items from this collection in groups of an exact 142 | # size (except for perhaps the last group). 143 | # 144 | # collection.in_groups_of (10, :limit => 30) do |group| 145 | # 146 | # # each group should be exactly 10 items unless 147 | # # fewer than 30 items are returned by the service 148 | # group.each do |item| 149 | # #... 150 | # end 151 | # 152 | # end 153 | # 154 | # @param [Integer] size Size each each group of objects 155 | # should be yielded in. 156 | # @param [Hash] options 157 | # @option (see #each) 158 | # @return (see #each) 159 | def in_groups_of size, options = {}, &block 160 | 161 | group = [] 162 | 163 | nil_or_next_token = each_batch(options) do |batch| 164 | batch.each do |item| 165 | group << item 166 | if group.size == size 167 | yield(group) 168 | group = [] 169 | end 170 | end 171 | end 172 | 173 | yield(group) unless group.empty? 174 | 175 | nil_or_next_token 176 | 177 | end 178 | 179 | # Returns a single page of results in a kind-of array ({PageResult}). 180 | # 181 | # items = collection.page(:per_page => 10) # defaults to 10 items 182 | # items.is_a?(Array) # => true 183 | # items.size # => 8 184 | # items.per_page # => 10 185 | # items.last_page? # => true 186 | # 187 | # If you need to display a "next page" link in a web view you can 188 | # use the #more? method. Just make sure the generated link 189 | # contains the +next_token+. 190 | # 191 | # <% if items.more? %> 192 | # <%= link_to('Next Page', params.merge(:next_token => items.next_token) %> 193 | # <% end %> 194 | # 195 | # Then in your controller you can find the next page of results: 196 | # 197 | # items = collection.page(:next_token => params[:next_token]) 198 | # 199 | # Given a {PageResult} you can also get more results directly: 200 | # 201 | # more_items = items.next_page 202 | # 203 | # @note This method does not accept a +:page+ option, which means you 204 | # can only start at the begining of the collection and request 205 | # the next page of results. You can not choose an offset 206 | # or know how many pages of results there will be. 207 | # 208 | # @params [Hash] options A hash of options that modifies the 209 | # items returned in the page of results. 210 | # 211 | # @option options [Integer] :per_page (10) The number of results 212 | # to return for each page. 213 | # 214 | # @option options [String] :next_token (nil) A token that indicates 215 | # an offset to use when paging items. Next tokens are returned 216 | # by {PageResult#next_token}. 217 | # 218 | # Next tokens should only be consumed by the same collection that 219 | # created them. 220 | # 221 | def page options = {} 222 | 223 | each_opts = options.dup 224 | 225 | per_page = each_opts.delete(:per_page) 226 | per_page = [nil,''].include?(per_page) ? 10 : per_page.to_i 227 | 228 | each_opts[:limit] = per_page 229 | 230 | items = [] 231 | next_token = each(each_opts) do |item| 232 | items << item 233 | end 234 | 235 | Core::PageResult.new(self, items, per_page, next_token) 236 | 237 | end 238 | 239 | end 240 | end 241 | end 242 | -------------------------------------------------------------------------------- /lib/aws/core/collection/batchable.rb: -------------------------------------------------------------------------------- 1 | # Copyright 2011 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). You 4 | # may not use this file except in compliance with the License. A copy of 5 | # the License is located at 6 | # 7 | # http://aws.amazon.com/apache2.0/ 8 | # 9 | # or in the "license" file accompanying this file. This file is 10 | # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF 11 | # ANY KIND, either express or implied. See the License for the specific 12 | # language governing permissions and limitations under the License. 13 | 14 | module AWS 15 | module Core 16 | module Collection 17 | 18 | module Batchable 19 | 20 | include Collection 21 | 22 | def each_batch options = {}, &block 23 | 24 | raise NotImplementedError 25 | 26 | each_opts = options.dup 27 | 28 | limit = each_opts.delete(:limit) 29 | 30 | next_token, skip = each_opts.delete(:next_token) 31 | 32 | total = 0 # count of items yeileded across all batches 33 | 34 | begin 35 | 36 | batch = [] 37 | 38 | next_token = _each_item(next_token, each_opts.dup) do |item| 39 | total += 1 40 | batch << item 41 | 42 | if limit and total == limit 43 | yield(batch) 44 | end 45 | 46 | end 47 | 48 | end until next_token.nil? or (limit and limit = total) 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | options = options.dup 59 | 60 | limit = options.delete(:limit) 61 | batch_size = options.delete(:batch_size) 62 | options.delete(:next_token) if [nil, ''].include?(options[:next_token]) 63 | 64 | total = 0 # count of items yeileded across all batches 65 | 66 | _each_response(options, limit, batch_size) do |response| 67 | 68 | batch = [] 69 | each_item(response) do |item| 70 | batch << item 71 | if limit and (total += 1) == limit 72 | yield(batch) 73 | return 74 | end 75 | end 76 | 77 | yield(batch) 78 | 79 | batch.size 80 | 81 | end 82 | 83 | end 84 | 85 | # @note +limit+ is ignored because Batchable colections do not 86 | # accept a +:limit+ option at the service level. 87 | # @note +batch_size+ is ignored because Batchable collections do not 88 | # accept a +:batch_size+ option at the service level. 89 | protected 90 | def _each_response options, limit, batch_size, &block 91 | 92 | next_token = nil 93 | 94 | begin 95 | 96 | page_opts = {} 97 | page_opts[next_token_key] = next_token if next_token 98 | 99 | response = client.send(request_method, options.merge(page_opts)) 100 | 101 | yield(response) 102 | 103 | next_token = _next_token_for(response) 104 | 105 | end until next_token.nil? 106 | 107 | nil 108 | 109 | end 110 | 111 | # Override this method in collections that have an alternate method 112 | # for finding the next token. 113 | protected 114 | def _next_token_for response 115 | method = _next_token_key 116 | response.respond_to?(method) ? response.send(method) : nil 117 | end 118 | 119 | protected 120 | def _next_token_key 121 | :next_token 122 | end 123 | 124 | protected 125 | def _each_item next_token, options = {}, &block 126 | raise NotImplementedError 127 | end 128 | 129 | end 130 | 131 | end 132 | end 133 | end 134 | -------------------------------------------------------------------------------- /lib/aws/core/collection/limitable.rb: -------------------------------------------------------------------------------- 1 | # Copyright 2011 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). You 4 | # may not use this file except in compliance with the License. A copy of 5 | # the License is located at 6 | # 7 | # http://aws.amazon.com/apache2.0/ 8 | # 9 | # or in the "license" file accompanying this file. This file is 10 | # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF 11 | # ANY KIND, either express or implied. See the License for the specific 12 | # language governing permissions and limitations under the License. 13 | 14 | module AWS 15 | module Core 16 | module Collection 17 | 18 | # AWS::Core::Collection::Limitable is used by collections that 19 | # may truncate responses but that also accept a upper limit of 20 | # results to return in a single request. 21 | # 22 | # See {AWS::Core::Collection} for documentation on the available 23 | # methods. 24 | module Limitable 25 | 26 | include Model 27 | include Collection 28 | include Enumerable 29 | 30 | def each_batch options = {}, &block 31 | 32 | each_opts = options.dup 33 | limit = each_opts.delete(:limit) || _limit 34 | next_token = each_opts.delete(:next_token) 35 | batch_size = each_opts.delete(:batch_size) 36 | 37 | total = 0 # count of items yeileded across all batches 38 | 39 | begin 40 | 41 | max = nil 42 | if limit or batch_size 43 | max = [] 44 | max << (limit - total) if limit 45 | max << batch_size if batch_size 46 | max = max.min 47 | end 48 | 49 | batch = [] 50 | next_token = _each_item(next_token, max, each_opts.dup) do |item| 51 | 52 | total += 1 53 | batch << item 54 | 55 | end 56 | 57 | yield(batch) 58 | 59 | end until next_token.nil? or (limit and limit == total) 60 | 61 | next_token 62 | 63 | end 64 | 65 | protected 66 | def _each_item next_token, limit, options = {}, &block 67 | raise NotImplementedError 68 | end 69 | 70 | # This method should be overriden in collection classes 71 | # when there is another method to provide a "limit" besides 72 | # as an option to the enumerable methods. 73 | # 74 | # SimpleDB::ItemCollection (for example) allows setting the limit 75 | # in a method chain, e.g. 76 | # 77 | # collection.limit(10).each {|item| ... } 78 | # 79 | # These collection classes can simply override #_limit and return 80 | # their prefered limit. This is only called in the abscense of 81 | # the +:limit+ option. 82 | # 83 | # @private 84 | protected 85 | def _limit 86 | nil 87 | end 88 | 89 | end 90 | end 91 | end 92 | end 93 | -------------------------------------------------------------------------------- /lib/aws/core/collection/simple.rb: -------------------------------------------------------------------------------- 1 | # Copyright 2011 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). You 4 | # may not use this file except in compliance with the License. A copy of 5 | # the License is located at 6 | # 7 | # http://aws.amazon.com/apache2.0/ 8 | # 9 | # or in the "license" file accompanying this file. This file is 10 | # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF 11 | # ANY KIND, either express or implied. See the License for the specific 12 | # language governing permissions and limitations under the License. 13 | 14 | module AWS 15 | module Core 16 | module Collection 17 | 18 | # AWS::Core::Collection::Simple is used by collections that always 19 | # recieve every matching items in a single response. 20 | # 21 | # This means: 22 | # 23 | # * Paging methods are simulated 24 | # 25 | # * Next tokens are artificial (guessable numeric offsets) 26 | # 27 | # AWS services generally return all items only for requests with a 28 | # small maximum number of results. 29 | # 30 | # See {AWS::Core::Collection} for documentation on the available 31 | # collection methods. 32 | module Simple 33 | 34 | include Model 35 | include Enumerable 36 | include Collection 37 | 38 | # (see AWS::Core::Collection#each_batch) 39 | def each_batch options = {}, &block 40 | 41 | each_opts = options.dup 42 | limit = each_opts.delete(:limit) 43 | next_token = each_opts.delete(:next_token) 44 | offset = next_token ? next_token.to_i - 1 : 0 45 | total = 0 46 | 47 | nil_or_next_token = nil 48 | 49 | batch = [] 50 | _each_item(each_opts.dup) do |item| 51 | 52 | total += 1 53 | 54 | # skip until we reach our offset (derived from the "next token") 55 | next if total <= offset 56 | 57 | if limit 58 | 59 | if batch.size < limit 60 | batch << item 61 | else 62 | # allow _each_item to yield one more item than needed 63 | # so we can determine if we should return a "next token" 64 | nil_or_next_token = total 65 | break 66 | end 67 | 68 | else 69 | batch << item 70 | end 71 | 72 | end 73 | 74 | yield(batch) 75 | 76 | nil_or_next_token 77 | 78 | end 79 | 80 | protected 81 | def _each_item options = {}, &block 82 | raise NotImplementedError 83 | end 84 | 85 | end 86 | 87 | end 88 | end 89 | end 90 | -------------------------------------------------------------------------------- /lib/aws/core/configured_client_methods.rb: -------------------------------------------------------------------------------- 1 | # Copyright 2011 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). You 4 | # may not use this file except in compliance with the License. A copy of 5 | # the License is located at 6 | # 7 | # http://aws.amazon.com/apache2.0/ 8 | # 9 | # or in the "license" file accompanying this file. This file is 10 | # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF 11 | # ANY KIND, either express or implied. See the License for the specific 12 | # language governing permissions and limitations under the License. 13 | 14 | module AWS 15 | module Core 16 | 17 | # @private 18 | module ConfiguredClientMethods 19 | 20 | # @private 21 | module ClassMethods 22 | 23 | include ApiConfig 24 | 25 | def configure_client 26 | 27 | super 28 | 29 | unless self::XML.include?(ConfiguredXmlGrammars) 30 | self::XML.module_eval do 31 | include(ConfiguredXmlGrammars) 32 | define_configured_grammars 33 | end 34 | end 35 | 36 | unless self::Options.include?(ConfiguredOptionGrammars) 37 | self::Options.module_eval do 38 | include(ConfiguredOptionGrammars) 39 | define_configured_grammars 40 | end 41 | end 42 | 43 | api_config[:operations].each do |name, customizations| 44 | option_grammar = self::Options.operation_grammar(name) 45 | add_client_request_method(Inflection.ruby_name(name).to_sym, 46 | :xml_grammar => 47 | self::XML.operation_grammar(name)) do 48 | configure_request do |request, options| 49 | request.add_param("Action", name) 50 | option_grammar.request_params(options).each do |param| 51 | request.add_param(param) 52 | end 53 | end 54 | end 55 | end 56 | 57 | end 58 | 59 | def operation_xml_grammar(name) 60 | customized_name = "Customized#{name}" 61 | if self::XML.const_defined?(customized_name) 62 | self::XML.const_get(customized_name) 63 | else 64 | self::XML.const_get(name) 65 | end 66 | end 67 | 68 | end 69 | 70 | def self.included(mod) 71 | mod.extend(ClassMethods) 72 | end 73 | 74 | end 75 | end 76 | end 77 | -------------------------------------------------------------------------------- /lib/aws/core/configured_grammars.rb: -------------------------------------------------------------------------------- 1 | # Copyright 2011 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). You 4 | # may not use this file except in compliance with the License. A copy of 5 | # the License is located at 6 | # 7 | # http://aws.amazon.com/apache2.0/ 8 | # 9 | # or in the "license" file accompanying this file. This file is 10 | # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF 11 | # ANY KIND, either express or implied. See the License for the specific 12 | # language governing permissions and limitations under the License. 13 | 14 | module AWS 15 | module Core 16 | 17 | # @private 18 | module ConfiguredGrammars 19 | 20 | # @private 21 | module ClassMethods 22 | 23 | include ApiConfig 24 | 25 | def base_grammar 26 | raise NotImplementedError 27 | end 28 | 29 | def operation_grammar(name) 30 | customized_name = "Customized#{name}" 31 | if const_defined?(customized_name) 32 | const_get(customized_name) 33 | else 34 | const_get(name) 35 | end 36 | end 37 | 38 | def input_or_output 39 | raise NotImplementedError 40 | end 41 | 42 | def process_customizations(name, customizations) 43 | customizations 44 | end 45 | 46 | def define_configured_grammars 47 | api_config[:operations].each do |name, data| 48 | customizations = process_customizations(name, 49 | data[input_or_output]) 50 | const_set(name, 51 | base_grammar.customize(customizations)) 52 | # BaseResponse.customize([{ 53 | # "#{name}Result" => 54 | # [:ignore, *data[:output]] 55 | # }])) 56 | end 57 | end 58 | 59 | end 60 | 61 | end 62 | end 63 | end 64 | -------------------------------------------------------------------------------- /lib/aws/core/configured_option_grammars.rb: -------------------------------------------------------------------------------- 1 | # Copyright 2011 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). You 4 | # may not use this file except in compliance with the License. A copy of 5 | # the License is located at 6 | # 7 | # http://aws.amazon.com/apache2.0/ 8 | # 9 | # or in the "license" file accompanying this file. This file is 10 | # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF 11 | # ANY KIND, either express or implied. See the License for the specific 12 | # language governing permissions and limitations under the License. 13 | 14 | module AWS 15 | module Core 16 | 17 | # @private 18 | module ConfiguredOptionGrammars 19 | 20 | module ClassMethods 21 | 22 | include ConfiguredGrammars::ClassMethods 23 | 24 | def base_grammar 25 | if const_defined?(:BaseOptions) 26 | const_get(:BaseOptions) 27 | else 28 | OptionGrammar 29 | end 30 | end 31 | 32 | def input_or_output 33 | :input 34 | end 35 | 36 | end 37 | 38 | def self.included(mod) 39 | mod.extend(ClassMethods) 40 | end 41 | 42 | end 43 | end 44 | end 45 | -------------------------------------------------------------------------------- /lib/aws/core/configured_xml_grammars.rb: -------------------------------------------------------------------------------- 1 | # Copyright 2011 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). You 4 | # may not use this file except in compliance with the License. A copy of 5 | # the License is located at 6 | # 7 | # http://aws.amazon.com/apache2.0/ 8 | # 9 | # or in the "license" file accompanying this file. This file is 10 | # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF 11 | # ANY KIND, either express or implied. See the License for the specific 12 | # language governing permissions and limitations under the License. 13 | 14 | module AWS 15 | module Core 16 | 17 | # @private 18 | module ConfiguredXmlGrammars 19 | 20 | # @private 21 | module ClassMethods 22 | 23 | include ConfiguredGrammars::ClassMethods 24 | 25 | def base_grammar 26 | if const_defined?(:BaseResponse) 27 | const_get(:BaseResponse) 28 | else 29 | XmlGrammar 30 | end 31 | end 32 | 33 | def input_or_output 34 | :output 35 | end 36 | 37 | end 38 | 39 | def self.included(mod) 40 | mod.extend(ClassMethods) 41 | end 42 | 43 | end 44 | end 45 | end 46 | -------------------------------------------------------------------------------- /lib/aws/core/default_signer.rb: -------------------------------------------------------------------------------- 1 | # Copyright 2011 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). You 4 | # may not use this file except in compliance with the License. A copy of 5 | # the License is located at 6 | # 7 | # http://aws.amazon.com/apache2.0/ 8 | # 9 | # or in the "license" file accompanying this file. This file is 10 | # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF 11 | # ANY KIND, either express or implied. See the License for the specific 12 | # language governing permissions and limitations under the License. 13 | 14 | require 'base64' 15 | require 'openssl' 16 | 17 | module AWS 18 | module Core 19 | 20 | # Computes signatures using credentials that are stored in memory. 21 | class DefaultSigner 22 | 23 | # @return [String] The Access Key ID used to sign requests. 24 | attr_reader :access_key_id 25 | 26 | # @return [String] The Secret Access Key used to sign requests. 27 | attr_reader :secret_access_key 28 | 29 | # @return [String] The Session Token used to sign requests. 30 | attr_reader :session_token 31 | 32 | # @param [String] access_key_id The Access Key ID used to sign 33 | # requests. 34 | # 35 | # @param [String] secret_access_key The Secret Access Key used to 36 | # sign requests. 37 | # 38 | # @param [String] session_token The Session Token used to sign 39 | # requests. You can get credentials that include a session 40 | # token using the {STS} class. 41 | def initialize(access_key_id, secret_access_key, session_token = nil) 42 | 43 | @access_key_id = access_key_id 44 | @secret_access_key = secret_access_key 45 | @session_token = session_token 46 | 47 | raise "Missing credentials" unless access_key_id and secret_access_key 48 | 49 | end 50 | 51 | # Signs a string using the credentials stored in memory. 52 | # 53 | # @param [String] string_to_sign The string to sign. 54 | # 55 | # @param [String] digest_method The digest method to use when 56 | # computing the HMAC digest. 57 | def sign(string_to_sign, digest_method = 'sha256') 58 | Base64.encode64( 59 | OpenSSL::HMAC.digest( 60 | OpenSSL::Digest::Digest.new(digest_method), 61 | secret_access_key, 62 | string_to_sign)).strip 63 | end 64 | 65 | end 66 | end 67 | end 68 | -------------------------------------------------------------------------------- /lib/aws/core/http/curb_handler.rb: -------------------------------------------------------------------------------- 1 | # Copyright 2011 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). You 4 | # may not use this file except in compliance with the License. A copy of 5 | # the License is located at 6 | # 7 | # http://aws.amazon.com/apache2.0/ 8 | # 9 | # or in the "license" file accompanying this file. This file is 10 | # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF 11 | # ANY KIND, either express or implied. See the License for the specific 12 | # language governing permissions and limitations under the License. 13 | 14 | require 'thread' 15 | 16 | module AWS 17 | module Core 18 | module Http 19 | 20 | # @private 21 | class CurbHandler 22 | 23 | def initialize 24 | @q = [] 25 | @sem = Mutex.new 26 | @multi = Curl::Multi.new 27 | 28 | start_processor 29 | end 30 | 31 | def handle request, response 32 | raise "unsupport http reqest method: #{request.http_method}" unless 33 | ['GET', 'HEAD', 'PUT', 'POST', 'DELETE'].include? request.http_method 34 | @sem.synchronize do 35 | @q << [request, response, Thread.current] 36 | @processor.wakeup 37 | end 38 | Thread.stop 39 | nil 40 | end 41 | 42 | # fills the Curl::Multi handle with the given array of queue 43 | # items, calling make_easy_handle on each one first 44 | private 45 | def fill_multi(items) 46 | items.each do |item| 47 | c = make_easy_handle(*item) 48 | @multi.add(c) 49 | end 50 | end 51 | 52 | # starts a background thread that waits for new items and 53 | # sends them through the Curl::Multi handle 54 | private 55 | def start_processor 56 | @processor = Thread.new do 57 | loop do 58 | items = nil 59 | @sem.synchronize do 60 | items = @q.slice!(0..-1) 61 | end 62 | unless items.empty? 63 | fill_multi(items) 64 | @multi.perform do 65 | # curl is idle, so process more items if we can get them 66 | # without blocking 67 | if !@q.empty? && @sem.try_lock 68 | begin 69 | fill_multi(@q.slice!(0..-1)) 70 | ensure 71 | @sem.unlock 72 | end 73 | end 74 | end 75 | end 76 | 77 | # wait for a new item to show up before continuing 78 | Thread.stop if @q.empty? 79 | end 80 | end 81 | end 82 | 83 | private 84 | def make_easy_handle request, response, thread = nil 85 | 86 | url = request.use_ssl? ? 87 | "https://#{request.host}:443#{request.uri}" : 88 | "http://#{request.host}#{request.uri}" 89 | 90 | curl = Curl::Easy.new(url) 91 | curl.headers = request.headers 92 | 93 | if proxy = request.proxy_uri 94 | curl.proxy_url = proxy.to_s 95 | curl.proxy_port = proxy.port 96 | end 97 | 98 | curl.on_header {|header_data| 99 | name, value = header_data.strip.split(/:\s+/, 2) 100 | response.headers[name] = value 101 | header_data.length 102 | } 103 | 104 | case request.http_method 105 | when 'GET' 106 | # .... 107 | when 'HEAD' 108 | curl.head = true 109 | when 'PUT' 110 | curl.put_data = request.body 111 | when 'POST' 112 | curl.post_body = request.body 113 | when 'DELETE' 114 | curl.delete = true 115 | end 116 | 117 | curl.on_complete do 118 | response.body = curl.body_str 119 | response.status = curl.response_code 120 | thread.run if thread 121 | end 122 | 123 | curl 124 | end 125 | 126 | end 127 | end 128 | end 129 | end 130 | -------------------------------------------------------------------------------- /lib/aws/core/http/handler.rb: -------------------------------------------------------------------------------- 1 | # Copyright 2011 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). You 4 | # may not use this file except in compliance with the License. A copy of 5 | # the License is located at 6 | # 7 | # http://aws.amazon.com/apache2.0/ 8 | # 9 | # or in the "license" file accompanying this file. This file is 10 | # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF 11 | # ANY KIND, either express or implied. See the License for the specific 12 | # language governing permissions and limitations under the License. 13 | 14 | module AWS 15 | module Core 16 | module Http 17 | 18 | # @private 19 | class Handler 20 | 21 | attr_reader :base 22 | 23 | def initialize(base, &block) 24 | @base = base 25 | if base.respond_to?(:handle) 26 | 27 | unless block.arity == 2 28 | raise ArgumentError, 'passed block must accept 2 arguments' 29 | end 30 | MetaUtils.extend_method(self, :handle, &block) 31 | 32 | elsif base.respond_to?(:handle_async) 33 | 34 | unless block.arity == 3 35 | raise ArgumentError, 'passed block must accept 3 arguments' 36 | end 37 | 38 | MetaUtils.extend_method(self, :handle_async) do |req, resp, handle| 39 | @base.handle_async(req, resp, handle) 40 | end 41 | MetaUtils.extend(self) do 42 | define_method(:handle) do |req, resp| 43 | raise "attempted to call #handle on an async handler" 44 | end 45 | define_method(:handle_async, &block) 46 | end 47 | 48 | else 49 | raise ArgumentError, 'base must respond to #handle or #handle_async' 50 | end 51 | end 52 | 53 | def handle(request, http_response) 54 | @base.handle(request, http_response) 55 | end 56 | 57 | def handle_async(request, http_response, handle) 58 | Thread.new do 59 | begin 60 | self.handle(request, http_response) 61 | rescue => e 62 | handle.signal_failure 63 | else 64 | handle.signal_success 65 | end 66 | end 67 | end 68 | 69 | def sleep_with_callback seconds, &block 70 | Kernel.sleep(seconds) 71 | yield 72 | end 73 | 74 | end 75 | end 76 | end 77 | end 78 | -------------------------------------------------------------------------------- /lib/aws/core/http/httparty_handler.rb: -------------------------------------------------------------------------------- 1 | # Copyright 2011 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). You 4 | # may not use this file except in compliance with the License. A copy of 5 | # the License is located at 6 | # 7 | # http://aws.amazon.com/apache2.0/ 8 | # 9 | # or in the "license" file accompanying this file. This file is 10 | # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF 11 | # ANY KIND, either express or implied. See the License for the specific 12 | # language governing permissions and limitations under the License. 13 | 14 | require 'httparty' 15 | 16 | module AWS 17 | module Core 18 | module Http 19 | 20 | # Makes HTTP requests using HTTParty. This is the default 21 | # handler, so you don't need to do anything special to configure 22 | # it. However, you can directly instantiate this class in order 23 | # to send extra options to HTTParty, for example to enable an HTTP 24 | # proxy: 25 | # 26 | # AWS.config( 27 | # :http_handler => AWS::Http::HTTPartyHandler.new( 28 | # :http_proxyaddr => "http://myproxy.com", 29 | # :http_proxyport => 80 30 | # ) 31 | # ) 32 | # 33 | class HTTPartyHandler 34 | 35 | # @return [Hash] The default options to send to HTTParty on each 36 | # request. 37 | attr_reader :default_request_options 38 | 39 | # Constructs a new HTTP handler using HTTParty. 40 | # 41 | # @param [Hash] options Default options to send to HTTParty on 42 | # each request. These options will be sent to +get+, +post+, 43 | # +head+, +put+, or +delete+ when a request is made. Note 44 | # that +:body+, +:headers+, +:parser+, and +:ssl_ca_file+ are 45 | # ignored. If you need to set the CA file, you should use the 46 | # +:ssl_ca_file+ option to {AWS.config} or 47 | # {AWS::Configuration} instead. 48 | def initialize options = {} 49 | @default_request_options = options 50 | end 51 | 52 | include HTTParty 53 | 54 | class NoOpParser < HTTParty::Parser 55 | SupportedFormats = {} 56 | end 57 | 58 | def handle(request, response) 59 | 60 | opts = default_request_options.merge({ 61 | :body => request.body, 62 | :parser => NoOpParser 63 | }) 64 | 65 | if request.proxy_uri 66 | opts[:http_proxyaddr] = request.proxy_uri.host 67 | opts[:http_proxyport] = request.proxy_uri.port 68 | end 69 | 70 | if request.use_ssl? 71 | url = "https://#{request.host}:443#{request.uri}" 72 | opts[:ssl_ca_file] = request.ssl_ca_file if 73 | request.ssl_verify_peer? 74 | else 75 | url = "http://#{request.host}#{request.uri}" 76 | end 77 | 78 | # get, post, put, delete, head 79 | method = request.http_method.downcase 80 | 81 | # Net::HTTP adds this header for us when the body is 82 | # provided, but it messes up signing 83 | headers = { 'content-type' => '' } 84 | 85 | # headers must have string values (net http calls .strip on them) 86 | request.headers.each_pair do |key,value| 87 | headers[key] = value.to_s 88 | end 89 | 90 | opts[:headers] = headers 91 | 92 | begin 93 | http_response = self.class.send(method, url, opts) 94 | rescue Timeout::Error, Errno::ETIMEDOUT => e 95 | response.timeout = true 96 | else 97 | response.body = http_response.body 98 | response.status = http_response.code.to_i 99 | response.headers = http_response.to_hash 100 | end 101 | end 102 | end 103 | end 104 | end 105 | 106 | # We move this from AWS::Http to AWS::Core::Http, but we want the 107 | # previous default handler to remain accessible from its old namesapce 108 | # @private 109 | module Http 110 | class HTTPartyHandler < Core::Http::HTTPartyHandler; end 111 | end 112 | 113 | end 114 | -------------------------------------------------------------------------------- /lib/aws/core/http/net_http_handler.rb: -------------------------------------------------------------------------------- 1 | # Copyright 2011 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). You 4 | # may not use this file except in compliance with the License. A copy of 5 | # the License is located at 6 | # 7 | # http://aws.amazon.com/apache2.0/ 8 | # 9 | # or in the "license" file accompanying this file. This file is 10 | # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF 11 | # ANY KIND, either express or implied. See the License for the specific 12 | # language governing permissions and limitations under the License. 13 | 14 | require 'net/http' 15 | require 'net/https' 16 | require 'openssl' 17 | 18 | module AWS 19 | module Core 20 | module Http 21 | 22 | # The default http request handler for the aws-sdk gem. It is based 23 | # on Net::Http. 24 | class NetHttpHandler 25 | 26 | def handle request, response 27 | http_session_for(request) do |http| 28 | begin 29 | http_resp = http.request(build_request(request)) 30 | response.body = http_resp.body 31 | response.status = http_resp.code.to_i 32 | response.headers = http_resp.to_hash 33 | rescue Timeout::Error, Errno::ETIMEDOUT => e 34 | response.timeout = true 35 | end 36 | end 37 | end 38 | 39 | # @private 40 | protected 41 | def http_session_for request, &block 42 | begin 43 | http = build_http(request) 44 | http.start 45 | yield(http) 46 | ensure 47 | http.finish if http.started? 48 | end 49 | end 50 | 51 | # @private 52 | protected 53 | def build_http request 54 | 55 | http_args = [] 56 | http_args << request.host 57 | http_args << (request.use_ssl? ? 443 : 80) 58 | 59 | ## add proxy arguments 60 | 61 | if proxy = request.proxy_uri 62 | proxy_addr = proxy.respond_to?(:request_uri) ? 63 | "#{proxy.host}#{proxy.request_uri}" : proxy.host 64 | http_args << proxy_addr 65 | http_args << proxy.port 66 | http_args << proxy.user 67 | http_args << proxy.password 68 | end 69 | 70 | # ruby 1.8.7 does not accept a hash of options at the end of 71 | # Net::HTTP.new like 1.9 does, therefore we have to set ssl 72 | # options on the returned http object 73 | http = Net::HTTP.new(*http_args) 74 | 75 | ## configure ssl 76 | 77 | if request.use_ssl? 78 | http.use_ssl = true 79 | if request.ssl_verify_peer? 80 | http.verify_mode = OpenSSL::SSL::VERIFY_PEER 81 | http.ca_file = request.ssl_ca_file 82 | else 83 | http.verify_mode = OpenSSL::SSL::VERIFY_NONE 84 | end 85 | else 86 | http.use_ssl = false 87 | end 88 | 89 | http 90 | 91 | end 92 | 93 | # @private 94 | protected 95 | def build_request request 96 | 97 | # Net::HTTP adds a content-type header automatically unless its set 98 | # and this messes with request signature signing. Also, it expects 99 | # all header values to be strings (it call strip on them). 100 | headers = { 'content-type' => '' } 101 | request.headers.each_pair do |key,value| 102 | headers[key] = value.to_s 103 | end 104 | 105 | req_class = case request.http_method 106 | when 'GET' then Net::HTTP::Get 107 | when 'PUT' then Net::HTTP::Put 108 | when 'POST' then Net::HTTP::Post 109 | when 'HEAD' then Net::HTTP::Head 110 | when 'DELETE' then Net::HTTP::Delete 111 | else raise "unsupported http method: #{request.http_method}" 112 | end 113 | 114 | net_http_req = req_class.new(request.uri, headers) 115 | net_http_req.body = request.body 116 | net_http_req 117 | 118 | end 119 | 120 | end 121 | 122 | end 123 | end 124 | end 125 | -------------------------------------------------------------------------------- /lib/aws/core/http/request.rb: -------------------------------------------------------------------------------- 1 | # Copyright 2011 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). You 4 | # may not use this file except in compliance with the License. A copy of 5 | # the License is located at 6 | # 7 | # http://aws.amazon.com/apache2.0/ 8 | # 9 | # or in the "license" file accompanying this file. This file is 10 | # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF 11 | # ANY KIND, either express or implied. See the License for the specific 12 | # language governing permissions and limitations under the License. 13 | 14 | module AWS 15 | module Core 16 | module Http 17 | 18 | # Base class for all service reqeusts. 19 | class Request 20 | 21 | # Returns a new empty http request object. 22 | def initialize 23 | @host = nil 24 | @http_method = 'POST' 25 | @path = '/' 26 | @headers = CaseInsensitiveHash.new 27 | @params = [] 28 | @use_ssl = true 29 | end 30 | 31 | # @return [String] hostname of the request 32 | attr_accessor :host 33 | 34 | # @return [CaseInsensitiveHash] request headers 35 | attr_reader :headers 36 | 37 | # @return [Array] An array of request params, each param responds to 38 | # #name and #value. 39 | attr_reader :params 40 | 41 | # @return [String] GET, PUT POST, HEAD or DELETE, defaults to POST 42 | attr_accessor :http_method 43 | 44 | # @return [String] path of the request URI, defaults to / 45 | attr_reader :path 46 | 47 | # @return [String] the AWS access key ID used to authorize the 48 | # request 49 | attr_accessor :access_key_id 50 | 51 | # @return [nil, URI] The URI to the proxy server requests are 52 | # sent through if configured. Returns nil if there is no proxy. 53 | attr_accessor :proxy_uri 54 | 55 | # @param [Boolean] ssl If the request should be sent over ssl or not. 56 | def use_ssl= use_ssl 57 | @use_ssl = use_ssl 58 | end 59 | 60 | # @return [Boolean] If this request should be sent over ssl or not. 61 | def use_ssl? 62 | @use_ssl 63 | end 64 | 65 | # @param [Boolean] verify_peer If the client should verify the 66 | # peer certificate or not. 67 | def ssl_verify_peer=(verify_peer) 68 | @ssl_verify_peer = verify_peer 69 | end 70 | 71 | # @return [Boolean] If the client should verify the peer 72 | # certificate or not. 73 | def ssl_verify_peer? 74 | @ssl_verify_peer 75 | end 76 | 77 | # @param [String] ca_file Path to a bundle of CA certs in PEM 78 | # format; the HTTP handler should use this to verify all HTTPS 79 | # requests if {#ssl_verify_peer?} is true. 80 | def ssl_ca_file=(ca_file) 81 | @ssl_ca_file = ca_file 82 | end 83 | 84 | # @return [String] Path to a bundle of CA certs in PEM format; 85 | # the HTTP handler should use this to verify all HTTPS 86 | # requests if {#ssl_verify_peer?} is true. 87 | def ssl_ca_file 88 | @ssl_ca_file 89 | end 90 | 91 | # Adds a request param. 92 | # 93 | # @overload add_param(param_name, param_value = nil) 94 | # Add a param (name/value) 95 | # @param [String] param_name 96 | # @param [String] param_value Leave blank for sub resources 97 | # 98 | # @overload add_param(param_obj) 99 | # Add a param (object) 100 | # @param [Param] param_obj 101 | # 102 | def add_param name_or_param, value = nil 103 | if name_or_param.kind_of?(Param) 104 | @params << name_or_param 105 | else 106 | @params << Param.new(name_or_param, value) 107 | end 108 | end 109 | 110 | # @private 111 | def get_param param_name 112 | @params.detect{|p| p.name == param_name } || 113 | raise("undefined param #{param_name}") 114 | end 115 | 116 | # @return [String] the request uri 117 | def uri 118 | querystring ? "#{path}?#{querystring}" : path 119 | end 120 | 121 | # @return [String] Returns the request params url encoded, or nil if 122 | # this request has no params. 123 | def url_encoded_params 124 | if @params.empty? 125 | nil 126 | else 127 | @params.sort.collect{|p| p.encoded }.join('&') 128 | end 129 | end 130 | 131 | # @return [String, nil] Returns the requesty querystring. 132 | def querystring 133 | nil 134 | end 135 | 136 | # @return [String, nil] Returns the request body. 137 | def body 138 | url_encoded_params 139 | end 140 | 141 | # @private 142 | class CaseInsensitiveHash < Hash 143 | 144 | def []= key, value 145 | super(key.to_s.downcase, value) 146 | end 147 | 148 | def [] key 149 | super(key.to_s.downcase) 150 | end 151 | 152 | def has_key?(key) 153 | super(key.to_s.downcase) 154 | end 155 | alias_method :key?, :has_key? 156 | alias_method :include?, :has_key? 157 | alias_method :member?, :has_key? 158 | 159 | end 160 | 161 | # Represents a single request paramater. Some services accept this 162 | # in a form encoded body string, others as query parameters. 163 | # It is up to each service's Request class to determine how to 164 | # consume these params. 165 | # @private 166 | class Param 167 | 168 | include UriEscape 169 | 170 | attr_accessor :name, :value 171 | 172 | def initialize name, value = nil 173 | @name = name 174 | @value = value 175 | end 176 | 177 | def <=> other 178 | @name <=> other.name 179 | end 180 | 181 | def to_s 182 | if value 183 | "#{name}=#{value}" 184 | else 185 | name 186 | end 187 | end 188 | 189 | def ==(other) 190 | other.kind_of?(Param) && 191 | to_s == other.to_s 192 | end 193 | 194 | def encoded 195 | if value 196 | "#{escape(name)}=#{escape(value)}" 197 | else 198 | escape(name) 199 | end 200 | end 201 | 202 | end 203 | 204 | end 205 | end 206 | end 207 | end 208 | -------------------------------------------------------------------------------- /lib/aws/core/http/response.rb: -------------------------------------------------------------------------------- 1 | # Copyright 2011 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). You 4 | # may not use this file except in compliance with the License. A copy of 5 | # the License is located at 6 | # 7 | # http://aws.amazon.com/apache2.0/ 8 | # 9 | # or in the "license" file accompanying this file. This file is 10 | # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF 11 | # ANY KIND, either express or implied. See the License for the specific 12 | # language governing permissions and limitations under the License. 13 | 14 | module AWS 15 | module Core 16 | module Http 17 | 18 | # Represents the http response from a service request. 19 | # 20 | # Responses have: 21 | # 22 | # * status (200, 404, 500, etc) 23 | # * headers (hash of response headers) 24 | # * body (the response body) 25 | class Response 26 | 27 | # @return [Integer] (200) response http status code 28 | attr_accessor :status 29 | 30 | # @return [Hash] ({}) response http headers 31 | attr_accessor :headers 32 | 33 | # @return [String] ('') response http body 34 | attr_accessor :body 35 | 36 | # @return [String] (false) set to true if the client gives up 37 | # before getting a response from the service. 38 | attr_accessor :timeout 39 | alias_method :timeout?, :timeout 40 | 41 | # @param [Hash] options 42 | # @option options [Integer] :status (200) HTTP status code 43 | # @option options [Hash] :headers ({}) HTTP response headers 44 | # @option options [String] :body ('') HTTP response body 45 | def initialize options = {}, &block 46 | @status = options[:status] || 200 47 | @headers = options[:headers] || {} 48 | @body = options[:body] || '' 49 | yield(self) if block_given? 50 | self 51 | end 52 | 53 | # Returns the header value with the given name. 54 | # 55 | # The value is matched case-insensitively so if the headers hash 56 | # contains a key like 'Date' and you request the value for 57 | # 'date' the 'Date' value will be returned. 58 | # 59 | # @param [String,Symbol] name The name of the header to fetch a value for. 60 | # @return [String,nil] The value of the given header 61 | def header name 62 | headers.each_pair do |header_name, header_value| 63 | if header_name.downcase == name.to_s.downcase 64 | return header_value.is_a?(Array) ? header_value.first : header_value 65 | end 66 | end 67 | nil 68 | end 69 | 70 | end 71 | end 72 | end 73 | end 74 | -------------------------------------------------------------------------------- /lib/aws/core/ignore_result_element.rb: -------------------------------------------------------------------------------- 1 | # Copyright 2011 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). You 4 | # may not use this file except in compliance with the License. A copy of 5 | # the License is located at 6 | # 7 | # http://aws.amazon.com/apache2.0/ 8 | # 9 | # or in the "license" file accompanying this file. This file is 10 | # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF 11 | # ANY KIND, either express or implied. See the License for the specific 12 | # language governing permissions and limitations under the License. 13 | 14 | module AWS 15 | module Core 16 | 17 | # @private 18 | # 19 | # Mixin for XML grammar modules to ignore the outer 20 | # "OperationNameResult" element that wraps the response fields in 21 | # Coral-like service responses. 22 | module IgnoreResultElement 23 | 24 | # @private 25 | # 26 | # Invoked by 27 | # ConfiguredXmlGrammars::ClassMethods#define_configured_grammars 28 | def process_customizations name, customizations 29 | [{ "#{name}Result" => [:ignore, *customizations] }] 30 | end 31 | 32 | end 33 | end 34 | end 35 | -------------------------------------------------------------------------------- /lib/aws/core/indifferent_hash.rb: -------------------------------------------------------------------------------- 1 | # Copyright 2011 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). You 4 | # may not use this file except in compliance with the License. A copy of 5 | # the License is located at 6 | # 7 | # http://aws.amazon.com/apache2.0/ 8 | # 9 | # or in the "license" file accompanying this file. This file is 10 | # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF 11 | # ANY KIND, either express or implied. See the License for the specific 12 | # language governing permissions and limitations under the License. 13 | 14 | module AWS 15 | module Core 16 | 17 | # A utility class to provide indifferent access to hash data. 18 | # 19 | # Inspired by ActiveSupport's HashWithIndifferentAccess, this class 20 | # has a few notable differences: 21 | # 22 | # * ALL keys are converted to strings (via #to_s) 23 | # * It does not deep merge/convert hashes indifferently, good for fla 24 | # * It will not perserve default value behaviours 25 | # 26 | # These features were omitted because our primary use for this class is to 27 | # wrap a 1-level hash as a return value, but we want the user to access 28 | # the values with string or symbol keys. 29 | # 30 | # @private 31 | class IndifferentHash < Hash 32 | 33 | def initialize *args 34 | if args.first.is_a?(Hash) 35 | super() 36 | merge!(*args) 37 | else 38 | super(*args) 39 | end 40 | end 41 | 42 | alias_method :_getter, :[] 43 | alias_method :_setter, :[]= 44 | 45 | def []=(key, value) 46 | _setter(_convert_key(key), value) 47 | end 48 | alias_method :store, :[]= 49 | 50 | def [] key 51 | _getter(_convert_key(key)) 52 | end 53 | 54 | def merge! hash 55 | hash.each_pair do |key,value| 56 | self[key] = value 57 | end 58 | self 59 | end 60 | alias_method :update, :merge! 61 | 62 | def merge hash 63 | self.dup.merge!(hash) 64 | end 65 | 66 | def has_key? key 67 | super(_convert_key(key)) 68 | end 69 | alias_method :key?, :has_key? 70 | alias_method :member?, :has_key? 71 | alias_method :include?, :has_key? 72 | 73 | def fetch key, *extras, &block 74 | super(_convert_key(key), *extras, &block) 75 | end 76 | 77 | def delete key 78 | super(_convert_key(key)) 79 | end 80 | 81 | private 82 | def _convert_key key 83 | key.is_a?(String) ? key : key.to_s 84 | end 85 | 86 | end 87 | end 88 | end 89 | -------------------------------------------------------------------------------- /lib/aws/core/inflection.rb: -------------------------------------------------------------------------------- 1 | # Copyright 2011 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). You 4 | # may not use this file except in compliance with the License. A copy of 5 | # the License is located at 6 | # 7 | # http://aws.amazon.com/apache2.0/ 8 | # 9 | # or in the "license" file accompanying this file. This file is 10 | # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF 11 | # ANY KIND, either express or implied. See the License for the specific 12 | # language governing permissions and limitations under the License. 13 | 14 | module AWS 15 | module Core 16 | 17 | # @private 18 | module Inflection 19 | 20 | def ruby_name(aws_name) 21 | 22 | #aws_name.sub(/^.*:/, ''). 23 | # gsub(/[A-Z]+[a-z]+/){|str| "_#{str.downcase}_" }. 24 | # gsub(/(^_|_$)/, ''). 25 | # gsub(/__/, '_'). 26 | # downcase 27 | 28 | return 'etag' if aws_name == 'ETag' 29 | 30 | aws_name. 31 | sub(/^.*:/, ''). # strip namespace 32 | gsub(/([A-Z0-9]+)([A-Z][a-z])/, '\1_\2'). # split acronyms from words 33 | scan(/[a-z]+|\d+|[A-Z0-9]+[a-z]*/). # split remaining words 34 | join('_').downcase # join parts _ and downcase 35 | 36 | end 37 | module_function :ruby_name 38 | 39 | def class_name(name) 40 | name.sub(/^(.)/) { |m| m.upcase }. 41 | gsub(/[-_]([a-z])/i) { |m| m[1,1].upcase } 42 | end 43 | module_function :class_name 44 | 45 | end 46 | end 47 | end 48 | -------------------------------------------------------------------------------- /lib/aws/core/lazy_error_classes.rb: -------------------------------------------------------------------------------- 1 | # Copyright 2011 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). You 4 | # may not use this file except in compliance with the License. A copy of 5 | # the License is located at 6 | # 7 | # http://aws.amazon.com/apache2.0/ 8 | # 9 | # or in the "license" file accompanying this file. This file is 10 | # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF 11 | # ANY KIND, either express or implied. See the License for the specific 12 | # language governing permissions and limitations under the License. 13 | 14 | module AWS 15 | module Core 16 | 17 | # @private 18 | module LazyErrorClasses 19 | 20 | # @private 21 | module ClassMethods 22 | 23 | def const_missing(name) 24 | base_error_grammar = self::BASE_ERROR_GRAMMAR 25 | const_missing_mutex.synchronize do 26 | return if const_defined?(name) 27 | const_set(name, 28 | Class.new(self::Base) do 29 | include Errors::ModeledError 30 | 31 | # so that MyService::Errors::Foo::Bar will work 32 | const_set(:BASE_ERROR_GRAMMAR, base_error_grammar) 33 | include LazyErrorClasses 34 | end) 35 | end 36 | end 37 | 38 | def error_class(code) 39 | module_eval(code.gsub(".","::")) 40 | end 41 | 42 | def included(mod) 43 | raise NotImplementedError.new("#{self} lazy-generates error classes; "+ 44 | "therefore it is not suitable for "+ 45 | "inclusion in other modules") 46 | end 47 | 48 | end 49 | 50 | def self.included(mod) 51 | unless mod.const_defined?(:BASE_ERROR_GRAMMAR) 52 | mod.const_set(:BASE_ERROR_GRAMMAR, XmlGrammar) 53 | end 54 | mutex = Mutex.new 55 | MetaUtils.extend_method(mod, :const_missing_mutex) { mutex } 56 | mod.send(:include, Errors) 57 | mod.extend(ClassMethods) 58 | end 59 | 60 | end 61 | end 62 | end 63 | -------------------------------------------------------------------------------- /lib/aws/core/meta_utils.rb: -------------------------------------------------------------------------------- 1 | # Copyright 2011 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). You 4 | # may not use this file except in compliance with the License. A copy of 5 | # the License is located at 6 | # 7 | # http://aws.amazon.com/apache2.0/ 8 | # 9 | # or in the "license" file accompanying this file. This file is 10 | # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF 11 | # ANY KIND, either express or implied. See the License for the specific 12 | # language governing permissions and limitations under the License. 13 | 14 | module AWS 15 | module Core 16 | 17 | # @private 18 | module MetaUtils 19 | 20 | def extend_method(object, name, &block) 21 | object.extend( 22 | Module.new do 23 | define_method(name, &block) 24 | end 25 | ) 26 | end 27 | module_function :extend_method 28 | 29 | def class_extend_method(klass, name, &block) 30 | klass.send(:include, 31 | Module.new do 32 | define_method(name, &block) 33 | end 34 | ) 35 | end 36 | module_function :class_extend_method 37 | 38 | def extend(object, &block) 39 | object.extend(Module.new(&block)) 40 | end 41 | module_function :extend 42 | 43 | end 44 | end 45 | end 46 | -------------------------------------------------------------------------------- /lib/aws/core/model.rb: -------------------------------------------------------------------------------- 1 | # Copyright 2011 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). You 4 | # may not use this file except in compliance with the License. A copy of 5 | # the License is located at 6 | # 7 | # http://aws.amazon.com/apache2.0/ 8 | # 9 | # or in the "license" file accompanying this file. This file is 10 | # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF 11 | # ANY KIND, either express or implied. See the License for the specific 12 | # language governing permissions and limitations under the License. 13 | 14 | module AWS 15 | module Core 16 | 17 | # @private 18 | module Model 19 | 20 | # @private 21 | def initialize(*args) 22 | options = args.last.kind_of?(Hash) ? args.last : {} 23 | @config = case 24 | when options[:config] then options[:config] 25 | when args.first.respond_to?(:config) then args.first.config 26 | else AWS.config 27 | end 28 | end 29 | 30 | # @return [Configuration] Returns the configuration for this object. 31 | attr_reader :config 32 | 33 | # Each class including this module has its own client class. 34 | # Generally it is the service namespace suffixed by client: 35 | # 36 | # * s3_client 37 | # * simple_db_client 38 | # 39 | # @return Retruns the proper client class for the given model. 40 | def client 41 | @config.send("#{config_prefix}_client") 42 | end 43 | 44 | # @return [String] The short name of the service as used in coniguration. 45 | # (e.g. SimpleDB::Client.config_prefix #=> 'simple_db') 46 | def config_prefix 47 | Inflection.ruby_name(self.class.to_s.split(/::/)[1]) 48 | end 49 | 50 | # @return [String] A sensible default inspect string. 51 | def inspect 52 | "<#{self.class}>" 53 | end 54 | 55 | end 56 | end 57 | end 58 | -------------------------------------------------------------------------------- /lib/aws/core/naming.rb: -------------------------------------------------------------------------------- 1 | # Copyright 2011 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). You 4 | # may not use this file except in compliance with the License. A copy of 5 | # the License is located at 6 | # 7 | # http://aws.amazon.com/apache2.0/ 8 | # 9 | # or in the "license" file accompanying this file. This file is 10 | # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF 11 | # ANY KIND, either express or implied. See the License for the specific 12 | # language governing permissions and limitations under the License. 13 | 14 | module AWS 15 | module Core 16 | 17 | # @private 18 | module Naming 19 | 20 | def service_name 21 | self.name.split(/::/)[1] 22 | end 23 | 24 | def service_ruby_name 25 | @service_ruby_name ||= Inflection.ruby_name(service_name) 26 | end 27 | 28 | end 29 | end 30 | end 31 | -------------------------------------------------------------------------------- /lib/aws/core/page_result.rb: -------------------------------------------------------------------------------- 1 | # Copyright 2011 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). You 4 | # may not use this file except in compliance with the License. A copy of 5 | # the License is located at 6 | # 7 | # http://aws.amazon.com/apache2.0/ 8 | # 9 | # or in the "license" file accompanying this file. This file is 10 | # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF 11 | # ANY KIND, either express or implied. See the License for the specific 12 | # language governing permissions and limitations under the License. 13 | 14 | module AWS 15 | module Core 16 | 17 | class PageResult < Array 18 | 19 | # @return [Collection] Returns the collection that was used to 20 | # populated this page of results. 21 | attr_reader :collection 22 | 23 | # @return [Integer] Returns the maximum number of results per page. 24 | # The final page in a collection may return fewer than +:per_page+ 25 | # items (e.g. +:per_page+ is 10 and there are only 7 items). 26 | attr_reader :per_page 27 | 28 | # @return [String] An opaque token that can be passed the #page method 29 | # of the collection that returned this page of results. This next 30 | # token behaves as a pseudo offset. If +next_token+ is +nil+ then 31 | # there are no more results for the collection. 32 | attr_reader :next_token 33 | 34 | # @param [Collection] collection The collection that was used to 35 | # request this page of results. The collection should respond to 36 | # #page and accept a :next_token option. 37 | # 38 | # @param [Array] items An array of result items that represent a 39 | # page of results. 40 | # 41 | # @param [Integer] per_page The number of requested items for this 42 | # page of results. If the count of items is smaller than +per_page+ 43 | # then this is the last page of results. 44 | # 45 | # @param [String] next_token (nil) A token that can be passed to the 46 | # 47 | def initialize collection, items, per_page, next_token 48 | @collection = collection 49 | @per_page = per_page 50 | @next_token = next_token 51 | super(items) 52 | end 53 | 54 | def next_page 55 | if last_page? 56 | raise 'unable to get the next page, already at the last page' 57 | end 58 | collection.page(:per_page => per_page, :next_token => next_token) 59 | end 60 | 61 | # @return [Boolean] Returns true if this is the last page of results. 62 | def last_page? 63 | next_token.nil? 64 | end 65 | 66 | # @return [Boolean] Returns true if there are more pages of results. 67 | def more? 68 | !!next_token 69 | end 70 | 71 | end 72 | end 73 | end 74 | -------------------------------------------------------------------------------- /lib/aws/core/resource_cache.rb: -------------------------------------------------------------------------------- 1 | # Copyright 2011 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). You 4 | # may not use this file except in compliance with the License. A copy of 5 | # the License is located at 6 | # 7 | # http://aws.amazon.com/apache2.0/ 8 | # 9 | # or in the "license" file accompanying this file. This file is 10 | # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF 11 | # ANY KIND, either express or implied. See the License for the specific 12 | # language governing permissions and limitations under the License. 13 | 14 | module AWS 15 | module Core 16 | 17 | # @private 18 | class ResourceCache 19 | 20 | def initialize 21 | @cache = {} 22 | end 23 | 24 | def store(key, attributes) 25 | (@cache[key] ||= {}).merge!(attributes) 26 | end 27 | 28 | def cached?(key, attribute) 29 | attributes = @cache[key] and attributes.has_key?(attribute) 30 | end 31 | 32 | def get(key, attribute) 33 | raise "No cached value for attribute :#{attribute} of #{key}" unless 34 | cached?(key, attribute) 35 | @cache[key][attribute] 36 | end 37 | 38 | end 39 | end 40 | end 41 | -------------------------------------------------------------------------------- /lib/aws/core/response.rb: -------------------------------------------------------------------------------- 1 | # Copyright 2011 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). You 4 | # may not use this file except in compliance with the License. A copy of 5 | # the License is located at 6 | # 7 | # http://aws.amazon.com/apache2.0/ 8 | # 9 | # or in the "license" file accompanying this file. This file is 10 | # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF 11 | # ANY KIND, either express or implied. See the License for the specific 12 | # language governing permissions and limitations under the License. 13 | 14 | module AWS 15 | module Core 16 | 17 | # @private 18 | class Response 19 | 20 | include AsyncHandle 21 | 22 | # @return [AWS::Error] Returns nil unless the request failed. 23 | # Normally this will be nil unless you are using the Asynchronous 24 | # interface. 25 | attr_accessor :error 26 | 27 | # @return [Hash] The hash of options passed to the low level request 28 | # method that generated this response. 29 | attr_accessor :request_options 30 | 31 | # @return [Symbol] The low-level request method that generated 32 | # this response 33 | attr_accessor :request_type 34 | 35 | # @return [Http::Request] the HTTP request object 36 | attr_accessor :http_request 37 | 38 | # @return [Http::Response] the HTTP response object 39 | attr_accessor :http_response 40 | 41 | # @return [Boolean] true if the response is cached 42 | attr_accessor :cached 43 | 44 | # @param [Http::Request] http_request 45 | # @param [Http::Response] http_request 46 | def initialize http_request = nil, http_response = nil, &block 47 | @http_request = http_request 48 | @http_response = http_response 49 | @request_builder = block 50 | rebuild_request if @request_builder && !http_request 51 | end 52 | 53 | # Rebuilds the HTTP request using the block passed to the initializer 54 | def rebuild_request 55 | @http_request = @request_builder.call 56 | end 57 | 58 | # @return [Boolean] Returns true unless there is a response error. 59 | def successful? 60 | error.nil? 61 | end 62 | 63 | # @return [Boolean] Returns true if the http request was throttled 64 | # by AWS. 65 | def throttled? 66 | !successful? and 67 | http_response.body and 68 | parsed_body = XmlGrammar.parse(http_response.body) and 69 | parsed_body.respond_to?(:code) and 70 | parsed_body.code == "Throttling" 71 | end 72 | 73 | # @return [Boolean] Returns true if the http request timed out. 74 | def timeout? 75 | http_response.timeout? 76 | end 77 | 78 | # @private 79 | def inspect 80 | if request_type 81 | "<#{self.class}:#{request_type}>" 82 | else 83 | "<#{self.class}>" 84 | end 85 | end 86 | 87 | def cache_key 88 | [http_request.access_key_id, 89 | http_request.host, 90 | request_type, 91 | serialized_options].join(":") 92 | end 93 | 94 | def serialized_options 95 | serialize_options_hash(request_options) 96 | end 97 | 98 | private 99 | def serialize_options_hash(hash) 100 | "(" + hash.keys.sort_by(&:to_s).map do |key| 101 | "#{key}=#{serialize_options_value(hash[key])}" 102 | end.join(" ") + ")" 103 | end 104 | 105 | private 106 | def serialize_options_value(value) 107 | case value 108 | when Hash 109 | serialize_options_hash(value) 110 | when Array 111 | serialize_options_array(value) 112 | else 113 | value.inspect 114 | end 115 | end 116 | 117 | private 118 | def serialize_options_array(ary) 119 | "[" + ary.map { |v| serialize_options_value(v) }.join(" ") + 120 | "]" 121 | end 122 | 123 | end 124 | end 125 | end 126 | -------------------------------------------------------------------------------- /lib/aws/core/response_cache.rb: -------------------------------------------------------------------------------- 1 | # Copyright 2011 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). You 4 | # may not use this file except in compliance with the License. A copy of 5 | # the License is located at 6 | # 7 | # http://aws.amazon.com/apache2.0/ 8 | # 9 | # or in the "license" file accompanying this file. This file is 10 | # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF 11 | # ANY KIND, either express or implied. See the License for the specific 12 | # language governing permissions and limitations under the License. 13 | 14 | module AWS 15 | module Core 16 | 17 | # @private 18 | class ResponseCache 19 | 20 | attr_reader :cached_responses 21 | 22 | attr_reader :resource_cache 23 | 24 | def initialize 25 | @cached_responses = [] 26 | @indexed_responses = {} 27 | @resource_cache = ResourceCache.new 28 | end 29 | 30 | def add(resp) 31 | cached_responses.unshift(resp) 32 | @indexed_responses[resp.cache_key] = resp if 33 | resp.respond_to?(:cache_key) 34 | @resource_cache = ResourceCache.new 35 | end 36 | 37 | def select(*types, &block) 38 | cached_responses.select do |resp| 39 | types.map{|t| t.to_s }.include?(resp.request_type.to_s) and 40 | (block.nil? || block.call(resp)) 41 | end 42 | end 43 | 44 | def cached(resp) 45 | @indexed_responses[resp.cache_key] 46 | end 47 | 48 | end 49 | end 50 | end 51 | -------------------------------------------------------------------------------- /lib/aws/core/service_interface.rb: -------------------------------------------------------------------------------- 1 | # Copyright 2011 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). You 4 | # may not use this file except in compliance with the License. A copy of 5 | # the License is located at 6 | # 7 | # http://aws.amazon.com/apache2.0/ 8 | # 9 | # or in the "license" file accompanying this file. This file is 10 | # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF 11 | # ANY KIND, either express or implied. See the License for the specific 12 | # language governing permissions and limitations under the License. 13 | 14 | module AWS 15 | module Core 16 | 17 | # @private 18 | module ServiceInterface 19 | 20 | def self.included base 21 | 22 | base.send(:attr_reader, :config) 23 | base.send(:attr_reader, :client) 24 | 25 | base.module_eval('module Errors; end') 26 | 27 | unless base::Errors.include?(Errors) 28 | base::Errors.module_eval { include Errors } 29 | end 30 | 31 | end 32 | 33 | # Returns a new interface object for this service. You can override 34 | # any of the global configuration parameters by passing them in as 35 | # hash options. They are merged with AWS.config or merged 36 | # with the provided +:config+ object. 37 | # 38 | # @ec2 = AWS::EC2.new(:max_retries => 2) 39 | # 40 | # @see AWS::Cofiguration 41 | # 42 | # @param [Hash] options 43 | # @option options [Configuration] :config An AWS::Configuration 44 | # object to initialize this service interface object with. Defaults 45 | # to AWS.config when not provided. 46 | def initialize options = {} 47 | @config = options[:config] 48 | @config ||= AWS.config 49 | @config = @config.with(options) 50 | @client = config.send(Inflection.ruby_name(self.class.to_s) + '_client') 51 | end 52 | 53 | # @private 54 | def inspect 55 | "<#{self.class}>" 56 | end 57 | 58 | end 59 | end 60 | end 61 | -------------------------------------------------------------------------------- /lib/aws/core/uri_escape.rb: -------------------------------------------------------------------------------- 1 | # Copyright 2011 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). You 4 | # may not use this file except in compliance with the License. A copy of 5 | # the License is located at 6 | # 7 | # http://aws.amazon.com/apache2.0/ 8 | # 9 | # or in the "license" file accompanying this file. This file is 10 | # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF 11 | # ANY KIND, either express or implied. See the License for the specific 12 | # language governing permissions and limitations under the License. 13 | 14 | require 'cgi' 15 | 16 | module AWS 17 | module Core 18 | 19 | # @private 20 | module UriEscape 21 | 22 | # Does URI escaping the way AWS expects it to be done. 23 | # 24 | # @private 25 | protected 26 | def escape value 27 | value = value.encode("UTF-8") if 28 | value.respond_to?(:encode) 29 | CGI::escape(value.to_s).gsub('+', '%20') 30 | end 31 | 32 | # URI-escapes a path without escaping the separators 33 | # 34 | # @private 35 | protected 36 | def escape_path value 37 | escaped = "" 38 | value.scan(%r{(/*)([^/]*)(/*)}) do |(leading, part, trailing)| 39 | escaped << leading + escape(part) + trailing 40 | end 41 | escaped 42 | end 43 | 44 | end 45 | end 46 | end 47 | -------------------------------------------------------------------------------- /lib/aws/errors.rb: -------------------------------------------------------------------------------- 1 | # Copyright 2011 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). You 4 | # may not use this file except in compliance with the License. A copy of 5 | # the License is located at 6 | # 7 | # http://aws.amazon.com/apache2.0/ 8 | # 9 | # or in the "license" file accompanying this file. This file is 10 | # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF 11 | # ANY KIND, either express or implied. See the License for the specific 12 | # language governing permissions and limitations under the License. 13 | 14 | module AWS 15 | module Errors 16 | 17 | # Base class for the two service error classes: 18 | # 19 | # * {ClientError} 20 | # * {ServerError} 21 | # 22 | # When interacting with Amazon AWS services, you will sometimes 23 | # receive a non-200 level response. These http responses are treated 24 | # as errors. 25 | # 26 | # == Client Errors 27 | # 28 | # Errors in the three and four hundreds are client errors ({ClientError}). 29 | # A client error should not be resent without changes. The body of the 30 | # http response (the error #message) should give more information about 31 | # the nature of the problem. 32 | # 33 | # == Server Errors 34 | # 35 | # A 500 level error typically indicates the service is having an issue. 36 | # 37 | # Requests that generate service errors are automatically retried with 38 | # an exponential backoff. If the service still fails to respond with 39 | # a 200 after 3 retries the error is raised. 40 | # 41 | class Base < StandardError 42 | 43 | # @return [Http::Request] The low level http request that caused the 44 | # error to be raised. 45 | attr_reader :http_request 46 | 47 | # @return [Http::Response] The low level http response from the service 48 | # that wrapped the service error. 49 | attr_reader :http_response 50 | 51 | def initialize http_request = nil, http_response = nil, message = nil 52 | message ||= http_response.body if http_response 53 | @http_request = http_request 54 | @http_response = http_response 55 | super(message) 56 | end 57 | 58 | end 59 | 60 | # @private 61 | module ExceptionMixinClassMethods 62 | def new(*args) 63 | e = Base.new(*args) 64 | e.extend(self) 65 | e 66 | end 67 | end 68 | 69 | # Raised when an error occurs as a result of bad client 70 | # behavior, most commonly when the parameters passed to a method 71 | # are somehow invalid. Other common cases: 72 | # 73 | # * Throttling errors 74 | # * Bad credentials 75 | # * No permission to do the requested operation 76 | # * Limits exceeded (e.g. too many buckets) 77 | # 78 | module ClientError 79 | extend ExceptionMixinClassMethods 80 | end 81 | 82 | # Raised when an AWS service is unable to handle the request. These 83 | # are automatically retired. If after 3 retries the request is still 84 | # failing, then the error is raised. 85 | module ServerError 86 | extend ExceptionMixinClassMethods 87 | end 88 | 89 | # @private 90 | module ModeledError 91 | 92 | # @return [String] The error message given by the AWS service. 93 | attr_accessor :message 94 | 95 | # @return [Integer] The HTTP status code returned by the AWS service. 96 | attr_reader :code 97 | 98 | def initialize(req = nil, resp = nil) 99 | if req.kind_of?(String) 100 | # makes it easier to test handling of modeled exceptions 101 | super(nil, nil, req) 102 | @message = req 103 | elsif req and resp 104 | super(req, resp, message) 105 | include_error_type 106 | parse_body(resp.body) 107 | else 108 | super() 109 | end 110 | end 111 | 112 | def include_error_type 113 | if http_response.status >= 500 114 | extend ServerError 115 | else 116 | extend ClientError 117 | end 118 | end 119 | 120 | def parse_body(body) 121 | error_grammar.parse(body, :context => self) 122 | end 123 | 124 | def extract_message(body) 125 | error_grammar.parse(body).message 126 | end 127 | 128 | def error_grammar 129 | self.class::BASE_ERROR_GRAMMAR 130 | end 131 | 132 | end 133 | 134 | end 135 | end 136 | -------------------------------------------------------------------------------- /lib/aws/s3.rb: -------------------------------------------------------------------------------- 1 | # Copyright 2011 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). You 4 | # may not use this file except in compliance with the License. A copy of 5 | # the License is located at 6 | # 7 | # http://aws.amazon.com/apache2.0/ 8 | # 9 | # or in the "license" file accompanying this file. This file is 10 | # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF 11 | # ANY KIND, either express or implied. See the License for the specific 12 | # language governing permissions and limitations under the License. 13 | 14 | require 'aws/core' 15 | require 'aws/s3/config' 16 | 17 | module AWS 18 | 19 | # Provides an expressive, object-oriented interface to Amazon S3. 20 | # 21 | # To use Amazon S3 you must first 22 | # {sign up here}[http://aws.amazon.com/s3/]. 23 | # 24 | # For more information about Amazon S3, see: 25 | # 26 | # * {Amazon S3}[http://aws.amazon.com/s3/] 27 | # * {Amazon S3 Documentation}[http://aws.amazon.com/documentation/s3/] 28 | # 29 | # == Credentials 30 | # 31 | # You can setup default credentials for all AWS services via 32 | # AWS.config: 33 | # 34 | # AWS.config( 35 | # :access_key_id => 'YOUR_ACCESS_KEY_ID', 36 | # :secret_access_key => 'YOUR_SECRET_ACCESS_KEY') 37 | # 38 | # Or you can set them directly on the S3 interface: 39 | # 40 | # s3 = AWS::S3.new( 41 | # :access_key_id => 'YOUR_ACCESS_KEY_ID', 42 | # :secret_access_key => 'YOUR_SECRET_ACCESS_KEY') 43 | # 44 | # == Buckets Keys and Objects 45 | # 46 | # S3 stores objects in buckets. 47 | # 48 | # To create a bucket: 49 | # 50 | # bucket = s3.buckets.create('mybucket') 51 | # 52 | # To get a bucket: 53 | # 54 | # bucket = s3.buckets[:mybucket] 55 | # bucket = s3.buckets['mybucket'] 56 | # 57 | # Listing buckets: 58 | # 59 | # s3.buckets.each do |bucket| 60 | # puts bucket.name 61 | # end 62 | # 63 | # See {Bucket} and {BucketCollection} for more information on working 64 | # with S3 buckets. 65 | # 66 | # == Listing Objects 67 | # 68 | # Enumerating objects in a bucket: 69 | # 70 | # bucket.objects.each do |object| 71 | # puts object.key #=> no data is fetched from s3, just a list of keys 72 | # end 73 | # 74 | # You can limit the list of objects with a prefix, or explore the objects 75 | # in a bucket as a tree. See {ObjectCollection} for more information. 76 | # 77 | # == Reading and Writing to S3 78 | # 79 | # Each object in a bucket has a unique key. 80 | # 81 | # photo = s3.buckets['mybucket'].objects['photo.jpg'] 82 | # 83 | # Writing to an S3Object: 84 | # 85 | # photo.write(File.read('/some/photo.jpg')) 86 | # 87 | # Reading from an S3Object: 88 | # 89 | # File.open("/some/path/on/disk.jpg", "w") do |f| 90 | # f.write(photo.read) 91 | # end 92 | # 93 | # See {S3Object} for more information on reading and writing to S3. 94 | # 95 | class S3 96 | 97 | AWS.register_autoloads(self) do 98 | autoload :AccessControlList, 'access_control_list' 99 | autoload :ACLObject, 'acl_object' 100 | autoload :Bucket, 'bucket' 101 | autoload :BucketCollection, 'bucket_collection' 102 | autoload :BucketVersionCollection, 'bucket_version_collection' 103 | autoload :Client, 'client' 104 | autoload :DataOptions, 'data_options' 105 | autoload :Errors, 'errors' 106 | autoload :MultipartUpload, 'multipart_upload' 107 | autoload :MultipartUploadCollection, 'multipart_upload_collection' 108 | autoload :ObjectCollection, 'object_collection' 109 | autoload :ObjectMetadata, 'object_metadata' 110 | autoload :ObjectUploadCollection, 'object_upload_collection' 111 | autoload :ObjectVersion, 'object_version' 112 | autoload :ObjectVersionCollection, 'object_version_collection' 113 | autoload :PaginatedCollection, 'paginated_collection' 114 | autoload :Policy, 'policy' 115 | autoload :PrefixAndDelimiterCollection, 'prefix_and_delimiter_collection' 116 | autoload :PrefixedCollection, 'prefixed_collection' 117 | autoload :PresignedPost, 'presigned_post' 118 | autoload :Request, 'request' 119 | autoload :S3Object, 's3_object' 120 | autoload :Tree, 'tree' 121 | autoload :UploadedPart, 'uploaded_part' 122 | autoload :UploadedPartCollection, 'uploaded_part_collection' 123 | end 124 | 125 | include Core::ServiceInterface 126 | 127 | # @return [BucketCollection] Returns a collection that represents all 128 | # buckets in the account. 129 | def buckets 130 | BucketCollection.new(:config => @config) 131 | end 132 | 133 | end 134 | end 135 | -------------------------------------------------------------------------------- /lib/aws/s3/access_control_list.rb: -------------------------------------------------------------------------------- 1 | # Copyright 2011 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). You 4 | # may not use this file except in compliance with the License. A copy of 5 | # the License is located at 6 | # 7 | # http://aws.amazon.com/apache2.0/ 8 | # 9 | # or in the "license" file accompanying this file. This file is 10 | # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF 11 | # ANY KIND, either express or implied. See the License for the specific 12 | # language governing permissions and limitations under the License. 13 | 14 | module AWS 15 | class S3 16 | 17 | # Represents an access control list for S3 objects and buckets. For example: 18 | # 19 | # acl = AccessControlList.new 20 | # acl.grant(:full_control). 21 | # to(:canonical_user_id => "8a6925ce4adf588a4f21c32aa379004fef") 22 | # acl.to_xml # => '...' 23 | # 24 | # You can also construct an AccessControlList from a hash: 25 | # 26 | # AccessControlList.new( 27 | # :owner => { :id => "8a6925ce4adf588a4f21c32aa379004fef" }, 28 | # :grants => [{ :grantee => { 29 | # :canonical_user_id => "8a6925ce4adf588a4f21c32aa379004fef", 30 | # }, 31 | # :permission => :full_control }] 32 | # ) 33 | # 34 | # @see ACLObject 35 | # 36 | # @attr [AccessControlList::Owner] owner The owner of the access 37 | # control list. You can set this as a hash, for example: 38 | # acl.owner = { :id => '8a6925ce4adf588a4f21c32aa379004fef' } 39 | # This attribute is required when setting an ACL. 40 | # 41 | # @attr [list of AccessControlList::Grant] grants The list of 42 | # grants. You can set this as a list of hashes, for example: 43 | # acl.grants = [{ :grantee => { :canonical_user_id => 44 | # "8a6925ce4adf588a4f21c32aa379004fef" }, 45 | # :permission => :full_control }] 46 | class AccessControlList 47 | 48 | # Represents an ACL owner. In the default ACL, this is the 49 | # bucket owner. 50 | # 51 | # @attr [String] id The canonical user ID of the ACL owner. 52 | # This attribute is required when setting an ACL. 53 | # 54 | # @attr [String] display_name The display name of the ACL 55 | # owner. This value is ignored when setting an ACL. 56 | class Owner 57 | include ACLObject 58 | 59 | string_attr "ID", :required => true 60 | string_attr "DisplayName" 61 | end 62 | 63 | # Represents a user who is granted some kind of permission 64 | # through a Grant. There are three ways to specify a grantee: 65 | # 66 | # * You can specify the canonical user ID, for example. When 67 | # you read an ACL from S3, all grantees will be identified 68 | # this way, and the display_name attribute will also be provided. 69 | # 70 | # Grantee.new(:canonical_user_id => "8a6925ce4adf588a4f21c32aa379004fef") 71 | # 72 | # * You can specify the e-mail address of an AWS customer, for example: 73 | # Grantee.new(:amazon_customer_email => 'foo@example.com') 74 | # 75 | # * You can specify a group URI, for example: 76 | # Grantee.new(:group_uri => 'http://acs.amazonaws.com/groups/global/AllUsers') 77 | # For more details about group URIs, see: 78 | # http://docs.amazonwebservices.com/AmazonS3/latest/dev/ACLOverview.html 79 | # 80 | # When constructing a grantee, you must provide a value for 81 | # exactly one of the following attributes: 82 | # 83 | # * +amazon_customer_email+ 84 | # * +canonical_user_id+ 85 | # * +group_uri+ 86 | # 87 | # @attr [String] amazon_customer_email The e-mail address of 88 | # an AWS customer. 89 | # 90 | # @attr [String] canonical_user_id The canonical user ID of an 91 | # AWS customer. 92 | # 93 | # @attr [String] group_uri A URI that identifies a particular 94 | # group of users. 95 | # 96 | # @attr [String] display_name The display name associated with 97 | # the grantee. This is provided by S3 when reading an ACL. 98 | class Grantee 99 | include ACLObject 100 | 101 | SIGNAL_ATTRIBUTES = [:amazon_customer_email, 102 | :canonical_user_id, 103 | :group_uri] 104 | 105 | string_attr "EmailAddress", :method_name => "amazon_customer_email" 106 | string_attr "ID", :method_name => "canonical_user_id" 107 | string_attr "URI", :method_name => "group_uri" 108 | string_attr "DisplayName" 109 | 110 | # (see ACLObject#validate!) 111 | def validate! 112 | attr = signal_attribute 113 | raise "missing amazon_customer_email, canonical_user_id, "+ 114 | "or group_uri" unless attr 115 | raise "display_name is invalid with #{attr}" if 116 | attr != :canonical_user_id and display_name 117 | end 118 | 119 | # @private 120 | def stag 121 | if attr = signal_attribute 122 | super + " xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\"" + 123 | " xsi:type=\"#{type_for_attr(attr)}\"" 124 | else 125 | super 126 | end 127 | end 128 | 129 | # @private 130 | def signal_attribute 131 | SIGNAL_ATTRIBUTES.find { |att| send(att) } 132 | end 133 | 134 | # @private 135 | def type_for_attr(attr) 136 | { :amazon_customer_email => "AmazonCustomerByEmail", 137 | :canonical_user_id => "CanonicalUser", 138 | :group_uri => "Group" }[attr] 139 | end 140 | 141 | end 142 | 143 | # Represents the permission being granted in a Grant object. 144 | # Typically you will not need to construct an instance of this 145 | # class directly. 146 | # @see Grant#permission 147 | class Permission 148 | include ACLObject 149 | 150 | # The permission expressed as a symbol following Ruby 151 | # conventions. For example, S3's FULL_CONTROL permission 152 | # will be returned as +:full_control+. 153 | attr_reader :name 154 | 155 | # @private 156 | def initialize(name) 157 | raise "expected string or symbol" unless 158 | name.respond_to?(:to_str) or name.respond_to?(:to_sym) 159 | @name = name.to_sym 160 | end 161 | 162 | def body_xml 163 | name.to_s.upcase 164 | end 165 | 166 | end 167 | 168 | # Represents a single grant in an ACL. Both +grantee+ and 169 | # +permission+ are required for each grant when setting an 170 | # ACL. 171 | # 172 | # See 173 | # http://docs.amazonwebservices.com/AmazonS3/latest/dev/ACLOverview.html 174 | # for more information on how grantees and permissions are 175 | # interpreted by S3. 176 | # 177 | # @attr [Grantee] grantee The user or users who are granted 178 | # access according to this grant. You can specify this as a 179 | # hash: 180 | # grant.grantee = { :amazon_customer_email => "foo@example.com" } 181 | # 182 | # @attr [Permission or Symbol] permission The type of 183 | # permission that is granted by this grant. Valid values are: 184 | # * +:read+ 185 | # * +:write+ 186 | # * +:read_acp+ 187 | # * +:write_acp+ 188 | # * +:full_control+ 189 | class Grant 190 | 191 | include ACLObject 192 | 193 | object_attr Grantee, :required => true 194 | object_attr Permission, :required => true, :cast => Symbol 195 | 196 | end 197 | 198 | include ACLObject 199 | 200 | # @private 201 | def stag 202 | super()+" xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\"" 203 | end 204 | 205 | # @private 206 | def element_name 207 | "AccessControlPolicy" 208 | end 209 | 210 | class GrantBuilder 211 | 212 | # @private 213 | def initialize(acl, grant) 214 | @acl = acl 215 | @grant = grant 216 | end 217 | 218 | # Specifies the grantee. 219 | # 220 | # @param [Grantee or Hash] grantee A Grantee object or hash; 221 | # for example: 222 | # acl.grant(:full_control).to(:amazon_customer_email => "foo@example.com") 223 | def to(grantee) 224 | @grant.grantee = grantee 225 | @acl.grants << @grant 226 | end 227 | 228 | end 229 | 230 | # Convenience method for constructing a new grant and adding 231 | # it to the ACL. Example usage: 232 | # 233 | # acl.grants.size # => 0 234 | # acl.grant(:full_control). 235 | # to(:canonical_user_id => "8a6925ce4adf588a4f21c32aa379004fef") 236 | # acl.grants.size # => 1 237 | # 238 | # @return [GrantBuilder] 239 | def grant(permission) 240 | GrantBuilder.new(self, Grant.new(:permission => permission)) 241 | end 242 | 243 | object_attr Owner, :required => true 244 | object_list_attr("AccessControlList", Grant, 245 | :required => true, :method_name => :grants) 246 | 247 | end 248 | 249 | end 250 | end 251 | -------------------------------------------------------------------------------- /lib/aws/s3/acl_object.rb: -------------------------------------------------------------------------------- 1 | # Copyright 2011 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). You 4 | # may not use this file except in compliance with the License. A copy of 5 | # the License is located at 6 | # 7 | # http://aws.amazon.com/apache2.0/ 8 | # 9 | # or in the "license" file accompanying this file. This file is 10 | # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF 11 | # ANY KIND, either express or implied. See the License for the specific 12 | # language governing permissions and limitations under the License. 13 | 14 | require 'rexml/text' 15 | 16 | module AWS 17 | class S3 18 | 19 | # Common methods for AccessControlList and related objects. 20 | module ACLObject 21 | 22 | # @private 23 | def initialize(opts = {}); end 24 | 25 | # @private 26 | def body_xml; ""; end 27 | 28 | # @private 29 | def stag 30 | element_name 31 | end 32 | 33 | # @private 34 | def element_name 35 | self.class.name[/::([^:]*)$/, 1] 36 | end 37 | 38 | # Returns the XML representation of the object. Generally 39 | # you'll want to call this on an AccessControlList object, 40 | # which will yield an XML representation of the ACL that you 41 | # can send to S3. 42 | def to_s 43 | if body_xml.empty? 44 | "<#{stag}/>" 45 | else 46 | "<#{stag}>#{body_xml}" 47 | end 48 | end 49 | 50 | # (see #to_s) 51 | def to_xml 52 | to_s 53 | end 54 | 55 | # Returns true if and only if this object is valid according 56 | # to S3's published ACL schema. In particular, this will 57 | # check that all required attributes are provided and that 58 | # they are of the correct type. 59 | def valid? 60 | validate! 61 | rescue => e 62 | false 63 | else 64 | true 65 | end 66 | 67 | # Raises an exception unless this object is valid according to 68 | # S3's published ACL schema. 69 | # @see #valid? 70 | def validate!; end 71 | 72 | # @private 73 | def validate_input(name, value, context = nil) 74 | send("validate_#{name}_input!", value, context) 75 | end 76 | 77 | # @private 78 | module ClassMethods 79 | 80 | def string_attr(element_name, options = {}) 81 | method_name = options[:method_name] || 82 | Core::Inflection.ruby_name(element_name) 83 | 84 | attr_accessor(method_name) 85 | setter_option(method_name) 86 | string_input_validator(method_name) 87 | validate_string(method_name) if options[:required] 88 | body_xml_string_content(element_name, method_name) 89 | end 90 | 91 | def object_attr(klass, options = {}) 92 | base_name = klass.name[/::([^:]*)$/, 1] 93 | method_name = Core::Inflection.ruby_name(base_name) 94 | cast = options[:cast] || Hash 95 | 96 | attr_reader(method_name) 97 | setter_option(method_name) 98 | object_setter(klass, method_name, cast) 99 | object_input_validator(klass, base_name, method_name, cast) 100 | validate_object(method_name) if options[:required] 101 | body_xml_content(method_name) 102 | end 103 | 104 | def object_list_attr(list_element, klass, options = {}) 105 | base_name = klass.name[/::([^:]*)$/, 1] 106 | method_name = Core::Inflection.ruby_name(options[:method_name].to_s || list_element) 107 | default_value = nil 108 | default_value = [] if options[:required] 109 | 110 | attr_reader(method_name) 111 | setter_option(method_name) { [] if options[:required] } 112 | object_list_setter(klass, method_name) 113 | object_list_input_validator(klass, base_name, method_name) 114 | validate_list(method_name) 115 | body_xml_list_content(list_element, method_name) 116 | end 117 | 118 | def setter_option(method_name) 119 | Core::MetaUtils.class_extend_method(self, :initialize) do |*args| 120 | opts = args.last || {} 121 | instance_variable_set("@#{method_name}", yield) if block_given? 122 | key = method_name.to_sym 123 | 124 | if opts.has_key?(key) 125 | value = opts[key] 126 | validate_input(method_name, value, "for #{method_name} option") 127 | self.send("#{method_name}=", value) 128 | end 129 | super(opts) 130 | end 131 | end 132 | 133 | def string_input_validator(method_name) 134 | input_validator(method_name) do |value, context| 135 | raise ArgumentError.new("expected string"+context) unless 136 | value.respond_to?(:to_str) 137 | end 138 | end 139 | 140 | def object_input_validator(klass, base_name, method_name, cast) 141 | input_validator(method_name) do |value, context| 142 | if value.kind_of?(cast) 143 | klass.new(value).validate! 144 | else 145 | raise ArgumentError.new("expected #{base_name} object or hash"+context) unless 146 | value.nil? or value.kind_of? klass 147 | end 148 | end 149 | end 150 | 151 | def object_list_input_validator(klass, base_name, method_name) 152 | input_validator(method_name) do |value, context| 153 | raise ArgumentError.new("expected array"+context) unless value.kind_of?(Array) 154 | value.each do |member| 155 | if member.kind_of?(Hash) 156 | klass.new(member).validate! 157 | else 158 | raise ArgumentError.new("expected array#{context} "+ 159 | "to contain #{base_name} objects "+ 160 | "or hashes") unless 161 | member.kind_of? klass 162 | end 163 | end 164 | end 165 | end 166 | 167 | def input_validator(method_name, &blk) 168 | validator = "__validator__#{blk.object_id}" 169 | Core::MetaUtils.class_extend_method(self, validator, &blk) 170 | Core::MetaUtils.class_extend_method(self, "validate_#{method_name}_input!") do |*args| 171 | (value, context) = args 172 | context = " "+context if context 173 | context ||= "" 174 | send(validator, value, context) 175 | end 176 | end 177 | 178 | def object_setter(klass, method_name, cast) 179 | define_method("#{method_name}=") do |value| 180 | validate_input(method_name, value) 181 | if value.kind_of?(cast) 182 | value = klass.new(value) 183 | end 184 | instance_variable_set("@#{method_name}", value) 185 | end 186 | end 187 | 188 | def object_list_setter(klass, method_name) 189 | define_method("#{method_name}=") do |value| 190 | validate_input(method_name, value) 191 | list = value.map do |member| 192 | if member.kind_of?(Hash) 193 | klass.new(member) 194 | else 195 | member 196 | end 197 | end 198 | instance_variable_set("@#{method_name}", list) 199 | end 200 | end 201 | 202 | def validate_string(method_name) 203 | Core::MetaUtils.class_extend_method(self, :validate!) do 204 | super() 205 | raise "missing #{method_name}" unless send(method_name) 206 | end 207 | end 208 | 209 | def validate_object(method_name) 210 | Core::MetaUtils.class_extend_method(self, :validate!) do 211 | super() 212 | raise "missing #{method_name}" unless send(method_name) 213 | send(method_name).validate! 214 | end 215 | end 216 | 217 | def validate_list(method_name) 218 | Core::MetaUtils.class_extend_method(self, :validate!) do 219 | super() 220 | raise "missing #{method_name}" unless send(method_name) 221 | send(method_name).each { |member| member.validate! } 222 | end 223 | end 224 | 225 | def body_xml_string_content(element_name, method_name) 226 | add_xml_child(method_name) do |value| 227 | normalized = REXML::Text.normalize(value.to_s) 228 | "<#{element_name}>#{normalized}" 229 | end 230 | end 231 | 232 | def body_xml_content(method_name) 233 | add_xml_child(method_name) { |value| value.to_s } 234 | end 235 | 236 | def body_xml_list_content(list_element, method_name) 237 | add_xml_child(method_name) do |list| 238 | list_content = list.map { |member| member.to_s }.join 239 | if list_content.empty? 240 | "<#{list_element}/>" 241 | else 242 | "<#{list_element}>#{list_content}" 243 | end 244 | end 245 | end 246 | 247 | def add_xml_child(method_name) 248 | Core::MetaUtils.class_extend_method(self, :body_xml) do 249 | xml = super() 250 | value = send(method_name) 251 | xml += yield(value) if value 252 | xml 253 | end 254 | end 255 | 256 | end 257 | 258 | def self.included(m) 259 | m.extend(ClassMethods) 260 | end 261 | 262 | end 263 | end 264 | end 265 | -------------------------------------------------------------------------------- /lib/aws/s3/bucket_collection.rb: -------------------------------------------------------------------------------- 1 | # Copyright 2011 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). You 4 | # may not use this file except in compliance with the License. A copy of 5 | # the License is located at 6 | # 7 | # http://aws.amazon.com/apache2.0/ 8 | # 9 | # or in the "license" file accompanying this file. This file is 10 | # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF 11 | # ANY KIND, either express or implied. See the License for the specific 12 | # language governing permissions and limitations under the License. 13 | 14 | module AWS 15 | class S3 16 | 17 | # Represents a collection of buckets. 18 | # 19 | # You can use this to create a bucket: 20 | # 21 | # s3.buckets.create(:name => "mybucket") 22 | # 23 | # You can get a handle for a specific bucket with indifferent 24 | # access: 25 | # 26 | # bucket = s3.buckets[:mybucket] 27 | # bucket = s3.buckets['mybucket'] 28 | # 29 | # You can also use it to find out which buckets are in your account: 30 | # 31 | # s3.buckets.collect(&:name) 32 | # #=> ['bucket1', 'bucket2', ...] 33 | # 34 | class BucketCollection 35 | 36 | include Core::Model 37 | include Enumerable 38 | 39 | # Creates and returns a new Bucket. For example: 40 | # 41 | # @note If your bucket name contains one or more periods and it 42 | # is hosted in a non-US region, you should make requests 43 | # against the bucket using the S3 endpoint specific to the 44 | # region in which your bucket resides. For example: 45 | # 46 | # s3 = AWS::S3.new(:s3_endpoint => "s3-eu-west-1.amazonaws.com") 47 | # bucket = s3.buckets.create("my.eu.bucket") 48 | # 49 | # For a full list of endpoints and regions, see 50 | # {http://docs.amazonwebservices.com/general/latest/gr/index.html?rande.html 51 | # Regions and Endpoints} in the Amazon Web Services General 52 | # Reference. 53 | # 54 | # @example 55 | # 56 | # bucket = s3.buckets.create('my-bucket') 57 | # bucket.name #=> "my-bucket" 58 | # bucket.exists? #=> true 59 | # 60 | # @param [String] bucket_name 61 | # @param [Hash] options 62 | # 63 | # @option options [String] :location_constraint (nil) The 64 | # location where the bucket should be created. Defaults to 65 | # the classic US region; however, if you configure a regional 66 | # endpoint for Amazon S3 this option will default to the 67 | # appropriate location constraint for the endpoint. For 68 | # example: 69 | # 70 | # s3 = AWS::S3.new(:s3_endpoint => "s3-us-west-1.amazonaws.com") 71 | # bucket = s3.buckets.create("my-us-west-bucket") 72 | # bucket.location_constraint # => "us-west-1" 73 | # 74 | # @option options [String] :acl (:private) Sets the ACL of the bucket 75 | # you are creating. Valid Values include :private, :public_read, 76 | # :public_read_write, :authenticated_read, :bucket_owner_read and 77 | # :bucket_owner_full_control 78 | # @return [Bucket] 79 | def create bucket_name, options = {} 80 | 81 | # auto set the location constraint for the user if it is not 82 | # passed in and the endpoint is not the us-standard region. don't 83 | # override the location constraint though, even it is wrong, 84 | unless 85 | config.s3_endpoint == 's3.amazonaws.com' or 86 | options[:location_constraint] 87 | then 88 | constraint = 89 | case config.s3_endpoint 90 | when 's3-eu-west-1.amazonaws.com' then 'EU' 91 | when /^s3-(.*)\.amazonaws\.com$/ then $1 92 | end 93 | options[:location_constraint] = constraint if constraint 94 | end 95 | 96 | client.create_bucket(options.merge(:bucket_name => bucket_name)) 97 | bucket_named(bucket_name) 98 | 99 | end 100 | 101 | # Returns the Bucket with the given name. 102 | # 103 | # Makes no requests. The returned bucket object can 104 | # be used to make requets for the bucket and its objects. 105 | # 106 | # @example 107 | # 108 | # bucket = s3.buckets[:mybucket], 109 | # bucket = s3.buckets['mybucket'], 110 | # 111 | # @param [String] bucket_name 112 | # @return [Bucket] 113 | def [] bucket_name 114 | bucket_named(bucket_name) 115 | end 116 | 117 | # Iterates the buckets in this collection. 118 | # 119 | # @example 120 | # 121 | # s3.buckets.each do |bucket| 122 | # puts bucket.name 123 | # end 124 | # 125 | # @return [nil] 126 | def each &block 127 | response = client.list_buckets 128 | response.buckets.each do |b| 129 | yield(bucket_named(b.name, response.owner)) 130 | end 131 | nil 132 | end 133 | 134 | # @private 135 | private 136 | def bucket_named name, owner = nil 137 | S3::Bucket.new(name.to_s, :owner => owner, :config => config) 138 | end 139 | 140 | end 141 | 142 | end 143 | end 144 | -------------------------------------------------------------------------------- /lib/aws/s3/bucket_version_collection.rb: -------------------------------------------------------------------------------- 1 | # Copyright 2011 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). You 4 | # may not use this file except in compliance with the License. A copy of 5 | # the License is located at 6 | # 7 | # http://aws.amazon.com/apache2.0/ 8 | # 9 | # or in the "license" file accompanying this file. This file is 10 | # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF 11 | # ANY KIND, either express or implied. See the License for the specific 12 | # language governing permissions and limitations under the License. 13 | 14 | module AWS 15 | class S3 16 | 17 | # A collection of versioned objects for the entire bucket. 18 | # 19 | # @see PrefixedCollection 20 | class BucketVersionCollection 21 | 22 | include Core::Model 23 | include Enumerable 24 | include PrefixAndDelimiterCollection 25 | 26 | # @param [Bucket] bucket 27 | def initialize bucket, options = {} 28 | @bucket = bucket 29 | super 30 | end 31 | 32 | # @return [Bucket] The bucket this collection belongs to. 33 | attr_reader :bucket 34 | 35 | # @return [ObjectVersion] Returns the most recently created object 36 | # version in the entire bucket. 37 | def latest 38 | self.find{|version| true } 39 | end 40 | 41 | # Yields once for each version in the bucket. 42 | # 43 | # @yield [object_version] 44 | # @yieldparam [ObjectVersion] object_version 45 | # @return nil 46 | def each options = {}, █ super; end 47 | 48 | # @private 49 | protected 50 | def each_member_in_page(page, &block) 51 | super 52 | page.versions.each do |version| 53 | object_version = 54 | ObjectVersion.new(bucket.objects[version.key], 55 | version.version_id, 56 | :delete_marker => version.delete_marker?) 57 | yield(object_version) 58 | end 59 | end 60 | 61 | # @private 62 | protected 63 | def list_request(options) 64 | client.list_object_versions(options) 65 | end 66 | 67 | # @private 68 | protected 69 | def limit_param; :max_keys; end 70 | 71 | # @private 72 | protected 73 | def pagination_markers; super + [:version_id_marker]; end 74 | 75 | # @private 76 | protected 77 | def page_size(resp); super + resp.versions.size; end 78 | 79 | end 80 | end 81 | end 82 | -------------------------------------------------------------------------------- /lib/aws/s3/client/xml.rb: -------------------------------------------------------------------------------- 1 | # Copyright 2011 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). You 4 | # may not use this file except in compliance with the License. A copy of 5 | # the License is located at 6 | # 7 | # http://aws.amazon.com/apache2.0/ 8 | # 9 | # or in the "license" file accompanying this file. This file is 10 | # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF 11 | # ANY KIND, either express or implied. See the License for the specific 12 | # language governing permissions and limitations under the License. 13 | 14 | module AWS 15 | class S3 16 | class Client < Core::Client 17 | 18 | # @private 19 | module XML 20 | 21 | Error = Core::XmlGrammar.customize { } 22 | 23 | module HasCommonPrefixes 24 | 25 | def self.included(mod) 26 | mod.module_eval do 27 | element "CommonPrefixes" do 28 | collect_values 29 | format_value {|value| value.prefix } 30 | end 31 | end 32 | end 33 | 34 | end 35 | 36 | ListBuckets = Core::XmlGrammar.customize do 37 | element "Buckets" do 38 | element "Bucket" do 39 | collect_values 40 | end 41 | format_value { |value| super(value.bucket) } 42 | end 43 | end 44 | 45 | GetBucketAcl = GetObjectAcl = Core::XmlGrammar.customize do 46 | wrapper(:acl, 47 | :for => ["Owner", 48 | "AccessControlList"]) do 49 | construct_value { AccessControlList.new } 50 | end 51 | 52 | element "Owner" do 53 | construct_value { AccessControlList::Owner.new } 54 | end 55 | 56 | element "AccessControlList" do 57 | element "Grant" do 58 | collect_values 59 | construct_value { AccessControlList::Grant.new } 60 | 61 | element "Grantee" do 62 | construct_value { AccessControlList::Grantee.new } 63 | element "ID" do 64 | rename :canonical_user_id 65 | end 66 | end 67 | 68 | element "Permission" do 69 | symbol_value 70 | end 71 | end 72 | 73 | format_value { |value| super(value.grant) } 74 | rename :grants 75 | 76 | end 77 | end 78 | 79 | ListObjects = Core::XmlGrammar.customize do 80 | 81 | element("Name") { rename "bucket_name" } 82 | element("MaxKeys") { integer_value } 83 | element("IsTruncated") { rename "truncated"; boolean_value } 84 | element("Delimiter") { force } 85 | 86 | element("Contents") do 87 | list 88 | element("Owner") do 89 | element("ID") { } 90 | element("DisplayName") { } 91 | end 92 | element("Key") { } 93 | element("Size") { } 94 | element("StorageClass") { } 95 | element("ETag") { rename "etag" } 96 | 97 | # DateTime is more general, Time is much faster to construct 98 | element("LastModified") { time_value } 99 | end 100 | 101 | include HasCommonPrefixes 102 | 103 | end 104 | 105 | GetBucketVersioning = Core::XmlGrammar.customize do 106 | element("Status") do 107 | symbol_value 108 | format_value {|value| super(value) || :unversioned } 109 | force 110 | end 111 | end 112 | 113 | ListObjectVersions = Core::XmlGrammar.customize do 114 | 115 | element("MaxKeys") { integer_value } 116 | element("IsTruncated") { rename "Truncated"; boolean_value } 117 | element("NextKeyMarker") { force } 118 | element("NextVersionIdMarker") { force } 119 | 120 | %w(DeleteMarker Version).each do |element_name| 121 | element(element_name) do 122 | collect_values 123 | rename("versions") 124 | element("IsLatest") { rename "latest"; boolean_value } 125 | element("LastModified") { datetime_value } 126 | element("ETag") { rename "etag" } 127 | element("Size") { integer_value } 128 | element("StorageClass") { symbol_value } 129 | end 130 | end 131 | 132 | element "DeleteMarker" do 133 | add_method(:delete_marker?) { true } 134 | add_method(:version?) { false } 135 | end 136 | 137 | element "Version" do 138 | add_method(:delete_marker?) { false } 139 | add_method(:version?) { true } 140 | end 141 | 142 | include HasCommonPrefixes 143 | 144 | end 145 | 146 | # default behavior is good enough 147 | InitiateMultipartUpload = Core::XmlGrammar.customize do 148 | element("UploadId") { force } 149 | end 150 | 151 | ListMultipartUploads = Core::XmlGrammar.customize do 152 | element("IsTruncated") { rename "Truncated"; boolean_value } 153 | element("MaxUploads") { integer_value } 154 | element("NextKeyMarker") { force } 155 | element("NextUploadIdMarker") { force } 156 | element("Upload") do 157 | collect_values 158 | rename :uploads 159 | element("StorageClass") { symbol_value } 160 | element("Initiated") { datetime_value } 161 | end 162 | 163 | include HasCommonPrefixes 164 | end 165 | 166 | # keep default behavior 167 | CompleteMultipartUpload = Core::XmlGrammar.customize 168 | 169 | ListParts = Core::XmlGrammar.customize do 170 | element("StorageClass") { symbol_value } 171 | element("IsTruncated") { rename "Truncated"; boolean_value } 172 | element("MaxParts") { integer_value } 173 | element("PartNumberMarker") { integer_value } 174 | element("NextPartNumberMarker") { integer_value } 175 | element("Part") do 176 | collect_values 177 | rename :parts 178 | element("PartNumber") { integer_value } 179 | element("LastModified") { datetime_value } 180 | element("Size") { integer_value } 181 | end 182 | end 183 | 184 | end 185 | end 186 | end 187 | end 188 | -------------------------------------------------------------------------------- /lib/aws/s3/config.rb: -------------------------------------------------------------------------------- 1 | # Copyright 2011 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). You 4 | # may not use this file except in compliance with the License. A copy of 5 | # the License is located at 6 | # 7 | # http://aws.amazon.com/apache2.0/ 8 | # 9 | # or in the "license" file accompanying this file. This file is 10 | # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF 11 | # ANY KIND, either express or implied. See the License for the specific 12 | # language governing permissions and limitations under the License. 13 | 14 | AWS::Core::Configuration.module_eval do 15 | 16 | add_service 'S3', 's3', 's3.amazonaws.com' 17 | 18 | add_option :s3_multipart_threshold, 16 * 1024 * 1024 19 | 20 | add_option :s3_multipart_min_part_size, 5 * 1024 * 1024 21 | 22 | add_option :s3_multipart_max_parts, 10000 23 | 24 | add_option :s3_server_side_encryption, nil 25 | 26 | end 27 | -------------------------------------------------------------------------------- /lib/aws/s3/data_options.rb: -------------------------------------------------------------------------------- 1 | # Copyright 2011 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). You 4 | # may not use this file except in compliance with the License. A copy of 5 | # the License is located at 6 | # 7 | # http://aws.amazon.com/apache2.0/ 8 | # 9 | # or in the "license" file accompanying this file. This file is 10 | # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF 11 | # ANY KIND, either express or implied. See the License for the specific 12 | # language governing permissions and limitations under the License. 13 | 14 | require 'pathname' 15 | 16 | module AWS 17 | class S3 18 | module DataOptions 19 | 20 | protected 21 | def data_stream_from options, &block 22 | 23 | validate_data!(options, block) 24 | 25 | # block format 26 | if block_given? 27 | buffer = StringIO.new 28 | yield(buffer) 29 | buffer.rewind 30 | return buffer 31 | end 32 | 33 | # string, pathname, file, io-like object, etc 34 | data = options[:data] 35 | file_opts = ["rb"] 36 | file_opts << { :encoding => "BINARY" } if Object.const_defined?(:Encoding) 37 | case 38 | when data.is_a?(String) 39 | data.force_encoding("BINARY") if data.respond_to?(:force_encoding) 40 | StringIO.new(data) 41 | when data.is_a?(Pathname) then File.open(data.to_s, *file_opts) 42 | when options[:file] then File.open(options[:file], *file_opts) 43 | else data 44 | end 45 | 46 | end 47 | 48 | protected 49 | def content_length_from options 50 | data = options[:data] 51 | case 52 | when options[:content_length] then options[:content_length] 53 | when options[:file] then File.size(options[:file]) 54 | when data.is_a?(Pathname) then File.size(data.to_s) 55 | when data.is_a?(File) then File.size(data.path) 56 | when data.respond_to?(:bytesize) then data.bytesize 57 | when data.respond_to?(:size) then data.size 58 | when data.respond_to?(:length) then data.length 59 | else raise ArgumentError, 'content_length was not provided ' + 60 | 'and could not be determined' 61 | end 62 | end 63 | 64 | protected 65 | def validate_data! options, block 66 | 67 | data = options[:data] 68 | filename = options[:file] 69 | 70 | raise ArgumentError, 'data passed multiple ways' if 71 | [data, filename, block].compact.size > 1 72 | 73 | # accepting block format 74 | return if block and block.arity == 1 75 | 76 | # accepting file path 77 | return if filename.kind_of?(String) 78 | 79 | # accepting strings 80 | return if data.kind_of?(String) 81 | 82 | # accepting pathname 83 | return if data.kind_of?(Pathname) 84 | 85 | # accepts io-like objects (responds to read and eof?) 86 | if data.respond_to?(:read) and 87 | data.method(:read).arity != 0 and 88 | data.respond_to?(:eof?) then 89 | return true 90 | end 91 | 92 | raise ArgumentError, 'data must be provided as a String, ' + 93 | 'Pathname, file path, or an object that responds to #read and #eof?' 94 | 95 | end 96 | 97 | end 98 | end 99 | end 100 | -------------------------------------------------------------------------------- /lib/aws/s3/errors.rb: -------------------------------------------------------------------------------- 1 | # Copyright 2011 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). You 4 | # may not use this file except in compliance with the License. A copy of 5 | # the License is located at 6 | # 7 | # http://aws.amazon.com/apache2.0/ 8 | # 9 | # or in the "license" file accompanying this file. This file is 10 | # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF 11 | # ANY KIND, either express or implied. See the License for the specific 12 | # language governing permissions and limitations under the License. 13 | 14 | module AWS 15 | class S3 16 | 17 | # This module contains exception classes for each of the error 18 | # types that S3 can return. You can use these classes to rescue 19 | # specific errors, for example: 20 | # 21 | # begin 22 | # S3.new.buckets.mybucket. 23 | # objects.myobj.write("HELLO") 24 | # rescue S3::Errors::NoSuchBucket => e 25 | # S3.new.buckets.create("mybucket") 26 | # retry 27 | # end 28 | # 29 | # All errors raised as a result of error responses from the 30 | # service are instances of either {ClientError} or {ServerError}. 31 | # @private 32 | module Errors 33 | 34 | # This error is special, because S3 does not (and must not 35 | # according to RFC 2616) return a body with the HTTP response. 36 | # The interface is the same as for any other client error. 37 | class NotModified < AWS::Errors::Base 38 | 39 | include AWS::Errors::ClientError 40 | 41 | def code; "NotModified"; end 42 | 43 | def initialize(req, resp) 44 | super(req, resp, "Not Modified") 45 | end 46 | 47 | end 48 | 49 | # This error is special, because S3 does not return a body with 50 | # the HTTP response. The interface is the same as for any other 51 | # client error. 52 | class NoSuchKey < AWS::Errors::Base 53 | 54 | include AWS::Errors::ClientError 55 | 56 | def code; "NoSuchKey"; end 57 | 58 | def initialize(req, resp) 59 | super(req, resp, "No Such Key") 60 | end 61 | 62 | end 63 | 64 | BASE_ERROR_GRAMMAR = Client::XML::Error 65 | 66 | include Core::LazyErrorClasses 67 | 68 | end 69 | end 70 | end 71 | -------------------------------------------------------------------------------- /lib/aws/s3/multipart_upload_collection.rb: -------------------------------------------------------------------------------- 1 | # Copyright 2011 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). You 4 | # may not use this file except in compliance with the License. A copy of 5 | # the License is located at 6 | # 7 | # http://aws.amazon.com/apache2.0/ 8 | # 9 | # or in the "license" file accompanying this file. This file is 10 | # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF 11 | # ANY KIND, either express or implied. See the License for the specific 12 | # language governing permissions and limitations under the License. 13 | 14 | module AWS 15 | class S3 16 | 17 | # Represents the uploads in progress for a bucket. 18 | # 19 | # @example Finding uploads by prefix 20 | # bucket.multipart_uploads.with_prefix("photos/"). 21 | # map { |upload| upload.object.key } 22 | # # => ["photos/1.jpg", "photos/2.jpg", ...] 23 | # 24 | # @example Browsing with a tree interface 25 | # bucket.multipart_uploads.with_prefix("photos").as_tree. 26 | # children.select(&:branch?).map(&:prefix) 27 | # # => ["photos/2010", "photos/2011", ...] 28 | # 29 | # @see Tree 30 | class MultipartUploadCollection 31 | 32 | include Enumerable 33 | include Core::Model 34 | include PrefixAndDelimiterCollection 35 | 36 | # @return [Bucket] The bucket in which the uploads are taking 37 | # place. 38 | attr_reader :bucket 39 | 40 | # @private 41 | def initialize(bucket, opts = {}) 42 | @bucket = bucket 43 | super 44 | end 45 | 46 | protected 47 | def each_member_in_page(page, &block) 48 | super 49 | page.uploads.each do |u| 50 | object = S3Object.new(bucket, u.key) 51 | upload = MultipartUpload.new(object, u.upload_id) 52 | yield(upload) 53 | end 54 | end 55 | 56 | protected 57 | def list_request(options) 58 | client.list_multipart_uploads(options) 59 | end 60 | 61 | protected 62 | def limit_param; :max_uploads; end 63 | 64 | protected 65 | def pagination_markers; super + [:upload_id_marker]; end 66 | 67 | protected 68 | def page_size(resp); super + resp.uploads.size; end 69 | 70 | end 71 | 72 | end 73 | end 74 | -------------------------------------------------------------------------------- /lib/aws/s3/object_collection.rb: -------------------------------------------------------------------------------- 1 | # Copyright 2011 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). You 4 | # may not use this file except in compliance with the License. A copy of 5 | # the License is located at 6 | # 7 | # http://aws.amazon.com/apache2.0/ 8 | # 9 | # or in the "license" file accompanying this file. This file is 10 | # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF 11 | # ANY KIND, either express or implied. See the License for the specific 12 | # language governing permissions and limitations under the License. 13 | 14 | module AWS 15 | class S3 16 | 17 | # Represents a collection of S3 objects. 18 | # 19 | # == Getting an S3Object by Key 20 | # 21 | # If you know the key of the object you want, you can reference it this way: 22 | # 23 | # # this will not make any requests against S3 24 | # object = bucket.objects['foo.jpg'] 25 | # object.key #=> 'foo.jpg' 26 | # 27 | # == Finding objects with a Prefix 28 | # 29 | # Given a bucket with the following keys: 30 | # 31 | # photos/sunset.jpg 32 | # photos/sunrise.jpg 33 | # photos/winter.jpg 34 | # videos/comedy.mpg 35 | # videos/dancing.mpg 36 | # 37 | # You can list objects that share a prefix: 38 | # 39 | # bucket.objects.with_prefix('videos').collect(&:key) 40 | # #=> ['videos/comedy.mpg', 'videos/dancing.mpg'] 41 | # 42 | # == Exploring Objects with a Tree Interface 43 | # 44 | # Given a bucket with the following keys: 45 | # 46 | # README.txt 47 | # videos/wedding.mpg 48 | # videos/family_reunion.mpg 49 | # photos/2010/house.jpg 50 | # photos/2011/fall/leaves.jpg 51 | # photos/2011/summer/vacation.jpg 52 | # photos/2011/summer/family.jpg 53 | # 54 | # tree = bucket.objects.with_prefix.prefix('photos').as_tree 55 | # 56 | # directories = tree.children.select(&:branch?).collect(&:prefix) 57 | # #=> ['photos/2010', 'photos/2011'] 58 | # 59 | class ObjectCollection 60 | 61 | include Core::Model 62 | include Enumerable 63 | include PrefixAndDelimiterCollection 64 | 65 | # @param [Bucket] The S3 bucket this object collection belongs to. 66 | def initialize(bucket, options = {}) 67 | @bucket = bucket 68 | super 69 | end 70 | 71 | # @return [Bucket] The bucket this collection belongs to. 72 | attr_reader :bucket 73 | 74 | # Writes a new object to S3. 75 | # 76 | # The first param is the key you want to write this object to. 77 | # All other params/options are documented in {S3Object#write}. 78 | # 79 | # @see S3Object#write 80 | # 81 | # @param [String] key Where in S3 to write the object. 82 | # @return [S3Object] 83 | def create key, *args 84 | self[key].write(*args) 85 | end 86 | 87 | # Returns an S3Object given its name. For example: 88 | # 89 | # @example 90 | # 91 | # object = bucket.objects['file.txt'] 92 | # object.class #=> S3Object 93 | # 94 | # @param [String] key The object key. 95 | # @return [S3Object] 96 | def [] key 97 | S3Object.new(bucket, key.to_s) 98 | end 99 | 100 | # (see PrefixedCollection#with_prefix) 101 | def with_prefix prefix, mode = :replace 102 | super(prefix, mode) 103 | end 104 | 105 | # Iterates the collection, yielding instances of S3Object. 106 | # 107 | # Use break or raise an exception to terminate the enumeration. 108 | # 109 | # @param [Hash] options 110 | # @option options [Integer] :limit (nil) The maximum number of 111 | # objects to yield. 112 | # @option options [Integer] :batch_size (1000) The number of objects to 113 | # fetch each request to S3. Maximum is 1000 keys at time. 114 | # @return [nil] 115 | def each options = {}, &block 116 | super 117 | end 118 | 119 | # @private 120 | protected 121 | def each_member_in_page(page, &block) 122 | super 123 | page.contents.each do |content| 124 | yield(S3Object.new(bucket, content.key)) 125 | end 126 | end 127 | 128 | # @private 129 | protected 130 | def list_request(options) 131 | client.list_objects(options) 132 | end 133 | 134 | # @private 135 | protected 136 | def limit_param 137 | :max_keys 138 | end 139 | 140 | # @private 141 | protected 142 | def page_size resp 143 | super + resp.contents.size 144 | end 145 | 146 | # @private 147 | protected 148 | def next_markers page 149 | { :marker => (last = page.contents.last and last.key) } 150 | end 151 | 152 | end 153 | 154 | end 155 | end 156 | -------------------------------------------------------------------------------- /lib/aws/s3/object_metadata.rb: -------------------------------------------------------------------------------- 1 | # Copyright 2011 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). You 4 | # may not use this file except in compliance with the License. A copy of 5 | # the License is located at 6 | # 7 | # http://aws.amazon.com/apache2.0/ 8 | # 9 | # or in the "license" file accompanying this file. This file is 10 | # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF 11 | # ANY KIND, either express or implied. See the License for the specific 12 | # language governing permissions and limitations under the License. 13 | 14 | module AWS 15 | class S3 16 | 17 | # Returns an object that represents the metadata for an S3 object. 18 | class ObjectMetadata 19 | 20 | include Core::Model 21 | 22 | # @param [S3Object] 23 | # @param [Hash] options 24 | # @option options [String] :version_id A specific version of the object 25 | # to get metadata for 26 | def initialize(object, options = {}) 27 | @object = object 28 | @version_id = options[:version_id] 29 | super 30 | end 31 | 32 | # @return [S3Object] 33 | attr_reader :object 34 | 35 | # Returns the value for the given name stored in the S3Object's 36 | # metadata: 37 | # 38 | # bucket.objects['myobject'].metadata['purpose'] 39 | # # returns nil if the given metadata key has not been set 40 | # 41 | # @param [String,Symbol] name The name of the metadata field to 42 | # get. 43 | # 44 | # @return [String,nil] Returns the metadata for the given name. 45 | def [] name 46 | to_h[name.to_s] 47 | end 48 | 49 | # Changes the value of the given name stored in the S3Object's 50 | # metadata: 51 | # 52 | # object = bucket.object['myobject'] 53 | # object.metadata['purpose'] = 'research' 54 | # object.metadata['purpose'] # => 'research' 55 | # 56 | # @note The name and value of each metadata field must conform 57 | # to US-ASCII. 58 | # 59 | # @param [String,Symbol] name The name of the metadata field to 60 | # set. 61 | # 62 | # @param [String] value The new value of the metadata field. 63 | # 64 | # @return [String,nil] Returns the value that was set. 65 | def []= name, value 66 | raise "cannot change the metadata of an object version; "+ 67 | "use S3Object#write to create a new version with different metadata" if 68 | @version_id 69 | metadata = to_h.dup 70 | metadata[name.to_s] = value 71 | object.copy_from(object.key, 72 | :metadata => metadata) 73 | value 74 | end 75 | 76 | # Proxies the method to {#[]}. 77 | # @return (see #[]) 78 | def method_missing name, *args, &blk 79 | return super if !args.empty? || blk 80 | self[name] 81 | end 82 | 83 | # @return [Hash] Returns the user-generated metadata stored with 84 | # this S3 Object. 85 | def to_h 86 | options = {} 87 | options[:bucket_name] = object.bucket.name 88 | options[:key] = object.key 89 | options[:version_id] = @version_id if @version_id 90 | client.head_object(options).meta 91 | end 92 | 93 | end 94 | 95 | end 96 | end 97 | -------------------------------------------------------------------------------- /lib/aws/s3/object_upload_collection.rb: -------------------------------------------------------------------------------- 1 | # Copyright 2011 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). You 4 | # may not use this file except in compliance with the License. A copy of 5 | # the License is located at 6 | # 7 | # http://aws.amazon.com/apache2.0/ 8 | # 9 | # or in the "license" file accompanying this file. This file is 10 | # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF 11 | # ANY KIND, either express or implied. See the License for the specific 12 | # language governing permissions and limitations under the License. 13 | 14 | module AWS 15 | class S3 16 | 17 | # Represents uploads in progress for a single object. 18 | # 19 | # @example Cancel all uploads for an object 20 | # object.multipart_uploads.each(&:abort) 21 | # 22 | # @example Get an upload by ID 23 | # object.multipart_uploads[id] 24 | class ObjectUploadCollection 25 | 26 | include Enumerable 27 | include Core::Model 28 | 29 | # @return [S3Object] The object to which the uploads belong. 30 | attr_reader :object 31 | 32 | # @private 33 | def initialize(object, opts = {}) 34 | @all_uploads = 35 | MultipartUploadCollection.new(object.bucket). 36 | with_prefix(object.key) 37 | @object = object 38 | super 39 | end 40 | 41 | # Creates a new multipart upload. It is usually more 42 | # convenient to use {S3Object#multipart_upload}. 43 | def create(options = {}) 44 | options[:storage_class] = :reduced_redundancy if 45 | options.delete(:reduced_redundancy) 46 | initiate_opts = { 47 | :bucket_name => object.bucket.name, 48 | :key => object.key 49 | }.merge(options) 50 | id = client.initiate_multipart_upload(initiate_opts).upload_id 51 | MultipartUpload.new(object, id) 52 | end 53 | 54 | # Iterates the uploads in the collection. 55 | # 56 | # @yieldparam [MultipartUpload] upload An upload in the 57 | # collection. 58 | # @return [nil] 59 | def each(options = {}, &block) 60 | @all_uploads.each(options) do |upload| 61 | yield(upload) if upload.object.key == @object.key 62 | end 63 | nil 64 | end 65 | 66 | # @return [MultipartUpload] An object representing the upload 67 | # with the given ID. 68 | # 69 | # @param [String] id The ID of an upload to get. 70 | def [] id 71 | MultipartUpload.new(object, id) 72 | end 73 | 74 | end 75 | 76 | end 77 | end 78 | -------------------------------------------------------------------------------- /lib/aws/s3/object_version.rb: -------------------------------------------------------------------------------- 1 | # Copyright 2011 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). You 4 | # may not use this file except in compliance with the License. A copy of 5 | # the License is located at 6 | # 7 | # http://aws.amazon.com/apache2.0/ 8 | # 9 | # or in the "license" file accompanying this file. This file is 10 | # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF 11 | # ANY KIND, either express or implied. See the License for the specific 12 | # language governing permissions and limitations under the License. 13 | 14 | module AWS 15 | class S3 16 | 17 | # Represents a single version of an S3Object. 18 | # 19 | # When you enable versioning on a S3 bucket, writing to an object 20 | # will create an object version instead of replacing the existing 21 | # object. 22 | class ObjectVersion 23 | 24 | include Core::Model 25 | 26 | # @param [S3Object] object The object this is a version of. 27 | # @param [String] version_id The unique id for this version. 28 | # @param [Hash] options 29 | # @option options [Boolean] :delete_marker Is this version a 30 | # delete marker? 31 | def initialize(object, version_id, options = {}) 32 | @object = object 33 | @version_id = version_id 34 | @delete_marker = options[:delete_marker] 35 | super 36 | end 37 | 38 | # @return [S3Object] the object this is a version of. 39 | attr_reader :object 40 | 41 | def bucket 42 | object.bucket 43 | end 44 | 45 | # @return [String] The unique version identifier. 46 | attr_reader :version_id 47 | 48 | # @return (see S3Object#key) 49 | def key 50 | object.key 51 | end 52 | 53 | # @see S3Object#head 54 | # @return (see S3Object#head) 55 | def head 56 | object.head(:version_id => @version_id) 57 | end 58 | 59 | # @see S3Object#etag 60 | # @return (see S3Object#etag) 61 | def etag 62 | head.etag 63 | end 64 | 65 | # @return (see S3Object#content_length) 66 | def content_length 67 | head.content_length 68 | end 69 | 70 | # @note (see S3Object#content_type) 71 | # @see S3Object#content_type 72 | # @return (see S3Object#content_type) 73 | def content_type 74 | head.content_type 75 | end 76 | 77 | # @see S3Object#metadata 78 | # @return (see S3Object#metadata) 79 | def metadata 80 | object.metadata(:version_id => @version_id) 81 | end 82 | 83 | # Reads the data from this object version. 84 | # @see S3Object#read 85 | # @options (see S3Object#read) 86 | # @return (see S3Object#read) 87 | def read options = {}, &block 88 | object.read(options.merge(:version_id => @version_id), &block) 89 | end 90 | 91 | # Deletes this object version from S3. 92 | # @return (see S3Object#delete) 93 | def delete 94 | object.delete(:version_id => @version_id) 95 | end 96 | 97 | # @return [Boolean] Returns this if this is the latest version of 98 | # the object, false if the object has been written to since 99 | # this version was created. 100 | def latest? 101 | object.versions.latest.version_id == self.version_id 102 | end 103 | 104 | # If you delete an object in a versioned bucket, a delete marker 105 | # is created. 106 | # @return [Boolean] Returns true if this version is a delete marker. 107 | def delete_marker? 108 | if @delete_marker.nil? 109 | begin 110 | # S3 responds with a 405 (method not allowed) when you try 111 | # to HEAD an s3 object version that is a delete marker 112 | metadata['foo'] 113 | @delete_marker = false 114 | rescue Errors::MethodNotAllowed => error 115 | @delete_marker = true 116 | end 117 | end 118 | @delete_marker 119 | end 120 | 121 | # @return [Boolean] Returns true if the other object version has 122 | # the same s3 object key and version id. 123 | def ==(other) 124 | other.kind_of?(ObjectVersion) and 125 | other.object == object and 126 | other.version_id == version_id 127 | end 128 | 129 | alias_method :eql?, :== 130 | 131 | # @private 132 | def inspect 133 | "<#{self.class}:#{object.bucket.name}:#{object.key}:#{version_id}>" 134 | end 135 | 136 | end 137 | end 138 | end 139 | -------------------------------------------------------------------------------- /lib/aws/s3/object_version_collection.rb: -------------------------------------------------------------------------------- 1 | # Copyright 2011 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). You 4 | # may not use this file except in compliance with the License. A copy of 5 | # the License is located at 6 | # 7 | # http://aws.amazon.com/apache2.0/ 8 | # 9 | # or in the "license" file accompanying this file. This file is 10 | # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF 11 | # ANY KIND, either express or implied. See the License for the specific 12 | # language governing permissions and limitations under the License. 13 | 14 | module AWS 15 | class S3 16 | 17 | # For S3 buckets with versioning enabled, objects will store versions 18 | # each time you write to them. 19 | # 20 | # object = bucket.objects['myobj'] 21 | # object.write('1') 22 | # object.write('2') 23 | # object.write('3') 24 | # 25 | # object.versions.collect(&:read) 26 | # #=> ['1', '2', '3'] 27 | # 28 | # If you know the id of a particular version you can get that object. 29 | # 30 | # bucket.objets['myobj'].version[version_id].delete 31 | # 32 | class ObjectVersionCollection 33 | 34 | include Core::Model 35 | include Enumerable 36 | 37 | # @return [S3Object] The object this collection belongs to. 38 | attr_reader :object 39 | 40 | # @param [S3Object] object 41 | def initialize object, options = {} 42 | @object = object 43 | super(options) 44 | end 45 | 46 | # Returns an object that represents a single version of the {#object}. 47 | # @param [String] version_id 48 | # @return [ObjectVersion] 49 | def [] version_id 50 | ObjectVersion.new(object, version_id) 51 | end 52 | 53 | # @note Generally you will just want to grab the object key its key. 54 | # @return [ObjectVersion] Returns the latest version of this object. 55 | def latest 56 | self.find{|version| true } 57 | end 58 | 59 | # Yields once for each version of the {#object}. 60 | # 61 | # @yield [object_version] 62 | # @yieldparam [ObectVersion] object_version 63 | # @return [nil] 64 | def each &block 65 | object.bucket.versions.with_prefix(object.key).each do |version| 66 | if version.key == object.key 67 | yield(version) 68 | end 69 | end 70 | nil 71 | end 72 | 73 | end 74 | end 75 | end 76 | -------------------------------------------------------------------------------- /lib/aws/s3/paginated_collection.rb: -------------------------------------------------------------------------------- 1 | # Copyright 2011 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). You 4 | # may not use this file except in compliance with the License. A copy of 5 | # the License is located at 6 | # 7 | # http://aws.amazon.com/apache2.0/ 8 | # 9 | # or in the "license" file accompanying this file. This file is 10 | # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF 11 | # ANY KIND, either express or implied. See the License for the specific 12 | # language governing permissions and limitations under the License. 13 | 14 | module AWS 15 | class S3 16 | 17 | # @private 18 | module PaginatedCollection 19 | 20 | def each(options = {}, &block) 21 | each_page(options) do |page| 22 | each_member_in_page(page, &block) 23 | end 24 | nil 25 | end 26 | 27 | protected 28 | def each_member_in_page(page, &block); end 29 | 30 | protected 31 | def each_page(options = {}, &block) 32 | opts = list_options(options) 33 | limit = options[:limit] 34 | batch_size = options[:batch_size] || 1000 35 | markers = {} 36 | received = 0 37 | 38 | loop do 39 | page_opts = opts.dup 40 | page_opts.merge!(markers) 41 | page_opts[limit_param] = 42 | limit ? [limit - received, batch_size].min : batch_size 43 | 44 | page = list_request(page_opts) 45 | markers = next_markers(page) 46 | received += page_size(page) 47 | 48 | yield(page) 49 | 50 | return unless page.truncated? 51 | end 52 | end 53 | 54 | protected 55 | def list_request(options) 56 | raise NotImplementedError 57 | end 58 | 59 | protected 60 | def list_options(options) 61 | opts = {} 62 | opts[:bucket_name] = bucket.name if respond_to?(:bucket) 63 | opts 64 | end 65 | 66 | protected 67 | def limit_param 68 | raise NotImplementedError 69 | end 70 | 71 | protected 72 | def pagination_markers 73 | [:key_marker] 74 | end 75 | 76 | protected 77 | def next_markers(page) 78 | pagination_markers.inject({}) do |markers, marker| 79 | markers[marker] = page.send("next_#{marker}") 80 | markers 81 | end 82 | end 83 | 84 | protected 85 | def page_size(resp) 86 | (resp.respond_to?(:common_prefixes) and 87 | prefixes = resp.common_prefixes and 88 | prefixes.size) or 0 89 | end 90 | 91 | end 92 | 93 | end 94 | end 95 | -------------------------------------------------------------------------------- /lib/aws/s3/policy.rb: -------------------------------------------------------------------------------- 1 | # Copyright 2011 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). You 4 | # may not use this file except in compliance with the License. A copy of 5 | # the License is located at 6 | # 7 | # http://aws.amazon.com/apache2.0/ 8 | # 9 | # or in the "license" file accompanying this file. This file is 10 | # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF 11 | # ANY KIND, either express or implied. See the License for the specific 12 | # language governing permissions and limitations under the License. 13 | 14 | module AWS 15 | class S3 16 | 17 | # @see Core::Policy 18 | class Policy < Core::Policy 19 | 20 | class Statement < Core::Policy::Statement 21 | 22 | ACTION_MAPPING = { 23 | :list_buckets => "s3:ListAllMyBuckets", 24 | :create_bucket => "s3:CreateBucket", 25 | :delete_bucket => "s3:DeleteBucket", 26 | :list_objects => "s3:ListBucket", 27 | :list_object_versions => "s3:ListBucketVersions", 28 | :list_multipart_uploads => "s3:ListBucketMultipartUploads", 29 | :get_object => "s3:GetObject", 30 | :get_object_version => "s3:GetObjectVersion", 31 | :put_object => "s3:PutObject", 32 | :get_object_acl => "s3:GetObjectAcl", 33 | :get_object_version_acl => "s3:GetObjectVersionAcl", 34 | :set_object_acl => "s3:PutObjectAcl", 35 | :set_object_acl_version => "s3:PutObjectAclVersion", 36 | :delete_object => "s3:DeleteObject", 37 | :delete_object_version => "s3:DeleteObjectVersion", 38 | :list_multipart_upload_parts => "s3:ListMultipartUploadParts", 39 | :abort_multipart_upload => "s3:AbortMultipartUpload", 40 | :get_bucket_acl => "s3:GetBucketAcl", 41 | :set_bucket_acl => "s3:PutBucketAcl", 42 | :get_bucket_versioning => "s3:GetBucketVersioning", 43 | :set_bucket_versioning => "s3:PutBucketVersioning", 44 | :get_bucket_requester_pays => "s3:GetBucketRequesterPays", 45 | :set_bucket_requester_pays => "s3:PutBucketRequesterPays", 46 | :get_bucket_location => "s3:GetBucketLocation", 47 | :get_bucket_policy => "s3:GetBucketPolicy", 48 | :set_bucket_policy => "s3:PutBucketPolicy", 49 | :get_bucket_notification => "s3:GetBucketNotification", 50 | :set_bucket_notification => "s3:PutBucketNotification" 51 | } 52 | 53 | protected 54 | def resource_arn resource 55 | prefix = 'arn:aws:s3:::' 56 | case resource 57 | when Bucket 58 | "#{prefix}#{resource.name}" 59 | when S3Object 60 | "#{prefix}#{resource.bucket.name}/#{resource.key}" 61 | when ObjectCollection 62 | "#{prefix}#{resource.bucket.name}/#{resource.prefix}*" 63 | when /^arn:/ 64 | resource 65 | else 66 | "arn:aws:s3:::#{resource}" 67 | end 68 | end 69 | 70 | end 71 | 72 | end 73 | end 74 | end 75 | -------------------------------------------------------------------------------- /lib/aws/s3/prefix_and_delimiter_collection.rb: -------------------------------------------------------------------------------- 1 | # Copyright 2011 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). You 4 | # may not use this file except in compliance with the License. A copy of 5 | # the License is located at 6 | # 7 | # http://aws.amazon.com/apache2.0/ 8 | # 9 | # or in the "license" file accompanying this file. This file is 10 | # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF 11 | # ANY KIND, either express or implied. See the License for the specific 12 | # language governing permissions and limitations under the License. 13 | 14 | module AWS 15 | class S3 16 | 17 | # @private 18 | module PrefixAndDelimiterCollection 19 | 20 | include PrefixedCollection 21 | 22 | def each(options = {}, &block) 23 | each_page(options) do |page| 24 | each_member_in_page(page, &block) 25 | end 26 | nil 27 | end 28 | 29 | # @see Bucket#as_tree 30 | def as_tree options = {} 31 | Tree.new(self, { :prefix => prefix }.merge(options)) 32 | end 33 | 34 | # @private 35 | protected 36 | def each_member_in_page(page, &block) 37 | super 38 | page.common_prefixes.each do |p| 39 | yield(with_prefix(p)) 40 | end 41 | end 42 | 43 | # @private 44 | protected 45 | def list_options(options) 46 | opts = super 47 | opts[:delimiter] = options[:delimiter] if options.key?(:delimiter) 48 | opts 49 | end 50 | 51 | end 52 | 53 | end 54 | end 55 | -------------------------------------------------------------------------------- /lib/aws/s3/prefixed_collection.rb: -------------------------------------------------------------------------------- 1 | # Copyright 2011 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). You 4 | # may not use this file except in compliance with the License. A copy of 5 | # the License is located at 6 | # 7 | # http://aws.amazon.com/apache2.0/ 8 | # 9 | # or in the "license" file accompanying this file. This file is 10 | # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF 11 | # ANY KIND, either express or implied. See the License for the specific 12 | # language governing permissions and limitations under the License. 13 | 14 | module AWS 15 | class S3 16 | 17 | module PrefixedCollection 18 | 19 | include PaginatedCollection 20 | 21 | # @private 22 | def initialize *args 23 | options = args.last.is_a?(Hash) ? args.pop : {} 24 | @prefix = options[:prefix] 25 | args.push(options) 26 | super(*args) 27 | end 28 | 29 | # @return [String,nil] The prefix of this collection. 30 | attr_reader :prefix 31 | 32 | # Returns a new collection with a different prefix 33 | # 34 | # @example 35 | # objects = collection.with_prefix('photos') 36 | # objects.prefix #=> 'photos' 37 | # 38 | # @example Chaining with_prefix replaces previous prefix 39 | # objects = collection.with_prefix('photos').with_prefix('videos') 40 | # objects.prefix #=> 'videos' 41 | # 42 | # @example Chaining with_prefix with :append 43 | # objects = collection.with_prefix('a/').with_prefix('b/', :append) 44 | # objects.prefix #=> 'a/b/' 45 | # 46 | # @example Chaining with_prefix with :prepend 47 | # objects = collection.with_prefix('a/').with_prefix('b/', :prepend) 48 | # objects.prefix #=> 'b/a/' 49 | # 50 | # @param [String] prefix The prefix condition that limits what objects 51 | # are returned by this collection. 52 | # @param [Symbol] mode (:replace) If you chain calls to #with_prefix 53 | # the +mode+ affects if the prefix prepends, appends, or replaces. 54 | # Valid modes are: 55 | # * +:replace+ 56 | # * +:append+ 57 | # * +:prepend+ 58 | # @return [Collection] Returns a new collection with a modified prefix. 59 | def with_prefix prefix, mode = :replace 60 | new_prefix = case mode 61 | when :replace then prefix 62 | when :append then "#{@prefix}#{prefix}" 63 | when :prepend then "#{prefix}#{@prefix}" 64 | else 65 | raise ArgumentError, "invalid prefix mode `#{mode}`, it must be " + 66 | ":replace, :append or :prepend" 67 | end 68 | self.class.new(bucket, :prefix => new_prefix) 69 | end 70 | 71 | # @private 72 | protected 73 | def list_options(options) 74 | opts = super 75 | opts[:prefix] = prefix if prefix 76 | opts 77 | end 78 | 79 | end 80 | end 81 | end 82 | -------------------------------------------------------------------------------- /lib/aws/s3/request.rb: -------------------------------------------------------------------------------- 1 | # Copyright 2011 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). You 4 | # may not use this file except in compliance with the License. A copy of 5 | # the License is located at 6 | # 7 | # http://aws.amazon.com/apache2.0/ 8 | # 9 | # or in the "license" file accompanying this file. This file is 10 | # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF 11 | # ANY KIND, either express or implied. See the License for the specific 12 | # language governing permissions and limitations under the License. 13 | 14 | require 'uri' 15 | require 'time' 16 | 17 | module AWS 18 | class S3 19 | 20 | # @private 21 | class Request < Core::Http::Request 22 | 23 | include Core::UriEscape 24 | 25 | # @param [bucket] S3 bucket name 26 | attr_accessor :bucket 27 | 28 | # @param [String] S3 object key 29 | attr_accessor :key 30 | 31 | attr_accessor :body_stream 32 | 33 | def metadata= metadata 34 | Array(metadata).each do |name, value| 35 | headers["x-amz-meta-#{name}"] = value 36 | end 37 | end 38 | 39 | def canned_acl= acl 40 | if acl.kind_of?(Symbol) 41 | headers["x-amz-acl"] = acl.to_s.gsub("_", "-") 42 | elsif acl 43 | headers["x-amz-acl"] = acl 44 | end 45 | end 46 | 47 | def storage_class= storage_class 48 | if storage_class.kind_of?(Symbol) 49 | headers["x-amz-storage-class"] = storage_class.to_s.upcase 50 | elsif storage_class 51 | headers["x-amz-storage-class"] = storage_class 52 | end 53 | end 54 | 55 | def host 56 | Client.path_style_bucket_name?(bucket) ? @host : "#{bucket}.#{@host}" 57 | end 58 | 59 | def path 60 | parts = [] 61 | parts << bucket if bucket and Client.path_style_bucket_name?(bucket) 62 | parts << escape_path(key) if key 63 | "/#{parts.join('/')}" 64 | end 65 | 66 | def querystring 67 | url_encoded_params 68 | end 69 | 70 | # @param [String, IO] The http request body. This can be a string or 71 | # any object that responds to #read and #eof? (like an IO object). 72 | def body= body 73 | @body_stream = StringIO.new(body) 74 | end 75 | 76 | # @return [String, nil] The http request body. 77 | def body 78 | if @body_stream 79 | string = @body_stream.read 80 | @body_stream.rewind 81 | string 82 | else 83 | nil 84 | end 85 | end 86 | 87 | # From the S3 developer guide: 88 | # 89 | # StringToSign = 90 | # HTTP-Verb + "\n" + 91 | # content-md5 + "\n" + 92 | # content-type + "\n" + 93 | # date + "\n" + 94 | # CanonicalizedAmzHeaders + CanonicalizedResource; 95 | # 96 | def string_to_sign 97 | [ 98 | http_method, 99 | headers.values_at('content-md5', 'content-type').join("\n"), 100 | signing_string_date, 101 | canonicalized_headers, 102 | canonicalized_resource, 103 | ].flatten.compact.join("\n") 104 | end 105 | 106 | def signing_string_date 107 | # if a date is provided via x-amz-date then we should omit the 108 | # Date header from the signing string (should appear as a blank line) 109 | if headers.detect{|k,v| k.to_s =~ /^x-amz-date$/i } 110 | '' 111 | else 112 | headers['date'] ||= Time.now.rfc822 113 | end 114 | end 115 | 116 | # From the S3 developer guide 117 | # 118 | # CanonicalizedResource = 119 | # [ "/" + Bucket ] + 120 | # + 121 | # [ sub-resource, if present. e.g. "?acl", "?location", 122 | # "?logging", or "?torrent"]; 123 | # 124 | def canonicalized_resource 125 | 126 | parts = [] 127 | 128 | # virtual hosted-style requests require the hostname to appear 129 | # in the canonicalized resource prefixed by a forward slash. 130 | if 131 | Client.dns_compatible_bucket_name?(bucket) and 132 | !Client.path_style_bucket_name?(bucket) 133 | then 134 | parts << "/#{bucket}" 135 | end 136 | 137 | # all requests require the portion of the un-decoded uri up to 138 | # but not including the query string 139 | parts << path 140 | 141 | # lastly any sub resource querystring params need to be appened 142 | # in lexigraphical ordered joined by '&' and prefixed by '?' 143 | params = (sub_resource_params + query_parameters_for_signature) 144 | unless params.empty? 145 | parts << '?' 146 | parts << params.sort.collect{|p| p.to_s }.join('&') 147 | end 148 | 149 | parts.join 150 | end 151 | 152 | # CanonicalizedAmzHeaders 153 | # 154 | # See the developer guide for more information on how this element 155 | # is generated. 156 | # 157 | def canonicalized_headers 158 | x_amz = headers.select{|name, value| name.to_s =~ /^x-amz-/i } 159 | x_amz = x_amz.collect{|name, value| [name.downcase, value] } 160 | x_amz = x_amz.sort_by{|name, value| name } 161 | x_amz = x_amz.collect{|name, value| "#{name}:#{value}" }.join("\n") 162 | x_amz == '' ? nil : x_amz 163 | end 164 | 165 | def sub_resource_params 166 | params.select{|p| self.class.sub_resources.include?(p.name) } 167 | end 168 | 169 | def query_parameters_for_signature 170 | params.select { |p| self.class.query_parameters.include?(p.name) } 171 | end 172 | 173 | def add_authorization!(signer) 174 | if signer.respond_to?(:session_token) and 175 | token = signer.session_token 176 | headers["x-amz-security-token"] = token 177 | end 178 | signature = URI.escape(signer.sign(string_to_sign, 'sha1')) 179 | headers["authorization"] = "AWS #{signer.access_key_id}:#{signature}" 180 | end 181 | 182 | class << self 183 | 184 | def sub_resources 185 | %w(acl location logging notification partNumber policy 186 | requestPayment torrent uploadId uploads versionId 187 | versioning versions) 188 | end 189 | 190 | def query_parameters 191 | %w(response-content-type response-content-language 192 | response-expires response-cache-control 193 | response-content-disposition response-content-encoding) 194 | end 195 | 196 | end 197 | 198 | end 199 | end 200 | end 201 | -------------------------------------------------------------------------------- /lib/aws/s3/tree.rb: -------------------------------------------------------------------------------- 1 | # Copyright 2011 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). You 4 | # may not use this file except in compliance with the License. A copy of 5 | # the License is located at 6 | # 7 | # http://aws.amazon.com/apache2.0/ 8 | # 9 | # or in the "license" file accompanying this file. This file is 10 | # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF 11 | # ANY KIND, either express or implied. See the License for the specific 12 | # language governing permissions and limitations under the License. 13 | 14 | module AWS 15 | class S3 16 | 17 | # A utility class that supports exploring an S3 {Bucket} like a 18 | # tree. 19 | # 20 | # Frequently objects stored in S3 have keys that look like a filesystem 21 | # directory structure. 22 | # 23 | # Given you have a bucket with the following keys: 24 | # 25 | # README.txt 26 | # videos/wedding.mpg 27 | # videos/family_reunion.mpg 28 | # photos/2010/house.jpg 29 | # photos/2011/fall/leaves.jpg 30 | # photos/2011/summer/vacation.jpg 31 | # photos/2011/summer/family.jpg 32 | # 33 | # You might like to explore the contents of this bucket as a tree: 34 | # 35 | # tree = bucket.as_tree 36 | # 37 | # directories = tree.children.select(&:branch?).collect(&:prefix) 38 | # #=> ['photos', 'videos'] 39 | # 40 | # files = tree.children.select(&:leaf?).collect(&:key) 41 | # #=> ['README.txt'] 42 | # 43 | # If you want to start further down, pass a prefix to {Bucket#as_tree}: 44 | # 45 | # tree = bucket.as_tree(:prefix => 'photos/2011') 46 | # 47 | # directories = tree.children.select(&:branch?).collect(&:prefix) 48 | # #=> ['photos/20011/fall', 'photos/20011/summer'] 49 | # 50 | # files = tree.children.select(&:leaf?).collect(&:key) 51 | # #=> [] 52 | # 53 | # All non-leaf nodes ({Tree} and {Tree::BranchNode} instances) 54 | # have a {Tree::Parent#children} method that provides access to 55 | # the next level of the tree, and all nodes ({Tree}, 56 | # {Tree::BranchNode}, and {Tree::LeafNode}) have a {#parent} 57 | # method that returns the parent node. In our examples above, the 58 | # non-leaf nodes are common prefixes to multiple keys 59 | # (directories) and leaf nodes are object keys. 60 | # 61 | # You can continue crawling the tree using the +children+ 62 | # collection on each branch node, which will contain the branch 63 | # nodes and leaf nodes below it. 64 | # 65 | # You can construct a Tree object using the +as_tree+ method of 66 | # any of the following classes: 67 | # 68 | # * {Bucket} or {ObjectCollection} (for {S3Object} leaf nodes) 69 | # 70 | # * {BucketVersionCollection} (for {ObjectVersion} leaf nodes) 71 | # 72 | # * {MultipartUploadCollection} (for {MultipartUpload} leaf nodes) 73 | # 74 | # The methods to explore the tree are the same for each kind of 75 | # leaf node, but {Tree::LeafNode#member} will return a different 76 | # type of object depending on which kind of collection the tree is 77 | # using. 78 | class Tree 79 | 80 | AWS.register_autoloads(self) do 81 | autoload :BranchNode, 'branch_node' 82 | autoload :ChildCollection, 'child_collection' 83 | autoload :LeafNode, 'leaf_node' 84 | autoload :Node, 'node' 85 | autoload :Parent, 'parent' 86 | end 87 | 88 | include Parent 89 | 90 | # @param [ObjectCollection, BucketVersionCollection, 91 | # MultipartUploadCollection] collection The collection whose 92 | # members will be explored using the tree. 93 | # 94 | # @param [Hash] options Additional options for constructing the 95 | # tree. 96 | # 97 | # @option options [String] :prefix (nil) Set prefix to choose 98 | # where the top of the tree will be. A value of +nil+ means 99 | # that the tree will include all objects in the collection. 100 | # 101 | # @option options [String] :delimiter ('/') The string that 102 | # separates each level of the tree. This is usually a 103 | # directory separator. 104 | # 105 | # @option options [Boolean] :append (true) If true, the delimiter is 106 | # appended to the prefix when the prefix does not already end 107 | # with the delimiter. 108 | def initialize collection, options = {} 109 | super 110 | end 111 | 112 | # @return The parent node in the tree. In the case of a Tree, 113 | # the parent is always nil. 114 | def parent; nil; end 115 | 116 | end 117 | end 118 | end 119 | -------------------------------------------------------------------------------- /lib/aws/s3/tree/branch_node.rb: -------------------------------------------------------------------------------- 1 | # Copyright 2011 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). You 4 | # may not use this file except in compliance with the License. A copy of 5 | # the License is located at 6 | # 7 | # http://aws.amazon.com/apache2.0/ 8 | # 9 | # or in the "license" file accompanying this file. This file is 10 | # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF 11 | # ANY KIND, either express or implied. See the License for the specific 12 | # language governing permissions and limitations under the License. 13 | 14 | module AWS 15 | class S3 16 | class Tree 17 | 18 | # Represents a branch in an {S3::Tree}. From a branch node you 19 | # can descend deeper into the tree using {Parent#children} or go 20 | # back to the parent node using {#parent}. 21 | # 22 | # When enumerating nodes in an S3 tree keys grouped by a common 23 | # prefix are represented as a branch node. 24 | # 25 | # Branch nodes are often treated like directories. 26 | # 27 | # @see Tree 28 | # @note Generally you do not need to create branch nodes. 29 | class BranchNode < Node 30 | 31 | include Parent 32 | 33 | # @private 34 | def initialize parent, collection, options = {} 35 | @parent = parent 36 | super(collection, 37 | options.merge(:prefix => collection.prefix)) 38 | end 39 | 40 | # @return [Tree, BranchNode] The parent node in the tree. 41 | attr_reader :parent 42 | 43 | # @return [true] 44 | def branch? 45 | true 46 | end 47 | 48 | # @return [false] 49 | def leaf? 50 | false 51 | end 52 | 53 | # Returns a new Tree object that starts at this branch node. 54 | # The returned tree will have the same prefix, delimiter and 55 | # append mode as the tree the branch belongs to. 56 | # 57 | # @return [Tree] 58 | def as_tree 59 | Tree.new(collection, 60 | :prefix => prefix, 61 | :delimiter => delimiter, 62 | :append => append?) 63 | end 64 | 65 | end 66 | end 67 | end 68 | end 69 | -------------------------------------------------------------------------------- /lib/aws/s3/tree/child_collection.rb: -------------------------------------------------------------------------------- 1 | # Copyright 2011 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). You 4 | # may not use this file except in compliance with the License. A copy of 5 | # the License is located at 6 | # 7 | # http://aws.amazon.com/apache2.0/ 8 | # 9 | # or in the "license" file accompanying this file. This file is 10 | # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF 11 | # ANY KIND, either express or implied. See the License for the specific 12 | # language governing permissions and limitations under the License. 13 | 14 | module AWS 15 | class S3 16 | class Tree 17 | 18 | class ChildCollection 19 | 20 | include Core::Model 21 | include Enumerable 22 | 23 | # @private 24 | def initialize parent, collection, options = {} 25 | 26 | options = { 27 | :prefix => nil, 28 | :delimiter => '/', 29 | :append => true, 30 | }.merge(options) 31 | 32 | @parent = parent 33 | @collection = collection 34 | @prefix = options[:prefix] 35 | @delimiter = options[:delimiter] 36 | @append = options[:append] 37 | 38 | super 39 | 40 | end 41 | 42 | # @return [Tree, BranchNode] The parent node in the tree. 43 | attr_reader :parent 44 | 45 | # @return [ObjectCollection, ObjectVersionCollection, 46 | # MultipartUploadCollection] Returns the collection this 47 | # tree is based on. 48 | attr_reader :collection 49 | 50 | # A tree may have a prefix of where in the bucket to be based from. 51 | # @return [String,nil] 52 | attr_reader :prefix 53 | 54 | # When looking at S3 keys as a tree, the delimiter defines what 55 | # string pattern seperates each level of the tree. The delimiter 56 | # defaults to '/' (like in a file system). 57 | # @return [String] 58 | attr_reader :delimiter 59 | 60 | # @return [Boolean] Returns true if the tree is set to auto-append 61 | # the delimiter to the prefix when the prefix does not end with 62 | # the delimiter. 63 | def append? 64 | @append 65 | end 66 | 67 | # Yields up branches and leaves. 68 | # 69 | # A branch node represents a common prefix (like a directory) 70 | # and a leaf node represents a key (S3 object). 71 | # 72 | # @yield [tree_node] Yields up a mixture of branches and leafs. 73 | # @yieldparam [BranchNode,LeafNode] tree_node A branch or a leaf. 74 | # @return [nil] 75 | def each &block 76 | collection = self.collection 77 | if prefix = prefix_with_delim 78 | collection = collection.with_prefix(prefix) 79 | end 80 | collection.each(:delimiter => delimiter) do |member| 81 | case 82 | when member.respond_to?(:key) 83 | yield LeafNode.new(parent, member) 84 | when member.respond_to?(:prefix) 85 | yield BranchNode.new(parent, member, 86 | :delimiter => delimiter, 87 | :append => append?) 88 | end 89 | end 90 | nil 91 | end 92 | 93 | protected 94 | def prefix_with_delim 95 | return prefix unless append? 96 | return nil if prefix.nil? 97 | prefix =~ /#{delimiter}$/ ? prefix : "#{prefix}#{delimiter}" 98 | end 99 | 100 | end 101 | 102 | end 103 | end 104 | end 105 | -------------------------------------------------------------------------------- /lib/aws/s3/tree/leaf_node.rb: -------------------------------------------------------------------------------- 1 | # Copyright 2011 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). You 4 | # may not use this file except in compliance with the License. A copy of 5 | # the License is located at 6 | # 7 | # http://aws.amazon.com/apache2.0/ 8 | # 9 | # or in the "license" file accompanying this file. This file is 10 | # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF 11 | # ANY KIND, either express or implied. See the License for the specific 12 | # language governing permissions and limitations under the License. 13 | 14 | module AWS 15 | class S3 16 | class Tree 17 | 18 | # Represents a leaf in an {S3::Tree}. 19 | # 20 | # When enumerating nodes in an S3 tree, keys are yielded 21 | # as leaf nodes (they have no children beneath them). 22 | # 23 | # @see Tree 24 | # @note Generally you do not need to create leaf nodes 25 | class LeafNode < Node 26 | 27 | # @private 28 | def initialize parent, member 29 | @parent = parent 30 | @member = member 31 | super() 32 | end 33 | 34 | # @return [Tree, BranchNode] The parent node in the tree. 35 | attr_reader :parent 36 | 37 | # @return [mixed] Returns the object this leaf node represents. 38 | # @see #object 39 | # @see #version 40 | # @see #upload 41 | attr_reader :member 42 | 43 | # @return [String] the key this leaf node represents. 44 | def key 45 | @member.key 46 | end 47 | 48 | # @return [false] 49 | def branch? 50 | false 51 | end 52 | 53 | # @return [true] 54 | def leaf? 55 | true 56 | end 57 | 58 | # @return [S3Object] The object this leaf node represents. 59 | def object 60 | if @member.kind_of?(S3Object) 61 | @member 62 | else 63 | @member.object 64 | end 65 | end 66 | 67 | # @return [ObjectVersion] Returns the object version this leaf 68 | # node represents. 69 | def version 70 | if @member.kind_of?(ObjectVersion) 71 | @member 72 | else 73 | raise "This leaf does not represent a version" 74 | end 75 | end 76 | 77 | # @return [MultipartUpload] Returns the object version this leaf 78 | # node represents. 79 | def upload 80 | if @member.kind_of?(MultipartUpload) 81 | @member 82 | else 83 | raise "This leaf does not represent an upload" 84 | end 85 | end 86 | 87 | def inspect 88 | "<#{self.class}:#{@member.bucket.name}:#{key}>" 89 | end 90 | 91 | end 92 | end 93 | end 94 | end 95 | -------------------------------------------------------------------------------- /lib/aws/s3/tree/node.rb: -------------------------------------------------------------------------------- 1 | # Copyright 2011 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). You 4 | # may not use this file except in compliance with the License. A copy of 5 | # the License is located at 6 | # 7 | # http://aws.amazon.com/apache2.0/ 8 | # 9 | # or in the "license" file accompanying this file. This file is 10 | # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF 11 | # ANY KIND, either express or implied. See the License for the specific 12 | # language governing permissions and limitations under the License. 13 | 14 | module AWS 15 | class S3 16 | class Tree 17 | # @private 18 | class Node 19 | end 20 | end 21 | end 22 | end 23 | -------------------------------------------------------------------------------- /lib/aws/s3/tree/parent.rb: -------------------------------------------------------------------------------- 1 | # Copyright 2011 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). You 4 | # may not use this file except in compliance with the License. A copy of 5 | # the License is located at 6 | # 7 | # http://aws.amazon.com/apache2.0/ 8 | # 9 | # or in the "license" file accompanying this file. This file is 10 | # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF 11 | # ANY KIND, either express or implied. See the License for the specific 12 | # language governing permissions and limitations under the License. 13 | 14 | module AWS 15 | class S3 16 | 17 | class Tree 18 | 19 | # Common methods for tree nodes that are parents to other nodes 20 | # ({Tree} and {BranchNode}). 21 | module Parent 22 | 23 | include Core::Model 24 | 25 | # @private 26 | def initialize collection, options = {} 27 | 28 | options = { 29 | :prefix => nil, 30 | :delimiter => '/', 31 | :append => true, 32 | }.merge(options) 33 | 34 | @collection = collection 35 | @prefix = options[:prefix] 36 | @delimiter = options[:delimiter] 37 | @append = options[:append] 38 | 39 | super 40 | 41 | end 42 | 43 | # @return [ObjectCollection, BucketVersionCollection, 44 | # MultipartUploadCollection] The collection whose members 45 | # will be explored using the tree. 46 | attr_reader :collection 47 | 48 | # A tree may have a prefix of where in the bucket to be based 49 | # from. A value of +nil+ means that the tree will include all 50 | # objects in the collection. 51 | # 52 | # @return [String,nil] 53 | attr_reader :prefix 54 | 55 | # When looking at S3 keys as a tree, the delimiter defines what 56 | # string pattern seperates each level of the tree. The delimiter 57 | # defaults to '/' (like in a file system). 58 | # 59 | # @return [String] 60 | attr_reader :delimiter 61 | 62 | # @return [Boolean] Returns true if the tree is set to auto-append 63 | # the delimiter to the prefix when the prefix does not end with 64 | # the delimiter. 65 | def append? 66 | @append 67 | end 68 | 69 | # @return [Tree::ChildCollection] A collection representing all 70 | # the child nodes of this node. These may be either 71 | # {Tree::BranchNode} objects or {Tree::LeafNode} objects. 72 | def children 73 | Tree::ChildCollection.new(self, collection, 74 | :delimiter => delimiter, 75 | :prefix => prefix, 76 | :append => append?) 77 | end 78 | 79 | def inspect 80 | "<#{self.class}:#{collection.bucket.name}:#{prefix}>" 81 | end 82 | 83 | end 84 | 85 | end 86 | end 87 | end 88 | -------------------------------------------------------------------------------- /lib/aws/s3/uploaded_part.rb: -------------------------------------------------------------------------------- 1 | # Copyright 2011 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). You 4 | # may not use this file except in compliance with the License. A copy of 5 | # the License is located at 6 | # 7 | # http://aws.amazon.com/apache2.0/ 8 | # 9 | # or in the "license" file accompanying this file. This file is 10 | # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF 11 | # ANY KIND, either express or implied. See the License for the specific 12 | # language governing permissions and limitations under the License. 13 | 14 | module AWS 15 | class S3 16 | 17 | # Represents a part of a multipart upload that has been uploaded 18 | # to S3. 19 | # 20 | # @example Get the total size of the uploaded parts 21 | # upload.parts.inject(0) { |sum, part| sum + part.size } 22 | class UploadedPart 23 | 24 | include Core::Model 25 | 26 | # @return [MultipartUpload] The upload to which this belongs. 27 | attr_reader :upload 28 | 29 | # @return [Integer] The part number. 30 | attr_reader :part_number 31 | 32 | # @private 33 | def initialize(upload, part_number, opts = {}) 34 | @upload = upload 35 | @part_number = part_number 36 | super 37 | end 38 | 39 | def ==(other) 40 | other.kind_of?(UploadedPart) and 41 | other.upload == upload and 42 | other.part_number == part_number 43 | end 44 | alias_method :eql?, :== 45 | 46 | # @return [Integer] The size of the part as it currently 47 | # exists in S3. 48 | def size 49 | get_attribute(:size) 50 | end 51 | 52 | # @return [DateTime] The time at which the part was last 53 | # modified. 54 | def last_modified 55 | get_attribute(:last_modified) 56 | end 57 | 58 | # @return [String] The ETag of the part. 59 | def etag 60 | get_attribute(:etag) 61 | end 62 | 63 | # @private 64 | private 65 | def get_attribute(name) 66 | (resp = client.list_parts(:bucket_name => upload.object.bucket.name, 67 | :key => upload.object.key, 68 | :upload_id => upload.id, 69 | :part_number_marker => part_number-1, 70 | :max_parts => 1) and 71 | part = resp.parts.first and 72 | part.part_number == part_number and 73 | part.send(name)) or 74 | raise "part 3 of upload abc123 does not exist" 75 | end 76 | 77 | end 78 | 79 | end 80 | end 81 | -------------------------------------------------------------------------------- /lib/aws/s3/uploaded_part_collection.rb: -------------------------------------------------------------------------------- 1 | # Copyright 2011 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). You 4 | # may not use this file except in compliance with the License. A copy of 5 | # the License is located at 6 | # 7 | # http://aws.amazon.com/apache2.0/ 8 | # 9 | # or in the "license" file accompanying this file. This file is 10 | # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF 11 | # ANY KIND, either express or implied. See the License for the specific 12 | # language governing permissions and limitations under the License. 13 | 14 | module AWS 15 | class S3 16 | 17 | # Represents the collection of parts that have been uploaded for 18 | # a given multipart upload. You can get an instance of this 19 | # class by calling {MultipartUpload#parts}. 20 | # 21 | # @example Get the total size of the uploaded parts 22 | # upload.parts.inject(0) { |sum, part| sum + part.size } 23 | class UploadedPartCollection 24 | 25 | include Enumerable 26 | include Core::Model 27 | include PaginatedCollection 28 | 29 | # @return [MultipartUpload] The upload to which the parts belong. 30 | attr_reader :upload 31 | 32 | # @private 33 | def initialize(upload, opts = {}) 34 | @upload = upload 35 | super 36 | end 37 | 38 | # @return [UploadedPart] An object representing the part with 39 | # the given part number. 40 | # 41 | # @param [Integer] number The part number. 42 | def [](number) 43 | UploadedPart.new(upload, number) 44 | end 45 | 46 | # @private 47 | protected 48 | def each_member_in_page(page, &block) 49 | page.parts.each do |part_info| 50 | part = UploadedPart.new(upload, part_info.part_number) 51 | yield(part) 52 | end 53 | end 54 | 55 | # @private 56 | protected 57 | def list_options(options) 58 | opts = super 59 | opts.merge!(:bucket_name => upload.object.bucket.name, 60 | :key => upload.object.key, 61 | :upload_id => upload.id) 62 | opts 63 | end 64 | 65 | # @private 66 | protected 67 | def limit_param; :max_parts; end 68 | 69 | # @private 70 | protected 71 | def list_request(options) 72 | client.list_parts(options) 73 | end 74 | 75 | # @private 76 | protected 77 | def pagination_markers; [:part_number_marker]; end 78 | 79 | end 80 | 81 | end 82 | end 83 | -------------------------------------------------------------------------------- /lib/redmine_s3/attachment_patch.rb: -------------------------------------------------------------------------------- 1 | module RedmineS3 2 | module AttachmentPatch 3 | def self.included(base) # :nodoc: 4 | base.extend(ClassMethods) 5 | base.send(:include, InstanceMethods) 6 | 7 | # Same as typing in the class 8 | base.class_eval do 9 | unloadable # Send unloadable so it will not be unloaded in development 10 | attr_accessor :s3_access_key_id, :s3_secret_acces_key, :s3_bucket, :s3_bucket 11 | after_validation :put_to_s3 12 | before_destroy :delete_from_s3 13 | end 14 | end 15 | 16 | module ClassMethods 17 | end 18 | 19 | module InstanceMethods 20 | def put_to_s3 21 | if @temp_file && (@temp_file.size > 0) 22 | logger.debug("Uploading to #{path_to_file}") 23 | RedmineS3::Connection.put(path_to_file, @temp_file.read) 24 | md5 = Digest::MD5.new 25 | self.digest = md5.hexdigest 26 | end 27 | @temp_file = nil # so that the model's original after_save block skips writing to the fs 28 | end 29 | 30 | def delete_from_s3 31 | logger.debug("Deleting #{path_to_file}") 32 | RedmineS3::Connection.delete(path_to_file) 33 | end 34 | 35 | def path_to_file 36 | # obscure the filename by using the timestamp for the 'directory' 37 | disk_filename.split('_').first + '/' + disk_filename 38 | end 39 | end 40 | end 41 | end 42 | -------------------------------------------------------------------------------- /lib/redmine_s3/attachments_controller_patch.rb: -------------------------------------------------------------------------------- 1 | module RedmineS3 2 | module AttachmentsControllerPatch 3 | def self.included(base) # :nodoc: 4 | base.extend(ClassMethods) 5 | base.send(:include, InstanceMethods) 6 | 7 | # Same as typing in the class 8 | base.class_eval do 9 | unloadable # Send unloadable so it will not be unloaded in development 10 | before_filter :redirect_to_s3, :except => :destroy 11 | skip_before_filter :file_readable 12 | end 13 | end 14 | 15 | module ClassMethods 16 | end 17 | 18 | module InstanceMethods 19 | def redirect_to_s3 20 | if @attachment.container.is_a?(Version) || @attachment.container.is_a?(Project) 21 | @attachment.increment_download 22 | end 23 | redirect_to(RedmineS3::Connection.object_url(@attachment.path_to_file)) 24 | end 25 | end 26 | end 27 | end 28 | -------------------------------------------------------------------------------- /lib/redmine_s3/connection.rb: -------------------------------------------------------------------------------- 1 | require 'aws-sdk' 2 | module RedmineS3 3 | class Connection 4 | @@conn = nil 5 | @@s3_options = { 6 | :access_key_id => nil, 7 | :secret_access_key => nil, 8 | :bucket => nil, 9 | :endpoint => nil, 10 | :private => false, 11 | :expires => nil, 12 | :secure => false 13 | } 14 | 15 | class << self 16 | def load_options 17 | file = ERB.new( File.read(File.join(Rails.root, 'config', 's3.yml')) ).result 18 | YAML::load( file )[Rails.env].each do |key, value| 19 | @@s3_options[key.to_sym] = value 20 | end 21 | end 22 | 23 | def establish_connection 24 | load_options unless @@s3_options[:access_key_id] && @@s3_options[:secret_access_key] 25 | options = { 26 | :access_key_id => @@s3_options[:access_key_id], 27 | :secret_access_key => @@s3_options[:secret_access_key] 28 | } 29 | options[:s3_endpoint] = self.endpoint unless self.endpoint.nil? 30 | @conn = AWS::S3.new(options) 31 | end 32 | 33 | def conn 34 | @@conn || establish_connection 35 | end 36 | 37 | def bucket 38 | load_options unless @@s3_options[:bucket] 39 | @@s3_options[:bucket] 40 | end 41 | 42 | def create_bucket 43 | bucket = self.conn.buckets[self.bucket] 44 | bucket.create unless bucket.exists? 45 | end 46 | 47 | def endpoint 48 | @@s3_options[:endpoint] 49 | end 50 | 51 | def expires 52 | @@s3_options[:expires] 53 | end 54 | 55 | def private? 56 | @@s3_options[:private] 57 | end 58 | 59 | def secure? 60 | @@s3_options[:secure] 61 | end 62 | 63 | def put(filename, data) 64 | objects = self.conn.buckets[self.bucket].objects 65 | object = objects[filename] 66 | object = objects.create(filename) unless object.exists? 67 | options = {} 68 | options[:acl] = :public_read unless self.private? 69 | object.write(data, options) 70 | end 71 | 72 | def delete(filename) 73 | object = self.conn.buckets[self.bucket].objects[filename] 74 | object.delete if object.exists? 75 | end 76 | 77 | def object_url(filename) 78 | object = self.conn.buckets[self.bucket].objects[filename] 79 | if self.private? 80 | options = {:secure => self.secure?} 81 | options[:expires] = self.expires unless self.expires.nil? 82 | object.url_for(:read, options).to_s 83 | else 84 | object.public_url(:secure => self.secure?).to_s 85 | end 86 | end 87 | end 88 | end 89 | end 90 | -------------------------------------------------------------------------------- /lib/tasks/files_to_s3.rake: -------------------------------------------------------------------------------- 1 | namespace :redmine_s3 do 2 | task :files_to_s3 => :environment do 3 | require 'thread' 4 | 5 | def s3_file_path(file_path) 6 | File.basename(file_path).split('_').first + '/' + File.basename(file_path) 7 | end 8 | 9 | # updates a single file on s3 10 | def update_file_on_s3(file, objects) 11 | file_path = s3_file_path(file) 12 | conn = RedmineS3::Connection.conn 13 | object = objects[file_path] 14 | 15 | # get the file modified time, which will stay nil if the file doesn't exist yet 16 | # we could check if the file exists, but this saves a head request 17 | s3_mtime = object.last_modified rescue nil 18 | 19 | # put it on s3 if the file has been updated or it doesn't exist on s3 yet 20 | if s3_mtime.nil? || s3_mtime < File.mtime(file) 21 | fileObj = File.open(file, 'r') 22 | RedmineS3::Connection.put(file_path, fileObj.read) 23 | fileObj.close 24 | 25 | puts "Put file " + File.basename(file) 26 | else 27 | puts File.basename(file) + ' is up-to-date on S3' 28 | end 29 | end 30 | 31 | # enqueue all of the files to be "worked" on 32 | fileQ = Queue.new 33 | Dir.glob(RAILS_ROOT + '/files/*').each do |file| 34 | fileQ << file 35 | end 36 | 37 | # init the connection, and grab the ObjectCollection object for the bucket 38 | conn = RedmineS3::Connection.establish_connection 39 | objects = conn.buckets[RedmineS3::Connection.bucket].objects 40 | 41 | # create some threads to start syncing all of the queued files with s3 42 | threads = Array.new 43 | 8.times do 44 | threads << Thread.new do 45 | while !fileQ.empty? 46 | update_file_on_s3(fileQ.pop, objects) 47 | end 48 | end 49 | end 50 | 51 | # wait on all of the threads to finish 52 | threads.each do |thread| 53 | thread.join 54 | end 55 | 56 | end 57 | end 58 | -------------------------------------------------------------------------------- /rails/init.rb: -------------------------------------------------------------------------------- 1 | require File.dirname(__FILE__) + "/../init" 2 | -------------------------------------------------------------------------------- /redmine_s3.gemspec: -------------------------------------------------------------------------------- 1 | # Generated by jeweler 2 | # DO NOT EDIT THIS FILE DIRECTLY 3 | # Instead, edit Jeweler::Tasks in Rakefile, and run the gemspec command 4 | # -*- encoding: utf-8 -*- 5 | 6 | Gem::Specification.new do |s| 7 | s.name = %q{redmine_s3} 8 | s.version = "0.0.3" 9 | 10 | s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version= 11 | s.authors = ["Christopher Dell"] 12 | s.date = %q{2010-01-04} 13 | s.description = %q{Plugin to have Redmine store uploads on S3} 14 | s.email = %q{edavis@littlestreamsoftware.com} 15 | s.extra_rdoc_files = [ 16 | "README.rdoc" 17 | ] 18 | s.files = [ 19 | "README.rdoc", 20 | "Rakefile", 21 | "VERSION", 22 | "config/locales/en.yml", 23 | "config/s3.yml.example", 24 | "init.rb", 25 | "lang/en.yml", 26 | "lib/S3.rb", 27 | "lib/redmine_s3/attachment_patch.rb", 28 | "lib/redmine_s3/attachments_controller_patch.rb", 29 | "lib/redmine_s3/connection.rb", 30 | "lib/s3_helper.rb", 31 | "rails/init.rb", 32 | "test/test_helper.rb" 33 | ] 34 | s.homepage = %q{http://projects.tigrish.com/projects/redmine-s3} 35 | s.rdoc_options = ["--charset=UTF-8"] 36 | s.require_paths = ["lib"] 37 | s.rubyforge_project = %q{TODO} 38 | s.rubygems_version = %q{1.3.5} 39 | s.summary = %q{Plugin to have Redmine store uploads on S3} 40 | s.test_files = [ 41 | "test/test_helper.rb" 42 | ] 43 | 44 | if s.respond_to? :specification_version then 45 | current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION 46 | s.specification_version = 3 47 | 48 | if Gem::Version.new(Gem::RubyGemsVersion) >= Gem::Version.new('1.2.0') then 49 | else 50 | end 51 | else 52 | end 53 | end 54 | 55 | -------------------------------------------------------------------------------- /test/test_helper.rb: -------------------------------------------------------------------------------- 1 | # Load the normal Rails helper 2 | require File.expand_path(File.dirname(__FILE__) + '/../../../../test/test_helper') 3 | 4 | # Ensure that we are using the temporary fixture path 5 | Engines::Testing.set_fixture_path 6 | --------------------------------------------------------------------------------