├── .rspec
├── lib
    ├── middleman_extension.rb
    ├── middleman
    │   ├── s3_sync
    │   │   ├── version.rb
    │   │   ├── status.rb
    │   │   ├── options.rb
    │   │   └── resource.rb
    │   └── s3_sync.rb
    ├── middleman-s3_sync.rb
    └── middleman-s3_sync
    │   ├── extension.rb
    │   └── commands.rb
├── spec
    ├── resource_spec.rb
    ├── spec_helper.rb
    └── options_spec.rb
├── .s3_sync.sample
├── Gemfile
├── Rakefile
├── .gitignore
├── .travis.yml
├── LICENSE.txt
├── middleman-s3_sync.gemspec
├── Changelog.md
└── README.md


/.rspec:
--------------------------------------------------------------------------------
1 | --color
2 | --format progress
3 | 


--------------------------------------------------------------------------------
/lib/middleman_extension.rb:
--------------------------------------------------------------------------------
1 | require 'middleman-s3_sync'
2 | 


--------------------------------------------------------------------------------
/spec/resource_spec.rb:
--------------------------------------------------------------------------------
1 | require 'spec_helper'
2 | 
3 | describe Middleman::S3Sync::Resource do
4 | 
5 | end
6 | 


--------------------------------------------------------------------------------
/.s3_sync.sample:
--------------------------------------------------------------------------------
1 | ---
2 | aws_access_key_id: <AWS Access Key>
3 | aws_secret_access_key: <AWS Secret Access Key>
4 | 


--------------------------------------------------------------------------------
/lib/middleman/s3_sync/version.rb:
--------------------------------------------------------------------------------
1 | module Middleman
2 |   module S3Sync
3 |     VERSION = "3.0.24"
4 |   end
5 | end
6 | 


--------------------------------------------------------------------------------
/Gemfile:
--------------------------------------------------------------------------------
1 | source 'https://rubygems.org'
2 | 
3 | # Specify your gem's dependencies in middleman-s3_sync.gemspec
4 | gemspec
5 | 


--------------------------------------------------------------------------------
/Rakefile:
--------------------------------------------------------------------------------
1 | require "bundler/gem_tasks"
2 | require "rspec/core/rake_task"
3 | 
4 | RSpec::Core::RakeTask.new(:spec)
5 | 
6 | task :default => :spec
7 | 


--------------------------------------------------------------------------------
/lib/middleman-s3_sync.rb:
--------------------------------------------------------------------------------
1 | require 'middleman-core'
2 | require 'middleman/s3_sync'
3 | 
4 | ::Middleman::Extensions.register(:s3_sync, '>= 3.0.0') do
5 |   ::Middleman::S3Sync
6 | end
7 | 
8 | 


--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
 1 | *.gem
 2 | *.rbc
 3 | .bundle
 4 | .config
 5 | .yardoc
 6 | Gemfile.lock
 7 | InstalledFiles
 8 | _yardoc
 9 | coverage
10 | doc/
11 | lib/bundler/man
12 | pkg
13 | rdoc
14 | spec/reports
15 | test/tmp
16 | test/version_tmp
17 | tmp
18 | 


--------------------------------------------------------------------------------
/lib/middleman/s3_sync/status.rb:
--------------------------------------------------------------------------------
 1 | require 'colorize'
 2 | 
 3 | module Middleman
 4 |   module S3Sync
 5 |     module Status
 6 |       def say_status(status)
 7 |         puts :s3_sync.to_s.rjust(12).light_green + "  #{status}"
 8 |       end
 9 |     end
10 |   end
11 | end
12 | 


--------------------------------------------------------------------------------
/.travis.yml:
--------------------------------------------------------------------------------
 1 | ---
 2 | language: ruby
 3 | rvm:
 4 | - 1.9.3
 5 | - 2.0.0
 6 | env:
 7 |   global:
 8 |   - secure: ! 'dQ/vYDmUZEW9LD8IyTRFoRFo24w2yJTYQHuAS24vfKToDZ0BO2/Tpq5lh0vb
 9 | 
10 |       EJ25M7+IwJgo0PC++c1cbssurpNvNeMmDMKuBd6IjnSoJSzK6AszK5wA8dRs
11 | 
12 |       gd3RC+G5lJ2sKPip28t6bIf5K6KFPD/3d45rE+qY9rXZxCwMJHw='
13 |   - secure: ! 'N26NTcIxZa6kUVMRPe2HONMjcAKTJUAFBMOXu9CC6FAyeIMFbldUMDs2Dk3D
14 | 
15 |       CWPYiWQmqXywTbqHsnPfv9LwqQ9zntsURlq8I5JPVGNvf7QWqKdojI05cIiS
16 | 
17 |       kT7C8IqnK3RzVmx4WZ5qFYxG6ypwilYq1px0OKjB4JHbFR5bj18='
18 | 


--------------------------------------------------------------------------------
/spec/spec_helper.rb:
--------------------------------------------------------------------------------
 1 | # This file was generated by the `rspec --init` command. Conventionally, all
 2 | # specs live under a `spec` directory, which RSpec adds to the `$LOAD_PATH`.
 3 | # Require this file using `require "spec_helper"` to ensure that it is only
 4 | # loaded once.
 5 | #
 6 | # See http://rubydoc.info/gems/rspec-core/RSpec/Core/Configuration
 7 | 
 8 | require 'middleman-s3_sync'
 9 | require 'timerizer'
10 | 
11 | RSpec.configure do |config|
12 |   config.treat_symbols_as_metadata_keys_with_true_values = true
13 |   config.run_all_when_everything_filtered = true
14 |   config.filter_run :focus
15 | 
16 |   # Run specs in random order to surface order dependencies. If you find an
17 |   # order dependency and want to debug it, you can fix the order by providing
18 |   # the seed, which is printed after each run.
19 |   #     --seed 1234
20 |   config.order = 'random'
21 | 
22 |   config.before :all do
23 |     Fog.mock!
24 |   end
25 | end
26 | 


--------------------------------------------------------------------------------
/LICENSE.txt:
--------------------------------------------------------------------------------
 1 | Copyright (c) 2012 Frederic Jean
 2 | 
 3 | MIT License
 4 | 
 5 | Permission is hereby granted, free of charge, to any person obtaining
 6 | a copy of this software and associated documentation files (the
 7 | "Software"), to deal in the Software without restriction, including
 8 | without limitation the rights to use, copy, modify, merge, publish,
 9 | distribute, sublicense, and/or sell copies of the Software, and to
10 | permit persons to whom the Software is furnished to do so, subject to
11 | the following conditions:
12 | 
13 | The above copyright notice and this permission notice shall be
14 | included in all copies or substantial portions of the Software.
15 | 
16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
17 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
18 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
19 | NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
20 | LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
21 | OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
22 | WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.


--------------------------------------------------------------------------------
/lib/middleman-s3_sync/extension.rb:
--------------------------------------------------------------------------------
 1 | require 'middleman-core'
 2 | require 'map'
 3 | 
 4 | module Middleman
 5 |   module S3Sync
 6 |     class << self
 7 |       def registered(app, options_hash = {}, &block)
 8 |         options = Options.new
 9 |         yield options if block_given?
10 | 
11 |         @@options = options
12 | 
13 |         app.send :include, Helpers
14 | 
15 |         app.after_configuration do |config|
16 | 
17 |           # Define the after_build step after during configuration so
18 |           # that it's pushed to the end of the callback chain
19 |           app.after_build do |builder|
20 |             ::Middleman::S3Sync.sync if options.after_build
21 |           end
22 | 
23 |           options.build_dir ||= build_dir
24 |         end
25 |       end
26 |       alias :included :registered
27 | 
28 |       def s3_sync_options
29 |         @@options
30 |       end
31 | 
32 |       module Helpers
33 |         def s3_sync_options
34 |           ::Middleman::S3Sync.s3_sync_options
35 |         end
36 | 
37 |         def default_caching_policy(policy = {})
38 |           s3_sync_options.add_caching_policy(:default, policy)
39 |         end
40 | 
41 |         def caching_policy(content_type, policy = {})
42 |           s3_sync_options.add_caching_policy(content_type, policy)
43 |         end
44 |       end
45 |     end
46 |   end
47 | end
48 | 


--------------------------------------------------------------------------------
/lib/middleman-s3_sync/commands.rb:
--------------------------------------------------------------------------------
 1 | require 'middleman-core/cli'
 2 | require 'middleman-s3_sync/extension'
 3 | 
 4 | module Middleman
 5 |   module Cli
 6 |     class S3Sync < Thor
 7 |       include Thor::Actions
 8 |       namespace :s3_sync
 9 | 
10 |       def self.exit_on_failure?
11 |         true
12 |       end
13 | 
14 |       desc "s3_sync", "Pushes the minimum set of files needed to S3"
15 |       option :force, type: :boolean,
16 |                      desc: "Push all local files to the server",
17 |                      aliases: :f
18 |       option :bucket, type: :string,
19 |                       desc: "Specify which bucket to use, overrides the configured bucket.",
20 |                       aliases: :b
21 |       option :verbose, type: :boolean,
22 |                        desc: "Adds more verbosity...",
23 |                        aliases: :v
24 | 
25 |       def s3_sync
26 |         shared_inst = ::Middleman::Application.server.inst
27 |         bucket = shared_inst.s3_sync_options.bucket rescue nil
28 |         unless bucket
29 |           raise Thor::Error.new "You need to activate the s3_sync extension."
30 |         end
31 | 
32 |         # Override options based on what was passed on the command line...
33 |         shared_inst.s3_sync_options.force = options[:force] if options[:force]
34 |         shared_inst.s3_sync_options.bucket = options[:bucket] if options[:bucket]
35 |         shared_inst.s3_sync_options.verbose = options[:verbose] if options[:verbose]
36 | 
37 |         ::Middleman::S3Sync.sync
38 |       end
39 |     end
40 |   end
41 | end
42 | 


--------------------------------------------------------------------------------
/middleman-s3_sync.gemspec:
--------------------------------------------------------------------------------
 1 | # -*- encoding: utf-8 -*-
 2 | lib = File.expand_path('../lib', __FILE__)
 3 | $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 4 | require 'middleman/s3_sync/version'
 5 | 
 6 | Gem::Specification.new do |gem|
 7 |   gem.name          = "middleman-s3_sync"
 8 |   gem.version       = Middleman::S3Sync::VERSION
 9 |   gem.authors       = ["Frederic Jean", "Will Koehler"]
10 |   gem.email         = ["fred@fredjean.net"]
11 |   gem.description   = %q{Only syncs files that have been updated to S3.}
12 |   gem.summary       = %q{Tries really, really hard not to push files to S3.}
13 |   gem.homepage      = "http://github.com/fredjean/middleman-s3_sync"
14 |   gem.license       = 'MIT'
15 | 
16 |   gem.files         = `git ls-files`.split($/)
17 |   gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
18 |   gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
19 |   gem.require_paths = ["lib"]
20 | 
21 |   gem.add_runtime_dependency 'middleman-core', '>= 3.0.0'
22 |   gem.add_runtime_dependency 'unf'
23 |   gem.add_runtime_dependency 'fog', '>= 1.10.1'
24 |   gem.add_runtime_dependency 'map'
25 |   gem.add_runtime_dependency 'pmap'
26 |   gem.add_runtime_dependency 'ruby-progressbar'
27 |   gem.add_runtime_dependency 'colorize'
28 | 
29 |   gem.add_development_dependency 'rake'
30 |   gem.add_development_dependency 'pry'
31 |   gem.add_development_dependency 'pry-nav'
32 |   gem.add_development_dependency 'rspec'
33 |   gem.add_development_dependency 'timerizer'
34 |   gem.add_development_dependency 'travis'
35 |   gem.add_development_dependency 'travis-lint'
36 | end
37 | 


--------------------------------------------------------------------------------
/Changelog.md:
--------------------------------------------------------------------------------
 1 | # Middleman::S3Sync Changelog
 2 | 
 3 | The gem that tries really hard not to push files to S3.
 4 | 
 5 | ## v3.0.22
 6 | 
 7 | * Fixes a bug where files were not closed, leading to an exhaustion of 
 8 |   file handles with large web sites.
 9 | * Internal fixes.
10 | 
11 | ## v3.0.17
12 | 
13 | * Limits the number of concurrent threads used while processing the
14 |   resources and files. (#21)
15 | * Adds the option to use reduced redundancy storage for the bucket. (#8)
16 | * Adds the license to the gem specs. (#20)
17 | * Makes sure tha the .s3_sync file is read when the sync occures within
18 |   a build. (#22, #23)
19 | 
20 | ## v3.0.16
21 | 
22 | * Adds the ignore directory and redirects logic to the --force option as
23 |   well.
24 | 
25 | ## v3.0.15
26 | 
27 | * Ignore objects that look like directories. In some cases, S3 objects
28 |   where created to simulate directories. S3 Sync would crash when
29 |   processing these and a matching local directory was present.
30 | 
31 | ## v3.0.14
32 | 
33 | * No longer deletes redirects from the S3 bucket. This prevents a
34 |   situation where the redirect is first removed then added back through
35 |   [middleman-s3_redirect](https://github.com/fredjean/middleman-s3_redirect).
36 | 
37 | ## v3.0.13
38 | 
39 | * Fails gracefully when the extension isn't activated
40 | 
41 | ## v3.0.12
42 | 
43 | * Remove S3 objects that look like directories. Addresses [issue
44 | #13](https://github.com/fredjean/middleman-s3_sync/issues/13)
45 | 
46 | ## v3.0.11
47 | 
48 | * Adds support for GZipped resources (fixes #3)
49 | * Quiets Fog's warning messages (fixes #10)
50 | * Rename the options method to s3_sync_options to remove a method name collision (fixes #9)
51 | * Colorize the output.
52 | 
53 | 
54 | 


--------------------------------------------------------------------------------
/spec/options_spec.rb:
--------------------------------------------------------------------------------
  1 | require 'spec_helper'
  2 | require 'middleman/s3_sync/options'
  3 | 
  4 | describe Middleman::S3Sync::Options do
  5 |   subject(:options) { Middleman::S3Sync::Options.new }
  6 | 
  7 |   its(:delete) { should be_true }
  8 |   its(:after_build) { should be_false }
  9 |   its(:prefer_gzip) { should be_true }
 10 |   its(:aws_secret_access_key) { should == ENV['AWS_SECRET_ACCESS_KEY'] }
 11 |   its(:aws_access_key_id) { should == ENV['AWS_ACCESS_KEY_ID'] }
 12 |   its(:caching_policies) { should be_empty }
 13 |   its(:default_caching_policy) { should be_nil }
 14 | 
 15 |   context "browser caching policy" do
 16 |     let(:policy) { options.default_caching_policy }
 17 | 
 18 |     it "should have a blank default caching policy" do
 19 |       options.add_caching_policy :default, {}
 20 | 
 21 |       policy.should_not be_nil
 22 | 
 23 |       policy.to_s.should_not =~ /max-age=/
 24 |       policy.to_s.should_not =~ /s-maxage=/
 25 |       policy.to_s.should_not =~ /public/
 26 |       policy.to_s.should_not =~ /private/
 27 |       policy.to_s.should_not =~ /no-cache/
 28 |       policy.to_s.should_not =~ /no-store/
 29 |       policy.to_s.should_not =~ /must-revalidate/
 30 |       policy.to_s.should_not =~ /proxy-revalidate/
 31 |       policy.expires.should be_nil
 32 |     end
 33 | 
 34 |     it "should set the max-age policy" do
 35 |       options.add_caching_policy :default, :max_age => 300
 36 | 
 37 |       policy.to_s.should =~ /max-age=300/
 38 |     end
 39 | 
 40 |     it "should set the s-maxage policy" do
 41 |       options.add_caching_policy :default, :s_maxage => 300
 42 | 
 43 |       policy.to_s.should =~ /s-maxage=300/
 44 |     end
 45 | 
 46 |     it "should set the public flag on the policy if set to true" do
 47 |       options.add_caching_policy :default, :public => true
 48 | 
 49 |       policy.to_s.should =~ /public/
 50 |     end
 51 | 
 52 |     it "should not set the public flag on the policy if it is set to false" do
 53 |       options.add_caching_policy :default, :public => false
 54 | 
 55 |       policy.to_s.should_not =~ /public/
 56 |     end
 57 | 
 58 |     it "should set the private flag on the policy if it is set to true" do
 59 |       options.add_caching_policy :default, :private => true
 60 | 
 61 |       policy.to_s.should =~ /private/
 62 |     end
 63 | 
 64 |     it "should set the no-cache flag on the policy if it is set to true" do
 65 |       options.add_caching_policy :default, :no_cache => true
 66 | 
 67 |       policy.to_s.should =~ /no-cache/
 68 |     end
 69 | 
 70 |     it "should set the no-store flag if it is set to true" do
 71 |       options.add_caching_policy :default, :no_store => true
 72 | 
 73 |       policy.to_s.should =~ /no-store/
 74 |     end
 75 | 
 76 |     it "should set the must-revalidate policy if it is set to true" do
 77 |       options.add_caching_policy :default, :must_revalidate => true
 78 | 
 79 |       policy.to_s.should =~ /must-revalidate/
 80 |     end
 81 | 
 82 |     it "should set the proxy-revalidate policy if it is set to true" do
 83 |       options.add_caching_policy :default, :proxy_revalidate => true
 84 | 
 85 |       policy.to_s.should =~ /proxy-revalidate/
 86 |     end
 87 | 
 88 |     it "should divide caching policies with commas and a space" do
 89 |       options.add_caching_policy :default, :max_age => 300, :public => true
 90 | 
 91 |       policies = policy.to_s.split(/, /)
 92 |       policies.length.should == 2
 93 |       policies.first.should == 'max-age=300'
 94 |       policies.last.should == 'public'
 95 |     end
 96 | 
 97 |     it "should set the expiration date" do
 98 |       expiration = 1.years.from_now
 99 | 
100 |       options.add_caching_policy :default, :expires => expiration
101 |       policy.expires.should == CGI.rfc1123_date(expiration)
102 |     end
103 |   end
104 | 
105 |   context "#read_config" do
106 |     let(:aws_access_key_id) { "foo" }
107 |     let(:aws_secret_access_key) { "bar" }
108 |     let(:config) { { "aws_access_key_id" => aws_access_key_id, "aws_secret_access_key" => aws_secret_access_key } }
109 |     let(:file) { StringIO.new(YAML.dump(config)) }
110 | 
111 |     before do
112 |       options.read_config(file)
113 |     end
114 | 
115 |     its(:aws_access_key_id) { should eq(aws_access_key_id) }
116 |     its(:aws_secret_access_key) { should eq(aws_secret_access_key) }
117 |   end
118 | end
119 | 


--------------------------------------------------------------------------------
/lib/middleman/s3_sync.rb:
--------------------------------------------------------------------------------
  1 | require 'fog'
  2 | require 'pmap'
  3 | require 'digest/md5'
  4 | require 'middleman/s3_sync/version'
  5 | require 'middleman/s3_sync/options'
  6 | require 'middleman-s3_sync/commands'
  7 | require 'middleman/s3_sync/status'
  8 | require 'middleman/s3_sync/resource'
  9 | require 'middleman-s3_sync/extension'
 10 | require 'ruby-progressbar'
 11 | 
 12 | module Middleman
 13 |   module S3Sync
 14 |     class << self
 15 |       include Status
 16 | 
 17 |       def sync
 18 |         unless work_to_be_done?
 19 |           say_status "\nAll S3 files are up to date."
 20 |           return
 21 |         end
 22 | 
 23 |         say_status "\nReady to apply updates to #{s3_sync_options.bucket}."
 24 | 
 25 |         ignore_resources
 26 |         create_resources
 27 |         update_resources
 28 |         delete_resources
 29 |       end
 30 | 
 31 |       def bucket
 32 |         @bucket ||= connection.directories.get(s3_sync_options.bucket)
 33 |       end
 34 | 
 35 |       protected
 36 |       def connection
 37 |         @connection ||= Fog::Storage.new({
 38 |           :provider => 'AWS',
 39 |           :aws_access_key_id => s3_sync_options.aws_access_key_id,
 40 |           :aws_secret_access_key => s3_sync_options.aws_secret_access_key,
 41 |           :region => s3_sync_options.region,
 42 |           :path_style => s3_sync_options.path_style
 43 |         })
 44 |       end
 45 | 
 46 |       def resources
 47 |         @resources ||= paths.pmap(32) do |p|
 48 |           progress_bar.increment
 49 |           S3Sync::Resource.new(p, bucket_files.find { |f| f.key == p }).tap(&:status)
 50 |         end
 51 |       end
 52 | 
 53 |       def progress_bar
 54 |         @progress_bar ||= ProgressBar.create(total: paths.length)
 55 |       end
 56 | 
 57 |       def paths
 58 |         @paths ||= begin
 59 |                      say_status "Gathering the paths to evaluate."
 60 |                      (remote_paths + local_paths).uniq.sort
 61 |                    end
 62 |       end
 63 | 
 64 |       def local_paths
 65 |         @local_paths ||= begin
 66 |                            local_paths = (Dir[build_dir + "/**/*"] + Dir[build_dir + "/**/.*"])
 67 |                                            .reject { |p| File.directory?(p) }
 68 | 
 69 |                            if s3_sync_options.prefer_gzip
 70 |                              local_paths.reject! { |p| p =~ /\.gz$/ && File.exist?(p.gsub(/\.gz$/, '')) }
 71 |                            end
 72 | 
 73 |                            local_paths.pmap(32) { |p| p.sub(/^#{build_dir}\//, '') }
 74 |                          end
 75 |       end
 76 | 
 77 |       def remote_paths
 78 |         @remote_paths ||= bucket_files.map(&:key)
 79 |       end
 80 |       
 81 |       def bucket_files
 82 |         @bucket_files ||= [].tap { |files|
 83 |           bucket.files.each { |f|
 84 |             files << f
 85 |           }
 86 |         }
 87 |       end
 88 | 
 89 |       def create_resources
 90 |         files_to_create.each do |r|
 91 |           r.create!
 92 |         end
 93 |       end
 94 | 
 95 |       def update_resources
 96 |         files_to_update.each do |r|
 97 |           r.update!
 98 |         end
 99 |       end
100 | 
101 |       def delete_resources
102 |         files_to_delete.each do |r|
103 |           r.destroy!
104 |         end
105 |       end
106 | 
107 |       def ignore_resources
108 |         files_to_ignore.each do |r|
109 |           r.ignore!
110 |         end
111 |       end
112 | 
113 |       def work_to_be_done?
114 |         !(files_to_create.empty? && files_to_update.empty? && files_to_delete.empty?)
115 |       end
116 | 
117 |       def files_to_delete
118 |         @files_to_delete ||= if s3_sync_options.delete
119 |                                  resources.select { |r| r.to_delete? }
120 |                              else
121 |                                []
122 |                              end
123 |       end
124 | 
125 |       def files_to_create
126 |         @files_to_create ||= resources.select { |r| r.to_create? }
127 |       end
128 | 
129 |       def files_to_update
130 |         return resources.select { |r| r.local? && !r.to_ignore? } if s3_sync_options.force
131 | 
132 |         @files_to_update ||= resources.select { |r| r.to_update? }
133 |       end
134 | 
135 |       def files_to_ignore
136 |         @files_to_ignore ||= resources.select { |r| r.to_ignore? }
137 |       end
138 | 
139 |       def build_dir
140 |         @build_dir ||= s3_sync_options.build_dir
141 |       end
142 |     end
143 |   end
144 | end
145 | 
146 | 


--------------------------------------------------------------------------------
/lib/middleman/s3_sync/options.rb:
--------------------------------------------------------------------------------
  1 | module Middleman
  2 |   module S3Sync
  3 |     class Options
  4 |       attr_accessor \
  5 |         :prefix,
  6 |         :acl,
  7 |         :bucket,
  8 |         :region,
  9 |         :aws_access_key_id,
 10 |         :aws_secret_access_key,
 11 |         :after_build,
 12 |         :delete,
 13 |         :encryption,
 14 |         :existing_remote_file,
 15 |         :build_dir,
 16 |         :force,
 17 |         :prefer_gzip,
 18 |         :reduced_redundancy_storage,
 19 |         :path_style,
 20 |         :verbose
 21 | 
 22 |       def initialize
 23 |         # read config from .s3_sync on initialization
 24 |         self.read_config
 25 |       end
 26 | 
 27 |       def acl
 28 |         @acl || 'public-read'
 29 |       end
 30 | 
 31 |       def add_caching_policy(content_type, options)
 32 |         caching_policies[content_type.to_s] = BrowserCachePolicy.new(options)
 33 |       end
 34 | 
 35 |       def caching_policy_for(content_type)
 36 |         caching_policies.fetch(content_type.to_s, caching_policies[:default])
 37 |       end
 38 | 
 39 |       def default_caching_policy
 40 |         caching_policies[:default]
 41 |       end
 42 | 
 43 |       def caching_policies
 44 |         @caching_policies ||= Map.new
 45 |       end
 46 | 
 47 |       def aws_access_key_id=(aws_access_key_id)
 48 |         @aws_access_key_id = aws_access_key_id if aws_access_key_id
 49 |       end
 50 | 
 51 |       def aws_access_key_id
 52 |         @aws_access_key_id || ENV['AWS_ACCESS_KEY_ID']
 53 |       end
 54 | 
 55 |       def aws_secret_access_key=(aws_secret_access_key)
 56 |         @aws_secret_access_key = aws_secret_access_key if aws_secret_access_key
 57 |       end
 58 | 
 59 |       def aws_secret_access_key
 60 |         @aws_secret_access_key || ENV['AWS_SECRET_ACCESS_KEY']
 61 |       end
 62 | 
 63 |       def encryption
 64 |         @encryption.nil? ? false : @encryption
 65 |       end
 66 | 
 67 |       def delete
 68 |         @delete.nil? ? true : @delete
 69 |       end
 70 | 
 71 |       def after_build
 72 |         @after_build.nil? ? false : @after_build
 73 |       end
 74 | 
 75 |       def prefer_gzip
 76 |         (@prefer_gzip.nil? ? true : @prefer_gzip)
 77 |       end
 78 | 
 79 |       def path_style
 80 |         (@path_style.nil? ? true : @path_style)
 81 |       end
 82 | 
 83 |       # Read config options from an IO stream and set them on `self`. Defaults
 84 |       # to reading from the `.s3_sync` file in the MM project root if it exists.
 85 |       #
 86 |       # @param io [IO] an IO stream to read from
 87 |       # @return [void]
 88 |       def read_config(io = nil)
 89 |         unless io
 90 |           root_path = ::Middleman::Application.root
 91 |           config_file_path = File.join(root_path, ".s3_sync")
 92 | 
 93 |           # skip if config file does not exist
 94 |           return unless File.exists?(config_file_path)
 95 | 
 96 |           io = File.open(config_file_path, "r")
 97 |         end
 98 | 
 99 |         config = YAML.load(io)
100 | 
101 |         self.aws_access_key_id = config["aws_access_key_id"] if config["aws_access_key_id"]
102 |         self.aws_secret_access_key = config["aws_secret_access_key"] if config["aws_secret_access_key"]
103 |       end
104 | 
105 |       protected
106 |       class BrowserCachePolicy
107 |         attr_accessor :policies
108 | 
109 |         def initialize(options)
110 |           @policies = Map.from_hash(options)
111 |         end
112 | 
113 |         def cache_control
114 |           policy = []
115 |           policy << "max-age=#{policies.max_age}" if policies.has_key?(:max_age)
116 |           policy << "s-maxage=#{policies.s_maxage}" if policies.has_key?(:s_maxage)
117 |           policy << "public" if policies.fetch(:public, false)
118 |           policy << "private" if policies.fetch(:private, false)
119 |           policy << "no-cache" if policies.fetch(:no_cache, false)
120 |           policy << "no-store" if policies.fetch(:no_store, false)
121 |           policy << "must-revalidate" if policies.fetch(:must_revalidate, false)
122 |           policy << "proxy-revalidate" if policies.fetch(:proxy_revalidate, false)
123 |           if policy.empty?
124 |             nil
125 |           else
126 |             policy.join(", ")
127 |           end
128 |         end
129 | 
130 |         def to_s
131 |           cache_control
132 |         end
133 | 
134 |         def expires
135 |           if expiration = policies.fetch(:expires, nil)
136 |             CGI.rfc1123_date(expiration)
137 |           end
138 |         end
139 |       end
140 |     end
141 |   end
142 | end
143 | 


--------------------------------------------------------------------------------
/lib/middleman/s3_sync/resource.rb:
--------------------------------------------------------------------------------
  1 | module Middleman
  2 |   module S3Sync
  3 |     class Resource
  4 |       attr_accessor :path, :partial_s3_resource, :content_type, :gzipped
  5 | 
  6 |       CONTENT_MD5_KEY = 'x-amz-meta-content-md5'
  7 | 
  8 |       include Status
  9 |       
 10 |       def s3_resource
 11 |         @full_s3_resource || @partial_s3_resource
 12 |       end
 13 | 
 14 |       # S3 resource as returned by a HEAD request
 15 |       def full_s3_resource
 16 |         @full_s3_resource ||= bucket.files.head(path)
 17 |       end
 18 | 
 19 |       def initialize(path, partial_s3_resource)
 20 |         @path = path
 21 |         @partial_s3_resource = partial_s3_resource
 22 |       end
 23 | 
 24 |       def remote_path
 25 |         s3_resource ? s3_resource.key : path
 26 |       end
 27 |       alias :key :remote_path
 28 | 
 29 |       def to_h
 30 |         attributes = {
 31 |           :key => key,
 32 |           :acl => options.acl,
 33 |           :content_type => content_type,
 34 |           CONTENT_MD5_KEY => local_content_md5
 35 |         }
 36 | 
 37 |         if caching_policy
 38 |           attributes[:cache_control] = caching_policy.cache_control
 39 |           attributes[:expires] = caching_policy.expires
 40 |         end
 41 | 
 42 |         if options.prefer_gzip && gzipped
 43 |           attributes[:content_encoding] = "gzip"
 44 |         end
 45 | 
 46 |         if options.reduced_redundancy_storage
 47 |           attributes[:storage_class] = 'REDUCED_REDUNDANCY'
 48 |         end
 49 | 
 50 |         if options.encryption
 51 |           attributes[:encryption] = 'AES256'
 52 |         end
 53 | 
 54 |         attributes
 55 |       end
 56 |       alias :attributes :to_h
 57 | 
 58 |       def update!
 59 |         body { |body|
 60 |           say_status "Updating".blue + " #{path}#{ gzipped ? ' (gzipped)'.white : ''}"
 61 |           if options.verbose
 62 |             say_status "Original:    #{original_path.white}"
 63 |             say_status "Local Path:  #{local_path.white}"
 64 |             say_status "remote md5:  #{remote_object_md5.white} / #{remote_content_md5}"
 65 |             say_status "content md5: #{local_object_md5.white} / #{local_content_md5}"
 66 |           end
 67 |           s3_resource.body = body
 68 | 
 69 |           s3_resource.acl = options.acl
 70 |           s3_resource.content_type = content_type
 71 |           s3_resource.metadata = { CONTENT_MD5_KEY => local_content_md5 }
 72 | 
 73 |           if caching_policy
 74 |             s3_resource.cache_control = caching_policy.cache_control
 75 |             s3_resource.expires = caching_policy.expires
 76 |           end
 77 | 
 78 |           if options.prefer_gzip && gzipped
 79 |             s3_resource.content_encoding = "gzip"
 80 |           end
 81 | 
 82 |           if options.reduced_redundancy_storage
 83 |             s3_resource.storage_class = 'REDUCED_REDUNDANCY'
 84 |           end
 85 | 
 86 |           if options.encryption
 87 |             s3_resource.encryption = 'AES256'
 88 |           end
 89 | 
 90 |           s3_resource.save
 91 |         }
 92 |       end
 93 | 
 94 |       def local_path
 95 |         local_path = build_dir + '/' + path
 96 |         if options.prefer_gzip && File.exist?(local_path + ".gz")
 97 |           @gzipped = true
 98 |           local_path += ".gz"
 99 |         end
100 |         local_path
101 |       end
102 | 
103 |       def destroy!
104 |         say_status "Deleting".red + " #{path}"
105 |         bucket.files.destroy remote_path
106 |       end
107 | 
108 |       def create!
109 |         say_status "Creating".green + " #{path}#{ gzipped ? ' (gzipped)'.white : ''}"
110 |         if options.verbose
111 |           say_status "Original:    #{original_path.white}"
112 |           say_status "Local Path:  #{local_path.white}"
113 |           say_status "content md5: #{local_content_md5.white}"
114 |         end
115 |         body { |body|
116 |           bucket.files.create(to_h.merge(:body => body))
117 |         }
118 |       end
119 | 
120 |       def ignore!
121 |         reason = if redirect?
122 |                    :redirect
123 |                  elsif directory?
124 |                    :directory
125 |                  elsif alternate_encoding?
126 |                    'alternate encoding'
127 |                  end
128 |         say_status "Ignoring".yellow + " #{path} #{ reason ? "(#{reason})".white : "" }"
129 |       end
130 | 
131 |       def to_delete?
132 |         status == :deleted
133 |       end
134 | 
135 |       def to_create?
136 |         status == :new
137 |       end
138 |       
139 |       def alternate_encoding?
140 |         status == :alternate_encoding
141 |       end
142 | 
143 |       def identical?
144 |         status == :identical
145 |       end
146 | 
147 |       def to_update?
148 |         status == :updated
149 |       end
150 | 
151 |       def to_ignore?
152 |         status == :ignored || status == :alternate_encoding
153 |       end
154 | 
155 |       def body(&block)
156 |         File.open(local_path, &block)
157 |       end
158 | 
159 |       def status
160 |         @status ||= if directory?
161 |                       if remote?
162 |                         :deleted
163 |                       else
164 |                         :ignored
165 |                       end
166 |                     elsif local? && remote?
167 |                       if local_object_md5 == remote_object_md5
168 |                         :identical
169 |                       else
170 |                         if !gzipped
171 |                           # we're not gzipped, object hashes being different indicates updated content
172 |                           :updated
173 |                         elsif (local_content_md5 != remote_content_md5)
174 |                           # we're gzipped, so we checked the content MD5, and it also changed
175 |                           :updated
176 |                         else
177 |                           # we're gzipped, the object hashes differ, but the content hashes are equal
178 |                           # this means the gzipped bits changed while the compressed bits did not
179 |                           # what's more, we spent a HEAD request to find out
180 |                           :alternate_encoding
181 |                         end
182 |                       end
183 |                     elsif local?
184 |                       :new
185 |                     elsif remote? && redirect?
186 |                       :ignored
187 |                     else
188 |                       :deleted
189 |                     end
190 |       end
191 |       
192 |       def local?
193 |         File.exist?(local_path)
194 |       end
195 | 
196 |       def remote?
197 |         s3_resource
198 |       end
199 | 
200 |       def redirect?
201 |         full_s3_resource.metadata.has_key?('x-amz-website-redirect-location')
202 |       end
203 | 
204 |       def directory?
205 |         File.directory?(local_path)
206 |       end
207 | 
208 |       def relative_path
209 |         @relative_path ||= local_path.gsub(/#{build_dir}/, '')
210 |       end
211 |       
212 |       def remote_object_md5
213 |         s3_resource.etag
214 |       end
215 |       
216 |       def remote_content_md5
217 |         full_s3_resource.metadata[CONTENT_MD5_KEY]
218 |       end
219 | 
220 |       def local_object_md5
221 |         @local_object_md5 ||= Digest::MD5.hexdigest(File.read(local_path))
222 |       end
223 |       
224 |       def local_content_md5
225 |         @local_content_md5 ||= Digest::MD5.hexdigest(File.read(original_path))
226 |       end
227 | 
228 |       def original_path
229 |         gzipped ? local_path.gsub(/\.gz$/, '') : local_path
230 |       end
231 | 
232 |       def content_type
233 |         @content_type ||= MIME::Types.of(path).first
234 |       end
235 | 
236 |       def caching_policy
237 |         @caching_policy ||= options.caching_policy_for(content_type)
238 |       end
239 | 
240 |       protected
241 |       def bucket
242 |         Middleman::S3Sync.bucket
243 |       end
244 | 
245 |       def build_dir
246 |         options.build_dir
247 |       end
248 | 
249 |       def options
250 |         Middleman::S3Sync.s3_sync_options
251 |       end
252 |     end
253 |   end
254 | end
255 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
  1 | # Middleman::S3Sync
  2 | 
  3 | [![Code Climate](https://codeclimate.com/github/fredjean/middleman-s3_sync.png)](https://codeclimate.com/github/fredjean/middleman-s3_sync) [![Build Status](https://travis-ci.org/fredjean/middleman-s3_sync.png?branch=master)](https://travis-ci.org/fredjean/middleman-s3_sync)
  4 | 
  5 | This gem determines which files need to be added, updated and optionally deleted
  6 | and only transfer these files up. This reduces the impact of an update
  7 | on a web site hosted on S3.
  8 | 
  9 | ## Why not Middleman Sync?
 10 | 
 11 | [Middleman Sync](https://github.com/karlfreeman/middleman-sync) does a
 12 | great job to push [Middleman](http://middlemanapp.com)  generated
 13 | websites to S3. The only issue I have with it is that it pushes
 14 | every files under build to S3 and doesn't seem to properly delete files
 15 | that are no longer needed.
 16 | 
 17 | ## Installation
 18 | 
 19 | Add this line to your application's Gemfile:
 20 | 
 21 |     gem 'middleman-s3_sync'
 22 | 
 23 | And then execute:
 24 | 
 25 |     $ bundle
 26 | 
 27 | Or install it yourself as:
 28 | 
 29 |     $ gem install middleman-s3_sync
 30 | 
 31 | ## Usage
 32 | 
 33 | You need to add the following code to your ```config.rb``` file:
 34 | 
 35 | ```ruby
 36 | activate :s3_sync do |s3_sync|
 37 |   s3_sync.bucket                     = 'my.bucket.com' # The name of the S3 bucket you are targetting. This is globally unique.
 38 |   s3_sync.region                     = 'us-west-1'     # The AWS region for your bucket.
 39 |   s3_sync.aws_access_key_id          = 'AWS KEY ID'
 40 |   s3_sync.aws_secret_access_key      = 'AWS SECRET KEY'
 41 |   s3_sync.delete                     = false # We delete stray files by default.
 42 |   s3_sync.after_build                = false # We do not chain after the build step by default. 
 43 |   s3_sync.prefer_gzip                = true
 44 |   s3_sync.path_style                 = true
 45 |   s3_sync.reduced_redundancy_storage = false
 46 |   s3_sync.acl                        = 'public-read'
 47 |   s3_sync.encryption                 = false
 48 | end
 49 | ```
 50 | 
 51 | You can then start synchronizing files with S3 through ```middleman s3_sync```.
 52 | 
 53 | ### Configuration Defaults
 54 | 
 55 | The following defaults apply to the configuration items:
 56 | 
 57 | | Setting                    | Default                            |
 58 | | -----------------          | ----------------------------       |
 59 | | aws_access_key_id          | ```ENV['AWS_ACCESS_KEY_ID']```     |
 60 | | aws_secret_access_key      | ```ENV['AWS_SECRET_ACCESS_KEY']``` |
 61 | | delete                     | ```true```                         |
 62 | | after_build                | ```false```                        |
 63 | | prefer_gzip                | ```true```                         |
 64 | | reduced_redundancy_storage | ```false```                        |
 65 | | path_style                 | ```true```                         |
 66 | | encryption                 | ```false```                        |
 67 | | acl                        | ```'public-read'```                |
 68 | 
 69 | You do not need to specify the settings that match the defaults. This
 70 | simplify the configuration of the extension:
 71 | 
 72 | ```ruby
 73 | activate :s3_sync do |s3_sync|
 74 |   s3_sync.bucket = 'my.bucket.com'
 75 | end
 76 | ```
 77 | 
 78 | ### Providing AWS Credentials
 79 | 
 80 | There are a few ways to provide the AWS credentials for s3_sync:
 81 | 
 82 | #### Through ```config.rb```
 83 | 
 84 | You can set the aws_access_key_id and aws_secret_access_key in the block
 85 | that is passed to the activate method.
 86 | 
 87 | #### Through ```.s3_sync``` File
 88 | 
 89 | You can create a ```.s3_sync``` at the root of your middleman project.
 90 | The credentials are passed in the YAML format. The keys match the
 91 | options keys.
 92 | 
 93 | The .s3_sync file takes precedence to the configuration passed in the
 94 | activate method.
 95 | 
 96 | A sample ```.s3_sync``` file is included at the root of this repo.
 97 | 
 98 | #### Through Environment
 99 | 
100 | You can also pass the credentials through environment variables. They
101 | map to the following values:
102 | 
103 | | Setting               | Environment Variable               |
104 | | --------------------- | ---------------------------------- |
105 | | aws_access_key_id     | ```ENV['AWS_ACCESS_KEY_ID```       |
106 | | aws_secret_access_key | ```ENV['AWS_SECRET_ACCESS_KEY']``` |
107 | 
108 | The environment is used when the credentials are not set in the activate
109 | method or passed through the ```.s3_sync``` configuration file.
110 | 
111 | ## Push All Content to S3
112 | 
113 | There are situations where you might need to push the files to S3. In
114 | such case, you can pass the ```--force``` option:
115 | 
116 |     $ middleman s3_sync --force
117 | 
118 | ## Overriding the destination bucket
119 | 
120 | You can now override the destination bucket using the --bucket switch.
121 | The command is:
122 | 
123 |     $ middleman s3_sync --bucket=my.new.bucket
124 | 
125 | ## HTTP Caching
126 | 
127 | By default, ```middleman-s3_sync``` does not set caching headers. In
128 | general, the default settings are sufficient. However, there are
129 | situations where you might want to set a different HTTP caching policy.
130 | This may be very helpful if you are using the ```asset_hash```
131 | extension.
132 | 
133 | ### Setting a policy based on the mime-type of a file
134 | 
135 | You can set a caching policy for every files that match a certain
136 | mime-type. For example, setting max-age to 0 and kindly asking the
137 | browser to revalidate the content for HTML files would take the
138 | following form:
139 | 
140 | ```ruby
141 | caching_policy 'text/html', max_age: 0, must_revalidate: true
142 | ```
143 | 
144 | As a result, the following ```Cache-Control``` header would be set to ```max-age:0, must-revalidate```
145 | 
146 | ### Setting a Default Policy
147 | 
148 | You can set the default policy by passing an options hash to ```default_caching_policy``` in your ```config.rb``` file:
149 | 
150 | ```ruby
151 | default_caching_policy max_age:(60 * 60 * 24 * 365)
152 | ```
153 | 
154 | This will apply the policy to any file that do not have a mime-type
155 | specific policy.
156 | 
157 | ### Caching Policies
158 | 
159 | The [Caching Tutorial](http://www.mnot.net/cache_docs/) is a great
160 | introduction to HTTP caching. The caching policy code in this gem is
161 | based on it.
162 | 
163 | The following keys can be set:
164 | 
165 | | Key                | Value   | Header             | Description                                                                                                                            |
166 | | ---                | ----    | ------             | -----------                                                                                                                            |
167 | | `max_age`          | seconds | `max-age`          | Specifies the maximum amount of time that a representation will be considered fresh. This value is relative to the time of the request |
168 | | `s_maxage`         | seconds | `s-maxage`         | Only applies to shared (proxies) caches                                                                                                |
169 | | `public`           | boolean | `public`           | Marks authenticated responses as cacheable.                                                                                            |
170 | | `private`          | boolean | `private`          | Allows caches that are specific to one user to store the response. Shared caches (proxies) may not.                                    |
171 | | `no_cache`         | boolean | `no-cache`         | Forces caches to submit the request to the origin server for validation before releasing a cached copy, every time.                    |
172 | | `no_store`         | boolean | `no-store`         | Instructs caches not to keep a copy of the representation under any conditions.                                                        |
173 | | `must_revalidate`  | boolean | `must-revalidate`  | Tells the caches that they must obey any freshness information you give them about a representation.                                   |
174 | | `proxy_revalidate` | boolean | `proxy-revalidate` | Similar as `must-revalidate`, but only for proxies.                                                                                    |
175 | 
176 | ### Setting `Expires` Header
177 | 
178 | You can pass the `expires` key to the `caching_policy` and
179 | `default_caching_policy` methods if you insist on setting the expires
180 | header on a results. You will need to pass it a Time object indicating
181 | when the resourse is set to expire.
182 | 
183 | > Note that the `Cache-Control` header will take precedence over the
184 | > `Expires` header if both are present.
185 | 
186 | ### A Note About Browser Caching
187 | 
188 | Browser caching is well specified. It hasn't always been the case.
189 | Still, even modern browsers have different behaviors if it suits it's
190 | developers or their employers. Specs are meant to be ignored and so they
191 | are (I'm looking at you Chrome!). Setting the `Cache-Control` or
192 | `Expires` headers are not a guarrantie that the browsers and the proxies
193 | that stand between them and your content will behave the way you want
194 | them to. YMMV.
195 | 
196 | ### ACLs
197 | 
198 | ```middleman-s3_sync``` will set the resources's ACL to ```public-read``` by default. You
199 | can specificy a different ACL via the ```acl``` configuration option.
200 | The valid values are:
201 | 
202 | * ```private```
203 | * ```public-read```
204 | * ```public-read-write```
205 | * ```authenticated-read```
206 | * ```bucket-owner-read```
207 | * ```bucket-owner-full-control```
208 | 
209 | The full values and their semantics are [documented on AWS's
210 | documentation
211 | site](http://docs.aws.amazon.com/AmazonS3/latest/dev/ACLOverview.html#CannedACL).
212 | 
213 | ### Encryption
214 | 
215 | You can ask Amazon to encrypt your files at rest by setting the
216 | ```encryption``` option to true. [Server side encryption is documented
217 | on the AWS documentation
218 | site](http://docs.aws.amazon.com/AmazonS3/latest/dev/UsingServerSideEncryption.html)
219 | .
220 | 
221 | ### GZipped Content Encoding
222 | 
223 | You can set the ```prefer_gzip``` option to look for a gzipped version
224 | of a resource. The gzipped version of the resource will be pushed to S3
225 | instead of the original and the ```Content-Encoding``` and ```Content-Type```
226 | headers will be set correctly. This will cause Amazon to serve the
227 | compressed version of the resource.
228 | 
229 | ## A Debt of Gratitude
230 | 
231 | I used Middleman Sync as a template for building a Middleman extension.
232 | The code is well structured and easy to understand and it was easy to
233 | extend it to add my synchronization code. My gratitude goes to @karlfreeman
234 | and is work on Middleman sync.
235 | 
236 | Many thanks to [Gnip](http://gnip.com) and [dojo4](http://dojo4.com) for
237 | supporting and sponsoring work on middleman-s3_sync.
238 | 
239 | ## Contributing
240 | 
241 | 1. Fork it
242 | 2. Create your feature branch (`git checkout -b my-new-feature`)
243 | 3. Commit your changes (`git commit -am 'Add some feature'`)
244 | 4. Push to the branch (`git push origin my-new-feature`)
245 | 5. Create new Pull Request
246 | 


--------------------------------------------------------------------------------