├── spec ├── dummy_app │ ├── app │ │ └── assets │ │ │ └── javascripts │ │ │ └── application.js │ └── Rakefile ├── fixtures │ ├── azure_rm_with_yml │ │ └── config │ │ │ └── asset_sync.yml │ ├── backblaze_with_yml │ │ └── config │ │ │ └── asset_sync.yml │ ├── google_with_yml │ │ └── config │ │ │ └── asset_sync.yml │ ├── google_with_service_account_yml │ │ └── config │ │ │ └── asset_sync.yml │ ├── rackspace_with_yml │ │ └── config │ │ │ └── asset_sync.yml │ ├── with_invalid_yml │ │ └── config │ │ │ └── asset_sync.yml │ └── aws_with_yml │ │ └── config │ │ └── asset_sync.yml ├── spec_helper.rb ├── integration │ ├── aws_integration_spec.rb │ ├── backblaze_intergration_spec.rb │ └── azure_rm_integration_spec.rb └── unit │ ├── railsless_spec.rb │ ├── rackspace_spec.rb │ ├── multi_mime_spec.rb │ ├── azure_rm_spec.rb │ ├── backblaze_spec.rb │ ├── google_spec.rb │ ├── asset_sync_spec.rb │ └── storage_spec.rb ├── lib ├── asset_sync │ ├── version.rb │ ├── railtie.rb │ ├── multi_mime.rb │ ├── asset_sync.rb │ ├── engine.rb │ ├── storage.rb │ └── config.rb ├── asset_sync.rb ├── tasks │ └── asset_sync.rake └── generators │ └── asset_sync │ ├── install_generator.rb │ └── templates │ ├── asset_sync.yml │ └── asset_sync.rb ├── Appraisals ├── .gitignore ├── Gemfile ├── gemfiles ├── rails_5_2.gemfile ├── rails_6_0.gemfile └── rails_6_1.gemfile ├── .editorconfig ├── UPGRADING.md ├── .travis.yml ├── Rakefile ├── asset_sync.gemspec ├── docs └── heroku.md ├── .github └── workflows │ └── tests.yaml ├── README.md └── CHANGELOG.md /spec/dummy_app/app/assets/javascripts/application.js: -------------------------------------------------------------------------------- 1 | console.log("hello"); -------------------------------------------------------------------------------- /lib/asset_sync/version.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module AssetSync 4 | VERSION = "2.15.1" 5 | end 6 | -------------------------------------------------------------------------------- /lib/asset_sync/railtie.rb: -------------------------------------------------------------------------------- 1 | class Rails::Railtie::Configuration 2 | def asset_sync 3 | AssetSync.config 4 | end 5 | end -------------------------------------------------------------------------------- /Appraisals: -------------------------------------------------------------------------------- 1 | 2 | appraise "rails_5_2" do 3 | gem "rails", "~> 5.2.0" 4 | end 5 | 6 | appraise "rails_6_0" do 7 | gem "rails", "~> 6.0.0" 8 | end 9 | 10 | appraise "rails_6_1" do 11 | gem "rails", "~> 6.1.0" 12 | end 13 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | *.gem 3 | .bundle 4 | Gemfile.lock 5 | pkg/* 6 | *.swp 7 | coverage/* 8 | .env 9 | spec/dummy_app/public/* 10 | spec/dummy_app/log/* 11 | spec/dummy_app/tmp/* 12 | .rbx 13 | gemfiles/*.lock 14 | .rvmrc 15 | .idea 16 | -------------------------------------------------------------------------------- /lib/asset_sync.rb: -------------------------------------------------------------------------------- 1 | require "asset_sync/asset_sync" 2 | require 'asset_sync/config' 3 | require 'asset_sync/storage' 4 | require 'asset_sync/multi_mime' 5 | 6 | 7 | require 'asset_sync/railtie' if defined?(Rails) 8 | require 'asset_sync/engine' if defined?(Rails) 9 | -------------------------------------------------------------------------------- /Gemfile: -------------------------------------------------------------------------------- 1 | source 'https://rubygems.org' 2 | gemspec 3 | gem 'rcov', :platforms => :mri_18, :group => [:development, :test] 4 | gem 'simplecov', :platforms => [:jruby, :mri_19, :ruby_19, :mri_20, :rbx], :group => [:development, :test], :require => false 5 | gem 'jruby-openssl', :platform => :jruby 6 | gem 'rails' 7 | -------------------------------------------------------------------------------- /gemfiles/rails_5_2.gemfile: -------------------------------------------------------------------------------- 1 | # This file was generated by Appraisal 2 | 3 | source "https://rubygems.org" 4 | 5 | gem "rcov", platforms: :mri_18, group: [:development, :test] 6 | gem "simplecov", platforms: [:jruby, :mri_19, :ruby_19, :mri_20, :rbx], group: [:development, :test], require: false 7 | gem "jruby-openssl", platform: :jruby 8 | gem "rails", "~> 5.2.0" 9 | 10 | gemspec path: "../" 11 | -------------------------------------------------------------------------------- /gemfiles/rails_6_0.gemfile: -------------------------------------------------------------------------------- 1 | # This file was generated by Appraisal 2 | 3 | source "https://rubygems.org" 4 | 5 | gem "rcov", platforms: :mri_18, group: [:development, :test] 6 | gem "simplecov", platforms: [:jruby, :mri_19, :ruby_19, :mri_20, :rbx], group: [:development, :test], require: false 7 | gem "jruby-openssl", platform: :jruby 8 | gem "rails", "~> 6.0.0" 9 | 10 | gemspec path: "../" 11 | -------------------------------------------------------------------------------- /gemfiles/rails_6_1.gemfile: -------------------------------------------------------------------------------- 1 | # This file was generated by Appraisal 2 | 3 | source "https://rubygems.org" 4 | 5 | gem "rcov", platforms: :mri_18, group: [:development, :test] 6 | gem "simplecov", platforms: [:jruby, :mri_19, :ruby_19, :mri_20, :rbx], group: [:development, :test], require: false 7 | gem "jruby-openssl", platform: :jruby 8 | gem "rails", "~> 6.1.0" 9 | 10 | gemspec path: "../" 11 | -------------------------------------------------------------------------------- /spec/fixtures/azure_rm_with_yml/config/asset_sync.yml: -------------------------------------------------------------------------------- 1 | defaults: &defaults 2 | fog_provider: "AzureRM" 3 | azure_storage_account_name: 'xxxx' 4 | azure_storage_access_key: 'zzzz' 5 | 6 | development: 7 | <<: *defaults 8 | fog_directory: "rails_app_development" 9 | existing_remote_files: keep 10 | 11 | test: 12 | <<: *defaults 13 | fog_directory: "rails_app_test" 14 | existing_remote_files: keep 15 | 16 | production: 17 | <<: *defaults 18 | fog_directory: "rails_app_production" 19 | existing_remote_files: delete 20 | -------------------------------------------------------------------------------- /spec/fixtures/backblaze_with_yml/config/asset_sync.yml: -------------------------------------------------------------------------------- 1 | defaults: &defaults 2 | fog_provider: "Backblaze" 3 | b2_key_id: 'xxxx' 4 | b2_key_token: 'zzzz' 5 | b2_bucket_id: '1234' 6 | 7 | development: 8 | <<: *defaults 9 | fog_directory: "rails_app_development" 10 | existing_remote_files: keep 11 | 12 | test: 13 | <<: *defaults 14 | fog_directory: "rails_app_test" 15 | existing_remote_files: keep 16 | 17 | production: 18 | <<: *defaults 19 | fog_directory: "rails_app_production" 20 | existing_remote_files: delete 21 | -------------------------------------------------------------------------------- /spec/fixtures/google_with_yml/config/asset_sync.yml: -------------------------------------------------------------------------------- 1 | defaults: &defaults 2 | fog_provider: "Google" 3 | google_storage_access_key_id: 'xxxx' 4 | google_storage_secret_access_key: 'zzzz' 5 | 6 | development: 7 | <<: *defaults 8 | fog_directory: "rails_app_development" 9 | existing_remote_files: keep 10 | 11 | test: 12 | <<: *defaults 13 | fog_directory: "rails_app_test" 14 | existing_remote_files: keep 15 | 16 | production: 17 | <<: *defaults 18 | fog_directory: "rails_app_production" 19 | existing_remote_files: delete 20 | -------------------------------------------------------------------------------- /spec/fixtures/google_with_service_account_yml/config/asset_sync.yml: -------------------------------------------------------------------------------- 1 | defaults: &defaults 2 | fog_provider: "Google" 3 | google_json_key_location: 'gcs.json' 4 | google_project: 'some-project' 5 | 6 | development: 7 | <<: *defaults 8 | fog_directory: "rails_app_development" 9 | existing_remote_files: keep 10 | 11 | test: 12 | <<: *defaults 13 | fog_directory: "rails_app_test" 14 | existing_remote_files: keep 15 | 16 | production: 17 | <<: *defaults 18 | fog_directory: "rails_app_production" 19 | existing_remote_files: delete 20 | -------------------------------------------------------------------------------- /spec/fixtures/rackspace_with_yml/config/asset_sync.yml: -------------------------------------------------------------------------------- 1 | defaults: &defaults 2 | fog_provider: "Rackspace" 3 | rackspace_username: "xxxx" 4 | rackspace_api_key: "zzzz" 5 | region: "eu-west-1" 6 | 7 | development: 8 | <<: *defaults 9 | fog_directory: "rails_app_development" 10 | existing_remote_files: keep 11 | 12 | test: 13 | <<: *defaults 14 | fog_directory: "rails_app_test" 15 | existing_remote_files: keep 16 | 17 | production: 18 | <<: *defaults 19 | fog_directory: "rails_app_production" 20 | existing_remote_files: delete 21 | -------------------------------------------------------------------------------- /.editorconfig: -------------------------------------------------------------------------------- 1 | # EditorConfig is awesome: http://EditorConfig.org 2 | 3 | # top-most EditorConfig file 4 | root = true 5 | 6 | # default configuration 7 | [*] 8 | indent_style = space 9 | indent_size = 2 10 | insert_final_newline = true 11 | trim_trailing_whitespace = true 12 | 13 | # Unix-style newlines with a newline ending every file 14 | end_of_line = lf 15 | 16 | # Set default charset 17 | charset = utf-8 18 | 19 | # Tab indentation (no size specified) 20 | [Makefile] 21 | indent_style = tab 22 | 23 | [*.{md,markdown}] 24 | trim_trailing_whitespace = false 25 | -------------------------------------------------------------------------------- /spec/fixtures/with_invalid_yml/config/asset_sync.yml: -------------------------------------------------------------------------------- 1 | defaults: &defaults 2 | fog_provider= "AWS" 3 | aws_access_key_id: "xxxx" 4 | aws_secret_access_key: "zzzz" 5 | region: "eu-west-1" 6 | 7 | development: 8 | <<: *defaults 9 | fog_directory: "rails_app_development" 10 | existing_remote_files: keep 11 | 12 | test: 13 | <<: *defaults 14 | fog_directory: "rails_app_test" 15 | existing_remote_files: keep 16 | 17 | production: 18 | <<: *defaults 19 | fog_directory: "rails_app_production" 20 | existing_remote_files: delete 21 | 22 | hybrid: 23 | <<: *defaults 24 | enabled: false 25 | -------------------------------------------------------------------------------- /spec/fixtures/aws_with_yml/config/asset_sync.yml: -------------------------------------------------------------------------------- 1 | defaults: &defaults 2 | fog_provider: "AWS" 3 | aws_access_key_id: "xxxx" 4 | aws_secret_access_key: "zzzz" 5 | region: "eu-west-1" 6 | run_on_precompile: false 7 | fog_path_style: true 8 | cache_asset_regexps: ['cache_me.js', !ruby/regexp '/cache_some\.\d{8}\.css/'] 9 | 10 | development: 11 | <<: *defaults 12 | fog_directory: "rails_app_development" 13 | existing_remote_files: keep 14 | 15 | test: 16 | <<: *defaults 17 | fog_directory: "rails_app_test" 18 | existing_remote_files: keep 19 | 20 | production: 21 | <<: *defaults 22 | fog_directory: "rails_app_production" 23 | existing_remote_files: delete 24 | 25 | hybrid: 26 | <<: *defaults 27 | enabled: false 28 | -------------------------------------------------------------------------------- /lib/asset_sync/multi_mime.rb: -------------------------------------------------------------------------------- 1 | require 'mime/types' 2 | 3 | module AssetSync 4 | class MultiMime 5 | 6 | def self.lookup(ext) 7 | overrides = 8 | ::AssetSync.config.file_ext_to_mime_type_overrides 9 | if overrides.key?(ext) 10 | return overrides.fetch(ext) 11 | end 12 | 13 | if defined?(::MIME::Types) 14 | ::MIME::Types.type_for(ext).first.to_s 15 | elsif defined?(::Mime::Type) 16 | ::Mime::Type.lookup_by_extension(ext).to_s 17 | elsif defined?(::Rack::Mime) 18 | ext_with_dot = ".#{ext}" 19 | ::Rack::Mime.mime_type(ext_with_dot) 20 | else 21 | raise "No library found for mime type lookup" 22 | end 23 | 24 | end 25 | 26 | end 27 | end 28 | -------------------------------------------------------------------------------- /UPGRADING.md: -------------------------------------------------------------------------------- 1 | # Guide to upgrading from AssetSync 1.x to 2.x 2 | 3 | Make sure that you're running the latest AssetSync 1.x release. 4 | 5 | This upgrading guide touches on: 6 | - Changed dependencies 7 | 8 | 9 | ## Changed dependencies 10 | Asset Sync now depends on gem `fog-core` instead of `fog`. 11 | This is due to `fog` is including many unused storage provider gems as its dependencies. 12 | 13 | Asset Sync has no idea about what provider will be used, 14 | so you are responsible for bundling the right gem for the provider to be used. 15 | 16 | For example, when using AWS as fog provider: 17 | ```ruby 18 | # Gemfile 19 | gem "asset_sync" 20 | gem "fog-aws" 21 | ``` 22 | 23 | If you don't install the required gem, 24 | Fog will complain (by exception) about it when provider is set by Asset Sync. 25 | 26 | -------------------------------------------------------------------------------- /spec/dummy_app/Rakefile: -------------------------------------------------------------------------------- 1 | require 'rubygems' 2 | # require "rake" 3 | ENV['RAILS_ROOT'] = File.dirname(__FILE__) 4 | # Set up gems listed in the Gemfile. 5 | ENV['BUNDLE_GEMFILE'] ||= File.expand_path('../../Gemfile', __FILE__) 6 | require 'bundler/setup' if File.exist?(ENV['BUNDLE_GEMFILE']) 7 | # require 'rails/all' 8 | require "action_controller/railtie" 9 | require "sprockets/railtie" 10 | if defined?(Bundler) 11 | Bundler.require(*Rails.groups(:assets => %w(development test))) 12 | end 13 | module AssetSyncTest 14 | class Application < Rails::Application 15 | config.encoding = "utf-8" 16 | config.filter_parameters += [:password] 17 | config.eager_load = false 18 | config.assets.enabled = true 19 | config.assets.version = '1.0' 20 | config.secret_token = 'bf196b4383deefa4e0120a6ef1d9af1cc45f5c4ebd04405' 21 | config.session_store :cookie_store, :key => '_asset_sync_test_session', :secret => 'xxxx' 22 | config.active_support.deprecation = :log 23 | config.assets.compress = true 24 | config.assets.digest = true 25 | config.assets.prefix = ENV['ASSET_SYNC_PREFIX'] 26 | end 27 | end 28 | AssetSyncTest::Application.initialize! 29 | AssetSyncTest::Application.load_tasks 30 | # Rake::Task['assets:precompile:all'].invoke 31 | -------------------------------------------------------------------------------- /lib/tasks/asset_sync.rake: -------------------------------------------------------------------------------- 1 | namespace :assets do 2 | 3 | desc 'Synchronize assets to remote (assumes assets are already compiled)' 4 | task :sync => :environment do 5 | AssetSync.sync 6 | end 7 | namespace :sync do 8 | desc 'Delete out-of-sync files on remote' 9 | task :clean => :environment do 10 | AssetSync.clean 11 | end 12 | end 13 | 14 | end 15 | 16 | if Rake::Task.task_defined?("assets:precompile:nondigest") 17 | Rake::Task["assets:precompile:nondigest"].enhance do 18 | # Conditional execution needs to be inside the enhance block because the enhance block 19 | # will get executed before yaml or Rails initializers. 20 | Rake::Task["assets:sync"].invoke if defined?(AssetSync) && AssetSync.config.run_on_precompile 21 | end 22 | elsif Rake::Task.task_defined?("assets:precompile") 23 | Rake::Task["assets:precompile"].enhance do 24 | # rails 3.1.1 will clear out Rails.application.config if the env vars 25 | # RAILS_GROUP and RAILS_ENV are not defined. We need to reload the 26 | # assets environment in this case. 27 | # Rake::Task["assets:environment"].invoke if Rake::Task.task_defined?("assets:environment") 28 | Rake::Task["assets:sync"].invoke if defined?(AssetSync) && AssetSync.config.run_on_precompile 29 | end 30 | else 31 | # Nothing to be enhanced 32 | end 33 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | # Send builds to container-based infrastructure 2 | # http://docs.travis-ci.com/user/workers/container-based-infrastructure/ 3 | sudo: false 4 | language: ruby 5 | arch: 6 | - amd64 7 | - ppc64le 8 | cache: 9 | bundler: true 10 | rvm: 11 | - 2.5 12 | - 2.6 13 | - 2.7 14 | - ruby-head 15 | - jruby 16 | - jruby-head 17 | gemfile: 18 | - gemfiles/rails_5_2.gemfile 19 | - gemfiles/rails_6_0.gemfile 20 | - gemfiles/rails_6_1.gemfile 21 | before_install: 22 | # Cannot use bundler 2.x due to dependency (mainly rails 4.2) 23 | # Solution from https://github.com/rails/rails/blob/4-2-stable/.travis.yml 24 | - "travis_retry gem update --system --no-doc || travis_retry gem update --system --no-rdoc --no-ri" 25 | - "travis_retry gem install bundler -v '<2'" 26 | env: 27 | global: 28 | - FOG_DIRECTORY=asset-sync-travis 29 | - FOG_PROVIDER=AWS 30 | - secure: "dy8Fqlg3b1ZMK1BY5z6NMQLbzAVd7GWVYY0MeCQALz76zRac0z8MyU8hkv6h\nozFry7DSdbGehGT9foOnkWTwzGzf1rzdd5cmWrUPk1wDTRgMM9SrwodPj1TU\nzsq2EFx0a79vADQN8JXkpLC1YD6kEb9aWkTxrIT9KBgw+J5H32o=" 31 | - secure: "Hmx7D7/p2LlA2ya/xBIz21s/8MLIQCjvfYB7RWBNlWk1PfqRLAz8wX6TUVWy\nfAFktMjLnpRLRYO7AgQS4jcfQ/k0HYK9IXzqXzeI00TNm0Vwp0TCXhawiOFT\nSvUMhs2/1vRfjN0HOJ75XlWRhJzV/G5rOMiafAZLsVzN/0iiB8g=" 32 | matrix: 33 | fast_finish: true 34 | allow_failures: 35 | - rvm: ruby-head 36 | - rvm: jruby-head 37 | notifications: 38 | webhooks: 39 | urls: 40 | - https://www.travisbuddy.com/ 41 | on_success: never 42 | -------------------------------------------------------------------------------- /Rakefile: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env rake 2 | begin 3 | require 'bundler/setup' 4 | require 'appraisal' 5 | rescue LoadError 6 | puts 'You must `gem install bundler` and `bundle install` to run rake tasks' 7 | end 8 | require 'bundler/gem_tasks' 9 | require 'rspec/core/rake_task' 10 | 11 | namespace :spec do 12 | RSpec::Core::RakeTask.new(:unit) do |spec| 13 | spec.pattern = 'spec/unit/*_spec.rb' 14 | spec.rspec_opts = ['--backtrace'] 15 | 16 | rbx = defined?(RUBY_ENGINE) && RUBY_ENGINE == 'rbx' 17 | jruby = defined?(RUBY_ENGINE) && RUBY_ENGINE == 'jruby' 18 | if RUBY_VERSION == '1.8.7' && !(rbx || jruby) 19 | spec.rcov = true 20 | spec.rcov_opts = %w{--exclude gems\/,spec\/} 21 | end 22 | end 23 | RSpec::Core::RakeTask.new(:integration) do |spec| 24 | spec.pattern = 'spec/integration/*_spec.rb' 25 | spec.rspec_opts = ['--backtrace'] 26 | end 27 | desc "run spec:unit and spec:integration tasks" 28 | task :all do 29 | Rake::Task['spec:unit'].execute 30 | 31 | # Travis CI does not expose encrypted ENV variables for pull requests, so 32 | # do not run integration specs 33 | # http://about.travis-ci.org/docs/user/build-configuration/#Set-environment-variables 34 | # 35 | pull_request = ENV['TRAVIS_PULL_REQUEST'] == 'true' 36 | ci_build = ENV['TRAVIS'] == 'true' 37 | if !ci_build || (ci_build && pull_request) 38 | Rake::Task['spec:integration'].execute 39 | end 40 | end 41 | end 42 | 43 | if !ENV["APPRAISAL_INITIALIZED"] && !ENV["TRAVIS"] 44 | task :default do 45 | sh "appraisal install && rake appraisal spec:unit" 46 | end 47 | else 48 | task default: ["spec:unit"] 49 | end 50 | -------------------------------------------------------------------------------- /lib/asset_sync/asset_sync.rb: -------------------------------------------------------------------------------- 1 | require "yaml" 2 | 3 | module AssetSync 4 | 5 | class << self 6 | 7 | def config=(data) 8 | @config = data 9 | end 10 | 11 | def config 12 | @config ||= Config.new 13 | @config 14 | end 15 | 16 | def reset_config! 17 | remove_instance_variable :@config if defined?(@config) 18 | end 19 | 20 | def configure(&proc) 21 | @config ||= Config.new 22 | yield @config 23 | end 24 | 25 | def storage 26 | @storage ||= Storage.new(self.config) 27 | end 28 | 29 | def sync 30 | with_config do 31 | self.storage.sync 32 | end 33 | end 34 | 35 | def clean 36 | with_config do 37 | self.storage.delete_extra_remote_files 38 | end 39 | end 40 | 41 | def with_config(&block) 42 | return unless AssetSync.enabled? 43 | 44 | errors = config.valid? ? "" : config.errors.full_messages.join(', ') 45 | 46 | if !(config && config.valid?) 47 | if config.fail_silently? 48 | self.warn(errors) 49 | else 50 | raise Config::Invalid.new(errors) 51 | end 52 | else 53 | block.call 54 | end 55 | end 56 | 57 | def warn(msg) 58 | stderr.puts msg 59 | end 60 | 61 | def log(msg) 62 | stdout.puts msg unless config.log_silently? 63 | end 64 | 65 | def load_yaml(yaml) 66 | if YAML.respond_to?(:unsafe_load) 67 | YAML.unsafe_load(yaml) 68 | else 69 | YAML.load(yaml) 70 | end 71 | end 72 | 73 | def enabled? 74 | config.enabled? 75 | end 76 | 77 | # easier to stub 78 | def stderr ; STDERR ; end 79 | def stdout ; STDOUT ; end 80 | 81 | end 82 | 83 | end 84 | -------------------------------------------------------------------------------- /asset_sync.gemspec: -------------------------------------------------------------------------------- 1 | # -*- encoding: utf-8 -*- 2 | $:.push File.expand_path("../lib", __FILE__) 3 | 4 | require "asset_sync/version" 5 | 6 | Gem::Specification.new do |s| 7 | s.name = "asset_sync" 8 | s.version = AssetSync::VERSION 9 | s.platform = Gem::Platform::RUBY 10 | s.authors = ["Simon Hamilton", "David Rice", "Phil McClure", "Toby Osbourn"] 11 | s.email = ["shamilton@rumblelabs.com", "me@davidjrice.co.uk", "pmcclure@rumblelabs.com", "tosbourn@rumblelabs.com"] 12 | s.homepage = "https://github.com/rumblelabs/asset_sync" 13 | s.summary = %q{Synchronises Assets in a Rails 3 application and Amazon S3/Cloudfront and Rackspace Cloudfiles} 14 | s.description = %q{After you run assets:precompile your compiled assets will be synchronised with your S3 bucket.} 15 | 16 | s.license = 'MIT' 17 | 18 | s.add_dependency("fog-core") 19 | s.add_dependency('unf') 20 | s.add_dependency('activemodel', ">= 4.1.0") 21 | s.add_dependency('mime-types', ">= 2.99") 22 | 23 | s.add_development_dependency "rspec" 24 | s.add_development_dependency "bundler" 25 | s.add_development_dependency "coveralls", ">= 0.7" 26 | 27 | s.add_development_dependency('mime-types', ">= 3.0") 28 | 29 | s.add_development_dependency "fog-aws" 30 | s.add_development_dependency "gitlab-fog-azure-rm" 31 | s.add_development_dependency "fog-backblaze" 32 | 33 | s.add_development_dependency "uglifier" 34 | s.add_development_dependency "appraisal" 35 | 36 | s.add_development_dependency "gem-release" 37 | 38 | s.files = `git ls-files`.split("\n") 39 | s.test_files = `git ls-files -- {test,spec,features}/*`.split("\n") 40 | s.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) } 41 | s.require_paths = ["lib"] 42 | end 43 | -------------------------------------------------------------------------------- /docs/heroku.md: -------------------------------------------------------------------------------- 1 | The following issues are currently present in Heroku if you are not following the steps outlined in the main [README](http://github.com/rumblelabs/asset_sync) 2 | 3 | ## KNOWN ISSUES (IMPORTANT) 4 | 5 | We are currently trying to talk with Heroku to iron these out. 6 | 7 | 1. Will not work on heroku on an application with a *RAILS_ENV* configured as anything other than production 8 | 2. Will not work on heroku using ENV variables with the configuration as described below, you must hardcode all variables 9 | 10 | ### 1. RAILS_ENV 11 | 12 | When you see `rake assets:precompile` during deployment. Heroku is actually running something like 13 | 14 | env RAILS_ENV=production DATABASE_URL=scheme://user:pass@127.0.0.1/dbname bundle exec rake assets:precompile 2>&1 15 | 16 | This means the *RAILS_ENV* you have set via *heroku:config* is not used. 17 | 18 | **Workaround:** you could have just one S3 bucket dedicated to assets and configure `AssetSync` to not delete existing files: 19 | 20 | AssetSync.configure do |config| 21 | ... 22 | config.fog_directory = 'app-assets' 23 | config.existing_remote_files = "keep" 24 | end 25 | 26 | ### 2. ENV varables not available 27 | 28 | Currently when heroku runs `rake assets:precompile` during deployment. It does not load your Rails application's environment config. This means using any **ENV** variables you could normally depend on are not available. For now you can just run `heroku run rake assets:precompile` after deploy. 29 | 30 | **Workaround:** you could just hardcode your AWS credentials in the initializer or yml 31 | 32 | AssetSync.configure do |config| 33 | config.aws_access_key_id = 'xxx' 34 | config.aws_secret_access_key = 'xxx' 35 | config.fog_directory = 'mybucket' 36 | end 37 | -------------------------------------------------------------------------------- /spec/spec_helper.rb: -------------------------------------------------------------------------------- 1 | require 'rubygems' 2 | require 'bundler' 3 | 4 | begin 5 | require 'simplecov' 6 | SimpleCov.start do 7 | add_filter 'spec' 8 | end 9 | rescue LoadError 10 | # SimpleCov ain't available - continue 11 | end 12 | 13 | if ENV["TRAVIS"] 14 | require "coveralls" 15 | Coveralls.wear!("rails") 16 | end 17 | 18 | 19 | begin 20 | Bundler.setup(:default, :development) 21 | rescue Bundler::BundlerError => e 22 | $stderr.puts e.message 23 | $stderr.puts "Run `bundle install` to install missing gems" 24 | exit e.status_code 25 | end 26 | 27 | $LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib')) 28 | $LOAD_PATH.unshift(File.dirname(__FILE__)) 29 | require 'asset_sync' 30 | 31 | require 'rspec' 32 | RSpec.configure do |config| 33 | config.mock_framework = :rspec 34 | end 35 | 36 | shared_context "mock without Rails" do 37 | before(:each) do 38 | if defined? Rails 39 | Object.send(:remove_const, :Rails) 40 | end 41 | allow(AssetSync).to receive(:log) 42 | end 43 | end 44 | 45 | 46 | shared_context "mock Rails" do 47 | before(:each) do 48 | Object.send(:remove_const, :Rails) if defined? Rails 49 | Rails = double 'Rails' 50 | allow(Rails).to receive(:env).and_return('test') 51 | allow(Rails).to receive_messages :application => double('application') 52 | allow(Rails.application).to receive_messages :config => double('config') 53 | allow(Rails.application.config).to receive_messages :assets => ActiveSupport::OrderedOptions.new 54 | Rails.application.config.assets.prefix = '/assets' 55 | allow(AssetSync).to receive(:log) 56 | end 57 | end 58 | 59 | shared_context "mock Rails without_yml" do 60 | include_context "mock Rails" 61 | 62 | before(:each) do 63 | set_rails_root('without_yml') 64 | allow(Rails).to receive(:public_path).and_return(Rails.root.join('public').to_s) 65 | end 66 | end 67 | 68 | def set_rails_root(path) 69 | allow(Rails).to receive(:root).and_return(Pathname.new(File.join(File.dirname(__FILE__), 'fixtures', path))) 70 | end 71 | -------------------------------------------------------------------------------- /.github/workflows/tests.yaml: -------------------------------------------------------------------------------- 1 | name: Tests 2 | 3 | on: 4 | pull_request: 5 | branches: 6 | - master 7 | paths-ignore: 8 | - 'README.md' 9 | - 'CHANGELOG.md' 10 | push: 11 | branches: 12 | - master 13 | paths-ignore: 14 | - 'README.md' 15 | - 'CHANGELOG.md' 16 | 17 | jobs: 18 | unit_tests: 19 | name: Unit Tests 20 | # Homemade support for [ci skip] no longer needed 21 | # https://github.blog/changelog/2021-02-08-github-actions-skip-pull-request-and-push-workflows-with-skip-ci/ 22 | # if: "contains(github.event.commits[0].message, '[ci skip]') == false" 23 | strategy: 24 | fail-fast: false 25 | matrix: 26 | os: 27 | - ubuntu 28 | ruby: 29 | - 2.5 30 | - 2.6 31 | - 2.7 32 | - 3.0 33 | - jruby 34 | gemfile: 35 | - gemfiles/rails_5_2.gemfile 36 | - gemfiles/rails_6_0.gemfile 37 | - gemfiles/rails_6_1.gemfile 38 | allow_failures: 39 | - false 40 | include: 41 | - os: ubuntu 42 | ruby: ruby-head 43 | gemfile: gemfiles/rails_6_1.gemfile 44 | allow_failures: true 45 | - os: ubuntu 46 | ruby: jruby-head 47 | gemfile: gemfiles/rails_6_1.gemfile 48 | allow_failures: true 49 | exclude: 50 | - os: ubuntu 51 | ruby: 3.0 52 | gemfile: gemfiles/rails_5_2.gemfile 53 | allow_failures: false 54 | env: 55 | BUNDLE_GEMFILE: "${{ matrix.gemfile }}" 56 | ALLOW_FAILURES: "${{ matrix.allow_failures }}" 57 | runs-on: ${{ matrix.os }}-latest 58 | continue-on-error: ${{ endsWith(matrix.ruby, 'head') || matrix.ruby == 'debug' }} 59 | steps: 60 | - name: Checkout 61 | uses: actions/checkout@v2 62 | - name: Setup Ruby 63 | uses: ruby/setup-ruby@v1 64 | with: 65 | ruby-version: ${{ matrix.ruby }} 66 | bundler-cache: true 67 | - name: Test 68 | run: bundle exec rake spec:unit || $ALLOW_FAILURES 69 | -------------------------------------------------------------------------------- /spec/integration/aws_integration_spec.rb: -------------------------------------------------------------------------------- 1 | require File.dirname(__FILE__) + '/../spec_helper' 2 | require "fog/aws" 3 | 4 | def bucket(name) 5 | options = { 6 | :provider => 'AWS', 7 | :aws_access_key_id => ENV['AWS_ACCESS_KEY_ID'], 8 | :aws_secret_access_key => ENV['AWS_SECRET_ACCESS_KEY'] 9 | } 10 | 11 | connection = Fog::Storage.new(options) 12 | connection.directories.get(ENV['FOG_DIRECTORY'], :prefix => name) 13 | end 14 | 15 | def execute(command) 16 | app_path = File.expand_path("../../dummy_app", __FILE__) 17 | Dir.chdir app_path 18 | `#{command}` 19 | end 20 | 21 | describe "AssetSync" do 22 | 23 | before(:each) do 24 | @prefix = SecureRandom.hex(6) 25 | end 26 | 27 | let(:app_js_regex){ 28 | /#{@prefix}\/application-[a-zA-Z0-9]*.js$/ 29 | } 30 | 31 | let(:app_js_gz_regex){ 32 | /#{@prefix}\/application-[a-zA-Z0-9]*.js.gz$/ 33 | } 34 | 35 | let(:files){ bucket(@prefix).files } 36 | 37 | 38 | after(:each) do 39 | @directory = bucket(@prefix) 40 | @directory.files.each do |f| 41 | f.destroy 42 | end 43 | end 44 | 45 | it "sync" do 46 | execute "rake ASSET_SYNC_PREFIX=#{@prefix} assets:precompile" 47 | 48 | files = bucket(@prefix).files 49 | 50 | app_js_path = files.select{ |f| f.key =~ app_js_regex }.first 51 | app_js_gz_path = files.select{ |f| f.key =~ app_js_gz_regex }.first 52 | 53 | app_js = files.get( app_js_path.key ) 54 | expect(app_js.content_type).to eq("text/javascript") 55 | 56 | app_js_gz = files.get( app_js_gz_path.key ) 57 | expect(app_js_gz.content_type).to eq("text/javascript") 58 | expect(app_js_gz.content_encoding).to eq("gzip") 59 | end 60 | 61 | it "sync with enabled=false" do 62 | execute "rake ASSET_SYNC_PREFIX=#{@prefix} ASSET_SYNC_ENABLED=false assets:precompile" 63 | expect(bucket(@prefix).files.size).to eq(0) 64 | end 65 | 66 | it "sync with gzip_compression=true" do 67 | execute "rake ASSET_SYNC_PREFIX=#{@prefix} ASSET_SYNC_GZIP_COMPRESSION=true assets:precompile" 68 | # bucket(@prefix).files.size.should == 3 69 | 70 | app_js_path = files.select{ |f| f.key =~ app_js_regex }.first 71 | app_js = files.get( app_js_path.key ) 72 | expect(app_js.content_type).to eq("text/javascript") 73 | end 74 | 75 | end 76 | 77 | -------------------------------------------------------------------------------- /spec/integration/backblaze_intergration_spec.rb: -------------------------------------------------------------------------------- 1 | require File.dirname(__FILE__) + '/../spec_helper' 2 | require "fog/backblaze" 3 | 4 | def bucket(name) 5 | options = { 6 | :provider => 'Backblaze', 7 | :b2_key_id => ENV['B2_KEY_ID'], 8 | :b2_key_token => ENV['B2_KEY_TOKEN'], 9 | :b2_bucket_id => ENV['B2_BUCKET_ID'] 10 | } 11 | options.merge!({ :environment => ENV['FOG_REGION'] }) if ENV.has_key?('FOG_REGION') 12 | 13 | connection = Fog::Storage.new(options) 14 | connection.directories.get(ENV['FOG_DIRECTORY']) 15 | end 16 | 17 | def execute(command) 18 | app_path = File.expand_path("../../dummy_app", __FILE__) 19 | Dir.chdir app_path 20 | `#{command}` 21 | end 22 | 23 | describe "AssetSync" do 24 | 25 | before(:each) do 26 | @prefix = SecureRandom.hex(6) 27 | end 28 | 29 | let(:app_js_regex){ 30 | /#{@prefix}\/application-[a-zA-Z0-9]*.js$/ 31 | } 32 | 33 | let(:app_js_gz_regex){ 34 | /#{@prefix}\/application-[a-zA-Z0-9]*.js.gz$/ 35 | } 36 | 37 | let(:files){ bucket(@prefix).files } 38 | 39 | 40 | after(:each) do 41 | @directory = bucket(@prefix) 42 | @directory.files.each do |f| 43 | f.destroy 44 | end 45 | end 46 | 47 | it "sync" do 48 | execute "rake ASSET_SYNC_PREFIX=#{@prefix} assets:precompile" 49 | 50 | files = bucket(@prefix).files 51 | 52 | app_js = files.select{ |f| f.key =~ app_js_regex }.first 53 | expect(app_js.content_type).to eq("application/javascript") 54 | 55 | app_js_gz = files.select{ |f| f.key =~ app_js_gz_regex }.first 56 | expect(app_js_gz.content_type).to eq("application/javascript") 57 | expect(app_js_gz.content_encoding).to eq("gzip") 58 | end 59 | 60 | it "sync with enabled=false" do 61 | execute "rake ASSET_SYNC_PREFIX=#{@prefix} ASSET_SYNC_ENABLED=false assets:precompile" 62 | expect(bucket(@prefix).files.size).to eq(0) 63 | end 64 | 65 | it "sync with gzip_compression=true" do 66 | execute "rake ASSET_SYNC_PREFIX=#{@prefix} ASSET_SYNC_GZIP_COMPRESSION=true assets:precompile" 67 | # bucket(@prefix).files.size.should == 3 68 | 69 | app_js_path = files.select{ |f| f.key =~ app_js_regex }.first 70 | app_js = files.get( app_js_path.key ) 71 | expect(app_js.content_type).to eq("application/javascript") 72 | end 73 | 74 | end 75 | -------------------------------------------------------------------------------- /spec/integration/azure_rm_integration_spec.rb: -------------------------------------------------------------------------------- 1 | require File.dirname(__FILE__) + '/../spec_helper' 2 | require "fog/azurerm" 3 | 4 | def bucket(name) 5 | options = { 6 | :provider => 'AzureRM', 7 | :azure_storage_account_name => ENV['AZURE_STORAGE_ACCOUNT_NAME'], 8 | :azure_storage_access_key => ENV['AZURE_STORAGE_ACCESS_KEY'] 9 | } 10 | options.merge!({ :environment => ENV['FOG_REGION'] }) if ENV.has_key?('FOG_REGION') 11 | 12 | connection = Fog::Storage.new(options) 13 | connection.directories.get(ENV['FOG_DIRECTORY'], :prefix => name) 14 | end 15 | 16 | def execute(command) 17 | app_path = File.expand_path("../../dummy_app", __FILE__) 18 | Dir.chdir app_path 19 | `#{command}` 20 | end 21 | 22 | describe "AssetSync" do 23 | 24 | before(:each) do 25 | @prefix = SecureRandom.hex(6) 26 | end 27 | 28 | let(:app_js_regex){ 29 | /#{@prefix}\/application-[a-zA-Z0-9]*.js$/ 30 | } 31 | 32 | let(:app_js_gz_regex){ 33 | /#{@prefix}\/application-[a-zA-Z0-9]*.js.gz$/ 34 | } 35 | 36 | let(:files){ bucket(@prefix).files } 37 | 38 | 39 | after(:each) do 40 | @directory = bucket(@prefix) 41 | @directory.files.each do |f| 42 | f.destroy 43 | end 44 | end 45 | 46 | it "sync" do 47 | execute "rake ASSET_SYNC_PREFIX=#{@prefix} assets:precompile" 48 | 49 | files = bucket(@prefix).files 50 | 51 | app_js = files.select{ |f| f.key =~ app_js_regex }.first 52 | expect(app_js.content_type).to eq("application/javascript") 53 | 54 | app_js_gz = files.select{ |f| f.key =~ app_js_gz_regex }.first 55 | expect(app_js_gz.content_type).to eq("application/javascript") 56 | expect(app_js_gz.content_encoding).to eq("gzip") 57 | end 58 | 59 | it "sync with enabled=false" do 60 | execute "rake ASSET_SYNC_PREFIX=#{@prefix} ASSET_SYNC_ENABLED=false assets:precompile" 61 | expect(bucket(@prefix).files.size).to eq(0) 62 | end 63 | 64 | it "sync with gzip_compression=true" do 65 | execute "rake ASSET_SYNC_PREFIX=#{@prefix} ASSET_SYNC_GZIP_COMPRESSION=true assets:precompile" 66 | # bucket(@prefix).files.size.should == 3 67 | 68 | app_js_path = files.select{ |f| f.key =~ app_js_regex }.first 69 | app_js = files.get( app_js_path.key ) 70 | expect(app_js.content_type).to eq("application/javascript") 71 | end 72 | 73 | end 74 | 75 | -------------------------------------------------------------------------------- /spec/unit/railsless_spec.rb: -------------------------------------------------------------------------------- 1 | require File.dirname(__FILE__) + '/../spec_helper' 2 | 3 | describe AssetSync do 4 | include_context "mock without Rails" 5 | 6 | describe 'with initializer' do 7 | before(:each) do 8 | AssetSync.config = AssetSync::Config.new 9 | AssetSync.configure do |config| 10 | config.fog_provider = 'AWS' 11 | config.aws_access_key_id = 'aaaa' 12 | config.aws_secret_access_key = 'bbbb' 13 | config.fog_directory = 'mybucket' 14 | config.fog_region = 'eu-west-1' 15 | config.existing_remote_files = "keep" 16 | config.prefix = "assets" 17 | config.public_path = "./public" 18 | end 19 | end 20 | 21 | it "should have prefix of assets" do 22 | expect(AssetSync.config.prefix).to eq("assets") 23 | end 24 | 25 | it "should have public_path" do 26 | expect(AssetSync.config.public_path.to_s).to be_end_with("/public") 27 | expect(AssetSync.config.public_path).to be_absolute 28 | end 29 | 30 | it "should default AssetSync to enabled" do 31 | expect(AssetSync.config.enabled?).to be_truthy 32 | expect(AssetSync.enabled?).to be_truthy 33 | end 34 | 35 | it "should configure provider as AWS" do 36 | expect(AssetSync.config.fog_provider).to eq('AWS') 37 | expect(AssetSync.config).to be_aws 38 | end 39 | 40 | it "should should keep existing remote files" do 41 | expect(AssetSync.config.existing_remote_files?).to eq(true) 42 | end 43 | 44 | it "should configure aws_access_key" do 45 | expect(AssetSync.config.aws_access_key_id).to eq("aaaa") 46 | end 47 | 48 | it "should configure aws_secret_access_key" do 49 | expect(AssetSync.config.aws_secret_access_key).to eq("bbbb") 50 | end 51 | 52 | it "should configure aws_access_key" do 53 | expect(AssetSync.config.fog_directory).to eq("mybucket") 54 | end 55 | 56 | it "should configure fog_region" do 57 | expect(AssetSync.config.fog_region).to eq("eu-west-1") 58 | end 59 | 60 | it "should configure existing_remote_files" do 61 | expect(AssetSync.config.existing_remote_files).to eq("keep") 62 | end 63 | 64 | it "should default gzip_compression to false" do 65 | expect(AssetSync.config.gzip_compression).to be_falsey 66 | end 67 | 68 | it "should default manifest to false" do 69 | expect(AssetSync.config.manifest).to be_falsey 70 | end 71 | 72 | end 73 | end 74 | -------------------------------------------------------------------------------- /lib/generators/asset_sync/install_generator.rb: -------------------------------------------------------------------------------- 1 | require 'rails/generators' 2 | module AssetSync 3 | class InstallGenerator < Rails::Generators::Base 4 | desc "Install a config/asset_sync.yml and the asset:precompile rake task enhancer" 5 | 6 | # Commandline options can be defined here using Thor-like options: 7 | class_option :use_yml, :type => :boolean, :default => false, :desc => "Use YML file instead of Rails Initializer" 8 | class_option :provider, :type => :string, :default => "AWS", :desc => "Generate with support for 'AWS', 'Rackspace', 'Google', 'AzureRM', or 'Backblaze'" 9 | 10 | def self.source_root 11 | @source_root ||= File.join(File.dirname(__FILE__), 'templates') 12 | end 13 | 14 | def aws? 15 | options[:provider] == 'AWS' 16 | end 17 | 18 | def google? 19 | options[:provider] == 'Google' 20 | end 21 | 22 | def rackspace? 23 | options[:provider] == 'Rackspace' 24 | end 25 | 26 | def azure_rm? 27 | options[:provider] == 'AzureRM' 28 | end 29 | 30 | def backblaze? 31 | options[:provider] == 'Backblaze' 32 | end 33 | 34 | def aws_access_key_id 35 | "<%= ENV['AWS_ACCESS_KEY_ID'] %>" 36 | end 37 | 38 | def aws_secret_access_key 39 | "<%= ENV['AWS_SECRET_ACCESS_KEY'] %>" 40 | end 41 | 42 | def aws_session_token 43 | "<%= ENV['AWS_SESSION_TOKEN'] %>" 44 | end 45 | 46 | def google_storage_access_key_id 47 | "<%= ENV['GOOGLE_STORAGE_ACCESS_KEY_ID'] %>" 48 | end 49 | 50 | def google_storage_secret_access_key 51 | "<%= ENV['GOOGLE_STORAGE_SECRET_ACCESS_KEY'] %>" 52 | end 53 | 54 | def rackspace_username 55 | "<%= ENV['RACKSPACE_USERNAME'] %>" 56 | end 57 | 58 | def rackspace_api_key 59 | "<%= ENV['RACKSPACE_API_KEY'] %>" 60 | end 61 | 62 | def azure_storage_account_name 63 | "<%= ENV['AZURE_STORAGE_ACCOUNT_NAME'] %>" 64 | end 65 | 66 | def azure_storage_access_key 67 | "<%= ENV['AZURE_STORAGE_ACCESS_KEY'] %>" 68 | end 69 | 70 | def b2_key_id 71 | "<%= ENV['B2_KEY_ID'] %>" 72 | end 73 | 74 | def b2_key_token 75 | "<%= ENV['B2_KEY_TOKEN'] %>" 76 | end 77 | 78 | def b2_bucket_id 79 | "<%= ENV['B2_BUCKET_ID'] %>" 80 | end 81 | 82 | def app_name 83 | @app_name ||= Rails.application.is_a?(Rails::Application) && Rails.application.class.name.sub(/::Application$/, "").downcase 84 | end 85 | 86 | def generate_config 87 | if options[:use_yml] 88 | template "asset_sync.yml", "config/asset_sync.yml" 89 | end 90 | end 91 | 92 | def generate_initializer 93 | unless options[:use_yml] 94 | template "asset_sync.rb", "config/initializers/asset_sync.rb" 95 | end 96 | end 97 | 98 | end 99 | end 100 | -------------------------------------------------------------------------------- /spec/unit/rackspace_spec.rb: -------------------------------------------------------------------------------- 1 | require File.dirname(__FILE__) + '/../spec_helper' 2 | 3 | describe AssetSync do 4 | include_context "mock Rails" 5 | 6 | describe 'using Rackspace with initializer' do 7 | before(:each) do 8 | set_rails_root('without_yml') 9 | AssetSync.config = AssetSync::Config.new 10 | AssetSync.configure do |config| 11 | config.fog_provider = 'Rackspace' 12 | config.fog_directory = 'mybucket' 13 | config.fog_region = 'dunno' 14 | config.rackspace_username = 'aaaa' 15 | config.rackspace_api_key = 'bbbb' 16 | config.existing_remote_files = 'keep' 17 | end 18 | end 19 | 20 | it "should configure provider as Rackspace" do 21 | expect(AssetSync.config.fog_provider).to eq('Rackspace') 22 | expect(AssetSync.config).to be_rackspace 23 | end 24 | 25 | it "should keep existing remote files" do 26 | expect(AssetSync.config.existing_remote_files?).to eq(true) 27 | end 28 | 29 | it "should configure rackspace_username" do 30 | expect(AssetSync.config.rackspace_username).to eq("aaaa") 31 | end 32 | 33 | it "should configure rackspace_api_key" do 34 | expect(AssetSync.config.rackspace_api_key).to eq("bbbb") 35 | end 36 | 37 | it "should configure fog_directory" do 38 | expect(AssetSync.config.fog_directory).to eq("mybucket") 39 | end 40 | 41 | it "should configure fog_region" do 42 | expect(AssetSync.config.fog_region).to eq("dunno") 43 | end 44 | 45 | it "should configure existing_remote_files" do 46 | expect(AssetSync.config.existing_remote_files).to eq("keep") 47 | end 48 | 49 | it "should configure existing_remote_files" do 50 | expect(AssetSync.config.existing_remote_files).to eq("keep") 51 | end 52 | 53 | it "should default rackspace_auth_url to false" do 54 | expect(AssetSync.config.rackspace_auth_url).to be_falsey 55 | end 56 | 57 | end 58 | 59 | describe 'using Rackspace from yml' do 60 | 61 | before(:each) do 62 | set_rails_root('rackspace_with_yml') 63 | AssetSync.config = AssetSync::Config.new 64 | end 65 | 66 | it "should keep existing remote files" do 67 | expect(AssetSync.config.existing_remote_files?).to eq(true) 68 | end 69 | 70 | it "should configure rackspace_username" do 71 | expect(AssetSync.config.rackspace_username).to eq("xxxx") 72 | end 73 | 74 | it "should configure rackspace_api_key" do 75 | expect(AssetSync.config.rackspace_api_key).to eq("zzzz") 76 | end 77 | 78 | it "should configure fog_directory" do 79 | expect(AssetSync.config.fog_directory).to eq("rails_app_test") 80 | end 81 | 82 | it "should configure fog_region" do 83 | expect(AssetSync.config.fog_region).to eq("eu-west-1") 84 | end 85 | 86 | it "should configure existing_remote_files" do 87 | expect(AssetSync.config.existing_remote_files).to eq("keep") 88 | end 89 | end 90 | end 91 | -------------------------------------------------------------------------------- /lib/generators/asset_sync/templates/asset_sync.yml: -------------------------------------------------------------------------------- 1 | defaults: &defaults 2 | <%- if aws? -%> 3 | fog_provider: 'AWS' 4 | aws_access_key_id: "<%= aws_access_key_id %>" 5 | aws_secret_access_key: "<%= aws_secret_access_key %>" 6 | 7 | # To use AWS reduced redundancy storage. 8 | # aws_reduced_redundancy: true 9 | # 10 | # Change AWS signature version. Default is 4 11 | # aws_signature_version: 4 12 | # 13 | # Change canned ACL of uploaded object. Default is unset. Will override fog_public if set. 14 | # Choose from: private | public-read | public-read-write | aws-exec-read | 15 | # authenticated-read | bucket-owner-read | bucket-owner-full-control 16 | # aws_acl: null 17 | # 18 | # Change host option in fog (only if you need to) 19 | # fog_host: "s3.amazonaws.com" 20 | # 21 | # Change port option in fog (only if you need to) 22 | # config.fog_port = "9000" 23 | # 24 | # Use http instead of https. Default should be "https" (at least for fog-aws) 25 | # fog_scheme: "http" 26 | <%- elsif google? -%> 27 | fog_provider: 'Google' 28 | google_storage_access_key_id: "<%= google_storage_access_key_id %>" 29 | google_storage_secret_access_key: "<%= google_storage_secret_access_key %>" 30 | <%- elsif rackspace? -%> 31 | fog_provider: 'Rackspace' 32 | rackspace_username: "<%= rackspace_username %>" 33 | rackspace_api_key: "<%= rackspace_api_key %>" 34 | # if you need to change rackspace_auth_url (e.g. if you need to use Rackspace London) 35 | # rackspace_auth_url: "https://lon.identity.api.rackspacecloud.com/v2.0" 36 | <%- elsif azure_rm? -%> 37 | fog_provider: 'AzureRM' 38 | azure_storage_account_name: "<%= azure_storage_account_name %>" 39 | azure_storage_access_key: "<%= azure_storage_access_key %>" 40 | # fog_directory specifies container name of Azure Blob storage 41 | <%- elsif backblaze? -%> 42 | fog_provider: Backblaze 43 | b2_key_id: "<%= b2_key_id %>" 44 | b2_key_token: "<%= b2_key_token %>" 45 | b2_bucket_id: "<%= b2_bucket_id %>" 46 | # fog_directory specifies container name of Backblaze B2 Bucket 47 | <%- end -%> 48 | fog_directory: "<%= app_name %>-assets" 49 | 50 | # You may need to specify what region your storage bucket is in 51 | # fog_region: "eu-west-1" 52 | 53 | # Set `public` option when uploading file depending on value, 54 | # Setting to "default" makes asset sync skip setting the option 55 | # Possible values: true, false, "default" (default: true) 56 | # config.fog_public = true 57 | 58 | existing_remote_files: keep 59 | # To delete existing remote files. 60 | # existing_remote_files: delete 61 | 62 | # Automatically replace files with their equivalent gzip compressed version 63 | # gzip_compression: true 64 | 65 | # Fail silently. Useful for environments such as Heroku 66 | # fail_silently: true 67 | 68 | # Allow custom assets to be cacheable. Note: The base filename will be matched 69 | # cache_asset_regexps: ['cache_me.js', !ruby/regexp '/cache_some\.\d{8}\.css/'] 70 | 71 | development: 72 | <<: *defaults 73 | enabled: false 74 | 75 | test: 76 | <<: *defaults 77 | enabled: false 78 | 79 | staging: 80 | <<: *defaults 81 | fog_directory: "<%= app_name %>-staging-assets" 82 | 83 | production: 84 | <<: *defaults 85 | -------------------------------------------------------------------------------- /spec/unit/multi_mime_spec.rb: -------------------------------------------------------------------------------- 1 | require File.dirname(__FILE__) + '/../spec_helper' 2 | 3 | describe AssetSync::MultiMime do 4 | 5 | before(:each) do 6 | # Object#remove_const does not remove the loaded 7 | # file from the $" variable 8 | # 9 | # So we need do both 10 | # 11 | # 1. Remove constant(s) to avoid warning messages 12 | # 2. Remove loaded file(s) 13 | Object.send(:remove_const, :Rails) if defined?(Rails) 14 | Object.send(:remove_const, :Mime) if defined?(Mime) 15 | Object.send(:remove_const, :Rack) if defined?(Rack) 16 | Object.send(:remove_const, :MIME) if defined?(MIME) 17 | 18 | $".grep(/mime\//).each do |file_path| 19 | $".delete(file_path) 20 | end 21 | end 22 | 23 | after(:each) do 24 | # Object#remove_const does not remove the loaded 25 | # file from the $" variable 26 | # 27 | # So we need do both 28 | # 29 | # 1. Remove constant(s) to avoid warning messages 30 | # 2. Remove loaded file(s) 31 | Object.send(:remove_const, :Rails) if defined?(Rails) 32 | Object.send(:remove_const, :Mime) if defined?(Mime) 33 | Object.send(:remove_const, :Rack) if defined?(Rack) 34 | Object.send(:remove_const, :MIME) if defined?(MIME) 35 | 36 | $".grep(/mime\//).each do |file_path| 37 | $".delete(file_path) 38 | end 39 | 40 | AssetSync.config = AssetSync::Config.new 41 | end 42 | 43 | after(:all) do 44 | require 'mime/types' 45 | end 46 | 47 | describe 'Mime::Type' do 48 | 49 | it 'should detect mime type' do 50 | require 'rails' 51 | expect(AssetSync::MultiMime.lookup('css')).to eq("text/css") 52 | end 53 | 54 | end 55 | 56 | describe 'Rack::Mime' do 57 | 58 | it 'should detect mime type' do 59 | require 'rack/mime' 60 | expect(AssetSync::MultiMime.lookup('css')).to eq("text/css") 61 | end 62 | 63 | end 64 | 65 | describe 'MIME::Types' do 66 | 67 | it 'should detect mime type' do 68 | require 'mime/types' 69 | expect(AssetSync::MultiMime.lookup('css')).to eq("text/css") 70 | end 71 | 72 | end 73 | 74 | describe "use of option file_ext_to_mime_type_overrides" do 75 | before(:each) do 76 | require 'mime/types' 77 | end 78 | 79 | context "with default value" do 80 | it "should return default value set by gem" do 81 | expect( 82 | AssetSync::MultiMime.lookup("js").to_s, 83 | ).to eq("application/javascript") 84 | end 85 | end 86 | context "with empty value" do 87 | before(:each) do 88 | AssetSync.config = AssetSync::Config.new 89 | AssetSync.configure do |config| 90 | config.file_ext_to_mime_type_overrides.clear 91 | end 92 | end 93 | 94 | it "should return value from mime-types gem" do 95 | expect( 96 | AssetSync::MultiMime.lookup("js").to_s, 97 | ).to eq(::MIME::Types.type_for("js").first.to_s) 98 | end 99 | end 100 | context "with custom value" do 101 | before(:each) do 102 | AssetSync.config = AssetSync::Config.new 103 | AssetSync.configure do |config| 104 | config.file_ext_to_mime_type_overrides.add( 105 | :js, 106 | :"application/x-javascript", 107 | ) 108 | end 109 | end 110 | 111 | it "should return custom value" do 112 | expect( 113 | AssetSync::MultiMime.lookup("js").to_s, 114 | ).to eq("application/x-javascript") 115 | end 116 | end 117 | end 118 | 119 | end 120 | -------------------------------------------------------------------------------- /lib/generators/asset_sync/templates/asset_sync.rb: -------------------------------------------------------------------------------- 1 | if defined?(AssetSync) 2 | AssetSync.configure do |config| 3 | <%- if aws? -%> 4 | config.fog_provider = 'AWS' 5 | config.aws_access_key_id = ENV['AWS_ACCESS_KEY_ID'] 6 | config.aws_secret_access_key = ENV['AWS_SECRET_ACCESS_KEY'] 7 | config.aws_session_token = ENV['AWS_SESSION_TOKEN'] if ENV.key?('AWS_SESSION_TOKEN') 8 | # To use AWS reduced redundancy storage. 9 | # config.aws_reduced_redundancy = true 10 | # 11 | # Change AWS signature version. Default is 4 12 | # config.aws_signature_version = 4 13 | # 14 | # Change canned ACL of uploaded object. Default is unset. Will override fog_public if set. 15 | # Choose from: private | public-read | public-read-write | aws-exec-read | 16 | # authenticated-read | bucket-owner-read | bucket-owner-full-control 17 | # config.aws_acl = nil 18 | # 19 | # Change host option in fog (only if you need to) 20 | # config.fog_host = "s3.amazonaws.com" 21 | # 22 | # Change port option in fog (only if you need to) 23 | # config.fog_port = "9000" 24 | # 25 | # Use http instead of https. Default should be "https" (at least for fog-aws) 26 | # config.fog_scheme = "http" 27 | <%- elsif google? -%> 28 | config.fog_provider = 'Google' 29 | config.google_storage_access_key_id = ENV['GOOGLE_STORAGE_ACCESS_KEY_ID'] 30 | config.google_storage_secret_access_key = ENV['GOOGLE_STORAGE_SECRET_ACCESS_KEY'] 31 | <%- elsif rackspace? -%> 32 | config.fog_provider = 'Rackspace' 33 | config.rackspace_username = ENV['RACKSPACE_USERNAME'] 34 | config.rackspace_api_key = ENV['RACKSPACE_API_KEY'] 35 | 36 | # if you need to change rackspace_auth_url (e.g. if you need to use Rackspace London) 37 | # config.rackspace_auth_url = "lon.auth.api.rackspacecloud.com" 38 | <%- elsif azure_rm? -%> 39 | config.fog_provider = 'AzureRM' 40 | config.azure_storage_account_name = ENV['AZURE_STORAGE_ACCOUNT_NAME'] 41 | config.azure_storage_access_key = ENV['AZURE_STORAGE_ACCESS_KEY'] 42 | 43 | <%- elsif backblaze? -%> 44 | config.fog_provider = 'Backblaze' 45 | config.b2_key_id = ENV['B2_KEY_ID'] 46 | config.b2_key_token = ENV['B2_KEY_TOKEN'] 47 | config.b2_bucket_id = ENV['B2_BUCKET_ID'] 48 | 49 | # config.fog_directory specifies container name of Azure Blob storage 50 | <%- end -%> 51 | config.fog_directory = ENV['FOG_DIRECTORY'] 52 | 53 | # Invalidate a file on a cdn after uploading files 54 | # config.cdn_distribution_id = "12345" 55 | # config.invalidate = ['file1.js'] 56 | 57 | # Increase upload performance by configuring your region 58 | # config.fog_region = 'eu-west-1' 59 | # 60 | # Set `public` option when uploading file depending on value, 61 | # Setting to "default" makes asset sync skip setting the option 62 | # Possible values: true, false, "default" (default: true) 63 | # config.fog_public = true 64 | # 65 | # Don't delete files from the store 66 | # config.existing_remote_files = "keep" 67 | # 68 | # Automatically replace files with their equivalent gzip compressed version 69 | # config.gzip_compression = true 70 | # 71 | # Use the Rails generated 'manifest.yml' file to produce the list of files to 72 | # upload instead of searching the assets directory. 73 | # config.manifest = true 74 | # 75 | # Upload the manifest file also. 76 | # config.include_manifest = false 77 | # 78 | # Upload files concurrently 79 | # config.concurrent_uploads = false 80 | # 81 | # Path to cache file to skip scanning remote 82 | # config.remote_file_list_cache_file_path = './.asset_sync_remote_file_list_cache.json' 83 | # 84 | # Fail silently. Useful for environments such as Heroku 85 | # config.fail_silently = true 86 | # 87 | # Log silently. Default is `true`. But you can set it to false if more logging message are preferred. 88 | # Logging messages are sent to `STDOUT` when `log_silently` is falsy 89 | # config.log_silently = true 90 | # 91 | # Allow custom assets to be cacheable. Note: The base filename will be matched 92 | # If you have an asset with name `app.0ba4d3.js`, only `app.0ba4d3` will need to be matched 93 | # config.cache_asset_regexps = [ /\.[a-f0-9]{8}$/i, /\.[a-f0-9]{20}$/i ] 94 | # config.cache_asset_regexp = /\.[a-f0-9]{8}$/i 95 | end 96 | end 97 | -------------------------------------------------------------------------------- /lib/asset_sync/engine.rb: -------------------------------------------------------------------------------- 1 | module AssetSync 2 | class Engine < Rails::Engine 3 | 4 | engine_name "asset_sync" 5 | 6 | initializer "asset_sync config", :group => :all do |app| 7 | app_initializer = Rails.root.join('config', 'initializers', 'asset_sync.rb').to_s 8 | app_yaml = Rails.root.join('config', 'asset_sync.yml').to_s 9 | 10 | if File.exist?( app_initializer ) 11 | AssetSync.log "AssetSync: using #{app_initializer}" 12 | load app_initializer 13 | elsif !File.exist?( app_initializer ) && !File.exist?( app_yaml ) 14 | AssetSync.log "AssetSync: using default configuration from built-in initializer" 15 | AssetSync.configure do |config| 16 | config.fog_provider = ENV['FOG_PROVIDER'] if ENV.has_key?('FOG_PROVIDER') 17 | config.fog_directory = ENV['FOG_DIRECTORY'] if ENV.has_key?('FOG_DIRECTORY') 18 | config.fog_region = ENV['FOG_REGION'] if ENV.has_key?('FOG_REGION') 19 | config.fog_host = ENV['FOG_HOST'] if ENV.has_key?('FOG_HOST') 20 | config.fog_port = ENV['FOG_PORT'] if ENV.has_key?('FOG_PORT') 21 | config.fog_scheme = ENV['FOG_SCHEMA'] if ENV.has_key?('FOG_SCHEMA') 22 | config.fog_path_style = ENV['FOG_PATH_STYLE'] if ENV.has_key?('FOG_PATH_STYLE') 23 | 24 | config.aws_access_key_id = ENV['AWS_ACCESS_KEY_ID'] if ENV.has_key?('AWS_ACCESS_KEY_ID') 25 | config.aws_secret_access_key = ENV['AWS_SECRET_ACCESS_KEY'] if ENV.has_key?('AWS_SECRET_ACCESS_KEY') 26 | config.aws_session_token = ENV['AWS_SESSION_TOKEN'] if ENV.has_key?('AWS_SESSION_TOKEN') 27 | config.aws_signature_version = ENV['AWS_SIGNATURE_VERSION'] if ENV.has_key?('AWS_SIGNATURE_VERSION') 28 | config.aws_acl = ENV['AWS_ACL'] if ENV.has_key?('AWS_ACL') 29 | config.aws_reduced_redundancy = ENV['AWS_REDUCED_REDUNDANCY'] == true if ENV.has_key?('AWS_REDUCED_REDUNDANCY') 30 | 31 | config.rackspace_username = ENV['RACKSPACE_USERNAME'] if ENV.has_key?('RACKSPACE_USERNAME') 32 | config.rackspace_api_key = ENV['RACKSPACE_API_KEY'] if ENV.has_key?('RACKSPACE_API_KEY') 33 | 34 | config.google_storage_access_key_id = ENV['GOOGLE_STORAGE_ACCESS_KEY_ID'] if ENV.has_key?('GOOGLE_STORAGE_ACCESS_KEY_ID') 35 | config.google_storage_secret_access_key = ENV['GOOGLE_STORAGE_SECRET_ACCESS_KEY'] if ENV.has_key?('GOOGLE_STORAGE_SECRET_ACCESS_KEY') 36 | 37 | config.azure_storage_account_name = ENV['AZURE_STORAGE_ACCOUNT_NAME'] if ENV.has_key?('AZURE_STORAGE_ACCOUNT_NAME') 38 | config.azure_storage_access_key = ENV['AZURE_STORAGE_ACCESS_KEY'] if ENV.has_key?('AZURE_STORAGE_ACCESS_KEY') 39 | 40 | config.b2_key_id = ENV['B2_KEY_ID'] if ENV.has_key?('B2_KEY_ID') 41 | config.b2_key_token = ENV['B2_KEY_TOKEN'] if ENV.has_key?('B2_KEY_TOKEN') 42 | config.b2_bucket_id = ENV['B2_BUCKET_ID'] if ENV.has_key?('B2_BUCKET_ID') 43 | 44 | config.enabled = (ENV['ASSET_SYNC_ENABLED'] == 'true') if ENV.has_key?('ASSET_SYNC_ENABLED') 45 | 46 | config.existing_remote_files = ENV['ASSET_SYNC_EXISTING_REMOTE_FILES'] || "keep" 47 | 48 | config.gzip_compression = (ENV['ASSET_SYNC_GZIP_COMPRESSION'] == 'true') if ENV.has_key?('ASSET_SYNC_GZIP_COMPRESSION') 49 | config.manifest = (ENV['ASSET_SYNC_MANIFEST'] == 'true') if ENV.has_key?('ASSET_SYNC_MANIFEST') 50 | config.include_manifest = (ENV['ASSET_SYNC_INCLUDE_MANIFEST'] == 'true') if ENV.has_key?('ASSET_SYNC_INCLUDE_MANIFEST') 51 | config.concurrent_uploads = (ENV['ASSET_SYNC_CONCURRENT_UPLOADS'] == 'true') if ENV.has_key?('ASSET_SYNC_CONCURRENT_UPLOADS') 52 | config.remote_file_list_cache_file_path = ENV['ASSET_SYNC_REMOTE_FILE_LIST_CACHE_FILE_PATH'] if ENV.has_key?('ASSET_SYNC_REMOTE_FILE_LIST_CACHE_FILE_PATH') 53 | end 54 | 55 | config.prefix = ENV['ASSET_SYNC_PREFIX'] if ENV.has_key?('ASSET_SYNC_PREFIX') 56 | 57 | config.existing_remote_files = ENV['ASSET_SYNC_EXISTING_REMOTE_FILES'] || "keep" 58 | 59 | config.gzip_compression = (ENV['ASSET_SYNC_GZIP_COMPRESSION'] == 'true') if ENV.has_key?('ASSET_SYNC_GZIP_COMPRESSION') 60 | config.manifest = (ENV['ASSET_SYNC_MANIFEST'] == 'true') if ENV.has_key?('ASSET_SYNC_MANIFEST') 61 | 62 | end 63 | 64 | if File.exist?( app_yaml ) 65 | AssetSync.log "AssetSync: YAML file found #{app_yaml} settings will be merged into the configuration" 66 | end 67 | end 68 | 69 | end 70 | end 71 | -------------------------------------------------------------------------------- /spec/unit/azure_rm_spec.rb: -------------------------------------------------------------------------------- 1 | require File.dirname(__FILE__) + '/../spec_helper' 2 | 3 | describe AssetSync do 4 | include_context "mock Rails without_yml" 5 | 6 | describe 'with initializer' do 7 | before(:each) do 8 | AssetSync.config = AssetSync::Config.new 9 | AssetSync.configure do |config| 10 | config.fog_provider = 'AzureRM' 11 | config.azure_storage_account_name = 'aaaa' 12 | config.azure_storage_access_key = 'bbbb' 13 | config.fog_directory = 'mybucket' 14 | config.existing_remote_files = "keep" 15 | end 16 | end 17 | 18 | it "should configure provider as AzureRM" do 19 | expect(AssetSync.config.fog_provider).to eq('AzureRM') 20 | expect(AssetSync.config).to be_azure_rm 21 | end 22 | 23 | it "should should keep existing remote files" do 24 | expect(AssetSync.config.existing_remote_files?).to eq(true) 25 | end 26 | 27 | it "should configure azure_storage_account_name" do 28 | expect(AssetSync.config.azure_storage_account_name).to eq("aaaa") 29 | end 30 | 31 | it "should configure azure_storage_access_key" do 32 | expect(AssetSync.config.azure_storage_access_key).to eq("bbbb") 33 | end 34 | 35 | it "should configure fog_directory" do 36 | expect(AssetSync.config.fog_directory).to eq("mybucket") 37 | end 38 | 39 | it "should configure existing_remote_files" do 40 | expect(AssetSync.config.existing_remote_files).to eq("keep") 41 | end 42 | 43 | it "should default gzip_compression to false" do 44 | expect(AssetSync.config.gzip_compression).to be_falsey 45 | end 46 | 47 | it "should default manifest to false" do 48 | expect(AssetSync.config.manifest).to be_falsey 49 | end 50 | end 51 | 52 | describe 'from yml' do 53 | before(:each) do 54 | set_rails_root('azure_rm_with_yml') 55 | AssetSync.config = AssetSync::Config.new 56 | end 57 | 58 | it "should configure azure_storage_account_name" do 59 | expect(AssetSync.config.azure_storage_account_name).to eq("xxxx") 60 | end 61 | 62 | it "should configure azure_storage_access_key" do 63 | expect(AssetSync.config.azure_storage_access_key).to eq("zzzz") 64 | end 65 | 66 | it "should configure fog_directory" do 67 | expect(AssetSync.config.fog_directory).to eq("rails_app_test") 68 | end 69 | 70 | it "should configure existing_remote_files" do 71 | expect(AssetSync.config.existing_remote_files).to eq("keep") 72 | end 73 | 74 | it "should default gzip_compression to false" do 75 | expect(AssetSync.config.gzip_compression).to be_falsey 76 | end 77 | 78 | it "should default manifest to false" do 79 | expect(AssetSync.config.manifest).to be_falsey 80 | end 81 | end 82 | 83 | describe 'with no configuration' do 84 | before(:each) do 85 | AssetSync.config = AssetSync::Config.new 86 | end 87 | 88 | it "should be invalid" do 89 | expect{ AssetSync.sync }.to raise_error(::AssetSync::Config::Invalid) 90 | end 91 | end 92 | 93 | describe 'with fail_silent configuration' do 94 | before(:each) do 95 | allow(AssetSync).to receive(:stderr).and_return(StringIO.new) 96 | AssetSync.config = AssetSync::Config.new 97 | AssetSync.configure do |config| 98 | config.fail_silently = true 99 | end 100 | end 101 | 102 | it "should not raise an invalid exception" do 103 | expect{ AssetSync.sync }.not_to raise_error 104 | end 105 | end 106 | 107 | describe 'with gzip_compression enabled' do 108 | before(:each) do 109 | AssetSync.config = AssetSync::Config.new 110 | AssetSync.config.gzip_compression = true 111 | end 112 | 113 | it "config.gzip? should be true" do 114 | expect(AssetSync.config.gzip?).to be_truthy 115 | end 116 | end 117 | 118 | describe 'with manifest enabled' do 119 | before(:each) do 120 | AssetSync.config = AssetSync::Config.new 121 | AssetSync.config.manifest = true 122 | end 123 | 124 | it "config.manifest should be true" do 125 | expect(AssetSync.config.manifest).to be_truthy 126 | end 127 | 128 | it "config.manifest_path should default to public/assets.." do 129 | expect(AssetSync.config.manifest_path).to match(/public\/assets\/manifest.yml/) 130 | end 131 | 132 | it "config.manifest_path should default to public/assets.." do 133 | Rails.application.config.assets.manifest = "/var/assets" 134 | expect(AssetSync.config.manifest_path).to eq("/var/assets/manifest.yml") 135 | end 136 | 137 | it "config.manifest_path should default to public/custom_assets.." do 138 | Rails.application.config.assets.prefix = 'custom_assets' 139 | expect(AssetSync.config.manifest_path).to match(/public\/custom_assets\/manifest.yml/) 140 | end 141 | end 142 | end 143 | -------------------------------------------------------------------------------- /spec/unit/backblaze_spec.rb: -------------------------------------------------------------------------------- 1 | require File.dirname(__FILE__) + '/../spec_helper' 2 | 3 | describe AssetSync do 4 | include_context "mock Rails without_yml" 5 | 6 | describe 'with initializer' do 7 | before(:each) do 8 | AssetSync.config = AssetSync::Config.new 9 | AssetSync.configure do |config| 10 | config.fog_provider = 'Backblaze' 11 | config.b2_key_id = 'aaaa' 12 | config.b2_key_token = 'bbbb' 13 | config.b2_bucket_id = '4567' 14 | config.fog_directory = 'mybucket' 15 | config.existing_remote_files = "keep" 16 | end 17 | end 18 | 19 | it "should configure provider as Backblaze" do 20 | expect(AssetSync.config.fog_provider).to eq('Backblaze') 21 | end 22 | 23 | it "should should keep existing remote files" do 24 | expect(AssetSync.config.existing_remote_files?).to eq(true) 25 | end 26 | 27 | it "should configure b2_key_id" do 28 | expect(AssetSync.config.b2_key_id).to eq("aaaa") 29 | end 30 | 31 | it "should configure b2_key_token" do 32 | expect(AssetSync.config.b2_key_token).to eq("bbbb") 33 | end 34 | 35 | it "should configure b2_bucket_id" do 36 | expect(AssetSync.config.b2_bucket_id).to eq("4567") 37 | end 38 | 39 | it "should configure fog_directory" do 40 | expect(AssetSync.config.fog_directory).to eq("mybucket") 41 | end 42 | 43 | it "should configure existing_remote_files" do 44 | expect(AssetSync.config.existing_remote_files).to eq("keep") 45 | end 46 | 47 | it "should default gzip_compression to false" do 48 | expect(AssetSync.config.gzip_compression).to be_falsey 49 | end 50 | 51 | it "should default manifest to false" do 52 | expect(AssetSync.config.manifest).to be_falsey 53 | end 54 | end 55 | 56 | describe 'from yml' do 57 | before(:each) do 58 | set_rails_root('backblaze_with_yml') 59 | AssetSync.config = AssetSync::Config.new 60 | end 61 | 62 | it "should configure b2_key_id" do 63 | expect(AssetSync.config.b2_key_id).to eq("xxxx") 64 | end 65 | 66 | it "should configure b2_key_token" do 67 | expect(AssetSync.config.b2_key_token).to eq("zzzz") 68 | end 69 | 70 | it "should configure b2_bucket_id" do 71 | expect(AssetSync.config.b2_bucket_id).to eq("1234") 72 | end 73 | 74 | it "should configure fog_directory" do 75 | expect(AssetSync.config.fog_directory).to eq("rails_app_test") 76 | end 77 | 78 | it "should configure existing_remote_files" do 79 | expect(AssetSync.config.existing_remote_files).to eq("keep") 80 | end 81 | 82 | it "should default gzip_compression to false" do 83 | expect(AssetSync.config.gzip_compression).to be_falsey 84 | end 85 | 86 | it "should default manifest to false" do 87 | expect(AssetSync.config.manifest).to be_falsey 88 | end 89 | end 90 | 91 | describe 'with no configuration' do 92 | before(:each) do 93 | AssetSync.config = AssetSync::Config.new 94 | end 95 | 96 | it "should be invalid" do 97 | expect{ AssetSync.sync }.to raise_error(::AssetSync::Config::Invalid) 98 | end 99 | end 100 | 101 | describe 'with fail_silent configuration' do 102 | before(:each) do 103 | allow(AssetSync).to receive(:stderr).and_return(StringIO.new) 104 | AssetSync.config = AssetSync::Config.new 105 | AssetSync.configure do |config| 106 | config.fail_silently = true 107 | end 108 | end 109 | 110 | it "should not raise an invalid exception" do 111 | expect{ AssetSync.sync }.not_to raise_error 112 | end 113 | end 114 | 115 | describe 'with gzip_compression enabled' do 116 | before(:each) do 117 | AssetSync.config = AssetSync::Config.new 118 | AssetSync.config.gzip_compression = true 119 | end 120 | 121 | it "config.gzip? should be true" do 122 | expect(AssetSync.config.gzip?).to be_truthy 123 | end 124 | end 125 | 126 | describe 'with manifest enabled' do 127 | before(:each) do 128 | AssetSync.config = AssetSync::Config.new 129 | AssetSync.config.manifest = true 130 | end 131 | 132 | it "config.manifest should be true" do 133 | expect(AssetSync.config.manifest).to be_truthy 134 | end 135 | 136 | it "config.manifest_path should default to public/assets.." do 137 | expect(AssetSync.config.manifest_path).to match(/public\/assets\/manifest.yml/) 138 | end 139 | 140 | it "config.manifest_path should default to public/assets.." do 141 | Rails.application.config.assets.manifest = "/var/assets" 142 | expect(AssetSync.config.manifest_path).to eq("/var/assets/manifest.yml") 143 | end 144 | 145 | it "config.manifest_path should default to public/custom_assets.." do 146 | Rails.application.config.assets.prefix = 'custom_assets' 147 | expect(AssetSync.config.manifest_path).to match(/public\/custom_assets\/manifest.yml/) 148 | end 149 | end 150 | end 151 | -------------------------------------------------------------------------------- /spec/unit/google_spec.rb: -------------------------------------------------------------------------------- 1 | require File.dirname(__FILE__) + '/../spec_helper' 2 | 3 | describe AssetSync do 4 | include_context "mock Rails without_yml" 5 | 6 | describe 'with initializer' do 7 | before(:each) do 8 | AssetSync.config = AssetSync::Config.new 9 | AssetSync.configure do |config| 10 | config.fog_provider = 'Google' 11 | config.fog_directory = 'mybucket' 12 | config.existing_remote_files = "keep" 13 | end 14 | end 15 | 16 | it "should configure provider as Google" do 17 | expect(AssetSync.config.fog_provider).to eq('Google') 18 | expect(AssetSync.config).to be_google 19 | end 20 | 21 | it "should should keep existing remote files" do 22 | expect(AssetSync.config.existing_remote_files?).to eq(true) 23 | end 24 | 25 | it "should configure fog_directory" do 26 | expect(AssetSync.config.fog_directory).to eq("mybucket") 27 | end 28 | 29 | it "should configure existing_remote_files" do 30 | expect(AssetSync.config.existing_remote_files).to eq("keep") 31 | end 32 | 33 | it "should default gzip_compression to false" do 34 | expect(AssetSync.config.gzip_compression).to be_falsey 35 | end 36 | 37 | it "should default manifest to false" do 38 | expect(AssetSync.config.manifest).to be_falsey 39 | end 40 | 41 | describe "when using S3 interop API" do 42 | before(:each) do 43 | AssetSync.configure do |config| 44 | config.google_storage_access_key_id = 'aaaa' 45 | config.google_storage_secret_access_key = 'bbbb' 46 | end 47 | end 48 | 49 | it "should configure google_storage_access_key_id" do 50 | expect(AssetSync.config.google_storage_access_key_id).to eq("aaaa") 51 | end 52 | 53 | it "should configure google_storage_secret_access_key" do 54 | expect(AssetSync.config.google_storage_secret_access_key).to eq("bbbb") 55 | end 56 | 57 | it "should return the correct fog_options" do 58 | expected_fog_options = { google_storage_access_key_id: "aaaa", 59 | google_storage_secret_access_key: "bbbb", 60 | provider: "Google"} 61 | expect(AssetSync.config.fog_options).to eq(expected_fog_options) 62 | end 63 | 64 | it "should not require that google_json_key_location be set" do 65 | expect(AssetSync.config.valid?).to eq(true) 66 | end 67 | 68 | it "should require that google_storage_secret_access_key or access_key_id be set" do 69 | 70 | AssetSync.configure do |config| 71 | config.google_storage_access_key_id = nil 72 | config.google_storage_secret_access_key = nil 73 | end 74 | 75 | expect(AssetSync.config.valid?).to eq(false) 76 | end 77 | end 78 | 79 | describe "when using service account" do 80 | before(:each) do 81 | AssetSync.configure do |config| 82 | config.google_json_key_location = '/path/to.json' 83 | config.google_project = 'a-google-project-name' 84 | end 85 | end 86 | 87 | it "should configure google_json_key_location" do 88 | expect(AssetSync.config.google_json_key_location).to eq("/path/to.json") 89 | end 90 | 91 | it "should return the correct fog_options" do 92 | expected_fog_options = { google_json_key_location: "/path/to.json", 93 | google_project: 'a-google-project-name', 94 | provider: "Google"} 95 | expect(AssetSync.config.fog_options).to eq(expected_fog_options) 96 | end 97 | it "should not require that google_storage_secret_access_key or access_key_id be set" do 98 | expect(AssetSync.config.valid?).to eq(true) 99 | end 100 | end 101 | 102 | describe "when using service account with JSON key string" do 103 | before(:each) do 104 | AssetSync.configure do |config| 105 | config.google_json_key_string = 'a-google-json-key-string' 106 | config.google_project = 'a-google-project-name' 107 | end 108 | end 109 | 110 | it "should configure google_json_key_string" do 111 | expect(AssetSync.config.google_json_key_string).to eq("a-google-json-key-string") 112 | end 113 | 114 | it "should return the correct fog_options" do 115 | expected_fog_options = { google_json_key_string: "a-google-json-key-string", 116 | google_project: 'a-google-project-name', 117 | provider: "Google"} 118 | expect(AssetSync.config.fog_options).to eq(expected_fog_options) 119 | end 120 | it "should not require that google_storage_secret_access_key or access_key_id be set" do 121 | expect(AssetSync.config.valid?).to eq(true) 122 | end 123 | end 124 | end 125 | 126 | describe 'from yml' do 127 | describe 'when using S3 interop API' do 128 | before(:each) do 129 | set_rails_root('google_with_yml') 130 | AssetSync.config = AssetSync::Config.new 131 | end 132 | 133 | it "should configure google_storage_access_key_id" do 134 | expect(AssetSync.config.google_storage_access_key_id).to eq("xxxx") 135 | end 136 | 137 | it "should configure google_storage_secret_access_key" do 138 | expect(AssetSync.config.google_storage_secret_access_key).to eq("zzzz") 139 | end 140 | 141 | it "should not configure google_json_key_location" do 142 | expect(AssetSync.config.google_json_key_location).to eq(nil) 143 | end 144 | 145 | it "should configure fog_directory" do 146 | expect(AssetSync.config.fog_directory).to eq("rails_app_test") 147 | end 148 | 149 | it "should configure existing_remote_files" do 150 | expect(AssetSync.config.existing_remote_files).to eq("keep") 151 | end 152 | 153 | it "should default gzip_compression to false" do 154 | expect(AssetSync.config.gzip_compression).to be_falsey 155 | end 156 | 157 | it "should default manifest to false" do 158 | expect(AssetSync.config.manifest).to be_falsey 159 | end 160 | end 161 | 162 | describe 'when using service account API' do 163 | before(:each) do 164 | set_rails_root('google_with_service_account_yml') 165 | AssetSync.config = AssetSync::Config.new 166 | end 167 | 168 | it "should configure google_json_key_location" do 169 | expect(AssetSync.config.google_json_key_location).to eq("gcs.json") 170 | end 171 | 172 | it "should not configure google_storage_secret_access_key" do 173 | expect(AssetSync.config.google_storage_secret_access_key).to eq(nil) 174 | end 175 | end 176 | end 177 | 178 | describe 'with no configuration' do 179 | before(:each) do 180 | AssetSync.config = AssetSync::Config.new 181 | end 182 | 183 | it "should be invalid" do 184 | expect{ AssetSync.sync }.to raise_error(::AssetSync::Config::Invalid) 185 | end 186 | end 187 | 188 | describe 'with fail_silent configuration' do 189 | before(:each) do 190 | allow(AssetSync).to receive(:stderr).and_return(StringIO.new) 191 | AssetSync.config = AssetSync::Config.new 192 | AssetSync.configure do |config| 193 | config.fail_silently = true 194 | end 195 | end 196 | 197 | it "should not raise an invalid exception" do 198 | expect{ AssetSync.sync }.not_to raise_error 199 | end 200 | end 201 | 202 | describe 'with gzip_compression enabled' do 203 | before(:each) do 204 | AssetSync.config = AssetSync::Config.new 205 | AssetSync.config.gzip_compression = true 206 | end 207 | 208 | it "config.gzip? should be true" do 209 | expect(AssetSync.config.gzip?).to be_truthy 210 | end 211 | end 212 | 213 | describe 'with manifest enabled' do 214 | before(:each) do 215 | AssetSync.config = AssetSync::Config.new 216 | AssetSync.config.manifest = true 217 | end 218 | 219 | it "config.manifest should be true" do 220 | expect(AssetSync.config.manifest).to be_truthy 221 | end 222 | 223 | it "config.manifest_path should default to public/assets.." do 224 | expect(AssetSync.config.manifest_path).to match(/public\/assets\/manifest.yml/) 225 | end 226 | 227 | it "config.manifest_path should default to public/assets.." do 228 | Rails.application.config.assets.manifest = "/var/assets" 229 | expect(AssetSync.config.manifest_path).to eq("/var/assets/manifest.yml") 230 | end 231 | 232 | it "config.manifest_path should default to public/custom_assets.." do 233 | Rails.application.config.assets.prefix = 'custom_assets' 234 | expect(AssetSync.config.manifest_path).to match(/public\/custom_assets\/manifest.yml/) 235 | end 236 | end 237 | end 238 | -------------------------------------------------------------------------------- /spec/unit/asset_sync_spec.rb: -------------------------------------------------------------------------------- 1 | require File.dirname(__FILE__) + '/../spec_helper' 2 | 3 | describe AssetSync do 4 | include_context "mock Rails without_yml" 5 | 6 | describe 'with initializer' do 7 | before(:each) do 8 | AssetSync.config = AssetSync::Config.new 9 | AssetSync.configure do |config| 10 | config.fog_provider = 'AWS' 11 | config.aws_access_key_id = 'aaaa' 12 | config.aws_secret_access_key = 'bbbb' 13 | config.fog_directory = 'mybucket' 14 | config.fog_region = 'eu-west-1' 15 | config.fog_path_style = 'true' 16 | config.existing_remote_files = "keep" 17 | end 18 | end 19 | 20 | it "should default to running on precompile" do 21 | expect(AssetSync.config.run_on_precompile).to be_truthy 22 | end 23 | 24 | it "should default AssetSync to enabled" do 25 | expect(AssetSync.config.enabled?).to be_truthy 26 | expect(AssetSync.enabled?).to be_truthy 27 | end 28 | 29 | it "should configure provider as AWS" do 30 | expect(AssetSync.config.fog_provider).to eq('AWS') 31 | expect(AssetSync.config).to be_aws 32 | end 33 | 34 | it "should should keep existing remote files" do 35 | expect(AssetSync.config.existing_remote_files?).to eq(true) 36 | end 37 | 38 | it "should configure aws_access_key" do 39 | expect(AssetSync.config.aws_access_key_id).to eq("aaaa") 40 | end 41 | 42 | it "should configure aws_secret_access_key" do 43 | expect(AssetSync.config.aws_secret_access_key).to eq("bbbb") 44 | end 45 | 46 | it "should configure aws_access_key" do 47 | expect(AssetSync.config.fog_directory).to eq("mybucket") 48 | end 49 | 50 | it "should configure fog_region" do 51 | expect(AssetSync.config.fog_region).to eq("eu-west-1") 52 | end 53 | 54 | it "should configure path_style" do 55 | expect(AssetSync.config.fog_path_style).to be_truthy 56 | end 57 | 58 | it "should configure existing_remote_files" do 59 | expect(AssetSync.config.existing_remote_files).to eq("keep") 60 | end 61 | 62 | it "should default gzip_compression to false" do 63 | expect(AssetSync.config.gzip_compression).to be_falsey 64 | end 65 | 66 | it "should default manifest to false" do 67 | expect(AssetSync.config.manifest).to be_falsey 68 | end 69 | 70 | it "should default log_silently to true" do 71 | expect(AssetSync.config.log_silently).to be_truthy 72 | end 73 | 74 | it "log_silently? should reflect the configuration" do 75 | AssetSync.config.log_silently = false 76 | expect(AssetSync.config.log_silently?).to eq(false) 77 | end 78 | 79 | it "log_silently? should always be true if ENV['RAILS_GROUPS'] == 'assets'" do 80 | AssetSync.config.log_silently = false 81 | allow(ENV).to receive(:[]).with('RAILS_GROUPS').and_return('assets') 82 | 83 | expect(AssetSync.config.log_silently?).to eq(false) 84 | end 85 | 86 | it "should default cdn_distribution_id to nil" do 87 | expect(AssetSync.config.cdn_distribution_id).to be_nil 88 | end 89 | 90 | it "should default invalidate to empty array" do 91 | expect(AssetSync.config.invalidate).to eq([]) 92 | end 93 | 94 | it "should default asset_regexps to empty array" do 95 | expect(AssetSync.config.cache_asset_regexps).to eq([]) 96 | end 97 | end 98 | 99 | describe 'from yml' do 100 | before(:each) do 101 | set_rails_root('aws_with_yml') 102 | AssetSync.config = AssetSync::Config.new 103 | end 104 | 105 | it "should default AssetSync to enabled" do 106 | expect(AssetSync.config.enabled?).to be_truthy 107 | expect(AssetSync.enabled?).to be_truthy 108 | end 109 | 110 | it "should configure run_on_precompile" do 111 | expect(AssetSync.config.run_on_precompile).to be_falsey 112 | end 113 | 114 | it "should configure aws_access_key_id" do 115 | expect(AssetSync.config.aws_access_key_id).to eq("xxxx") 116 | end 117 | 118 | it "should configure aws_secret_access_key" do 119 | expect(AssetSync.config.aws_secret_access_key).to eq("zzzz") 120 | end 121 | 122 | it "should configure fog_directory" do 123 | expect(AssetSync.config.fog_directory).to eq("rails_app_test") 124 | end 125 | 126 | it "should configure fog_region" do 127 | expect(AssetSync.config.fog_region).to eq("eu-west-1") 128 | end 129 | 130 | it "should configure path_style" do 131 | expect(AssetSync.config.fog_path_style).to be_truthy 132 | end 133 | 134 | it "should configure existing_remote_files" do 135 | expect(AssetSync.config.existing_remote_files).to eq("keep") 136 | end 137 | 138 | it "should default gzip_compression to false" do 139 | expect(AssetSync.config.gzip_compression).to be_falsey 140 | end 141 | 142 | it "should default manifest to false" do 143 | expect(AssetSync.config.manifest).to be_falsey 144 | end 145 | 146 | it "should default asset_regexps to match regexps" do 147 | expect(AssetSync.config.cache_asset_regexps).to eq(['cache_me.js', /cache_some\.\d{8}\.css/]) 148 | end 149 | end 150 | 151 | describe 'from yml, exporting to a mobile hybrid development directory' do 152 | before(:each) do 153 | Rails.env.replace('hybrid') 154 | set_rails_root('aws_with_yml') 155 | AssetSync.config = AssetSync::Config.new 156 | end 157 | 158 | it "should be disabled" do 159 | expect{ AssetSync.sync }.not_to raise_error 160 | end 161 | 162 | after(:each) do 163 | Rails.env.replace('test') 164 | end 165 | end 166 | 167 | describe 'with no configuration' do 168 | before(:each) do 169 | AssetSync.config = AssetSync::Config.new 170 | end 171 | 172 | it "should be invalid" do 173 | expect{ AssetSync.sync }.to raise_error(::AssetSync::Config::Invalid) 174 | end 175 | end 176 | 177 | describe "with no other configuration than enabled = false" do 178 | before(:each) do 179 | AssetSync.config = AssetSync::Config.new 180 | AssetSync.configure do |config| 181 | config.enabled = false 182 | end 183 | end 184 | 185 | it "should do nothing, without complaining" do 186 | expect{ AssetSync.sync }.not_to raise_error 187 | end 188 | end 189 | 190 | describe 'with fail_silent configuration' do 191 | before(:each) do 192 | allow(AssetSync).to receive(:stderr).and_return(@stderr = StringIO.new) 193 | AssetSync.config = AssetSync::Config.new 194 | AssetSync.configure do |config| 195 | config.fail_silently = true 196 | end 197 | end 198 | 199 | it "should not raise an invalid exception" do 200 | expect{ AssetSync.sync }.not_to raise_error 201 | end 202 | 203 | it "should output a warning to stderr" do 204 | AssetSync.sync 205 | expect(@stderr.string).to match(/can't be blank/) 206 | end 207 | end 208 | 209 | describe 'with disabled config' do 210 | before(:each) do 211 | allow(AssetSync).to receive(:stderr).and_return(@stderr = StringIO.new) 212 | AssetSync.config = AssetSync::Config.new 213 | AssetSync.configure do |config| 214 | config.enabled = false 215 | end 216 | end 217 | 218 | it "should not raise an invalid exception" do 219 | expect{ AssetSync.sync }.not_to raise_error 220 | end 221 | end 222 | 223 | describe 'with gzip_compression enabled' do 224 | before(:each) do 225 | AssetSync.config = AssetSync::Config.new 226 | AssetSync.config.gzip_compression = true 227 | end 228 | 229 | it "config.gzip? should be true" do 230 | expect(AssetSync.config.gzip?).to be_truthy 231 | end 232 | end 233 | 234 | describe 'with manifest enabled' do 235 | before(:each) do 236 | AssetSync.config = AssetSync::Config.new 237 | AssetSync.config.manifest = true 238 | end 239 | 240 | it "config.manifest should be true" do 241 | expect(AssetSync.config.manifest).to be_truthy 242 | end 243 | 244 | it "config.manifest_path should default to public/assets.." do 245 | expect(AssetSync.config.manifest_path).to match(/public\/assets\/manifest.yml/) 246 | end 247 | 248 | it "config.manifest_path should default to public/assets.." do 249 | Rails.application.config.assets.manifest = "/var/assets" 250 | expect(AssetSync.config.manifest_path).to eq("/var/assets/manifest.yml") 251 | end 252 | 253 | it "config.manifest_path should default to public/custom_assets.." do 254 | Rails.application.config.assets.prefix = 'custom_assets' 255 | expect(AssetSync.config.manifest_path).to match(/public\/custom_assets\/manifest.yml/) 256 | end 257 | end 258 | 259 | describe 'with cache_asset_regexps' do 260 | before(:each) do 261 | AssetSync.config = AssetSync::Config.new 262 | end 263 | 264 | it "config.cache_asset_regexp should set cache_asset_regexps" do 265 | AssetSync.config.cache_asset_regexp = /\.[a-f0-9]{8}/i 266 | expect(AssetSync.config.cache_asset_regexps.size).to eq(1) 267 | expect(AssetSync.config.cache_asset_regexps[0]).to eq(/\.[a-f0-9]{8}/i) 268 | end 269 | 270 | it "set cache_asset_regexps" do 271 | AssetSync.config.cache_asset_regexps = ["app.abc123.js", /\.[a-f0-9]{10}/i] 272 | expect(AssetSync.config.cache_asset_regexps.size).to eq(2) 273 | expect(AssetSync.config.cache_asset_regexps).to eq(["app.abc123.js", /\.[a-f0-9]{10}/i]) 274 | end 275 | end 276 | 277 | describe 'with invalid yml' do 278 | before(:each) do 279 | set_rails_root('with_invalid_yml') 280 | end 281 | 282 | it "an error" do 283 | expect{ AssetSync::Config.new }.to raise_error(Psych::SyntaxError) 284 | end 285 | end 286 | 287 | describe 'FogPublicValue' do 288 | describe "#to_bool" do 289 | it "true should be converted to true" do 290 | expect(AssetSync::Config::FogPublicValue.new(true).to_bool).to be_truthy 291 | end 292 | it "false should be converted to false" do 293 | expect(AssetSync::Config::FogPublicValue.new(false).to_bool).to be_falsey 294 | end 295 | it "nil should be converted to false" do 296 | expect(AssetSync::Config::FogPublicValue.new(nil).to_bool).to be_falsey 297 | end 298 | it "'default' should be converted to false" do 299 | expect(AssetSync::Config::FogPublicValue.new("default").to_bool).to be_truthy 300 | end 301 | end 302 | 303 | describe "#use_explicit_value?" do 304 | it "true should be converted to true" do 305 | expect(AssetSync::Config::FogPublicValue.new(true).use_explicit_value?).to be_truthy 306 | end 307 | it "false should be converted to true" do 308 | expect(AssetSync::Config::FogPublicValue.new(false).use_explicit_value?).to be_truthy 309 | end 310 | it "nil should be converted to true" do 311 | expect(AssetSync::Config::FogPublicValue.new(nil).use_explicit_value?).to be_truthy 312 | end 313 | it "'default' should be converted to false" do 314 | expect(AssetSync::Config::FogPublicValue.new("default").use_explicit_value?).to be_falsey 315 | end 316 | end 317 | end 318 | end 319 | -------------------------------------------------------------------------------- /lib/asset_sync/storage.rb: -------------------------------------------------------------------------------- 1 | require "fog/core" 2 | 3 | require "asset_sync/multi_mime" 4 | 5 | module AssetSync 6 | class Storage 7 | REGEXP_FINGERPRINTED_FILES = /\A(.*)\/(.+)-[^\.]+\.([^\.]+)\z/m 8 | REGEXP_ASSETS_TO_CACHE_CONTROL = /-[0-9a-fA-F]{32,}$/ 9 | 10 | class BucketNotFound < StandardError; 11 | end 12 | 13 | attr_accessor :config 14 | 15 | def initialize(cfg) 16 | @config = cfg 17 | end 18 | 19 | def connection 20 | @connection ||= Fog::Storage.new(self.config.fog_options) 21 | end 22 | 23 | def bucket 24 | # fixes: https://github.com/rumblelabs/asset_sync/issues/18 25 | 26 | @bucket ||= if self.config.backblaze? 27 | connection.directories.get(self.config.fog_directory) 28 | else 29 | connection.directories.get(self.config.fog_directory, :prefix => self.config.assets_prefix) 30 | end 31 | 32 | end 33 | 34 | def log(msg) 35 | AssetSync.log(msg) 36 | end 37 | 38 | def keep_existing_remote_files? 39 | self.config.existing_remote_files? 40 | end 41 | 42 | def path 43 | self.config.public_path 44 | end 45 | 46 | def remote_file_list_cache_file_path 47 | self.config.remote_file_list_cache_file_path 48 | end 49 | 50 | def ignored_files 51 | expand_file_names(self.config.ignored_files) 52 | end 53 | 54 | def get_manifest_path 55 | return [] unless self.config.include_manifest 56 | 57 | if ActionView::Base.respond_to?(:assets_manifest) 58 | manifest = Sprockets::Manifest.new(ActionView::Base.assets_manifest.environment, ActionView::Base.assets_manifest.dir) 59 | manifest_path = manifest.filename 60 | else 61 | manifest_path = self.config.manifest_path 62 | end 63 | [manifest_path.sub(/^#{path}\//, "")] # full path to relative path 64 | end 65 | 66 | def local_files 67 | @local_files ||= 68 | (get_local_files + config.additional_local_file_paths).uniq 69 | end 70 | 71 | def remote_files 72 | return [] if ignore_existing_remote_files? 73 | return @remote_files if @remote_files 74 | 75 | if remote_file_list_cache_file_path && File.file?(remote_file_list_cache_file_path) 76 | begin 77 | content = File.read(remote_file_list_cache_file_path) 78 | return @remote_files = JSON.parse(content) 79 | rescue JSON::ParserError 80 | warn "Failed to parse #{remote_file_list_cache_file_path} as json" 81 | end 82 | end 83 | 84 | @remote_files = get_remote_files 85 | end 86 | 87 | def update_remote_file_list_cache(local_files_to_upload) 88 | return unless remote_file_list_cache_file_path 89 | return if ignore_existing_remote_files? 90 | 91 | File.open(self.remote_file_list_cache_file_path, 'w') do |file| 92 | uploaded = local_files_to_upload + remote_files 93 | file.write(uploaded.to_json) 94 | end 95 | end 96 | 97 | def always_upload_files 98 | expand_file_names(self.config.always_upload) + get_manifest_path 99 | end 100 | 101 | def files_with_custom_headers 102 | self.config.custom_headers.inject({}) { |h,(k, v)| h[File.join(self.config.assets_prefix, k)] = v; h; } 103 | end 104 | 105 | def files_to_invalidate 106 | self.config.invalidate.map { |filename| File.join("/", self.config.assets_prefix, filename) } 107 | end 108 | 109 | # @api 110 | # To get a list of asset files indicated in a manifest file. 111 | # It makes sense if a user sets `config.manifest` is true. 112 | def get_asset_files_from_manifest 113 | if self.config.manifest 114 | if ActionView::Base.respond_to?(:assets_manifest) 115 | log "Using: Rails 4.0 manifest access" 116 | manifest = Sprockets::Manifest.new(ActionView::Base.assets_manifest.environment, ActionView::Base.assets_manifest.dir) 117 | return manifest.assets.values.map { |f| File.join(self.config.assets_prefix, f) } 118 | elsif File.exist?(self.config.manifest_path) 119 | log "Using: Manifest #{self.config.manifest_path}" 120 | yml = AssetSync.load_yaml(IO.read(self.config.manifest_path)) 121 | 122 | return yml.map do |original, compiled| 123 | # Upload font originals and compiled 124 | if original =~ /^.+(eot|svg|ttf|woff)$/ 125 | [original, compiled] 126 | else 127 | compiled 128 | end 129 | end.flatten.map { |f| File.join(self.config.assets_prefix, f) }.uniq! 130 | else 131 | log "Warning: Manifest could not be found" 132 | end 133 | end 134 | end 135 | 136 | def get_local_files 137 | if from_manifest = get_asset_files_from_manifest 138 | return from_manifest 139 | end 140 | 141 | log "Using: Directory Search of #{path}/#{self.config.assets_prefix}" 142 | Dir.chdir(path) do 143 | to_load = self.config.assets_prefix.present? ? "#{self.config.assets_prefix}/**/**" : '**/**' 144 | Dir[to_load] 145 | end 146 | end 147 | 148 | def get_remote_files 149 | raise BucketNotFound.new("#{self.config.fog_provider} Bucket: #{self.config.fog_directory} not found.") unless bucket 150 | # fixes: https://github.com/rumblelabs/asset_sync/issues/16 151 | # (work-around for https://github.com/fog/fog/issues/596) 152 | files = [] 153 | bucket.files.each { |f| files << f.key } 154 | return files 155 | end 156 | 157 | def delete_file(f, remote_files_to_delete) 158 | if remote_files_to_delete.include?(f.key) 159 | log "Deleting: #{f.key}" 160 | f.destroy 161 | end 162 | end 163 | 164 | def delete_extra_remote_files 165 | log "Fetching files to flag for delete" 166 | remote_files = get_remote_files 167 | # fixes: https://github.com/rumblelabs/asset_sync/issues/19 168 | from_remote_files_to_delete = remote_files - local_files - ignored_files - always_upload_files 169 | 170 | log "Flagging #{from_remote_files_to_delete.size} file(s) for deletion" 171 | # Delete unneeded remote files, if we are on aws delete in bulk else use sequential delete 172 | if self.config.aws? && connection.respond_to?(:delete_multiple_objects) 173 | from_remote_files_to_delete.each_slice(500) do |slice| 174 | connection.delete_multiple_objects(config.fog_directory, slice) 175 | end 176 | else 177 | bucket.files.each do |f| 178 | delete_file(f, from_remote_files_to_delete) 179 | end 180 | end 181 | end 182 | 183 | def upload_file(f) 184 | # TODO output files in debug logs as asset filename only. 185 | one_year = 31557600 186 | ext = File.extname(f)[1..-1] 187 | mime = MultiMime.lookup(ext) 188 | gzip_file_handle = nil 189 | file_handle = File.open("#{path}/#{f}") 190 | file = { 191 | :key => f, 192 | :body => file_handle, 193 | :content_type => mime 194 | } 195 | 196 | # region fog_public 197 | 198 | if config.aws? && config.aws_acl 199 | file[:acl] = config.aws_acl 200 | elsif config.fog_public.use_explicit_value? 201 | file[:public] = config.fog_public.to_bool 202 | end 203 | 204 | # endregion fog_public 205 | 206 | uncompressed_filename = f.sub(/\.gz\z/, '') 207 | basename = File.basename(uncompressed_filename, File.extname(uncompressed_filename)) 208 | 209 | assets_to_cache_control = Regexp.union([REGEXP_ASSETS_TO_CACHE_CONTROL] | config.cache_asset_regexps).source 210 | if basename.match(Regexp.new(assets_to_cache_control)).present? 211 | file.merge!({ 212 | :cache_control => "public, max-age=#{one_year}", 213 | :expires => CGI.rfc1123_date(Time.now + one_year) 214 | }) 215 | end 216 | 217 | # overwrite headers if applicable, you probably shouldn't specific key/body, but cache-control headers etc. 218 | 219 | if files_with_custom_headers.has_key? f 220 | file.merge! files_with_custom_headers[f] 221 | log "Overwriting #{f} with custom headers #{files_with_custom_headers[f].to_s}" 222 | elsif key = self.config.custom_headers.keys.detect {|k| f.match(Regexp.new(k))} 223 | headers = {} 224 | self.config.custom_headers[key].each do |k, value| 225 | headers[k.to_sym] = value 226 | end 227 | file.merge! headers 228 | log "Overwriting matching file #{f} with custom headers #{headers.to_s}" 229 | end 230 | 231 | 232 | gzipped = "#{path}/#{f}.gz" 233 | ignore = false 234 | 235 | if config.gzip? && File.extname(f) == ".gz" 236 | # Don't bother uploading gzipped assets if we are in gzip_compression mode 237 | # as we will overwrite file.css with file.css.gz if it exists. 238 | log "Ignoring: #{f}" 239 | ignore = true 240 | elsif config.gzip? && File.exist?(gzipped) 241 | original_size = File.size("#{path}/#{f}") 242 | gzipped_size = File.size(gzipped) 243 | 244 | if gzipped_size < original_size 245 | percentage = ((gzipped_size.to_f/original_size.to_f)*100).round(2) 246 | gzip_file_handle = File.open(gzipped) 247 | file.merge!({ 248 | :key => f, 249 | :body => gzip_file_handle, 250 | :content_encoding => 'gzip' 251 | }) 252 | log "Uploading: #{gzipped} in place of #{f} saving #{percentage}%" 253 | else 254 | percentage = ((original_size.to_f/gzipped_size.to_f)*100).round(2) 255 | log "Uploading: #{f} instead of #{gzipped} (compression increases this file by #{percentage}%)" 256 | end 257 | else 258 | if !config.gzip? && File.extname(f) == ".gz" 259 | # set content encoding for gzipped files this allows cloudfront to properly handle requests with Accept-Encoding 260 | # http://docs.amazonwebservices.com/AmazonCloudFront/latest/DeveloperGuide/ServingCompressedFiles.html 261 | uncompressed_filename = f[0..-4] 262 | ext = File.extname(uncompressed_filename)[1..-1] 263 | mime = MultiMime.lookup(ext) 264 | file.merge!({ 265 | :content_type => mime, 266 | :content_encoding => 'gzip' 267 | }) 268 | end 269 | log "Uploading: #{f}" 270 | end 271 | 272 | if config.aws? && config.aws_rrs? 273 | file.merge!({ 274 | :storage_class => 'REDUCED_REDUNDANCY' 275 | }) 276 | end 277 | 278 | bucket.files.create( file ) unless ignore 279 | file_handle.close 280 | gzip_file_handle.close if gzip_file_handle 281 | end 282 | 283 | def upload_files 284 | # fixes: https://github.com/rumblelabs/asset_sync/issues/19 285 | local_files_to_upload = local_files - ignored_files - remote_files + always_upload_files 286 | local_files_to_upload = (local_files_to_upload + get_non_fingerprinted(local_files_to_upload)).uniq 287 | # Only files. 288 | local_files_to_upload = local_files_to_upload.select { |f| File.file? "#{path}/#{f}" } 289 | 290 | if self.config.concurrent_uploads 291 | jobs = Queue.new 292 | local_files_to_upload.each { |f| jobs.push(f) } 293 | jobs.close 294 | 295 | num_threads = [self.config.concurrent_uploads_max_threads, local_files_to_upload.length].min 296 | # Upload new files 297 | workers = Array.new(num_threads) do 298 | Thread.new do 299 | while f = jobs.pop 300 | upload_file(f) 301 | end 302 | end 303 | end 304 | workers.map(&:join) 305 | else 306 | # Upload new files 307 | local_files_to_upload.each do |f| 308 | upload_file f 309 | end 310 | end 311 | 312 | if self.config.cdn_distribution_id && files_to_invalidate.any? 313 | log "Invalidating Files" 314 | cdn ||= Fog::CDN.new(self.config.fog_options.except(:region)) 315 | data = cdn.post_invalidation(self.config.cdn_distribution_id, files_to_invalidate) 316 | log "Invalidation id: #{data.body["Id"]}" 317 | end 318 | 319 | update_remote_file_list_cache(local_files_to_upload) 320 | end 321 | 322 | def sync 323 | # fixes: https://github.com/rumblelabs/asset_sync/issues/19 324 | log "AssetSync: Syncing." 325 | upload_files 326 | delete_extra_remote_files unless keep_existing_remote_files? 327 | log "AssetSync: Done." 328 | end 329 | 330 | private 331 | 332 | def ignore_existing_remote_files? 333 | self.config.existing_remote_files == 'ignore' 334 | end 335 | 336 | def get_non_fingerprinted(files) 337 | files.map do |file| 338 | match_data = file.match(REGEXP_FINGERPRINTED_FILES) 339 | match_data && "#{match_data[1]}/#{match_data[2]}.#{match_data[3]}" 340 | end.compact 341 | end 342 | 343 | def expand_file_names(names) 344 | files = [] 345 | Array(names).each do |name| 346 | case name 347 | when Regexp 348 | files += self.local_files.select do |file| 349 | file =~ name 350 | end 351 | when String 352 | files += self.local_files.select do |file| 353 | file.split('/').last == name 354 | end 355 | else 356 | log "Error: please define file names as string or regular expression. #{name} (#{name.class}) ignored." 357 | end 358 | end 359 | files.uniq 360 | end 361 | 362 | end 363 | end 364 | -------------------------------------------------------------------------------- /lib/asset_sync/config.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require "active_model" 4 | require "erb" 5 | 6 | module AssetSync 7 | class Config 8 | include ActiveModel::Validations 9 | 10 | class Invalid < StandardError; end 11 | 12 | # AssetSync 13 | attr_accessor :existing_remote_files # What to do with your existing remote files? (keep or delete) 14 | attr_accessor :gzip_compression 15 | attr_accessor :manifest 16 | attr_accessor :fail_silently 17 | attr_accessor :log_silently 18 | attr_accessor :always_upload 19 | attr_accessor :ignored_files 20 | attr_accessor :prefix 21 | attr_accessor :enabled 22 | attr_accessor :custom_headers 23 | attr_accessor :run_on_precompile 24 | attr_accessor :invalidate 25 | attr_accessor :cdn_distribution_id 26 | attr_accessor :cache_asset_regexps 27 | attr_accessor :include_manifest 28 | attr_accessor :concurrent_uploads 29 | attr_accessor :concurrent_uploads_max_threads 30 | attr_accessor :remote_file_list_cache_file_path 31 | 32 | # FOG configuration 33 | attr_accessor :fog_provider # Currently Supported ['AWS', 'Rackspace'] 34 | attr_accessor :fog_directory # e.g. 'the-bucket-name' 35 | attr_accessor :fog_region # e.g. 'eu-west-1' 36 | attr_reader :fog_public # e.g. true, false, "default" 37 | 38 | # Amazon AWS 39 | attr_accessor :aws_access_key_id 40 | attr_accessor :aws_secret_access_key 41 | attr_accessor :aws_session_token 42 | attr_accessor :aws_reduced_redundancy 43 | attr_accessor :aws_iam_roles 44 | attr_accessor :aws_signature_version 45 | attr_accessor :aws_acl 46 | 47 | # Fog 48 | attr_accessor :fog_host # e.g. 's3.amazonaws.com' 49 | attr_accessor :fog_port # e.g. '9000' 50 | attr_accessor :fog_path_style # e.g. true 51 | attr_accessor :fog_scheme # e.g. 'http' 52 | 53 | # Rackspace 54 | attr_accessor :rackspace_username, :rackspace_api_key, :rackspace_auth_url 55 | 56 | # Google Storage 57 | attr_accessor :google_storage_secret_access_key, :google_storage_access_key_id # when using S3 interop 58 | attr_accessor :google_json_key_location # when using service accounts 59 | attr_accessor :google_json_key_string # when using service accounts 60 | attr_accessor :google_project # when using service accounts 61 | 62 | # Azure Blob with Fog::AzureRM 63 | attr_accessor :azure_storage_account_name 64 | attr_accessor :azure_storage_access_key 65 | 66 | # Backblaze B2 with Fog::Backblaze 67 | attr_accessor :b2_key_id 68 | attr_accessor :b2_key_token 69 | attr_accessor :b2_bucket_id 70 | 71 | validates :existing_remote_files, :inclusion => { :in => %w(keep delete ignore) } 72 | 73 | validates :fog_provider, :presence => true 74 | validates :fog_directory, :presence => true 75 | 76 | validates :aws_access_key_id, :presence => true, :if => proc {aws? && !aws_iam?} 77 | validates :aws_secret_access_key, :presence => true, :if => proc {aws? && !aws_iam?} 78 | validates :rackspace_username, :presence => true, :if => :rackspace? 79 | validates :rackspace_api_key, :presence => true, :if => :rackspace? 80 | validates :google_storage_secret_access_key, :presence => true, :if => :google_interop? 81 | validates :google_storage_access_key_id, :presence => true, :if => :google_interop? 82 | validates :google_project, :presence => true, :if => :google_service_account? 83 | validate(:if => :google_service_account?) do 84 | unless google_json_key_location.present? || google_json_key_string.present? 85 | errors.add(:base, 'must provide either google_json_key_location or google_json_key_string if using Google service account') 86 | end 87 | end 88 | validates :concurrent_uploads, :inclusion => { :in => [true, false] } 89 | 90 | def initialize 91 | self.fog_region = nil 92 | self.fog_public = true 93 | self.existing_remote_files = 'keep' 94 | self.gzip_compression = false 95 | self.manifest = false 96 | self.fail_silently = false 97 | self.log_silently = true 98 | self.always_upload = [] 99 | self.ignored_files = [] 100 | self.custom_headers = {} 101 | self.enabled = true 102 | self.run_on_precompile = true 103 | self.cdn_distribution_id = nil 104 | self.invalidate = [] 105 | self.cache_asset_regexps = [] 106 | self.include_manifest = false 107 | self.concurrent_uploads = false 108 | self.concurrent_uploads_max_threads = 10 109 | self.remote_file_list_cache_file_path = nil 110 | @additional_local_file_paths_procs = [] 111 | 112 | load_yml! if defined?(::Rails) && yml_exists? 113 | end 114 | 115 | def manifest_path 116 | directory = 117 | ::Rails.application.config.assets.manifest || default_manifest_directory 118 | File.join(directory, "manifest.yml") 119 | end 120 | 121 | def gzip? 122 | self.gzip_compression 123 | end 124 | 125 | def existing_remote_files? 126 | ['keep', 'ignore'].include?(self.existing_remote_files) 127 | end 128 | 129 | def aws? 130 | fog_provider =~ /aws/i 131 | end 132 | 133 | def aws_rrs? 134 | aws_reduced_redundancy == true 135 | end 136 | 137 | def aws_iam? 138 | aws_iam_roles == true 139 | end 140 | 141 | def fail_silently? 142 | fail_silently || !enabled? 143 | end 144 | 145 | def log_silently? 146 | !!self.log_silently 147 | end 148 | 149 | def enabled? 150 | enabled == true 151 | end 152 | 153 | def rackspace? 154 | fog_provider =~ /rackspace/i 155 | end 156 | 157 | def google? 158 | fog_provider =~ /google/i 159 | end 160 | 161 | def google_interop? 162 | google? && google_json_key_location.nil? && google_json_key_string.nil? 163 | end 164 | 165 | def google_service_account? 166 | google? && (google_json_key_location || google_json_key_string) 167 | end 168 | 169 | def azure_rm? 170 | fog_provider =~ /azurerm/i 171 | end 172 | 173 | def backblaze? 174 | fog_provider =~ /backblaze/i 175 | end 176 | 177 | def cache_asset_regexp=(cache_asset_regexp) 178 | self.cache_asset_regexps = [cache_asset_regexp] 179 | end 180 | 181 | def yml_exists? 182 | defined?(::Rails.root) ? File.exist?(self.yml_path) : false 183 | end 184 | 185 | def yml 186 | @yml ||= ::AssetSync.load_yaml(::ERB.new(IO.read(yml_path)).result)[::Rails.env] || {} 187 | end 188 | 189 | def yml_path 190 | ::Rails.root.join("config", "asset_sync.yml").to_s 191 | end 192 | 193 | def assets_prefix 194 | # Fix for Issue #38 when Rails.config.assets.prefix starts with a slash 195 | self.prefix || ::Rails.application.config.assets.prefix.sub(/^\//, '') 196 | end 197 | 198 | def public_path 199 | @public_path || ::Rails.public_path 200 | end 201 | 202 | def public_path=(path) 203 | # Generate absolute path even when relative path passed in 204 | # Required for generating relative sprockets manifest path 205 | pathname = Pathname(path) 206 | @public_path = if pathname.absolute? 207 | pathname 208 | elsif defined?(::Rails.root) 209 | ::Rails.root.join(pathname) 210 | else 211 | Pathname(::Dir.pwd).join(pathname) 212 | end 213 | end 214 | 215 | def load_yml! 216 | self.enabled = yml["enabled"] if yml.has_key?('enabled') 217 | self.fog_provider = yml["fog_provider"] 218 | self.fog_host = yml["fog_host"] 219 | self.fog_port = yml["fog_port"] 220 | self.fog_directory = yml["fog_directory"] 221 | self.fog_region = yml["fog_region"] 222 | self.fog_public = yml["fog_public"] if yml.has_key?("fog_public") 223 | self.fog_path_style = yml["fog_path_style"] 224 | self.fog_scheme = yml["fog_scheme"] 225 | self.aws_access_key_id = yml["aws_access_key_id"] 226 | self.aws_secret_access_key = yml["aws_secret_access_key"] 227 | self.aws_session_token = yml["aws_session_token"] if yml.has_key?("aws_session_token") 228 | self.aws_reduced_redundancy = yml["aws_reduced_redundancy"] 229 | self.aws_iam_roles = yml["aws_iam_roles"] 230 | self.aws_signature_version = yml["aws_signature_version"] 231 | self.aws_acl = yml["aws_acl"] 232 | self.rackspace_username = yml["rackspace_username"] 233 | self.rackspace_auth_url = yml["rackspace_auth_url"] if yml.has_key?("rackspace_auth_url") 234 | self.rackspace_api_key = yml["rackspace_api_key"] 235 | self.google_json_key_location = yml["google_json_key_location"] if yml.has_key?("google_json_key_location") 236 | self.google_project = yml["google_project"] if yml.has_key?("google_project") 237 | self.google_storage_secret_access_key = yml["google_storage_secret_access_key"] if yml.has_key?("google_storage_secret_access_key") 238 | self.google_storage_access_key_id = yml["google_storage_access_key_id"] if yml.has_key?("google_storage_access_key_id") 239 | self.google_json_key_string = yml["google_json_key_string"] if yml.has_key?("google_json_key_string") 240 | self.existing_remote_files = yml["existing_remote_files"] if yml.has_key?("existing_remote_files") 241 | self.gzip_compression = yml["gzip_compression"] if yml.has_key?("gzip_compression") 242 | self.manifest = yml["manifest"] if yml.has_key?("manifest") 243 | self.fail_silently = yml["fail_silently"] if yml.has_key?("fail_silently") 244 | self.log_silently = yml["log_silently"] if yml.has_key?("log_silently") 245 | self.always_upload = yml["always_upload"] if yml.has_key?("always_upload") 246 | self.ignored_files = yml["ignored_files"] if yml.has_key?("ignored_files") 247 | self.custom_headers = yml["custom_headers"] if yml.has_key?("custom_headers") 248 | self.run_on_precompile = yml["run_on_precompile"] if yml.has_key?("run_on_precompile") 249 | self.invalidate = yml["invalidate"] if yml.has_key?("invalidate") 250 | self.cdn_distribution_id = yml['cdn_distribution_id'] if yml.has_key?("cdn_distribution_id") 251 | self.cache_asset_regexps = yml['cache_asset_regexps'] if yml.has_key?("cache_asset_regexps") 252 | self.include_manifest = yml['include_manifest'] if yml.has_key?("include_manifest") 253 | self.concurrent_uploads = yml['concurrent_uploads'] if yml.has_key?('concurrent_uploads') 254 | self.concurrent_uploads_max_threads = yml['concurrent_uploads_max_threads'] if yml.has_key?('concurrent_uploads_max_threads') 255 | self.remote_file_list_cache_file_path = yml['remote_file_list_cache_file_path'] if yml.has_key?('remote_file_list_cache_file_path') 256 | 257 | self.azure_storage_account_name = yml['azure_storage_account_name'] if yml.has_key?("azure_storage_account_name") 258 | self.azure_storage_access_key = yml['azure_storage_access_key'] if yml.has_key?("azure_storage_access_key") 259 | 260 | self.b2_key_id = yml['b2_key_id'] if yml.has_key?("b2_key_id") 261 | self.b2_key_token = yml['b2_key_token'] if yml.has_key?("b2_key_token") 262 | self.b2_bucket_id = yml['b2_bucket_id'] if yml.has_key?("b2_bucket_id") 263 | 264 | # TODO deprecate the other old style config settings. FML. 265 | self.aws_access_key_id = yml["aws_access_key"] if yml.has_key?("aws_access_key") 266 | self.aws_secret_access_key = yml["aws_access_secret"] if yml.has_key?("aws_access_secret") 267 | self.fog_directory = yml["aws_bucket"] if yml.has_key?("aws_bucket") 268 | self.fog_region = yml["aws_region"] if yml.has_key?("aws_region") 269 | 270 | # TODO deprecate old style config settings 271 | self.aws_access_key_id = yml["access_key_id"] if yml.has_key?("access_key_id") 272 | self.aws_secret_access_key = yml["secret_access_key"] if yml.has_key?("secret_access_key") 273 | self.fog_directory = yml["bucket"] if yml.has_key?("bucket") 274 | self.fog_region = yml["region"] if yml.has_key?("region") 275 | 276 | self.public_path = yml["public_path"] if yml.has_key?("public_path") 277 | end 278 | 279 | 280 | def fog_options 281 | options = { :provider => fog_provider } 282 | if aws? 283 | if aws_iam? 284 | options.merge!({ 285 | :use_iam_profile => true 286 | }) 287 | else 288 | options.merge!({ 289 | :aws_access_key_id => aws_access_key_id, 290 | :aws_secret_access_key => aws_secret_access_key 291 | }) 292 | options.merge!({:aws_session_token => aws_session_token}) if aws_session_token 293 | end 294 | options.merge!({:host => fog_host}) if fog_host 295 | options.merge!({:port => fog_port}) if fog_port 296 | options.merge!({:scheme => fog_scheme}) if fog_scheme 297 | options.merge!({:aws_signature_version => aws_signature_version}) if aws_signature_version 298 | options.merge!({:path_style => fog_path_style}) if fog_path_style 299 | options.merge!({:region => fog_region}) if fog_region 300 | elsif rackspace? 301 | options.merge!({ 302 | :rackspace_username => rackspace_username, 303 | :rackspace_api_key => rackspace_api_key 304 | }) 305 | options.merge!({ :rackspace_region => fog_region }) if fog_region 306 | options.merge!({ :rackspace_auth_url => rackspace_auth_url }) if rackspace_auth_url 307 | elsif google? 308 | if google_json_key_location 309 | options.merge!({:google_json_key_location => google_json_key_location, :google_project => google_project}) 310 | elsif google_json_key_string 311 | options.merge!({:google_json_key_string => google_json_key_string, :google_project => google_project}) 312 | else 313 | options.merge!({ 314 | :google_storage_secret_access_key => google_storage_secret_access_key, 315 | :google_storage_access_key_id => google_storage_access_key_id 316 | }) 317 | end 318 | options.merge!({:region => fog_region}) if fog_region 319 | elsif azure_rm? 320 | require 'fog/azurerm' 321 | options.merge!({ 322 | :azure_storage_account_name => azure_storage_account_name, 323 | :azure_storage_access_key => azure_storage_access_key, 324 | }) 325 | options.merge!({:environment => fog_region}) if fog_region 326 | elsif backblaze? 327 | require 'fog/backblaze' 328 | options.merge!({ 329 | :b2_key_id => b2_key_id, 330 | :b2_key_token => b2_key_token, 331 | :b2_bucket_id => b2_bucket_id, 332 | }) 333 | else 334 | raise ArgumentError, "AssetSync Unknown provider: #{fog_provider} only AWS, Rackspace and Google are supported currently." 335 | end 336 | 337 | options 338 | end 339 | 340 | # @api 341 | def add_local_file_paths(&block) 342 | @additional_local_file_paths_procs = 343 | additional_local_file_paths_procs + [block] 344 | end 345 | 346 | # @api private 347 | # This is to be called in Storage 348 | # Not to be called by user 349 | def additional_local_file_paths 350 | return [] if additional_local_file_paths_procs.empty? 351 | 352 | # Using `Array()` to ensure it works when single value is returned 353 | additional_local_file_paths_procs.each_with_object([]) do |proc, paths| 354 | paths.concat(Array(proc.call)) 355 | end 356 | end 357 | 358 | #@api 359 | def file_ext_to_mime_type_overrides 360 | @file_ext_to_mime_type_overrides ||= FileExtToMimeTypeOverrides.new 361 | end 362 | 363 | def fog_public=(new_val) 364 | @fog_public = FogPublicValue.new(new_val) 365 | end 366 | 367 | private 368 | 369 | # This is a proc to get additional local files paths 370 | # Since this is a proc it won't be able to be configured by a YAML file 371 | attr_reader :additional_local_file_paths_procs 372 | 373 | def default_manifest_directory 374 | File.join(::Rails.public_path, assets_prefix) 375 | end 376 | 377 | 378 | # @api private 379 | class FileExtToMimeTypeOverrides 380 | def initialize 381 | # The default is to prevent new mime type `application/ecmascript` to be returned 382 | # which disables compression on some CDNs 383 | @overrides = { 384 | "js" => "application/javascript", 385 | } 386 | end 387 | 388 | # @api 389 | def add(ext, mime_type) 390 | # Symbol / Mime type object might be passed in 391 | # But we want strings only 392 | @overrides.store( 393 | ext.to_s, mime_type.to_s, 394 | ) 395 | end 396 | 397 | # @api 398 | def clear 399 | @overrides = {} 400 | end 401 | 402 | 403 | # @api private 404 | def key?(key) 405 | @overrides.key?(key) 406 | end 407 | 408 | # @api private 409 | def fetch(key) 410 | @overrides.fetch(key) 411 | end 412 | end 413 | 414 | # @api private 415 | class FogPublicValue 416 | def initialize(val) 417 | @value = val 418 | end 419 | 420 | def use_explicit_value? 421 | @value.to_s != "default" 422 | end 423 | 424 | def to_bool 425 | !!@value 426 | end 427 | end 428 | end 429 | end 430 | -------------------------------------------------------------------------------- /spec/unit/storage_spec.rb: -------------------------------------------------------------------------------- 1 | require File.dirname(__FILE__) + '/../spec_helper' 2 | 3 | describe AssetSync::Storage do 4 | include_context "mock Rails without_yml" 5 | 6 | let(:file_like_object) do 7 | double("file like object").as_null_object 8 | end 9 | 10 | describe '#upload_files' do 11 | before(:each) do 12 | @local_files = ["local_image2.jpg", "local_image1.jpg", "local_stylesheet1.css", "local_stylesheet2.css"] 13 | @remote_files = ["local_image.jpg", "local_image3.svg", "local_image4.svg", "local_stylesheet1.css"] 14 | @config = AssetSync::Config.new 15 | end 16 | 17 | it 'should overwrite all remote files if set to ignore' do 18 | @config.existing_remote_files = 'ignore' 19 | storage = AssetSync::Storage.new(@config) 20 | allow(storage).to receive(:get_local_files).and_return(@local_files) 21 | allow(File).to receive(:file?).and_return(true) # Pretend they all exist 22 | 23 | @local_files.each do |file| 24 | expect(storage).to receive(:upload_file).with(file) 25 | end 26 | storage.upload_files 27 | end 28 | 29 | it 'should allow force overwriting of specific files' do 30 | @config.always_upload = ['local_image.jpg', /local_image\d\.svg/] 31 | 32 | storage = AssetSync::Storage.new(@config) 33 | allow(storage).to receive(:get_local_files).and_return(@local_files) 34 | allow(storage).to receive(:get_remote_files).and_return(@remote_files) 35 | allow(File).to receive(:file?).and_return(true) # Pretend they all exist 36 | 37 | (@local_files - @remote_files + storage.always_upload_files).each do |file| 38 | expect(storage).to receive(:upload_file).with(file) 39 | end 40 | storage.upload_files 41 | end 42 | 43 | it 'should allow to ignore files' do 44 | @config.ignored_files = ['local_image1.jpg', /local_stylesheet\d\.css/] 45 | 46 | storage = AssetSync::Storage.new(@config) 47 | allow(storage).to receive(:get_local_files).and_return(@local_files) 48 | allow(storage).to receive(:get_remote_files).and_return(@remote_files) 49 | allow(File).to receive(:file?).and_return(true) # Pretend they all exist 50 | 51 | (@local_files - @remote_files - storage.ignored_files + storage.always_upload_files).each do |file| 52 | expect(storage).to receive(:upload_file).with(file) 53 | end 54 | storage.upload_files 55 | end 56 | 57 | it 'should upload files concurrently if enabled' do 58 | @config.concurrent_uploads = true 59 | storage = AssetSync::Storage.new(@config) 60 | 61 | allow(storage).to receive(:get_local_files).and_return(@local_files) 62 | allow(storage).to receive(:get_remote_files).and_return(@remote_files) 63 | allow(File).to receive(:file?).and_return(true) # Pretend they all exist 64 | 65 | expect(Thread).to receive(:new).exactly(3).times.and_call_original 66 | (@local_files - @remote_files + storage.always_upload_files).each do |file| 67 | expect(storage).to receive(:upload_file).with(file) 68 | end 69 | 70 | storage.upload_files 71 | end 72 | 73 | it 'should allow custom number of threads' do 74 | @config.concurrent_uploads = true 75 | @config.concurrent_uploads_max_threads = 2 76 | storage = AssetSync::Storage.new(@config) 77 | 78 | allow(storage).to receive(:get_local_files).and_return(@local_files) 79 | allow(storage).to receive(:get_remote_files).and_return(@remote_files) 80 | allow(File).to receive(:file?).and_return(true) # Pretend they all exist 81 | 82 | expect(Thread).to receive(:new).exactly(2).times.and_call_original 83 | (@local_files - @remote_files + storage.always_upload_files).each do |file| 84 | expect(storage).to receive(:upload_file).with(file) 85 | end 86 | 87 | storage.upload_files 88 | end 89 | 90 | it 'should allow remote_file_list_cache_file_path configuration' do 91 | file_path = './foo.json' 92 | @config.remote_file_list_cache_file_path = file_path 93 | storage = AssetSync::Storage.new(@config) 94 | 95 | allow(storage).to receive(:get_local_files).and_return(@local_files) 96 | File.write(file_path, @remote_files.to_json) 97 | expect(storage).not_to receive(:get_remote_files) 98 | allow(File).to receive(:file?).and_return(true) # Pretend they all exist 99 | 100 | (@local_files - @remote_files + storage.always_upload_files).each do |file| 101 | expect(storage).to receive(:upload_file).with(file) 102 | end 103 | 104 | expect(storage).not_to receive(:warn) 105 | storage.upload_files 106 | 107 | # update remote_file_list_cache corretly 108 | updated = JSON.parse(File.read(file_path)) 109 | expect(updated.sort.uniq).to eq (@remote_files + @local_files + storage.always_upload_files).sort.uniq 110 | 111 | File.delete(file_path) 112 | end 113 | 114 | it 'should work with broken cache' do 115 | file_path = './foo.json' 116 | @config.remote_file_list_cache_file_path = file_path 117 | 118 | storage = AssetSync::Storage.new(@config) 119 | 120 | File.write(file_path, 'some non-json text file content') 121 | 122 | allow(storage).to receive(:get_local_files).and_return(@local_files) 123 | allow(storage).to receive(:get_remote_files).and_return(@remote_files) 124 | allow(File).to receive(:file?).and_return(true) # Pretend they all exist 125 | 126 | (@local_files - @remote_files + storage.always_upload_files).each do |file| 127 | expect(storage).to receive(:upload_file).with(file) 128 | end 129 | 130 | # when broken, warning message should be prompted 131 | expect(storage).to receive(:warn) 132 | 133 | storage.upload_files 134 | 135 | File.delete(file_path) 136 | end 137 | 138 | it 'should upload updated non-fingerprinted files' do 139 | @local_files = [ 140 | 'public/great-image.png', 141 | 'public/great-image-82389298328.png', 142 | 'public/great-image-a8389f9h324.png', 143 | "public/new\nline.js", 144 | "public/new\nline-aaaaaaaaaaa.js", 145 | "public/new\nline-bbbbbbbbbbb.js", 146 | 'public/application.js', 147 | 'public/application-b3389d983k1.js', 148 | 'public/application-ac387d53f31.js', 149 | 'public', 150 | ] 151 | @remote_files = [ 152 | 'public/great-image.png', 153 | 'public/great-image-a8389f9h324.png', 154 | "public/new\nline.js", 155 | "public/new\nline-aaaaaaaaaaa.js", 156 | 'public/application.js', 157 | 'public/application-b3389d983k1.js', 158 | ] 159 | 160 | storage = AssetSync::Storage.new(@config) 161 | allow(storage).to receive(:get_local_files).and_return(@local_files) 162 | allow(storage).to receive(:get_remote_files).and_return(@remote_files) 163 | allow(File).to receive(:file?).and_return(true) # Pretend they all exist 164 | 165 | updated_nonfingerprinted_files = [ 166 | 'public/great-image.png', 167 | "public/new\nline.js", 168 | 'public/application.js', 169 | ] 170 | (@local_files - @remote_files + updated_nonfingerprinted_files).each do |file| 171 | expect(storage).to receive(:upload_file).with(file) 172 | end 173 | storage.upload_files 174 | end 175 | 176 | context "when config #add_local_file_paths is called" do 177 | let(:additional_local_file_paths) do 178 | ["webpack/example_asset.jpg"] 179 | end 180 | 181 | before(:each) do 182 | @config.add_local_file_paths do 183 | additional_local_file_paths 184 | end 185 | end 186 | 187 | let(:storage) do 188 | AssetSync::Storage.new(@config) 189 | end 190 | 191 | let(:file_paths_should_be_uploaded) do 192 | @local_files - 193 | @remote_files - 194 | storage.ignored_files + 195 | storage.always_upload_files + 196 | additional_local_file_paths 197 | end 198 | 199 | before do 200 | # Stubbing 201 | allow(storage).to receive(:get_local_files).and_return(@local_files) 202 | allow(storage).to receive(:get_remote_files).and_return(@remote_files) 203 | # Pretend the files all exist 204 | allow(File).to receive(:file?).and_return(true) 205 | end 206 | 207 | it "uploads additional files in additional to local files" do 208 | file_paths_should_be_uploaded.each do |file| 209 | expect(storage).to receive(:upload_file).with(file) 210 | end 211 | storage.upload_files 212 | end 213 | end 214 | 215 | it 'should upload additonal files' do 216 | @local_files = [ 217 | 'public/image.png', 218 | 'public/image-82389298328.png', 219 | 'public/image-a8389f9h324.png', 220 | 'public/application.js', 221 | 'public/application-b3389d983k1.js', 222 | 'public/application-ac387d53f31.js', 223 | 'public', 224 | ] 225 | @remote_files = [ 226 | 'public/image.png', 227 | 'public/image-a8389f9h324.png', 228 | 'public/application.js', 229 | 'public/application-b3389d983k1.js', 230 | ] 231 | 232 | storage = AssetSync::Storage.new(@config) 233 | allow(storage).to receive(:get_local_files).and_return(@local_files) 234 | allow(storage).to receive(:get_remote_files).and_return(@remote_files) 235 | allow(File).to receive(:file?).and_return(true) # Pretend they all exist 236 | 237 | updated_nonfingerprinted_files = [ 238 | 'public/image.png', 239 | 'public/application.js', 240 | ] 241 | (@local_files - @remote_files + updated_nonfingerprinted_files).each do |file| 242 | expect(storage).to receive(:upload_file).with(file) 243 | end 244 | storage.upload_files 245 | end 246 | 247 | 248 | it 'should correctly set expire date' do 249 | local_files = [ 250 | 'file1.jpg', 251 | 'file1-1234567890abcdef1234567890abcdef.jpg', 252 | 'file1-1234567890abcdef1234567890abcdef.jpg.gz', 253 | 'file1-1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef.jpg', 254 | 'file1-1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef.jpg.gz' 255 | ] 256 | local_files += [ 257 | 'dir1/dir2/file2.jpg', 258 | 'dir1/dir2/file2-1234567890abcdef1234567890abcdef.jpg', 259 | 'dir1/dir2/file2-1234567890abcdef1234567890abcdef.jpg.gz', 260 | 'dir1/dir2/file2-1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef.jpg', 261 | 'dir1/dir2/file2-1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef.jpg.gz' 262 | ] 263 | local_files += [ 264 | 'file3.png', 265 | 'file3.zabcde.png', 266 | 'file3.zab1cde2.png', 267 | 'file3.abcdef.jpg', 268 | 'file3.abc1def2.jpg', 269 | 'dir3/file3.abc123.jpg', 270 | 'dir3/file3.abcdf123.jpg' 271 | ] 272 | remote_files = [] 273 | @config.cache_asset_regexps = [/\.[a-f0-9]{6}$/i, /\.[a-f0-9]{8}$/i] 274 | storage = AssetSync::Storage.new(@config) 275 | allow(storage).to receive(:get_local_files).and_return(local_files) 276 | allow(storage).to receive(:get_remote_files).and_return(remote_files) 277 | allow(File).to receive(:file?).and_return(true) 278 | allow(File).to receive(:open).and_return(file_like_object) 279 | 280 | def check_file(file) 281 | case file[:key] 282 | when 'file1.jpg', 283 | 'dir1/dir2/file2.jpg', 284 | 'file3.png', 285 | 'file3.zabcde.png', 286 | 'file3.zab1cde2.png' 287 | !expect(file).not_to include(:cache_control, :expires) 288 | when 'file1-1234567890abcdef1234567890abcdef.jpg', 289 | 'file1-1234567890abcdef1234567890abcdef.jpg.gz', 290 | 'file1-1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef.jpg', 291 | 'file1-1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef.jpg.gz', 292 | 'dir1/dir2/file2-1234567890abcdef1234567890abcdef.jpg', 293 | 'dir1/dir2/file2-1234567890abcdef1234567890abcdef.jpg.gz', 294 | 'dir1/dir2/file2-1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef.jpg', 295 | 'dir1/dir2/file2-1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef.jpg.gz', 296 | 'file3.abcdef.jpg', 297 | 'file3.abc1def2.jpg', 298 | 'dir3/file3.abc123.jpg', 299 | 'dir3/file3.abcdf123.jpg' 300 | expect(file).to include(:cache_control, :expires) 301 | else 302 | fail 303 | end 304 | end 305 | 306 | files = double() 307 | local_files.count.times do 308 | expect(files).to receive(:create) { |file| check_file(file) } 309 | end 310 | allow(storage).to receive_message_chain(:bucket, :files).and_return(files) 311 | storage.upload_files 312 | end 313 | 314 | it "should invalidate files" do 315 | @config.cdn_distribution_id = "1234" 316 | @config.invalidate = ['local_image1.jpg'] 317 | @config.fog_provider = 'AWS' 318 | 319 | storage = AssetSync::Storage.new(@config) 320 | allow(storage).to receive(:get_local_files).and_return(@local_files) 321 | allow(storage).to receive(:get_remote_files).and_return(@remote_files) 322 | allow(storage).to receive(:upload_file).and_return(true) 323 | 324 | mock_cdn = double 325 | expect(Fog::CDN).to receive(:new).and_return(mock_cdn) 326 | expect(mock_cdn).to receive(:post_invalidation).with("1234", ["/assets/local_image1.jpg"]).and_return(double({:body => {:id => '1234'}})) 327 | 328 | storage.upload_files 329 | end 330 | end 331 | 332 | describe '#upload_file' do 333 | before(:each) do 334 | # Object#remove_const does not remove the loaded 335 | # file from the $" variable 336 | # 337 | # So we need do both 338 | # 339 | # 1. Remove constant(s) to avoid warning messages 340 | # 2. Remove loaded file(s) 341 | Object.send(:remove_const, :MIME) if defined?(MIME) 342 | 343 | $".grep(/mime\//).each do |file_path| 344 | $".delete(file_path) 345 | end 346 | require 'mime/types' 347 | 348 | @config = AssetSync::Config.new 349 | end 350 | 351 | it 'accepts custom headers per file' do 352 | @config.custom_headers = { 353 | "local_image2.jpg" => { 354 | :cache_control => 'max-age=0' 355 | } 356 | } 357 | storage = AssetSync::Storage.new(@config) 358 | allow(storage).to receive(:get_local_files).and_return(@local_files) 359 | allow(storage).to receive(:get_remote_files).and_return(@remote_files) 360 | # Pretend they all exist 361 | allow(File).to receive(:open).and_return(file_like_object) 362 | 363 | bucket = double 364 | files = double 365 | 366 | allow(storage).to receive(:bucket).and_return(bucket) 367 | allow(bucket).to receive(:files).and_return(files) 368 | 369 | expect(files).to receive(:create) do |argument| 370 | expect(argument[:cache_control]).to eq('max-age=0') 371 | end 372 | storage.upload_file('assets/local_image2.jpg') 373 | end 374 | 375 | it 'accepts custom headers with a regular expression' do 376 | @config.custom_headers = { 377 | ".*\.jpg" => { 378 | :cache_control => 'max-age=0' 379 | } 380 | } 381 | storage = AssetSync::Storage.new(@config) 382 | allow(storage).to receive(:get_local_files).and_return(@local_files) 383 | allow(storage).to receive(:get_remote_files).and_return(@remote_files) 384 | # Pretend they all exist 385 | allow(File).to receive(:open).and_return(file_like_object) 386 | bucket = double 387 | files = double 388 | allow(storage).to receive(:bucket).and_return(bucket) 389 | allow(bucket).to receive(:files).and_return(files) 390 | 391 | expect(files).to receive(:create) do |argument| 392 | expect(argument[:cache_control]).to eq('max-age=0') 393 | end 394 | storage.upload_file('assets/some_longer_path/local_image2.jpg') 395 | end 396 | end 397 | 398 | describe '#delete_extra_remote_files' do 399 | it 'should delete the files in bulk' do 400 | remote_files = ['public/image.png'] 401 | connection = double 402 | config = double 403 | 404 | storage = AssetSync::Storage.new(@config) 405 | 406 | [:local_files, :ignored_files, :always_upload_files].each do |method| 407 | expect(storage).to receive(method).and_return([]) 408 | end 409 | 410 | allow(storage).to receive(:get_remote_files).and_return(remote_files) 411 | allow(storage).to receive(:connection).and_return(connection).twice 412 | allow(storage).to receive(:config).and_return(config).twice 413 | allow(config).to receive(:aws?).and_return(true) 414 | allow(config).to receive(:fog_directory).and_return('foo') 415 | expect(connection).to receive(:delete_multiple_objects).with('foo', remote_files) 416 | 417 | storage.delete_extra_remote_files 418 | end 419 | 420 | context 'when not aws' do 421 | it 'deletes files sequentially' do 422 | remote_files = ['public/image.png'] 423 | connection = double 424 | config = double 425 | directories = double 426 | directory = double 427 | file = double 428 | 429 | storage = AssetSync::Storage.new(@config) 430 | 431 | [:local_files, :ignored_files, :always_upload_files].each do |method| 432 | expect(storage).to receive(method).and_return([]) 433 | end 434 | 435 | allow(storage).to receive(:get_remote_files).and_return(remote_files) 436 | allow(storage).to receive(:connection).and_return(connection).twice 437 | allow(storage).to receive(:config).and_return(config) 438 | allow(config).to receive(:aws?).and_return(false) 439 | allow(config).to receive(:fog_directory).and_return('foo') 440 | allow(config).to receive(:assets_prefix).and_return('foo') 441 | allow(directories).to receive(:get).and_return(directory) 442 | allow(directory).to receive(:files).and_return([file]) 443 | allow(file).to receive(:key).and_return('public/image.png') 444 | allow(connection).to receive(:directories).and_return(directories) 445 | allow(config).to receive(:backblaze?).and_return(false) 446 | expect(connection).not_to receive(:delete_multiple_objects) 447 | expect(file).to receive(:destroy) 448 | 449 | storage.delete_extra_remote_files 450 | end 451 | end 452 | end 453 | end 454 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | 2 | [![Gem Version](https://img.shields.io/gem/v/asset_sync.svg?style=flat-square)](http://badge.fury.io/rb/asset_sync) 3 | [![Build Status](https://img.shields.io/travis/AssetSync/asset_sync.svg?style=flat-square)](http://travis-ci.org/AssetSync/asset_sync) 4 | [![Coverage Status](http://img.shields.io/coveralls/AssetSync/asset_sync.svg?style=flat-square)](https://coveralls.io/r/AssetSync/asset_sync) 5 | 6 | 7 | # Asset Sync 8 | 9 | Synchronises Assets between Rails and S3. 10 | 11 | Asset Sync is built to run with the new Rails Asset Pipeline feature introduced in **Rails 3.1**. After you run __bundle exec rake assets:precompile__ your assets will be synchronised to your S3 12 | bucket, optionally deleting unused files and only uploading the files it needs to. 13 | 14 | This was initially built and is intended to work on [Heroku](http://heroku.com) but can work on any platform. 15 | 16 | ## Upgrading? 17 | 18 | Upgraded from 1.x? Read `UPGRADING.md` 19 | 20 | ## Installation 21 | 22 | Since 2.x, Asset Sync depends on gem `fog-core` instead of `fog`. 23 | This is due to `fog` is including many unused storage provider gems as its dependencies. 24 | 25 | Asset Sync has no idea about what provider will be used, 26 | so you are responsible for bundling the right gem for the provider to be used. 27 | 28 | In your Gemfile: 29 | ```ruby 30 | gem "asset_sync" 31 | gem "fog-aws" 32 | ``` 33 | 34 | Or, to use Azure Blob storage, configure as this. 35 | 36 | ``` ruby 37 | gem "asset_sync" 38 | gem "gitlab-fog-azure-rm" 39 | 40 | # This gem seems unmaintianed 41 | # gem "fog-azure-rm" 42 | ``` 43 | 44 | To use Backblaze B2, insert these. 45 | 46 | ``` ruby 47 | gem "asset_sync" 48 | gem "fog-backblaze" 49 | ``` 50 | 51 | 52 | ### Extended Installation (Faster sync with turbosprockets) 53 | 54 | It's possible to improve **asset:precompile** time if you are using Rails 3.2.x 55 | the main source of which being compilation of **non-digest** assets. 56 | 57 | [turbo-sprockets-rails3](https://github.com/ndbroadbent/turbo-sprockets-rails3) 58 | solves this by only compiling **digest** assets. Thus cutting compile time in half. 59 | 60 | > NOTE: It will be **deprecated in Rails 4** as sprockets-rails has been extracted 61 | out of Rails and will only compile **digest** assets by default. 62 | 63 | ## Configuration 64 | 65 | ### Rails 66 | 67 | Configure __config/environments/production.rb__ to use Amazon 68 | S3 as the asset host and ensure precompiling is enabled. 69 | 70 | 71 | ``` ruby 72 | #config/environments/production.rb 73 | config.action_controller.asset_host = "//#{ENV['FOG_DIRECTORY']}.s3.amazonaws.com" 74 | ``` 75 | 76 | Or, to use Google Storage Cloud, configure as this. 77 | 78 | ``` ruby 79 | #config/environments/production.rb 80 | config.action_controller.asset_host = "//#{ENV['FOG_DIRECTORY']}.storage.googleapis.com" 81 | ``` 82 | 83 | Or, to use Azure Blob storage, configure as this. 84 | 85 | ``` ruby 86 | #config/environments/production.rb 87 | config.action_controller.asset_host = "//#{ENV['AZURE_STORAGE_ACCOUNT_NAME']}.blob.core.windows.net/#{ENV['FOG_DIRECTORY']}" 88 | ``` 89 | 90 | Or, to use Backblaze B2, configure as this. 91 | 92 | ``` ruby 93 | #config/environments/production.rb 94 | config.action_controller.asset_host = "//f000.backblazeb2.com/file/#{ENV['FOG_DIRECTORY']}" 95 | ``` 96 | 97 | 98 | On **HTTPS**: the exclusion of any protocol in the asset host declaration above will allow browsers to choose the transport mechanism on the fly. So if your application is available under both HTTP and HTTPS the assets will be served to match. 99 | 100 | > The only caveat with this is that your S3 bucket name **must not contain any periods** so, mydomain.com.s3.amazonaws.com for example would not work under HTTPS as SSL certificates from Amazon would interpret our bucket name as **not** a subdomain of s3.amazonaws.com, but a multi level subdomain. To avoid this don't use a period in your subdomain or switch to the other style of S3 URL. 101 | 102 | ``` ruby 103 | config.action_controller.asset_host = "//s3.amazonaws.com/#{ENV['FOG_DIRECTORY']}" 104 | ``` 105 | 106 | Or, to use Google Storage Cloud, configure as this. 107 | 108 | ``` ruby 109 | config.action_controller.asset_host = "//storage.googleapis.com/#{ENV['FOG_DIRECTORY']}" 110 | ``` 111 | 112 | Or, to use Azure Blob storage, configure as this. 113 | 114 | ``` ruby 115 | #config/environments/production.rb 116 | config.action_controller.asset_host = "//#{ENV['AZURE_STORAGE_ACCOUNT_NAME']}.blob.core.windows.net/#{ENV['FOG_DIRECTORY']}" 117 | ``` 118 | 119 | On **non default S3 bucket region**: If your bucket is set to a region that is not the default US Standard (us-east-1) you must use the first style of url ``//#{ENV['FOG_DIRECTORY']}.s3.amazonaws.com`` or amazon will return a 301 permanently moved when assets are requested. Note the caveat above about bucket names and periods. 120 | 121 | If you wish to have your assets sync to a sub-folder of your bucket instead of into the root add the following to your ``production.rb`` file 122 | 123 | ``` ruby 124 | # store assets in a 'folder' instead of bucket root 125 | config.assets.prefix = "/production/assets" 126 | ``` 127 | 128 | Also, ensure the following are defined (in production.rb or application.rb) 129 | 130 | * **config.assets.digest** is set to **true**. 131 | * **config.assets.enabled** is set to **true**. 132 | 133 | Additionally, if you depend on any configuration that is setup in your `initializers` you will need to ensure that 134 | 135 | * **config.assets.initialize\_on\_precompile** is set to **true** 136 | 137 | ### AssetSync 138 | 139 | **AssetSync** supports the following methods of configuration. 140 | 141 | * [Built-in Initializer](https://github.com/AssetSync/asset_sync/blob/master/lib/asset_sync/engine.rb) (configured through environment variables) 142 | * Rails Initializer 143 | * A YAML config file 144 | 145 | 146 | Using the **Built-in Initializer** is the default method and is supposed to be used with **environment** variables. It's the recommended approach for deployments on Heroku. 147 | 148 | If you need more control over configuration you will want to use a **custom rails initializer**. 149 | 150 | Configuration using a **YAML** file (a common strategy for Capistrano deployments) is also supported. 151 | 152 | The recommend way to configure **asset_sync** is by using **environment variables** however it's up to you, it will work fine if you hard code them too. The main reason why using environment variables is recommended is so your access keys are not checked into version control. 153 | 154 | 155 | ### Built-in Initializer (Environment Variables) 156 | 157 | The Built-in Initializer will configure **AssetSync** based on the contents of your environment variables. 158 | 159 | Add your configuration details to **heroku** 160 | 161 | ``` bash 162 | heroku config:add AWS_ACCESS_KEY_ID=xxxx 163 | heroku config:add AWS_SECRET_ACCESS_KEY=xxxx 164 | heroku config:add FOG_DIRECTORY=xxxx 165 | heroku config:add FOG_PROVIDER=AWS 166 | # and optionally: 167 | heroku config:add FOG_REGION=eu-west-1 168 | heroku config:add ASSET_SYNC_GZIP_COMPRESSION=true 169 | heroku config:add ASSET_SYNC_MANIFEST=true 170 | heroku config:add ASSET_SYNC_EXISTING_REMOTE_FILES=keep 171 | ``` 172 | 173 | Or add to a traditional unix system 174 | 175 | ``` bash 176 | export AWS_ACCESS_KEY_ID=xxxx 177 | export AWS_SECRET_ACCESS_KEY=xxxx 178 | export FOG_DIRECTORY=xxxx 179 | ``` 180 | 181 | Rackspace configuration is also supported 182 | 183 | ``` bash 184 | heroku config:add RACKSPACE_USERNAME=xxxx 185 | heroku config:add RACKSPACE_API_KEY=xxxx 186 | heroku config:add FOG_DIRECTORY=xxxx 187 | heroku config:add FOG_PROVIDER=Rackspace 188 | ``` 189 | 190 | Google Storage Cloud configuration is supported as well. The preferred option is using the [GCS JSON API](https://github.com/fog/fog-google#storage) which requires that you create an appropriate service account, generate the signatures and make them accessible to asset sync at the prescribed location 191 | 192 | ```bash 193 | heroku config:add FOG_PROVIDER=Google 194 | heroku config:add GOOGLE_PROJECT=xxxx 195 | heroku config:add GOOGLE_JSON_KEY_LOCATION=xxxx 196 | heroku config:add FOG_DIRECTORY=xxxx 197 | ``` 198 | 199 | If using the S3 API the following config is required 200 | 201 | ``` bash 202 | heroku config:add FOG_PROVIDER=Google 203 | heroku config:add GOOGLE_STORAGE_ACCESS_KEY_ID=xxxx 204 | heroku config:add GOOGLE_STORAGE_SECRET_ACCESS_KEY=xxxx 205 | heroku config:add FOG_DIRECTORY=xxxx 206 | ``` 207 | 208 | The Built-in Initializer also sets the AssetSync default for **existing_remote_files** to **keep**. 209 | 210 | ### Custom Rails Initializer (config/initializers/asset_sync.rb) 211 | 212 | If you want to enable some of the advanced configuration options you will want to create your own initializer. 213 | 214 | Run the included Rake task to generate a starting point. 215 | 216 | rails g asset_sync:install --provider=Rackspace 217 | rails g asset_sync:install --provider=AWS 218 | rails g asset_sync:install --provider=AzureRM 219 | rails g asset_sync:install --provider=Backblaze 220 | 221 | The generator will create a Rails initializer at `config/initializers/asset_sync.rb`. 222 | 223 | ``` ruby 224 | AssetSync.configure do |config| 225 | config.fog_provider = 'AWS' 226 | config.fog_directory = ENV['FOG_DIRECTORY'] 227 | config.aws_access_key_id = ENV['AWS_ACCESS_KEY_ID'] 228 | config.aws_secret_access_key = ENV['AWS_SECRET_ACCESS_KEY'] 229 | config.aws_session_token = ENV['AWS_SESSION_TOKEN'] if ENV.key?('AWS_SESSION_TOKEN') 230 | 231 | # Don't delete files from the store 232 | # config.existing_remote_files = 'keep' 233 | # 234 | # Increase upload performance by configuring your region 235 | # config.fog_region = 'eu-west-1' 236 | # 237 | # Set `public` option when uploading file depending on value, 238 | # Setting to "default" makes asset sync skip setting the option 239 | # Possible values: true, false, "default" (default: true) 240 | # config.fog_public = true 241 | # 242 | # Change AWS signature version. Default is 4 243 | # config.aws_signature_version = 4 244 | # 245 | # Change canned ACL of uploaded object. Default is unset. Will override fog_public if set. 246 | # Choose from: private | public-read | public-read-write | aws-exec-read | 247 | # authenticated-read | bucket-owner-read | bucket-owner-full-control 248 | # config.aws_acl = nil 249 | # 250 | # Change host option in fog (only if you need to) 251 | # config.fog_host = 's3.amazonaws.com' 252 | # 253 | # Change port option in fog (only if you need to) 254 | # config.fog_port = "9000" 255 | # 256 | # Use http instead of https. 257 | # config.fog_scheme = 'http' 258 | # 259 | # Automatically replace files with their equivalent gzip compressed version 260 | # config.gzip_compression = true 261 | # 262 | # Use the Rails generated 'manifest.yml' file to produce the list of files to 263 | # upload instead of searching the assets directory. 264 | # config.manifest = true 265 | # 266 | # Upload the manifest file also. 267 | # config.include_manifest = false 268 | # 269 | # Upload files concurrently 270 | # config.concurrent_uploads = false 271 | # 272 | # Number of threads when concurrent_uploads is enabled 273 | # config.concurrent_uploads_max_threads = 10 274 | # 275 | # Path to cache file to skip scanning remote 276 | # config.remote_file_list_cache_file_path = './.asset_sync_remote_file_list_cache.json' 277 | # 278 | # Fail silently. Useful for environments such as Heroku 279 | # config.fail_silently = true 280 | # 281 | # Log silently. Default is `true`. But you can set it to false if more logging message are preferred. 282 | # Logging messages are sent to `STDOUT` when `log_silently` is falsy 283 | # config.log_silently = true 284 | # 285 | # Allow custom assets to be cacheable. Note: The base filename will be matched 286 | # If you have an asset with name `app.0b1a4cd3.js`, only `app.0b1a4cd3` will need to be matched 287 | # only one of `cache_asset_regexp` or `cache_asset_regexps` is allowed. 288 | # config.cache_asset_regexp = /\.[a-f0-9]{8}$/i 289 | # config.cache_asset_regexps = [ /\.[a-f0-9]{8}$/i, /\.[a-f0-9]{20}$/i ] 290 | end 291 | ``` 292 | 293 | ### YAML (config/asset_sync.yml) 294 | 295 | Run the included Rake task to generate a starting point. 296 | 297 | rails g asset_sync:install --use-yml --provider=Rackspace 298 | rails g asset_sync:install --use-yml --provider=AWS 299 | rails g asset_sync:install --use-yml --provider=AzureRM 300 | rails g asset_sync:install --use-yml --provider=Backblaze 301 | 302 | The generator will create a YAML file at `config/asset_sync.yml`. 303 | 304 | ``` yaml 305 | defaults: &defaults 306 | fog_provider: "AWS" 307 | fog_directory: "rails-app-assets" 308 | aws_access_key_id: "<%= ENV['AWS_ACCESS_KEY_ID'] %>" 309 | aws_secret_access_key: "<%= ENV['AWS_SECRET_ACCESS_KEY'] %>" 310 | 311 | # To use AWS reduced redundancy storage. 312 | # aws_reduced_redundancy: true 313 | # 314 | # You may need to specify what region your storage bucket is in 315 | # fog_region: "eu-west-1" 316 | # 317 | # Change AWS signature version. Default is 4 318 | # aws_signature_version: 4 319 | # 320 | # Change canned ACL of uploaded object. Default is unset. Will override fog_public if set. 321 | # Choose from: private | public-read | public-read-write | aws-exec-read | 322 | # authenticated-read | bucket-owner-read | bucket-owner-full-control 323 | # aws_acl: null 324 | # 325 | # Change host option in fog (only if you need to) 326 | # fog_host: "s3.amazonaws.com" 327 | # 328 | # Use http instead of https. Default should be "https" (at least for fog-aws) 329 | # fog_scheme: "http" 330 | 331 | existing_remote_files: keep # Existing pre-compiled assets on S3 will be kept 332 | # To delete existing remote files. 333 | # existing_remote_files: delete 334 | # To ignore existing remote files and overwrite. 335 | # existing_remote_files: ignore 336 | # Automatically replace files with their equivalent gzip compressed version 337 | # gzip_compression: true 338 | # Fail silently. Useful for environments such as Heroku 339 | # fail_silently: true 340 | # Always upload. Useful if you want to overwrite specific remote assets regardless of their existence 341 | # eg: Static files in public often reference non-fingerprinted application.css 342 | # note: You will still need to expire them from the CDN's edge cache locations 343 | # always_upload: ['application.js', 'application.css', !ruby/regexp '/application-/\d{32}\.css/'] 344 | # Ignored files. Useful if there are some files that are created dynamically on the server and you don't want to upload on deploy. 345 | # ignored_files: ['ignore_me.js', !ruby/regexp '/ignore_some/\d{32}\.css/'] 346 | # Allow custom assets to be cacheable. Note: The base filename will be matched 347 | # If you have an asset with name "app.0b1a4cd3.js", only "app.0b1a4cd3" will need to be matched 348 | # cache_asset_regexps: ['cache_me.js', !ruby/regexp '/cache_some\.\d{8}\.css/'] 349 | 350 | development: 351 | <<: *defaults 352 | 353 | test: 354 | <<: *defaults 355 | 356 | production: 357 | <<: *defaults 358 | ``` 359 | 360 | ### Available Configuration Options 361 | 362 | Most AssetSync configuration can be modified directly using environment variables with the **Built-in initializer**. e.g. 363 | 364 | ```ruby 365 | AssetSync.config.fog_provider == ENV['FOG_PROVIDER'] 366 | ``` 367 | 368 | Simply **upcase** the ruby attribute names to get the equivalent environment variable to set. The only exception to that rule are the internal **AssetSync** config variables, they must be prepended with `ASSET_SYNC_*` e.g. 369 | 370 | ```ruby 371 | AssetSync.config.gzip_compression == ENV['ASSET_SYNC_GZIP_COMPRESSION'] 372 | ``` 373 | 374 | #### AssetSync (optional) 375 | 376 | * **existing_remote_files**: (`'keep', 'delete', 'ignore'`) what to do with previously precompiled files. **default:** `'keep'` 377 | * **gzip\_compression**: (`true, false`) when enabled, will automatically replace files that have a gzip compressed equivalent with the compressed version. **default:** `'false'` 378 | * **manifest**: (`true, false`) when enabled, will use the `manifest.yml` generated by Rails to get the list of local files to upload. **experimental**. **default:** `'false'` 379 | * **include_manifest**: (`true, false`) when enabled, will upload the `manifest.yml` generated by Rails. **default:** `'false'` 380 | * **concurrent_uploads**: (`true, false`) when enabled, will upload the files in different Threads, this greatly improves the upload speed. **default:** `'false'` 381 | * **concurrent_uploads_max_threads**: when concurrent_uploads is enabled, this determines the number of threads that will be created. **default:** `10` 382 | * **remote_file_list_cache_file_path**: if present, use this path to cache remote file list to skip scanning remote **default:** `nil` 383 | * **enabled**: (`true, false`) when false, will disable asset sync. **default:** `'true'` (enabled) 384 | * **ignored\_files**: an array of files to ignore e.g. `['ignore_me.js', %r(ignore_some/\d{32}\.css)]` Useful if there are some files that are created dynamically on the server and you don't want to upload on deploy **default**: `[]` 385 | * **cache\_asset\_regexps**: an array of files to add cache headers e.g. `['cache_me.js', %r(cache_some\.\d{8}\.css)]` Useful if there are some files that are added to sprockets assets list and need to be set as 'Cacheable' on uploaded server. Only rails compiled regexp is matched internally **default**: `[]` 386 | 387 | ##### Config Method `add_local_file_paths` 388 | Adding local files by providing a block: 389 | ```ruby 390 | AssetSync.configure do |config| 391 | # The block should return an array of file paths 392 | config.add_local_file_paths do 393 | # Any code that returns paths of local asset files to be uploaded 394 | # Like Webpacker 395 | public_root = Rails.root.join("public") 396 | Dir.chdir(public_root) do 397 | packs_dir = Webpacker.config.public_output_path.relative_path_from(public_root) 398 | Dir[File.join(packs_dir, '/**/**')] 399 | end 400 | end 401 | end 402 | ``` 403 | The blocks are run when local files are being scanned and uploaded 404 | 405 | ##### Config Method `file_ext_to_mime_type_overrides` 406 | It's reported that `mime-types` 3.x returns `application/ecmascript` instead of `application/javascript` 407 | Such change of mime type might cause some CDN to disable asset compression 408 | So this gem has defined a default override for file ext `js` to be mapped to `application/javascript` by default 409 | 410 | To customize the overrides: 411 | ```ruby 412 | AssetSync.configure do |config| 413 | # Clear the default overrides 414 | config.file_ext_to_mime_type_overrides.clear 415 | 416 | # Add/Edit overrides 417 | # Will call `#to_s` for inputs 418 | config.file_ext_to_mime_type_overrides.add(:js, :"application/x-javascript") 419 | end 420 | ``` 421 | The blocks are run when local files are being scanned and uploaded 422 | 423 | #### Fog (Required) 424 | * **fog\_provider**: your storage provider *AWS* (S3) or *Rackspace* (Cloud Files) or *Google* (Google Storage) or *AzureRM* (Azure Blob) or *Backblaze* (Backblaze B2) 425 | * **fog\_directory**: your bucket name 426 | 427 | #### Fog (Optional) 428 | 429 | * **fog\_region**: the region your storage bucket is in e.g. *eu-west-1* (AWS), *ord* (Rackspace), *japanwest* (Azure Blob) 430 | * **fog\_path\_style**: To use buckets with dot in names, check https://github.com/fog/fog/issues/2381#issuecomment-28088524 431 | 432 | #### AWS 433 | 434 | * **aws\_access\_key\_id**: your Amazon S3 access key 435 | * **aws\_secret\_access\_key**: your Amazon S3 access secret 436 | * **aws\_acl**: set [canned ACL](https://docs.aws.amazon.com/AmazonS3/latest/userguide/acl-overview.html#canned-acl) of uploaded object, will override fog_public if set 437 | 438 | #### Rackspace 439 | 440 | * **rackspace\_username**: your Rackspace username 441 | * **rackspace\_api\_key**: your Rackspace API Key. 442 | 443 | #### Google Storage 444 | 445 | When using the JSON API 446 | 447 | - **google\_project**: your Google Cloud Project name where the Google Cloud Storage bucket resides 448 | - **google\_json\_key\_location**: path to the location of the service account key. The service account key must be a JSON type key 449 | 450 | When using the S3 API 451 | 452 | * **google\_storage\_access\_key\_id**: your Google Storage access key 453 | * **google\_storage\_secret\_access\_key**: your Google Storage access secret 454 | 455 | #### Azure Blob 456 | * **azure\_storage\_account\_name**: your Azure Blob access key 457 | * **azure\_storage\_access\_key**: your Azure Blob access secret 458 | 459 | #### Backblaze B2 460 | * **b2\_key\_id**: Your Backblaze B2 key ID 461 | * **b2\_key\_token**: Your Backblaze B2 key token 462 | * **b2\_bucket\_id**: Your Backblaze B2 bucket ID 463 | 464 | #### Rackspace (Optional) 465 | 466 | * **rackspace\_auth\_url**: Rackspace auth URL, for Rackspace London use: `https://lon.identity.api.rackspacecloud.com/v2.0` 467 | 468 | ## Amazon S3 Multiple Region Support 469 | 470 | If you are using anything other than the US buckets with S3 then you'll want to set the **region**. For example with an EU bucket you could set the following environment variable. 471 | 472 | ``` bash 473 | heroku config:add FOG_REGION=eu-west-1 474 | ``` 475 | 476 | Or via a custom initializer 477 | 478 | ``` ruby 479 | AssetSync.configure do |config| 480 | # ... 481 | config.fog_region = 'eu-west-1' 482 | end 483 | ``` 484 | 485 | Or via YAML 486 | 487 | ``` yaml 488 | production: 489 | # ... 490 | fog_region: 'eu-west-1' 491 | ``` 492 | 493 | ### Amazon (AWS) IAM Users 494 | 495 | Amazon has switched to the more secure IAM User security policy model. When generating a user & policy for asset_sync you **must** ensure the policy has the following permissions, or you'll see the error: 496 | 497 | ``` 498 | Expected(200) <=> Actual(403 Forbidden) 499 | ``` 500 | 501 | IAM User Policy Example with minimum require permissions (replace `bucket_name` with your bucket): 502 | 503 | ``` json 504 | { 505 | "Statement": [ 506 | { 507 | "Action": "s3:ListBucket", 508 | "Effect": "Allow", 509 | "Resource": "arn:aws:s3:::bucket_name" 510 | }, 511 | { 512 | "Action": "s3:PutObject*", 513 | "Effect": "Allow", 514 | "Resource": "arn:aws:s3:::bucket_name/*" 515 | } 516 | ] 517 | } 518 | ``` 519 | 520 | If you want to use IAM roles you must set ```config.aws_iam_roles = true``` in your initializers. 521 | 522 | ``` 523 | AssetSync.configure do |config| 524 | # ... 525 | config.aws_iam_roles = true 526 | end 527 | ``` 528 | 529 | 530 | ## Automatic gzip compression 531 | 532 | With the `gzip_compression` option enabled, when uploading your assets. If a file has a gzip compressed equivalent we will replace that asset with the compressed version and sets the correct headers for S3 to serve it. For example, if you have a file **master.css** and it was compressed to **master.css.gz** we will upload the **.gz** file to S3 in place of the uncompressed file. 533 | 534 | If the compressed file is actually larger than the uncompressed file we will ignore this rule and upload the standard uncompressed version. 535 | 536 | ## Fail Silently 537 | 538 | With the `fail_silently` option enabled, when running `rake assets:precompile` AssetSync will never throw an error due to missing configuration variables. 539 | 540 | With the new **user_env_compile** feature of Heroku (see above), this is no longer required or recommended. Yet was added for the following reasons: 541 | 542 | > With Rails 3.1 on the Heroku cedar stack, the deployment process automatically runs `rake assets:precompile`. If you are using **ENV** variable style configuration. Due to the methods with which Heroku compile slugs, there will be an error raised by asset\_sync as the environment is not available. This causes heroku to install the `rails31_enable_runtime_asset_compilation` plugin which is not necessary when using **asset_sync** and also massively slows down the first incoming requests to your app. 543 | 544 | > To prevent this part of the deploy from failing (asset_sync raising a config error), but carry on as normal set `fail_silently` to true in your configuration and ensure to run `heroku run rake assets:precompile` after deploy. 545 | 546 | ## Rake Task 547 | 548 | A rake task is included within the **asset_sync** gem to perform the sync: 549 | 550 | ``` ruby 551 | namespace :assets do 552 | desc "Synchronize assets to S3" 553 | task :sync => :environment do 554 | AssetSync.sync 555 | end 556 | end 557 | ``` 558 | 559 | If `AssetSync.config.run_on_precompile` is `true` (default), then assets will be uploaded to S3 automatically after the `assets:precompile` rake task is invoked: 560 | 561 | ``` ruby 562 | if Rake::Task.task_defined?("assets:precompile:nondigest") 563 | Rake::Task["assets:precompile:nondigest"].enhance do 564 | Rake::Task["assets:sync"].invoke if defined?(AssetSync) && AssetSync.config.run_on_precompile 565 | end 566 | else 567 | Rake::Task["assets:precompile"].enhance do 568 | Rake::Task["assets:sync"].invoke if defined?(AssetSync) && AssetSync.config.run_on_precompile 569 | end 570 | end 571 | ``` 572 | 573 | You can disable this behavior by setting `AssetSync.config.run_on_precompile = false`. 574 | 575 | ## Sinatra/Rack Support 576 | 577 | You can use the gem with any Rack application, but you must specify two 578 | additional options; `prefix` and `public_path`. 579 | 580 | ```ruby 581 | AssetSync.configure do |config| 582 | config.fog_provider = 'AWS' 583 | config.fog_directory = ENV['FOG_DIRECTORY'] 584 | config.aws_access_key_id = ENV['AWS_ACCESS_KEY_ID'] 585 | config.aws_secret_access_key = ENV['AWS_SECRET_ACCESS_KEY'] 586 | config.prefix = 'assets' 587 | # Can be a `Pathname` or `String` 588 | # Will be converted into an `Pathname` 589 | # If relative, will be converted into an absolute path 590 | # via `::Rails.root` or `::Dir.pwd` 591 | config.public_path = Pathname('./public') 592 | end 593 | ``` 594 | 595 | Then manually call `AssetSync.sync` at the end of your asset precompilation 596 | task. 597 | 598 | ```ruby 599 | namespace :assets do 600 | desc 'Precompile assets' 601 | task :precompile do 602 | target = Pathname('./public/assets') 603 | manifest = Sprockets::Manifest.new(sprockets, './public/assets/manifest.json') 604 | 605 | sprockets.each_logical_path do |logical_path| 606 | if (!File.extname(logical_path).in?(['.js', '.css']) || logical_path =~ /application\.(css|js)$/) && asset = sprockets.find_asset(logical_path) 607 | filename = target.join(logical_path) 608 | FileUtils.mkpath(filename.dirname) 609 | puts "Write asset: #{filename}" 610 | asset.write_to(filename) 611 | manifest.compile(logical_path) 612 | end 613 | end 614 | 615 | AssetSync.sync 616 | end 617 | end 618 | ``` 619 | 620 | ## Webpacker (> 2.0) support 621 | 622 | 1. Add webpacker files and disable `run_on_precompile`: 623 | ```ruby 624 | AssetSync.configure do |config| 625 | # Disable automatic run on precompile in order to attach to webpacker rake task 626 | config.run_on_precompile = false 627 | # The block should return an array of file paths 628 | config.add_local_file_paths do 629 | # Support webpacker assets 630 | public_root = Rails.root.join("public") 631 | Dir.chdir(public_root) do 632 | packs_dir = Webpacker.config.public_output_path.relative_path_from(public_root) 633 | Dir[File.join(packs_dir, '/**/**')] 634 | end 635 | end 636 | end 637 | ``` 638 | 639 | 2. Add a `asset_sync.rake` in your `lib/tasks` directory that enhances the correct task, otherwise asset_sync runs before `webpacker:compile` does: 640 | ``` 641 | if defined?(AssetSync) 642 | Rake::Task['webpacker:compile'].enhance do 643 | Rake::Task["assets:sync"].invoke 644 | end 645 | end 646 | ``` 647 | 648 | ### Caveat 649 | By adding local files outside the normal Rails `assets` directory, the uploading part works, however checking that the asset was previously uploaded is not working because asset_sync is only fetching the files in the `assets` directory on the remote bucket. This will mean additional time used to upload the same assets again on every precompilation. 650 | 651 | ## Running the specs 652 | 653 | Make sure you have a .env file with these details:- 654 | 655 | # for AWS provider 656 | AWS_ACCESS_KEY_ID= 657 | AWS_SECRET_ACCESS_KEY= 658 | FOG_DIRECTORY= 659 | FOG_REGION= 660 | 661 | # for AzureRM provider 662 | AZURE_STORAGE_ACCOUNT_NAME= 663 | AZURE_STORAGE_ACCESS_KEY= 664 | FOG_DIRECTORY= 665 | FOG_REGION= 666 | 667 | Make sure the bucket has read/write permissions. Then to run the tests:- 668 | 669 | foreman run rake 670 | 671 | ## Todo 672 | 673 | 1. Add some before and after filters for deleting and uploading 674 | 2. Support more cloud storage providers 675 | 3. Better test coverage 676 | 4. Add rake tasks to clean old assets from a bucket 677 | 678 | ## Credits 679 | 680 | Inspired by: 681 | 682 | - [https://github.com/moocode/asset_id](https://github.com/moocode/asset_id) 683 | - [https://gist.github.com/1053855](https://gist.github.com/1053855) 684 | 685 | ## License 686 | 687 | MIT License. Copyright 2011-2013 Rumble Labs Ltd. [rumblelabs.com](http://rumblelabs.com) 688 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Change Log 2 | All notable changes to this project will be documented in this file. 3 | This project adheres to [Semantic Versioning](http://semver.org/). 4 | 5 | 6 | ## [Unreleased] 7 | 8 | ### Added 9 | 10 | - Nothing 11 | 12 | ### Changed 13 | 14 | - Nothing 15 | 16 | ### Fixed 17 | 18 | - Nothing 19 | 20 | 21 | ## [2.15.1] - 2021-11-22 22 | 23 | ### Changed 24 | 25 | - Update dev dependency `fog-azure-rm` to become `gitlab-fog-azure-rm` 26 | 27 | ### Fixed 28 | 29 | - Fix YAML config file parsing with Psych v4 30 | (https://github.com/AssetSync/asset_sync/pull/422) 31 | 32 | 33 | ## [2.15.0] - 2021-08-05 34 | 35 | ### Added 36 | 37 | - Add support for option `aws_acl` 38 | (https://github.com/AssetSync/asset_sync/pull/420) 39 | 40 | 41 | ## [2.14.2] - 2021-05-31 42 | 43 | ### Added 44 | 45 | - Add support for setting option `google_json_key_string` in YML (not new option) 46 | (https://github.com/AssetSync/asset_sync/pull/419) 47 | 48 | 49 | ## [2.14.1] - 2021-05-14 50 | 51 | ### Added 52 | 53 | - Add support for setting option `log_silently` in YML (not new option) 54 | (https://github.com/AssetSync/asset_sync/pull/417) 55 | 56 | 57 | ## [2.14.0] - 2021-03-31 58 | 59 | ### Added 60 | 61 | - Add support for fog option `google_json_key_string` 62 | (https://github.com/AssetSync/asset_sync/pull/415) 63 | 64 | 65 | ## [2.13.1] - 2021-03-01 66 | 67 | ### Fixed 68 | 69 | - Fix "files to be uploaded list" generation for file names with dashes 70 | (https://github.com/AssetSync/asset_sync/pull/414) 71 | 72 | 73 | ## [2.13.0] - 2020-12-14 74 | 75 | ### Added 76 | 77 | - Add Backblaze B2 Cloud Storage support 78 | (https://github.com/AssetSync/asset_sync/pull/410) 79 | 80 | 81 | ## [2.12.1] - 2020-06-17 82 | 83 | ### Fixed 84 | 85 | - Fix initializer template in generator 86 | (https://github.com/AssetSync/asset_sync/pull/404) 87 | 88 | 89 | ## [2.12.0] - 2020-06-11 90 | 91 | ### Added 92 | 93 | - Add option `aws_session_token` to support AWS Temporary Security Credentials 94 | (https://github.com/AssetSync/asset_sync/pull/403) 95 | 96 | 97 | ## [2.11.0] - 2020-03-13 98 | 99 | ### Added 100 | 101 | - Add option `remote_file_list_cache_file_path` to skip scanning remote 102 | (https://github.com/AssetSync/asset_sync/pull/400) 103 | 104 | 105 | ## [2.10.0] - 2020-02-26 106 | 107 | ### Added 108 | 109 | - Add option `concurrent_uploads_max_threads` to limit number of threads for uploading files 110 | (https://github.com/AssetSync/asset_sync/pull/398) 111 | 112 | 113 | ## [2.9.1] - 2020-02-20 114 | 115 | ### Fixed 116 | 117 | - Fix uploading of sprockets manifest file 118 | (https://github.com/AssetSync/asset_sync/pull/397) 119 | 120 | 121 | ## [2.9.0] - 2020-01-15 122 | 123 | ### Added 124 | 125 | - Add option `concurrent_uploads` to improve speed of uploading 126 | (https://github.com/AssetSync/asset_sync/pull/393) 127 | 128 | 129 | ## [2.8.2] - 2019-12-27 130 | 131 | ### Changed 132 | 133 | - Use `delete_multiple_objects` when storage is `aws` 134 | (https://github.com/AssetSync/asset_sync/pull/392) 135 | 136 | 137 | ## [2.8.1] - 2019-07-25 138 | 139 | ### Changed 140 | 141 | - Removed `rubyforge_project` from gemspec 142 | (https://github.com/AssetSync/asset_sync/pull/386) 143 | 144 | ### Fixed 145 | 146 | - Fixed when `fog_public` set to `false`, file were still set to be public 147 | (https://github.com/AssetSync/asset_sync/pull/387) 148 | 149 | 150 | ## [2.8.0] - 2019-06-17 151 | 152 | ### Added 153 | 154 | - Add option `fog_port` 155 | (https://github.com/AssetSync/asset_sync/pull/385) 156 | 157 | 158 | ## [2.7.0] - 2019-03-15 159 | 160 | ### Added 161 | 162 | - Adds JSON API support when using Google Storage 163 | (https://github.com/AssetSync/asset_sync/pull/381) 164 | 165 | ### Changed 166 | 167 | - Update `AssetSync::MultiMime.lookup` to always return strings (kind of internal change) 168 | (https://github.com/AssetSync/asset_sync/pull/380) 169 | 170 | 171 | ## [2.6.0] - 2018-12-07 172 | 173 | ### Added 174 | 175 | - Add option `fog_public` 176 | (https://github.com/AssetSync/asset_sync/pull/377) 177 | 178 | 179 | ## [2.5.0] - 2018-10-25 180 | 181 | ### Added 182 | 183 | - Add ruby only option `file_ext_to_mime_type_overrides` 184 | (https://github.com/AssetSync/asset_sync/pull/374) 185 | 186 | ### Changed 187 | 188 | - Start testing against rails 5.2, stop testing against rails 4.1 189 | 190 | ### Fixed 191 | 192 | - Only enhance rake task assets:precompile if it's defined 193 | (https://github.com/AssetSync/asset_sync/commit/e1eb1a16b06fd39def1759428a2d94733915bbff) 194 | - Avoid ruby warning due to "method redefined" 195 | (https://github.com/AssetSync/asset_sync/pull/371) 196 | 197 | 198 | ## [2.4.0] - 2017-12-20 199 | 200 | ### Added 201 | 202 | - Add support for Azure Blob storage 203 | (https://github.com/AssetSync/asset_sync/pull/363) 204 | - Add option: `include_manifest` 205 | (https://github.com/AssetSync/asset_sync/pull/365) 206 | 207 | ### Changed 208 | 209 | - Add public API method `get_asset_files_from_manifest` split from `get_local_files` for another gem 210 | (https://github.com/AssetSync/asset_sync/pull/366) 211 | 212 | ### Fixed 213 | 214 | - Nothing 215 | 216 | 217 | ## [2.3.0] - 2017-12-05 218 | 219 | ### Added 220 | 221 | - Add options: `aws_signature_version`, `fog_host`, `fog_scheme` 222 | (https://github.com/AssetSync/asset_sync/pull/362) 223 | 224 | ### Changed 225 | 226 | - Change initializer template to only run when AssetSync const defined 227 | 228 | 229 | ## [2.2.0] - 2017-07-12 230 | 231 | ### Added 232 | 233 | - Add method `add_local_file_paths` to config for uploading additional files, like webpacker 234 | (https://github.com/AssetSync/asset_sync/pull/347) 235 | 236 | ### Changed 237 | 238 | - Nothing 239 | 240 | ### Fixed 241 | 242 | - Fix too many files open when uploading local files 243 | (https://github.com/AssetSync/asset_sync/pull/351) 244 | 245 | 246 | ## [2.1.0] - 2017-05-19 247 | 248 | ### Added 249 | 250 | - Allow customization of regexp of files on target bucket to be marked as 'Cacheable' 251 | so that browsers when serving the content would cache them. 252 | The value can be set by `cache_asset_regexps` 253 | 254 | ### Changed 255 | 256 | - Only support mime-type >= 2.99, 257 | which is released at the end of 2015 258 | - Only support activemodel >= 4.1, 259 | which is released in 2014 260 | 261 | 262 | ## [2.0.0] - 2016-12-21 263 | 264 | ### Changed 265 | 266 | - [BREAKING] require “fog-core” instead of “fog” as runtime requirement 267 | 268 | 269 | ## [1.3.0] - 2016-11-30 270 | 271 | ### Changed 272 | 273 | - Add regex support to always_upload (https://github.com/AssetSync/asset_sync/pull/333) 274 | - Stop failing sliently when YAML file does not contain key for rails environment (https://github.com/AssetSync/asset_sync/pull/270) 275 | - Stop failing sliently when YAML file cannot be parsed due to syntax error 276 | 277 | 278 | ## [1.2.1] - 2016-08-19 279 | 280 | ### Fixed 281 | 282 | - Respect value of option `log_silently` even when `ENV['RAILS_GROUPS'] == 'assets'` 283 | 284 | 285 | ## [1.2.0] - 2016-08-17 286 | 287 | ### Added 288 | 289 | - Support for `fog_path_style` config option (AWS only) (https://github.com/AssetSync/asset_sync/pull/302) 290 | 291 | ### Changed 292 | 293 | - Set Expires and Cache-Control headers for .gz files (https://github.com/AssetSync/asset_sync/pull/329) 294 | 295 | ### Fixed 296 | 297 | - Add missing runtime dependency declaration for `mime-types` to gemspec (https://github.com/AssetSync/asset_sync/pull/328) 298 | - Update outdated error message for unknown AssetSync provider (https://github.com/AssetSync/asset_sync/pull/298) 299 | - Allow hash digest in file name with over 32 chars (for sprockets 3+) (https://github.com/AssetSync/asset_sync/pull/315) 300 | - Fix `config.log_silently?` (https://github.com/AssetSync/asset_sync/pull/324) 301 | - Stop using deprecated Ruby API (https://github.com/AssetSync/asset_sync/pull/276) 302 | 303 | 304 | ## v1.1.0 / 2014-08-13 305 | 306 | Version 1.1.0 (Toby Osbourn ) 307 | 308 | Changes: 309 | 310 | * 1 Change 311 | 312 | * Bumping master to 1.1.0 - preparing to update RubyGems 313 | 314 | ## v0.5.6 / 2014-08-12 315 | 316 | Version 0.5.6 (Toby Osbourn ) 317 | 318 | Changes: 319 | 320 | * 1 Change 321 | 322 | * Merged in support for optimized fog loading 323 | 324 | ## v0.5.5 / 2014-08-12 325 | 326 | Version 0.5.5 (Toby Osbourn ) 327 | 328 | Changes: 329 | 330 | * 2 Nominal Changes 331 | 332 | * Merged some spec changes to get Travis to pass the build correctly 333 | * Support using AWS IAM Roles 334 | 335 | ## v0.5.1 / 2012-10-22 336 | 337 | Version 0.5.1 (David Rice ) 338 | 339 | Changes: 340 | 341 | * 5 Nominal Changes 342 | 343 | * Add a CHANGELOG.md (as generated by vclog -r -f markdown 344 | * Improve documentation on ignored_files config option 345 | * Allow failure of specs against ruby-head and jruby-head 346 | * Merge pull request #115 from demirhanaydin/patch-1 347 | * Merge support for always providing mime_type #93 from patdeegan/master 348 | 349 | 350 | 351 | ## v0.5.0 / 2012-08-23 352 | 353 | Version 0.5.0 (David Rice ) 354 | 355 | Changes: 356 | 357 | * 8 Nominal Changes 358 | 359 | * Merge branch 'sinatra' of github.com:rumblelabs/asset_sync into sinatra 360 | * Version 0.5.0, sinatra / rack support 361 | * Some refactoring to further remove dependency on Rails, add spec for railsless configuration 362 | * Update readme. 363 | * Add public_path and prefix config options so asset_sync can be used outside Rails. 364 | * Some refactoring to further remove dependency on Rails, add spec for railsless configuration 365 | * Merge branch 'ejholmes/sinatra' into sinatra 366 | * Version 0.4.3, removed dependency on Rails Time additions 367 | 368 | 369 | ## v0.4.3 / 2012-08-19 370 | 371 | Version 0.4.3 (David Rice ) 372 | 373 | Changes: 374 | 375 | * 21 Nominal Changes 376 | 377 | * Refactor cache control and expiry hearder definition to use same value of one year 378 | * Merge pull request #94 from karlfreeman/time 379 | 380 | Remove Rails time dependency 381 | * Allow failures in ruby-head 382 | * Merge pull request #88 from potomak/patch-1 383 | 384 | Fix defined? syntax 385 | * Merge pull request #95 from bbhoss/patch-1 386 | 387 | Fix syntax error in documentation 388 | * Describe using S3 HTTPS better 389 | * Fix syntax error 390 | * remove Rails time dependency 391 | * Update readme. 392 | * Add public_path and prefix config options so asset_sync can be used outside Rails. 393 | * Fix defined? syntax 394 | * Force build on travis 395 | * Get specs running under jruby and travis /cc @headius :) 396 | * Ignore ds_store 397 | * Add jruby-openssl gem to get tests passing on jruby 398 | * test all the things 399 | * Add travis config for rbx 400 | * Merge branch 'master' of github.com:rumblelabs/asset_sync 401 | * Update README for installing on heroku, labs is no-longer a plugin 402 | * Merge pull request #75 from mscottford/master 403 | 404 | Update asset_host configuration in README to not rely on request object 405 | * Version 0.4.2, allow configuration of an array of strings or regex for files to ignore uploading. #euruko 406 | 407 | 408 | ## v0.4.2 / 2012-06-02 409 | 410 | Version 0.4.2 (David Rice ) 411 | 412 | Changes: 413 | 414 | * 7 Nominal Changes 415 | 416 | * Remove errant puts from spec 417 | * Merge 418 | * Add option to configure ignored_files through YAML config file 419 | * Removes errant end in the asset_host config example. 420 | * Updates README to suggest a different asset_host configuration 421 | 422 | The previous version will fail in some cases because a request is not always available during asset compilation. 423 | * Fix Fog warnings when running specs. 424 | * Version 0.4.1, allow programatic disabling of AssetSync.config.enabled 425 | 426 | 427 | ## v0.4.1 / 2012-05-04 428 | 429 | Version 0.4.1 (David Rice ) 430 | 431 | Changes: 432 | 433 | * 10 Nominal Changes 434 | 435 | * Update docs 436 | * Don't default to disabled if ASSET_SYNC_ENABLED env var is not specified. 437 | * Add option to ignore files 438 | * Add support for ASSET_SYNC_ENABLED with env vars. 439 | * Oops, should have used the accessor 440 | * Add support for enabled in the yaml config. 441 | * Add specs for AssetSync.enabled? configured through the initializer. 442 | * Make it possible to turn off AssetSync... 443 | 444 | Useful when precompiling to export to a hybrid mobile app such as PhoneGap. 445 | Would fix issue #66. 446 | * How many times will I forget to update the release date? many 447 | * Version 0.4.0, google storage support. Allow force upload of all or selected files. Travis CI enabled 448 | 449 | 450 | ## v0.4.0 / 2012-04-26 451 | 452 | Version 0.4.0 (David Rice ) 453 | 454 | Changes: 455 | 456 | * 22 Nominal Changes 457 | 458 | * Add google storage options to built in initializer to allow config via ENV vars 459 | * Add google storage configuration to README 460 | * fix case on google provider in generator 461 | * added google storage tests 462 | * added google storage generators 463 | * add attr_accessor for google keys 464 | * add support for fog gems google storage option 465 | * Oh, travisci will build an orgs repos if you configure the webhookd. Use rumblelabs/asset_sync as build status 466 | * Merge branch 'master' into levent/feature/overwrite_specific_remote_files 467 | * Use my travis-ci build in README 468 | * Merge pull request #69 from levent/integrate_travis 469 | 470 | Integrate Travis CI 471 | * Merge branch 'integrate_travis' into feature/overwrite_specific_remote_files 472 | * Specs for uploading assets 473 | * Travis build logo 474 | * Setting up for Travis 475 | * Updated README 476 | * always_upload config option added 477 | * gitignore *.swp (for vim) 478 | * Add ability to ignore remote files 479 | * Correct name of specs 480 | * Allows specifying an array of assets to always upload to remote 481 | * Version 0.3.2, set content encoding header for .gz files 482 | 483 | 484 | ## v0.3.2 / 2012-04-18 485 | 486 | Version 0.3.2 (David Rice ) 487 | 488 | Changes: 489 | 490 | * 11 Nominal Changes 491 | 492 | * Remove trailing comma 493 | * Merge pull request #57 from nathanhumbert/master 494 | 495 | Set Content-Encoding for gzip files when config.gzip? is not true 496 | * Merge pull request #59 from kamui/master 497 | 498 | Use Rails.public_path and Pathname#join for path concat and string interpolation 499 | * Merge pull request #55 from manuelmeurer/patch-1 500 | 501 | Remove comments taken from another gem 502 | * Dir.chdir to path first to avoid a map call and path string slicing 503 | * add Rails.public_path stub and make Rails.root return Pathname class to match Rails behavior 504 | * Rails.root returns a Pathname, use Pathname#join instead of File.join and string interpolation 505 | * use Rails.public_path instead of concat Rails.root and 'public' 506 | * Set Content-Encoding for gzip files when config.gzip? is not true 507 | 508 | This allows a S3 bucket served via CloudFront to properly handle the 509 | Accept-Encoding request header. 510 | * Remove comments taken from another gem 511 | * Merge branch 'master' of github.com:rumblelabs/asset_sync 512 | 513 | 514 | ## v0.3.1 / 2012-03-07 515 | 516 | Version 0.3.1 (David Rice ) 517 | 518 | Changes: 519 | 520 | * 6 Nominal Changes 521 | 522 | * Version 0.3.1, improve logging of asset_sync configuration and sync events 523 | * Remove some debugging stuffs 524 | * Improve logging during asset_sync progress. 525 | * Separate log and warn message, should not mess up heroku precompile thread as it watches STDERR for output. 526 | * Improve logging, only log to STDERR if RAILS_GROUPS=assets. 527 | * Version 0.3.0, all configuration can be managed via env variables, improve docs on configuration 528 | 529 | 530 | ## v0.3.0 / 2012-03-07 531 | 532 | Version 0.3.0 (David Rice ) 533 | 534 | Changes: 535 | 536 | * 10 Nominal Changes 537 | 538 | * Merge pull request #50 from hampei/master 539 | 540 | made gzip_compression settable via ENV 541 | * namespaced the ENV gzip option: ASSET_SYNC_GZIP_COMPRESSION. added option to readme 542 | * made gzip_compression settable via ENV 543 | * Typo 544 | * Improve documentation 545 | * Version 0.2.12, fix the asset_sync rake task enhancement in Rails 3.2 (still supporting earlier releases) 546 | * Turns out this was an issue with Rails handling of the config.assets.digest parameter 547 | * When running rake assets:precompile this config variable is modified by Rails 548 | * So it therefore cannot be depended on to test wether to enhance the nondigest task or not 549 | * The solution is to always enhance assets:precompile:nondigest if it exists. 550 | 551 | 552 | ## v0.2.9 / 2012-01-30 553 | 554 | Version 0.2.9 (David Rice ) 555 | 556 | Changes: 557 | 558 | * 3 Nominal Changes 559 | 560 | * Merge pull request #42 from genuitytech/master 561 | 562 | Now correctly setting config.fog_region. 563 | * Now correctly setting config.fog_region. 564 | * Version 0.2.8, improve http headers. Add far future expires and cache control, public. 565 | 566 | 567 | ## v0.2.8 / 2012-01-27 568 | 569 | Version 0.2.8 (David Rice ) 570 | 571 | Changes: 572 | 573 | * 2 Nominal Changes 574 | 575 | * Add far future expires header 576 | * Version 0.2.7, Rails 3.2 compatibility, default Rake task improved 577 | 578 | 579 | ## v0.2.7 / 2012-01-25 580 | 581 | Version 0.2.7 (David Rice ) 582 | 583 | Changes: 584 | 585 | * 2 Nominal Changes 586 | 587 | * Merge branch 'rails-3-2' 588 | * Version 0.2.6, Rails 3.2 compatibility, default Rake task improved 589 | 590 | 591 | ## v0.2.6 / 2012-01-25 592 | 593 | Version 0.2.6 (David Rice ) 594 | 595 | Changes: 596 | 597 | * 3 Nominal Changes 598 | 599 | * Doc 600 | * Add Rails 3.2 compatible rake task 601 | * Fix issue #38 for when Rails.config.assets.prefix starts with a slash. 602 | 603 | 604 | ## v0.2.5 / 2012-01-10 605 | 606 | Version 0.2.5 (David Rice ) 607 | 608 | Changes: 609 | 610 | * 1 Nominal Changes 611 | 612 | * Version 0.2.4, Support for Rails.config.assets.prefix 613 | 614 | 615 | ## v0.2.4 / 2012-01-06 616 | 617 | Version 0.2.4 (David Rice ) 618 | 619 | Changes: 620 | 621 | * 5 Nominal Changes 622 | 623 | * Merge pull request #35 from siliconsalad/config_assets_prefix 624 | 625 | Rails.config.assets.prefix used for sync 626 | * added test with Rails.config.assets.prefix set 627 | * Rails.config.assets.prefix used for sync (instead of hardcoded 'assets' value) 628 | * specs now use shared context to mock Rails and fixed pending tests that were failing 629 | * Version 0.2.3, Rackspace London support 630 | 631 | 632 | ## v0.2.3 / 2011-12-06 633 | 634 | Version 0.2.3 (David Rice ) 635 | 636 | Changes: 637 | 638 | * 3 Nominal Changes 639 | 640 | * Merge pull request #28 from robink/master 641 | 642 | Rackspace London support 643 | * Only merge racksace_auth_url to fog config if defined 644 | * Bump date for release 645 | 646 | 647 | ## v0.2.2 / 2011-11-29 648 | 649 | Version 0.2.2 (David Rice ) 650 | 651 | Changes: 652 | 653 | * 10 Nominal Changes 654 | 655 | * Version 0.2.2: add fail_silently config option to avoid heroku installing the rails31_enable_runtime_asset_compilation, fixes issues #24, #29 656 | * Further explanation of fail_silently option 657 | * Merge pull request #29 from neilmiddleton/master 658 | 659 | Allow precompile to fail quietly on heroku 660 | * Update README, and generator templates 661 | * Changes as discussed in PR#29 662 | * Disable pre-compilation on Heroku. 663 | * Updated README and generators 664 | * Added support for specifying rackspace_auth_url (then the possibility to use Rackspace London) 665 | * Fixed typo in readme 666 | * Updated version and release date 667 | 668 | 669 | ## v0.2.12 / 2012-03-04 670 | 671 | Version 0.2.12 (David Rice ) 672 | 673 | Changes: 674 | 675 | * 1 Nominal Changes 676 | 677 | * Version 0.2.11, minor fix to YAML loading and improved docs 678 | 679 | 680 | ## v0.2.11 / 2012-03-04 681 | 682 | Version 0.2.11 (David Rice ) 683 | 684 | Changes: 685 | 686 | * 7 Nominal Changes 687 | 688 | * Merge pull request #48 from samsoffes/patch-1 689 | 690 | Fix Heroku Labs plugin URL and add code coloring to readme. 691 | * Fix Heroku Labs plugin URL and add code coloring to readme. 692 | * Merge pull request #47 from dbalatero/dont_read_yml_file_every_time 693 | 694 | Cache the YML config to avoid multiple file reads. 695 | * Cache the YML config to avoid multiple file reads. 696 | * Fix documentation typos 697 | * Move old known issues about heroku ENV variables to a docs folder, write new content referencing the recommended use of user_env_compile 698 | * Version 0.2.10, fix handling of non standard Rails.config.assets.manifest path 699 | 700 | 701 | ## v0.2.10 / 2012-02-16 702 | 703 | Version 0.2.10 (David Rice ) 704 | 705 | Changes: 706 | 707 | * 5 Nominal Changes 708 | 709 | * Add an AssetSync.log method for outputing sync config failure so we can stub it out easily in tests 710 | * Merge pull request #44 from dbalatero/fix_nonstandard_manifest_location 711 | 712 | Fixes asset_sync to correctly read manifest.yml files. 713 | * Fixes asset_sync to correctly read manifest.yml files. 714 | 715 | Rails.config.assets.manifest only points to the directory that contains 716 | the manifest.yml file: 717 | 718 | https://github.com/rails/rails/blob/226783d1e8891a38d4a61017952528970dba903d/actionpack/lib/sprockets/railtie.rb#L36 719 | * Add hack, seems required for some applications on push to Heroku, not for others 720 | * Version 0.2.9 fix bug in internal initializer 721 | 722 | 723 | ## v0.2.1 / 2011-11-21 724 | 725 | Version 0.2.1 (Phil ) 726 | 727 | Changes: 728 | 729 | * 4 Nominal Changes 730 | 731 | * Only configure with ENV vars if initializer and yml file do not exist 732 | * Typo in yaml, underscore need not be escaped here 733 | * Fix readme 734 | * Version 0.2.0 735 | 736 | 737 | ## v0.2.0 / 2011-11-15 738 | 739 | Version 0.2.0 (David Rice ) 740 | 741 | Changes: 742 | 743 | * 15 Nominal Changes 744 | 745 | * Add upgrade notice to README 746 | * Use fog directory 747 | * Merge 748 | * Fix readme 749 | * Tidy readme 750 | * Get AWS or Rackspace generators working correctly 751 | * Remove generated rake task, no need 752 | * Improve generators to generate AWS or Rackspace compatible initializer or yml 753 | * Prepare 0.2.0 for release 754 | * Convert readme and generators to new config options 755 | * Fix fog_options 756 | * Fix typo 757 | * Fix bug 758 | * Working on migrating the exposed config variables to reflect fog, add in a start on rackspace support. Write more specs, tidy up and document config 759 | * Add specs for manifest config 760 | 761 | 762 | ## v0.1.9 / 2011-11-06 763 | 764 | Version 0.1.9 (David Rice ) 765 | 766 | Changes: 767 | 768 | * 37 Nominal Changes 769 | 770 | * Document gzip compression 771 | * Add note about gzip_compression 772 | * Add spec to test config defaults gzip_compression to false 773 | * Add gzip compression info to generated asset_sync.rb or .yml. Fix .yml example with new config settings 774 | * Update gemspec 775 | * Update docs to note that rake task is no longer generated within the app. 776 | * Add todo 777 | * Add % symbol for clarity 778 | * Output % savings when uploading gzipped files. Only use gzipped files if the compressed version is actually smaller than the original. 779 | * Tidy readme 780 | * Get AWS or Rackspace generators working correctly 781 | * Remove generated rake task, no need 782 | * Improve generators to generate AWS or Rackspace compatible initializer or yml 783 | * Prepare 0.2.0 for release 784 | * Convert readme and generators to new config options 785 | * Fix fog_options 786 | * Fix bug 787 | * Fix typo 788 | * Working on migrating the exposed config variables to reflect fog, add in a start on rackspace support. Write more specs, tidy up and document config 789 | * Add spec for gzip? config method 790 | * Reorder logic to execute quicker if gzip? compression disabled and ignore .gz uploads correctly 791 | * Ignore .gz assets if we are in gzip_compression mode 792 | * Do not set a Vary: Accept-Encoding header, S3 does not support at all 793 | * Try setting vary header a different way 794 | * Set http header Vary: Accept-Encoding when storing gzipped assets to S3 795 | * Add todo 796 | * Refactor to computed path 797 | * Add path 798 | * Instead of overwriting the original file when processing the .gz, overwrite the original if a gz file exists to avoid any issues with whichever order files are processed in 799 | * Bump version (no release just yet) 800 | * Only handle gzip files specially if we have configured gzip_compression 801 | * Overwrite original files with gzipped equivalent, improve logging to show GZIP in action, make it a configurable option, config.gzip_compression that defaults to false 802 | * Upload GZIP compressed assets nicely to S3 with correct content type and encoding. 803 | * Refactor upload method to make enhancing nicer 804 | * Merge pull request #12 from bobbrez/master 805 | 806 | Minor correction to README for generated YAML file path. 807 | * Correcting location of generated yml in README 808 | * Comment out unnecessary logic for now 809 | 810 | 811 | ## v0.1.8 / 2011-10-17 812 | 813 | Version 0.1.8 (David Rice ) 814 | 815 | Changes: 816 | 817 | * 4 Nominal Changes 818 | 819 | * Don't log any debugging info v0.1.8 should add a debug mode in future 820 | * Fix specs, only require asset_sync engine and railtie if Rails is initialized 821 | * Improve docs 822 | * Tidy up for release of Rails 3.1.1 support. 823 | 824 | 825 | ## v0.1.7 / 2011-10-15 826 | 827 | Version 0.1.7 (David Rice ) 828 | 829 | Changes: 830 | 831 | * 6 Nominal Changes 832 | 833 | * Merge pull request #7 from hone/6_rails3.1.1 834 | 835 | Rails 3.1.1 Compatability 836 | * rails 3.1.1 support 837 | * fix typo 838 | * Update the generated yml config with a staging environment, use defaults more. Engine within asset sync doesn't appear to be ran even with :group => :assets in the definition. Add railtie to allow setting config.asset_sync configuration within a rails application.rb, this and moving the initializer style of config seems to work for Rails 3.1.1, also so does purely relying on the YAML config 839 | * New version of asset_sync to work around Rails 3.1.1 issues. Test if config/initializers/asset_sync.rb exists and load that, otherwise provide a default initializer that is configurable with environment variables. Then merge in settings if config/asset_sync.yml is found. Add the asset_sync.rake in to lib/tasks so it is autoloaded and don't bother generating it anymore 840 | * Bugfix 841 | 842 | 843 | ## v0.1.6 / 2011-09-26 844 | 845 | Version 0.1.6 (David Rice ) 846 | 847 | Changes: 848 | 849 | * 1 Nominal Changes 850 | 851 | * Fix gemfile 852 | 853 | 854 | ## v0.1.5 / 2011-09-26 855 | 856 | Version 0.1.5 (David Rice ) 857 | 858 | Changes: 859 | 860 | * 5 Nominal Changes 861 | 862 | * Should raise storage error if AWS S3 bucket is not found. Version 0.1.5 863 | * explain further 864 | * Merge branch 'master' of github.com:rumblelabs/asset_sync 865 | * List known issues with heroku and possible work arounds 866 | * Should raise error with no configuration 867 | 868 | 869 | ## v0.1.4 / 2011-08-30 870 | 871 | Version 0.1.4 (David Rice ) 872 | 873 | Changes: 874 | 875 | * 2 Nominal Changes 876 | 877 | * Require dependancy of active_model, add config validation, better specs, version 0.1.4 878 | * Tidied up read me with a DRYer use of AWS_BUCKET for asset_host. 879 | 880 | 881 | ## v0.1.3 / 2011-08-27 882 | 883 | Version 0.1.3 (Simon Hamilton ) 884 | 885 | Changes: 886 | 887 | * 1 Nominal Changes 888 | 889 | * Bump version for release 890 | 891 | 892 | ## v0.1.2 / 2011-08-25 893 | 894 | Version 0.1.2 (Simon Hamilton ) 895 | 896 | Changes: 897 | 898 | * 2 Nominal Changes 899 | 900 | * Removed public from cache control. May be causing a problem with uploads 901 | * Bump version for release 902 | 903 | 904 | ## v0.1.10 / 2011-11-15 905 | 906 | Version 0.1.10 (David Rice ) 907 | 908 | Changes: 909 | 910 | * 7 Nominal Changes 911 | 912 | * Improve manifest configuration by making it a boolean option only, it will automatically use the configured manifest path if different from the default. Add documentation to readme about the new option and upgrade generated configs. 913 | * Merge pull request #20 from agworld/e26f5ca36dee1c2196653268ed6bb38c0226e4d2 914 | 915 | Fixes issues #16, #17, #18 and #19 916 | * fixes https://github.com/rumblelabs/asset_sync/issues/19 917 | * Implements https://github.com/rumblelabs/asset_sync/issues/17 918 | * fixes https://github.com/rumblelabs/asset_sync/issues/18 919 | * fixes https://github.com/rumblelabs/asset_sync/issues/16 920 | * Merge branch 'gzip-compression' 921 | 922 | 923 | ## v0.1.1 / 2011-08-24 924 | 925 | Version 0.1.1 (Simon Hamilton ) 926 | 927 | Changes: 928 | 929 | * 5 Nominal Changes 930 | 931 | * Merge pull request #4 from jsmestad/patch-1 932 | 933 | [BUGFIX] Add support for 'existing_remote_files' configuration in YAML fi 934 | * Verbose output about the delete process. 935 | * Condense logic on keep 936 | * [BUGFIX] Add support for 'existing_remote_files' configuration in YAML file. 937 | * Version 0.1.0 ready 938 | 939 | 940 | ## v0.1.0 / 2011-08-22 941 | 942 | Version 0.1.0 (David Rice ) 943 | 944 | Changes: 945 | 946 | * 1 Nominal Changes 947 | 948 | * Merge 0.0.7 from master into new refactor branch 949 | 950 | 951 | ## v0.0.7 / 2011-08-22 952 | 953 | Version 0.0.7 (David Rice ) 954 | 955 | Changes: 956 | 957 | * 9 Nominal Changes 958 | 959 | * Added Cache-control header (1 year, public) on uploaded files 960 | * Update README to reflect new configuration styles 961 | * Extract all file manipulation methods to a storage class, update generator templates, fix a few bugs. 962 | * Config class working, specs added, still @wip 963 | * Refactoring 964 | * Get config working and loading yml or the initializer 965 | * small additions 966 | * @wip working on extracting out a configuration class and allow config via an initializer alone, also support yml file usage for when that is useful 967 | * merge config changes 968 | 969 | 970 | ## v0.0.6 / 2011-08-06 971 | 972 | Version 0.0.6 (Simon Hamilton ) 973 | 974 | Changes: 975 | 976 | * 1 Nominal Changes 977 | 978 | * Include ERB template rendering of yml. v0.0.5 979 | 980 | 981 | ## v0.0.5 / 2011-08-05 982 | 983 | Version 0.0.5 (David Rice ) 984 | 985 | Changes: 986 | 987 | * 3 Nominal Changes 988 | 989 | * now it parses the YAML file with ERB. 990 | * Set gem date for release 991 | * 0.0.4 Release 992 | 993 | 994 | ## v0.0.4 / 2011-08-05 995 | 996 | Version 0.0.4 (David Rice ) 997 | 998 | 999 | ## v0.0.3 / 2011-07-31 1000 | 1001 | Version 0.0.3 (David Rice ) 1002 | 1003 | Changes: 1004 | 1005 | * 1 Nominal Changes 1006 | 1007 | * Added homepage to gemspec 1008 | 1009 | 1010 | ## v0.0.2 / 2011-07-31 1011 | 1012 | Version 0.0.2 (Simon Hamilton ) 1013 | 1014 | Changes: 1015 | 1016 | * 7 Nominal Changes 1017 | 1018 | * Added a rails generator to install the rake task and the config. Just do "rails generate asset_sync:install" 1019 | * Updated readme 1020 | * Getting ready to release the gem 1021 | * Revert "remove version file" 1022 | 1023 | This reverts commit 7ebd853947b8d5f3b6e81f96535dfce843f2c855. 1024 | * remove version file 1025 | * Initial commit 1026 | * Initial commit 1027 | 1028 | 1029 | ## HEAD / 2012-08-27 1030 | 1031 | Current Development (David Rice) 1032 | 1033 | Changes: 1034 | 1035 | * 2 Nominal Changes 1036 | 1037 | * Improve documentation on ignored_files config option 1038 | * Merge branch 'sinatra' 1039 | 1040 | 1041 | [Unreleased]: https://github.com/AssetSync/asset_sync/compare/v2.15.1...HEAD 1042 | [2.15.1]: https://github.com/AssetSync/asset_sync/compare/v2.15.0...v2.15.1 1043 | [2.15.0]: https://github.com/AssetSync/asset_sync/compare/v2.14.2...v2.15.0 1044 | [2.14.2]: https://github.com/AssetSync/asset_sync/compare/v2.14.1...v2.14.2 1045 | [2.14.1]: https://github.com/AssetSync/asset_sync/compare/v2.14.0...v2.14.1 1046 | [2.14.0]: https://github.com/AssetSync/asset_sync/compare/v2.13.1...v2.14.0 1047 | [2.13.1]: https://github.com/AssetSync/asset_sync/compare/v2.13.0...v2.13.1 1048 | [2.13.0]: https://github.com/AssetSync/asset_sync/compare/v2.12.1...v2.13.0 1049 | [2.12.1]: https://github.com/AssetSync/asset_sync/compare/v2.12.0...v2.12.1 1050 | [2.12.0]: https://github.com/AssetSync/asset_sync/compare/v2.11.0...v2.12.0 1051 | [2.11.0]: https://github.com/AssetSync/asset_sync/compare/v2.10.0...v2.11.0 1052 | [2.10.0]: https://github.com/AssetSync/asset_sync/compare/v2.9.1...v2.10.0 1053 | [2.9.1]: https://github.com/AssetSync/asset_sync/compare/v2.9.0...v2.9.1 1054 | [2.9.0]: https://github.com/AssetSync/asset_sync/compare/v2.8.2...v2.9.0 1055 | [2.8.2]: https://github.com/AssetSync/asset_sync/compare/v2.8.1...v2.8.2 1056 | [2.8.1]: https://github.com/AssetSync/asset_sync/compare/v2.8.0...v2.8.1 1057 | [2.8.0]: https://github.com/AssetSync/asset_sync/compare/v2.7.0...v2.8.0 1058 | [2.7.0]: https://github.com/AssetSync/asset_sync/compare/v2.6.0...v2.7.0 1059 | [2.6.0]: https://github.com/AssetSync/asset_sync/compare/v2.5.0...v2.6.0 1060 | [2.5.0]: https://github.com/AssetSync/asset_sync/compare/v2.4.0...v2.5.0 1061 | [2.4.0]: https://github.com/AssetSync/asset_sync/compare/v2.3.0...v2.4.0 1062 | [2.3.0]: https://github.com/AssetSync/asset_sync/compare/v2.2.0...v2.3.0 1063 | [2.2.0]: https://github.com/AssetSync/asset_sync/compare/v2.1.0...v2.2.0 1064 | [2.1.0]: https://github.com/AssetSync/asset_sync/compare/v2.0.0...v2.1.0 1065 | [2.0.0]: https://github.com/AssetSync/asset_sync/compare/v1.3.0...v2.0.0 1066 | [1.3.0]: https://github.com/AssetSync/asset_sync/compare/v1.2.1...v1.3.0 1067 | [1.2.1]: https://github.com/AssetSync/asset_sync/compare/v1.2.0...v1.2.1 1068 | [1.2.0]: https://github.com/AssetSync/asset_sync/compare/v1.1.0...v1.2.0 1069 | --------------------------------------------------------------------------------