├── app ├── mailers │ └── .keep ├── models │ ├── .keep │ ├── concerns │ │ └── .keep │ ├── user.rb │ ├── redshift_user.rb │ └── redshift_base.rb ├── assets │ ├── images │ │ └── .keep │ ├── javascripts │ │ └── application.js │ └── stylesheets │ │ └── application.css ├── controllers │ ├── concerns │ │ └── .keep │ └── application_controller.rb ├── helpers │ └── application_helper.rb └── views │ └── layouts │ └── application.html.erb ├── lib ├── assets │ └── .keep ├── tasks │ └── .keep └── loader.rb ├── public ├── favicon.ico ├── robots.txt ├── 500.html ├── 422.html └── 404.html ├── test ├── helpers │ └── .keep ├── mailers │ └── .keep ├── models │ └── .keep ├── controllers │ └── .keep ├── fixtures │ └── .keep ├── integration │ └── .keep └── test_helper.rb ├── .ruby-version ├── vendor └── assets │ ├── javascripts │ └── .keep │ └── stylesheets │ └── .keep ├── bin ├── rake ├── bundle ├── rails └── setup ├── config ├── boot.rb ├── initializers │ ├── cookies_serializer.rb │ ├── session_store.rb │ ├── mime_types.rb │ ├── filter_parameter_logging.rb │ ├── redshift.rb │ ├── backtrace_silencers.rb │ ├── assets.rb │ ├── wrap_parameters.rb │ └── inflections.rb ├── environment.rb ├── database.yml ├── locales │ └── en.yml ├── secrets.yml ├── application.rb ├── environments │ ├── development.rb │ ├── test.rb │ └── production.rb └── routes.rb ├── config.ru ├── Rakefile ├── .gitignore ├── db ├── seeds.rb ├── migrate │ └── 20150827223207_create_users.rb └── schema.rb ├── LICENSE ├── Gemfile ├── Gemfile.lock └── README.md /app/mailers/.keep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /app/models/.keep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /lib/assets/.keep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /lib/tasks/.keep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /public/favicon.ico: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /test/helpers/.keep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /test/mailers/.keep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /test/models/.keep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /.ruby-version: -------------------------------------------------------------------------------- 1 | 2.2.2 2 | -------------------------------------------------------------------------------- /app/assets/images/.keep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /test/controllers/.keep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /test/fixtures/.keep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /test/integration/.keep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /app/models/concerns/.keep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /app/controllers/concerns/.keep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /vendor/assets/javascripts/.keep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /vendor/assets/stylesheets/.keep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /app/models/user.rb: -------------------------------------------------------------------------------- 1 | class User < ActiveRecord::Base 2 | end 3 | -------------------------------------------------------------------------------- /app/helpers/application_helper.rb: -------------------------------------------------------------------------------- 1 | module ApplicationHelper 2 | end 3 | -------------------------------------------------------------------------------- /app/models/redshift_user.rb: -------------------------------------------------------------------------------- 1 | class RedshiftUser < RedshiftBase 2 | self.table_name = :users 3 | end 4 | -------------------------------------------------------------------------------- /bin/rake: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | require_relative '../config/boot' 3 | require 'rake' 4 | Rake.application.run 5 | -------------------------------------------------------------------------------- /bin/bundle: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | ENV['BUNDLE_GEMFILE'] ||= File.expand_path('../../Gemfile', __FILE__) 3 | load Gem.bin_path('bundler', 'bundle') 4 | -------------------------------------------------------------------------------- /config/boot.rb: -------------------------------------------------------------------------------- 1 | ENV['BUNDLE_GEMFILE'] ||= File.expand_path('../../Gemfile', __FILE__) 2 | 3 | require 'bundler/setup' # Set up gems listed in the Gemfile. 4 | -------------------------------------------------------------------------------- /bin/rails: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | APP_PATH = File.expand_path('../../config/application', __FILE__) 3 | require_relative '../config/boot' 4 | require 'rails/commands' 5 | -------------------------------------------------------------------------------- /config.ru: -------------------------------------------------------------------------------- 1 | # This file is used by Rack-based servers to start the application. 2 | 3 | require ::File.expand_path('../config/environment', __FILE__) 4 | run Rails.application 5 | -------------------------------------------------------------------------------- /app/models/redshift_base.rb: -------------------------------------------------------------------------------- 1 | class RedshiftBase < ActiveRecord::Base 2 | establish_connection Rails.application.secrets.redshift_config 3 | self.abstract_class = true 4 | end 5 | -------------------------------------------------------------------------------- /config/initializers/cookies_serializer.rb: -------------------------------------------------------------------------------- 1 | # Be sure to restart your server when you modify this file. 2 | 3 | Rails.application.config.action_dispatch.cookies_serializer = :json 4 | -------------------------------------------------------------------------------- /config/initializers/session_store.rb: -------------------------------------------------------------------------------- 1 | # Be sure to restart your server when you modify this file. 2 | 3 | Rails.application.config.session_store :cookie_store, key: '_tutorial_session' 4 | -------------------------------------------------------------------------------- /config/environment.rb: -------------------------------------------------------------------------------- 1 | # Load the Rails application. 2 | require File.expand_path('../application', __FILE__) 3 | 4 | # Initialize the Rails application. 5 | Rails.application.initialize! 6 | -------------------------------------------------------------------------------- /config/initializers/mime_types.rb: -------------------------------------------------------------------------------- 1 | # Be sure to restart your server when you modify this file. 2 | 3 | # Add new mime types for use in respond_to blocks: 4 | # Mime::Type.register "text/richtext", :rtf 5 | -------------------------------------------------------------------------------- /public/robots.txt: -------------------------------------------------------------------------------- 1 | # See http://www.robotstxt.org/robotstxt.html for documentation on how to use the robots.txt file 2 | # 3 | # To ban all spiders from the entire site uncomment the next two lines: 4 | # User-agent: * 5 | # Disallow: / 6 | -------------------------------------------------------------------------------- /config/initializers/filter_parameter_logging.rb: -------------------------------------------------------------------------------- 1 | # Be sure to restart your server when you modify this file. 2 | 3 | # Configure sensitive parameters which will be filtered from the log file. 4 | Rails.application.config.filter_parameters += [:password] 5 | -------------------------------------------------------------------------------- /app/controllers/application_controller.rb: -------------------------------------------------------------------------------- 1 | class ApplicationController < ActionController::Base 2 | # Prevent CSRF attacks by raising an exception. 3 | # For APIs, you may want to use :null_session instead. 4 | protect_from_forgery with: :exception 5 | end 6 | -------------------------------------------------------------------------------- /Rakefile: -------------------------------------------------------------------------------- 1 | # Add your own tasks in files placed in lib/tasks ending in .rake, 2 | # for example lib/tasks/capistrano.rake, and they will automatically be available to Rake. 3 | 4 | require File.expand_path('../config/application', __FILE__) 5 | 6 | Rails.application.load_tasks 7 | -------------------------------------------------------------------------------- /config/initializers/redshift.rb: -------------------------------------------------------------------------------- 1 | Rails.application.secrets.redshift_config = { 2 | host: ENV['REDSHIFT_HOST'], 3 | port: ENV['REDSHIFT_PORT'], 4 | user: ENV['REDSHIFT_USER'], 5 | password: ENV['REDSHIFT_PASSWORD'], 6 | database: ENV['REDSHIFT_DATABASE'], 7 | adapter: 'redshift' 8 | } 9 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.rbc 2 | *.sassc 3 | .sass-cache 4 | capybara-*.html 5 | /.bundle 6 | /vendor/bundle 7 | /log/* 8 | /tmp/* 9 | /db/*.sqlite3 10 | /public/system/* 11 | /coverage/ 12 | /spec/tmp/* 13 | **.orig 14 | rerun.txt 15 | pickle-email-*.html 16 | .project 17 | config/initializers/secret_token.rb 18 | config_secure/* 19 | .DS_Store 20 | -------------------------------------------------------------------------------- /test/test_helper.rb: -------------------------------------------------------------------------------- 1 | ENV['RAILS_ENV'] ||= 'test' 2 | require File.expand_path('../../config/environment', __FILE__) 3 | require 'rails/test_help' 4 | 5 | class ActiveSupport::TestCase 6 | # Setup all fixtures in test/fixtures/*.yml for all tests in alphabetical order. 7 | fixtures :all 8 | 9 | # Add more helper methods to be used by all tests here... 10 | end 11 | -------------------------------------------------------------------------------- /app/views/layouts/application.html.erb: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | Tutorial 5 | <%= stylesheet_link_tag 'application', media: 'all', 'data-turbolinks-track' => true %> 6 | <%= javascript_include_tag 'application', 'data-turbolinks-track' => true %> 7 | <%= csrf_meta_tags %> 8 | 9 | 10 | 11 | <%= yield %> 12 | 13 | 14 | 15 | -------------------------------------------------------------------------------- /config/initializers/backtrace_silencers.rb: -------------------------------------------------------------------------------- 1 | # Be sure to restart your server when you modify this file. 2 | 3 | # You can add backtrace silencers for libraries that you're using but don't wish to see in your backtraces. 4 | # Rails.backtrace_cleaner.add_silencer { |line| line =~ /my_noisy_library/ } 5 | 6 | # You can also remove all the silencers if you're trying to debug a problem that might stem from framework code. 7 | # Rails.backtrace_cleaner.remove_silencers! 8 | -------------------------------------------------------------------------------- /config/initializers/assets.rb: -------------------------------------------------------------------------------- 1 | # Be sure to restart your server when you modify this file. 2 | 3 | # Version of your assets, change this if you want to expire all your assets. 4 | Rails.application.config.assets.version = '1.0' 5 | 6 | # Add additional assets to the asset load path 7 | # Rails.application.config.assets.paths << Emoji.images_path 8 | 9 | # Precompile additional assets. 10 | # application.js, application.css, and all non-JS/CSS in app/assets folder are already added. 11 | # Rails.application.config.assets.precompile += %w( search.js ) 12 | -------------------------------------------------------------------------------- /db/seeds.rb: -------------------------------------------------------------------------------- 1 | # This file should contain all the record creation needed to seed the database with its default values. 2 | # The data can then be loaded with the rake db:seed (or created alongside the db with db:setup). 3 | # 4 | # Examples: 5 | # 6 | # cities = City.create([{ name: 'Chicago' }, { name: 'Copenhagen' }]) 7 | # Mayor.create(name: 'Emanuel', city: cities.first) 8 | User.delete_all 9 | %w(Data Jordi Will Jean-Luc Beverley Worf).each do |x| 10 | user = User.new 11 | user.name = x 12 | user.email = "#{x.downcase}@enterprise.fed" 13 | user.save! 14 | end 15 | -------------------------------------------------------------------------------- /config/initializers/wrap_parameters.rb: -------------------------------------------------------------------------------- 1 | # Be sure to restart your server when you modify this file. 2 | 3 | # This file contains settings for ActionController::ParamsWrapper which 4 | # is enabled by default. 5 | 6 | # Enable parameter wrapping for JSON. You can disable this by setting :format to an empty array. 7 | ActiveSupport.on_load(:action_controller) do 8 | wrap_parameters format: [:json] if respond_to?(:wrap_parameters) 9 | end 10 | 11 | # To enable root element in JSON for ActiveRecord objects. 12 | # ActiveSupport.on_load(:active_record) do 13 | # self.include_root_in_json = true 14 | # end 15 | -------------------------------------------------------------------------------- /config/database.yml: -------------------------------------------------------------------------------- 1 | # SQLite version 3.x 2 | # gem install sqlite3 3 | # 4 | # Ensure the SQLite 3 gem is defined in your Gemfile 5 | # gem 'sqlite3' 6 | # 7 | default: &default 8 | adapter: sqlite3 9 | pool: 5 10 | timeout: 5000 11 | 12 | development: 13 | <<: *default 14 | database: db/development.sqlite3 15 | 16 | # Warning: The database defined as "test" will be erased and 17 | # re-generated from your development database when you run "rake". 18 | # Do not set this db to the same as development or production. 19 | test: 20 | <<: *default 21 | database: db/test.sqlite3 22 | 23 | production: 24 | <<: *default 25 | database: db/production.sqlite3 26 | -------------------------------------------------------------------------------- /config/initializers/inflections.rb: -------------------------------------------------------------------------------- 1 | # Be sure to restart your server when you modify this file. 2 | 3 | # Add new inflection rules using the following format. Inflections 4 | # are locale specific, and you may define rules for as many different 5 | # locales as you wish. All of these examples are active by default: 6 | # ActiveSupport::Inflector.inflections(:en) do |inflect| 7 | # inflect.plural /^(ox)$/i, '\1en' 8 | # inflect.singular /^(ox)en/i, '\1' 9 | # inflect.irregular 'person', 'people' 10 | # inflect.uncountable %w( fish sheep ) 11 | # end 12 | 13 | # These inflection rules are supported but not enabled by default: 14 | # ActiveSupport::Inflector.inflections(:en) do |inflect| 15 | # inflect.acronym 'RESTful' 16 | # end 17 | -------------------------------------------------------------------------------- /config/locales/en.yml: -------------------------------------------------------------------------------- 1 | # Files in the config/locales directory are used for internationalization 2 | # and are automatically loaded by Rails. If you want to use locales other 3 | # than English, add the necessary files in this directory. 4 | # 5 | # To use the locales, use `I18n.t`: 6 | # 7 | # I18n.t 'hello' 8 | # 9 | # In views, this is aliased to just `t`: 10 | # 11 | # <%= t('hello') %> 12 | # 13 | # To use a different locale, set it with `I18n.locale`: 14 | # 15 | # I18n.locale = :es 16 | # 17 | # This would use the information in config/locales/es.yml. 18 | # 19 | # To learn more, please read the Rails Internationalization guide 20 | # available at http://guides.rubyonrails.org/i18n.html. 21 | 22 | en: 23 | hello: "Hello world" 24 | -------------------------------------------------------------------------------- /app/assets/javascripts/application.js: -------------------------------------------------------------------------------- 1 | // This is a manifest file that'll be compiled into application.js, which will include all the files 2 | // listed below. 3 | // 4 | // Any JavaScript/Coffee file within this directory, lib/assets/javascripts, vendor/assets/javascripts, 5 | // or any plugin's vendor/assets/javascripts directory can be referenced here using a relative path. 6 | // 7 | // It's not advisable to add code directly here, but if you do, it'll appear at the bottom of the 8 | // compiled file. 9 | // 10 | // Read Sprockets README (https://github.com/rails/sprockets#sprockets-directives) for details 11 | // about supported directives. 12 | // 13 | //= require jquery 14 | //= require jquery_ujs 15 | //= require turbolinks 16 | //= require_tree . 17 | -------------------------------------------------------------------------------- /app/assets/stylesheets/application.css: -------------------------------------------------------------------------------- 1 | /* 2 | * This is a manifest file that'll be compiled into application.css, which will include all the files 3 | * listed below. 4 | * 5 | * Any CSS and SCSS file within this directory, lib/assets/stylesheets, vendor/assets/stylesheets, 6 | * or any plugin's vendor/assets/stylesheets directory can be referenced here using a relative path. 7 | * 8 | * You're free to add application-wide styles to this file and they'll appear at the bottom of the 9 | * compiled file so the styles you add here take precedence over styles defined in any styles 10 | * defined in the other CSS/SCSS files in this directory. It is generally better to create a new 11 | * file per style scope. 12 | * 13 | *= require_tree . 14 | *= require_self 15 | */ 16 | -------------------------------------------------------------------------------- /db/migrate/20150827223207_create_users.rb: -------------------------------------------------------------------------------- 1 | class CreateUsers < ActiveRecord::Migration 2 | def change 3 | create_table "users", force: :cascade do |t| 4 | t.string "name" 5 | t.string "email", default: "", null: false 6 | t.string "encrypted_password", default: "", null: false 7 | t.string "reset_password_token" 8 | t.datetime "reset_password_sent_at" 9 | t.datetime "remember_created_at" 10 | t.integer "sign_in_count", default: 0, null: false 11 | t.datetime "current_sign_in_at" 12 | t.datetime "last_sign_in_at" 13 | t.string "current_sign_in_ip" 14 | t.string "last_sign_in_ip" 15 | t.datetime "created_at" 16 | t.datetime "updated_at" 17 | end 18 | end 19 | end 20 | -------------------------------------------------------------------------------- /bin/setup: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | require 'pathname' 3 | 4 | # path to your application root. 5 | APP_ROOT = Pathname.new File.expand_path('../../', __FILE__) 6 | 7 | Dir.chdir APP_ROOT do 8 | # This script is a starting point to setup your application. 9 | # Add necessary setup steps to this file: 10 | 11 | puts "== Installing dependencies ==" 12 | system "gem install bundler --conservative" 13 | system "bundle check || bundle install" 14 | 15 | # puts "\n== Copying sample files ==" 16 | # unless File.exist?("config/database.yml") 17 | # system "cp config/database.yml.sample config/database.yml" 18 | # end 19 | 20 | puts "\n== Preparing database ==" 21 | system "bin/rake db:setup" 22 | 23 | puts "\n== Removing old logs and tempfiles ==" 24 | system "rm -f log/*" 25 | system "rm -rf tmp/cache" 26 | 27 | puts "\n== Restarting application server ==" 28 | system "touch tmp/restart.txt" 29 | end 30 | -------------------------------------------------------------------------------- /config/secrets.yml: -------------------------------------------------------------------------------- 1 | # Be sure to restart your server when you modify this file. 2 | 3 | # Your secret key is used for verifying the integrity of signed cookies. 4 | # If you change this key, all old signed cookies will become invalid! 5 | 6 | # Make sure the secret is at least 30 characters and all random, 7 | # no regular words or you'll be exposed to dictionary attacks. 8 | # You can use `rake secret` to generate a secure secret key. 9 | 10 | # Make sure the secrets in this file are kept private 11 | # if you're sharing your code publicly. 12 | 13 | development: 14 | secret_key_base: 18c5f3caab835689120af9f1e49eea110019be6b0f0be410fe4173aa2a8e4d944cf5038b314168380d45695b4a18438099c0accbbdb2169801f9b7dc86054479 15 | 16 | test: 17 | secret_key_base: ef2aac4e9cbf58e6d2a6bddcac03f998d252653bef07c6a19b783073e44ece08369b39a94c4a28b6c14e0866579cf023e68b42750b94f44e30e178f0ea9ff2f3 18 | 19 | # Do not keep production secrets in the repository, 20 | # instead read values from the environment. 21 | production: 22 | secret_key_base: <%= ENV["SECRET_KEY_BASE"] %> 23 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2015 Jimmy Zhang 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | 23 | -------------------------------------------------------------------------------- /config/application.rb: -------------------------------------------------------------------------------- 1 | require File.expand_path('../boot', __FILE__) 2 | 3 | require 'rails/all' 4 | 5 | # Require the gems listed in Gemfile, including any gems 6 | # you've limited to :test, :development, or :production. 7 | Bundler.require(*Rails.groups) 8 | 9 | module Tutorial 10 | class Application < Rails::Application 11 | # Settings in config/environments/* take precedence over those specified here. 12 | # Application configuration should go into files in config/initializers 13 | # -- all .rb files in that directory are automatically loaded. 14 | 15 | # Set Time.zone default to the specified zone and make Active Record auto-convert to this zone. 16 | # Run "rake -D time" for a list of tasks for finding time zone names. Default is UTC. 17 | # config.time_zone = 'Central Time (US & Canada)' 18 | 19 | # The default locale is :en and all translations from config/locales/*.rb,yml are auto loaded. 20 | # config.i18n.load_path += Dir[Rails.root.join('my', 'locales', '*.{rb,yml}').to_s] 21 | # config.i18n.default_locale = :de 22 | 23 | # Do not swallow errors in after_commit/after_rollback callbacks. 24 | config.active_record.raise_in_transactional_callbacks = true 25 | end 26 | end 27 | -------------------------------------------------------------------------------- /db/schema.rb: -------------------------------------------------------------------------------- 1 | # encoding: UTF-8 2 | # This file is auto-generated from the current state of the database. Instead 3 | # of editing this file, please use the migrations feature of Active Record to 4 | # incrementally modify your database, and then regenerate this schema definition. 5 | # 6 | # Note that this schema.rb definition is the authoritative source for your 7 | # database schema. If you need to create the application database on another 8 | # system, you should be using db:schema:load, not running all the migrations 9 | # from scratch. The latter is a flawed and unsustainable approach (the more migrations 10 | # you'll amass, the slower it'll run and the greater likelihood for issues). 11 | # 12 | # It's strongly recommended that you check this file into your version control system. 13 | 14 | ActiveRecord::Schema.define(version: 20150827223207) do 15 | 16 | create_table "users", force: :cascade do |t| 17 | t.string "name" 18 | t.string "email", default: "", null: false 19 | t.string "encrypted_password", default: "", null: false 20 | t.string "reset_password_token" 21 | t.datetime "reset_password_sent_at" 22 | t.datetime "remember_created_at" 23 | t.integer "sign_in_count", default: 0, null: false 24 | t.datetime "current_sign_in_at" 25 | t.datetime "last_sign_in_at" 26 | t.string "current_sign_in_ip" 27 | t.string "last_sign_in_ip" 28 | t.datetime "created_at" 29 | t.datetime "updated_at" 30 | end 31 | 32 | end 33 | -------------------------------------------------------------------------------- /Gemfile: -------------------------------------------------------------------------------- 1 | source 'https://rubygems.org' 2 | 3 | 4 | # Bundle edge Rails instead: gem 'rails', github: 'rails/rails' 5 | gem 'rails', '4.2.3' 6 | # Use sqlite3 as the database for Active Record 7 | gem 'sqlite3' 8 | # Use SCSS for stylesheets 9 | gem 'sass-rails', '~> 5.0' 10 | # Use Uglifier as compressor for JavaScript assets 11 | gem 'uglifier', '>= 1.3.0' 12 | # Use CoffeeScript for .coffee assets and views 13 | gem 'coffee-rails', '~> 4.1.0' 14 | # See https://github.com/rails/execjs#readme for more supported runtimes 15 | # gem 'therubyracer', platforms: :ruby 16 | 17 | # Use jquery as the JavaScript library 18 | gem 'jquery-rails' 19 | # Turbolinks makes following links in your web application faster. Read more: https://github.com/rails/turbolinks 20 | gem 'turbolinks' 21 | # Build JSON APIs with ease. Read more: https://github.com/rails/jbuilder 22 | gem 'jbuilder', '~> 2.0' 23 | # bundle exec rake doc:rails generates the API under doc/api. 24 | gem 'sdoc', '~> 0.4.0', group: :doc 25 | 26 | # Use ActiveModel has_secure_password 27 | # gem 'bcrypt', '~> 3.1.7' 28 | 29 | # Use Unicorn as the app server 30 | # gem 'unicorn' 31 | 32 | # Use Capistrano for deployment 33 | # gem 'capistrano-rails', group: :development 34 | gem 'pg' 35 | gem 'activerecord4-redshift-adapter', '~> 0.2.0' 36 | gem 'aws-sdk', '~> 2.1.0' 37 | 38 | group :development, :test do 39 | # Call 'byebug' anywhere in the code to stop execution and get a debugger console 40 | gem 'byebug' 41 | 42 | # Access an IRB console on exception pages or by using <%= console %> in views 43 | gem 'web-console', '~> 2.0' 44 | 45 | # Spring speeds up development by keeping your application running in the background. Read more: https://github.com/rails/spring 46 | gem 'spring' 47 | end 48 | 49 | -------------------------------------------------------------------------------- /public/500.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | We're sorry, but something went wrong (500) 5 | 6 | 55 | 56 | 57 | 58 | 59 |
60 |
61 |

We're sorry, but something went wrong.

62 |
63 |

If you are the application owner check the logs for more information.

64 |
65 | 66 | 67 | -------------------------------------------------------------------------------- /config/environments/development.rb: -------------------------------------------------------------------------------- 1 | Rails.application.configure do 2 | # Settings specified here will take precedence over those in config/application.rb. 3 | 4 | # In the development environment your application's code is reloaded on 5 | # every request. This slows down response time but is perfect for development 6 | # since you don't have to restart the web server when you make code changes. 7 | config.cache_classes = false 8 | 9 | # Do not eager load code on boot. 10 | config.eager_load = false 11 | 12 | # Show full error reports and disable caching. 13 | config.consider_all_requests_local = true 14 | config.action_controller.perform_caching = false 15 | 16 | # Don't care if the mailer can't send. 17 | config.action_mailer.raise_delivery_errors = false 18 | 19 | # Print deprecation notices to the Rails logger. 20 | config.active_support.deprecation = :log 21 | 22 | # Raise an error on page load if there are pending migrations. 23 | config.active_record.migration_error = :page_load 24 | 25 | # Debug mode disables concatenation and preprocessing of assets. 26 | # This option may cause significant delays in view rendering with a large 27 | # number of complex assets. 28 | config.assets.debug = true 29 | 30 | # Asset digests allow you to set far-future HTTP expiration dates on all assets, 31 | # yet still be able to expire them through the digest params. 32 | config.assets.digest = true 33 | 34 | # Adds additional error checking when serving assets at runtime. 35 | # Checks for improperly declared sprockets dependencies. 36 | # Raises helpful error messages. 37 | config.assets.raise_runtime_errors = true 38 | 39 | # Raises error for missing translations 40 | # config.action_view.raise_on_missing_translations = true 41 | end 42 | -------------------------------------------------------------------------------- /config/routes.rb: -------------------------------------------------------------------------------- 1 | Rails.application.routes.draw do 2 | # The priority is based upon order of creation: first created -> highest priority. 3 | # See how all your routes lay out with "rake routes". 4 | 5 | # You can have the root of your site routed with "root" 6 | # root 'welcome#index' 7 | 8 | # Example of regular route: 9 | # get 'products/:id' => 'catalog#view' 10 | 11 | # Example of named route that can be invoked with purchase_url(id: product.id) 12 | # get 'products/:id/purchase' => 'catalog#purchase', as: :purchase 13 | 14 | # Example resource route (maps HTTP verbs to controller actions automatically): 15 | # resources :products 16 | 17 | # Example resource route with options: 18 | # resources :products do 19 | # member do 20 | # get 'short' 21 | # post 'toggle' 22 | # end 23 | # 24 | # collection do 25 | # get 'sold' 26 | # end 27 | # end 28 | 29 | # Example resource route with sub-resources: 30 | # resources :products do 31 | # resources :comments, :sales 32 | # resource :seller 33 | # end 34 | 35 | # Example resource route with more complex sub-resources: 36 | # resources :products do 37 | # resources :comments 38 | # resources :sales do 39 | # get 'recent', on: :collection 40 | # end 41 | # end 42 | 43 | # Example resource route with concerns: 44 | # concern :toggleable do 45 | # post 'toggle' 46 | # end 47 | # resources :posts, concerns: :toggleable 48 | # resources :photos, concerns: :toggleable 49 | 50 | # Example resource route within a namespace: 51 | # namespace :admin do 52 | # # Directs /admin/products/* to Admin::ProductsController 53 | # # (app/controllers/admin/products_controller.rb) 54 | # resources :products 55 | # end 56 | end 57 | -------------------------------------------------------------------------------- /public/422.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | The change you wanted was rejected (422) 5 | 6 | 55 | 56 | 57 | 58 | 59 |
60 |
61 |

The change you wanted was rejected.

62 |

Maybe you tried to change something you didn't have access to.

63 |
64 |

If you are the application owner check the logs for more information.

65 |
66 | 67 | 68 | -------------------------------------------------------------------------------- /public/404.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | The page you were looking for doesn't exist (404) 5 | 6 | 55 | 56 | 57 | 58 | 59 |
60 |
61 |

The page you were looking for doesn't exist.

62 |

You may have mistyped the address or the page may have moved.

63 |
64 |

If you are the application owner check the logs for more information.

65 |
66 | 67 | 68 | -------------------------------------------------------------------------------- /config/environments/test.rb: -------------------------------------------------------------------------------- 1 | Rails.application.configure do 2 | # Settings specified here will take precedence over those in config/application.rb. 3 | 4 | # The test environment is used exclusively to run your application's 5 | # test suite. You never need to work with it otherwise. Remember that 6 | # your test database is "scratch space" for the test suite and is wiped 7 | # and recreated between test runs. Don't rely on the data there! 8 | config.cache_classes = true 9 | 10 | # Do not eager load code on boot. This avoids loading your whole application 11 | # just for the purpose of running a single test. If you are using a tool that 12 | # preloads Rails for running tests, you may have to set it to true. 13 | config.eager_load = false 14 | 15 | # Configure static file server for tests with Cache-Control for performance. 16 | config.serve_static_files = true 17 | config.static_cache_control = 'public, max-age=3600' 18 | 19 | # Show full error reports and disable caching. 20 | config.consider_all_requests_local = true 21 | config.action_controller.perform_caching = false 22 | 23 | # Raise exceptions instead of rendering exception templates. 24 | config.action_dispatch.show_exceptions = false 25 | 26 | # Disable request forgery protection in test environment. 27 | config.action_controller.allow_forgery_protection = false 28 | 29 | # Tell Action Mailer not to deliver emails to the real world. 30 | # The :test delivery method accumulates sent emails in the 31 | # ActionMailer::Base.deliveries array. 32 | config.action_mailer.delivery_method = :test 33 | 34 | # Randomize the order test cases are executed. 35 | config.active_support.test_order = :random 36 | 37 | # Print deprecation notices to the stderr. 38 | config.active_support.deprecation = :stderr 39 | 40 | # Raises error for missing translations 41 | # config.action_view.raise_on_missing_translations = true 42 | end 43 | -------------------------------------------------------------------------------- /lib/loader.rb: -------------------------------------------------------------------------------- 1 | # require 'loader'; Loader.load 2 | require 'csv' 3 | class Loader 4 | BUCKET = ENV['REDSHIFT_BUCKET'] 5 | BATCH_SIZE = 1000 6 | TABLE = 'users' 7 | COLUMNS = %w(id name email sign_in_count current_sign_in_at last_sign_in_at) 8 | 9 | class << self 10 | def load 11 | # setup AWS credentials 12 | Aws.config.update({ 13 | region: 'us-east-1', 14 | credentials: Aws::Credentials.new( 15 | ENV['AWS_ACCESS_KEY_ID'], 16 | ENV['AWS_SECRET_ACCESS_KEY']) 17 | }) 18 | 19 | # connect to Redshift 20 | db = PG.connect( 21 | host: ENV['REDSHIFT_HOST'], 22 | port: ENV['REDSHIFT_PORT'], 23 | user: ENV['REDSHIFT_USER'], 24 | password: ENV['REDSHIFT_PASSWORD'], 25 | dbname: ENV['REDSHIFT_DATABASE'], 26 | ) 27 | 28 | # extract data to CSV files and uplaod to S3 29 | User.find_in_batches(batch_size: BATCH_SIZE).with_index do |group, batch| 30 | Tempfile.open(TABLE) do |f| 31 | Zlib::GzipWriter.open(f) do |gz| 32 | csv_string = CSV.generate do |csv| 33 | group.each do |record| 34 | csv << COLUMNS.map{|x| record.send(x)} 35 | end 36 | end 37 | gz.write csv_string 38 | end 39 | # upload to s3 40 | s3 = Aws::S3::Resource.new 41 | key = "#{TABLE}/data-#{batch}.gz" 42 | obj = s3.bucket(BUCKET).object(key) 43 | obj.upload_file(f) 44 | end 45 | end 46 | 47 | 48 | # clear existing data for this table 49 | db.exec <<-EOS 50 | TRUNCATE #{TABLE} 51 | EOS 52 | 53 | # load the data, specifying the order of the fields 54 | db.exec <<-EOS 55 | COPY #{TABLE} (#{COLUMNS.join(', ')}) 56 | FROM 's3://#{BUCKET}/#{TABLE}/data' 57 | CREDENTIALS 'aws_access_key_id=#{ENV['AWS_ACCESS_KEY_ID']};aws_secret_access_key=#{ENV['AWS_SECRET_ACCESS_KEY']}' 58 | CSV 59 | EMPTYASNULL 60 | GZIP 61 | EOS 62 | end 63 | end 64 | end 65 | -------------------------------------------------------------------------------- /config/environments/production.rb: -------------------------------------------------------------------------------- 1 | Rails.application.configure do 2 | # Settings specified here will take precedence over those in config/application.rb. 3 | 4 | # Code is not reloaded between requests. 5 | config.cache_classes = true 6 | 7 | # Eager load code on boot. This eager loads most of Rails and 8 | # your application in memory, allowing both threaded web servers 9 | # and those relying on copy on write to perform better. 10 | # Rake tasks automatically ignore this option for performance. 11 | config.eager_load = true 12 | 13 | # Full error reports are disabled and caching is turned on. 14 | config.consider_all_requests_local = false 15 | config.action_controller.perform_caching = true 16 | 17 | # Enable Rack::Cache to put a simple HTTP cache in front of your application 18 | # Add `rack-cache` to your Gemfile before enabling this. 19 | # For large-scale production use, consider using a caching reverse proxy like 20 | # NGINX, varnish or squid. 21 | # config.action_dispatch.rack_cache = true 22 | 23 | # Disable serving static files from the `/public` folder by default since 24 | # Apache or NGINX already handles this. 25 | config.serve_static_files = ENV['RAILS_SERVE_STATIC_FILES'].present? 26 | 27 | # Compress JavaScripts and CSS. 28 | config.assets.js_compressor = :uglifier 29 | # config.assets.css_compressor = :sass 30 | 31 | # Do not fallback to assets pipeline if a precompiled asset is missed. 32 | config.assets.compile = false 33 | 34 | # Asset digests allow you to set far-future HTTP expiration dates on all assets, 35 | # yet still be able to expire them through the digest params. 36 | config.assets.digest = true 37 | 38 | # `config.assets.precompile` and `config.assets.version` have moved to config/initializers/assets.rb 39 | 40 | # Specifies the header that your server uses for sending files. 41 | # config.action_dispatch.x_sendfile_header = 'X-Sendfile' # for Apache 42 | # config.action_dispatch.x_sendfile_header = 'X-Accel-Redirect' # for NGINX 43 | 44 | # Force all access to the app over SSL, use Strict-Transport-Security, and use secure cookies. 45 | # config.force_ssl = true 46 | 47 | # Use the lowest log level to ensure availability of diagnostic information 48 | # when problems arise. 49 | config.log_level = :debug 50 | 51 | # Prepend all log lines with the following tags. 52 | # config.log_tags = [ :subdomain, :uuid ] 53 | 54 | # Use a different logger for distributed setups. 55 | # config.logger = ActiveSupport::TaggedLogging.new(SyslogLogger.new) 56 | 57 | # Use a different cache store in production. 58 | # config.cache_store = :mem_cache_store 59 | 60 | # Enable serving of images, stylesheets, and JavaScripts from an asset server. 61 | # config.action_controller.asset_host = 'http://assets.example.com' 62 | 63 | # Ignore bad email addresses and do not raise email delivery errors. 64 | # Set this to true and configure the email server for immediate delivery to raise delivery errors. 65 | # config.action_mailer.raise_delivery_errors = false 66 | 67 | # Enable locale fallbacks for I18n (makes lookups for any locale fall back to 68 | # the I18n.default_locale when a translation cannot be found). 69 | config.i18n.fallbacks = true 70 | 71 | # Send deprecation notices to registered listeners. 72 | config.active_support.deprecation = :notify 73 | 74 | # Use default logging formatter so that PID and timestamp are not suppressed. 75 | config.log_formatter = ::Logger::Formatter.new 76 | 77 | # Do not dump schema after migrations. 78 | config.active_record.dump_schema_after_migration = false 79 | end 80 | -------------------------------------------------------------------------------- /Gemfile.lock: -------------------------------------------------------------------------------- 1 | GEM 2 | remote: https://rubygems.org/ 3 | specs: 4 | actionmailer (4.2.3) 5 | actionpack (= 4.2.3) 6 | actionview (= 4.2.3) 7 | activejob (= 4.2.3) 8 | mail (~> 2.5, >= 2.5.4) 9 | rails-dom-testing (~> 1.0, >= 1.0.5) 10 | actionpack (4.2.3) 11 | actionview (= 4.2.3) 12 | activesupport (= 4.2.3) 13 | rack (~> 1.6) 14 | rack-test (~> 0.6.2) 15 | rails-dom-testing (~> 1.0, >= 1.0.5) 16 | rails-html-sanitizer (~> 1.0, >= 1.0.2) 17 | actionview (4.2.3) 18 | activesupport (= 4.2.3) 19 | builder (~> 3.1) 20 | erubis (~> 2.7.0) 21 | rails-dom-testing (~> 1.0, >= 1.0.5) 22 | rails-html-sanitizer (~> 1.0, >= 1.0.2) 23 | activejob (4.2.3) 24 | activesupport (= 4.2.3) 25 | globalid (>= 0.3.0) 26 | activemodel (4.2.3) 27 | activesupport (= 4.2.3) 28 | builder (~> 3.1) 29 | activerecord (4.2.3) 30 | activemodel (= 4.2.3) 31 | activesupport (= 4.2.3) 32 | arel (~> 6.0) 33 | activerecord4-redshift-adapter (0.2.0) 34 | activerecord (~> 4.2.0) 35 | pg 36 | activesupport (4.2.3) 37 | i18n (~> 0.7) 38 | json (~> 1.7, >= 1.7.7) 39 | minitest (~> 5.1) 40 | thread_safe (~> 0.3, >= 0.3.4) 41 | tzinfo (~> 1.1) 42 | arel (6.0.3) 43 | aws-sdk (2.1.16) 44 | aws-sdk-resources (= 2.1.16) 45 | aws-sdk-core (2.1.16) 46 | jmespath (~> 1.0) 47 | aws-sdk-resources (2.1.16) 48 | aws-sdk-core (= 2.1.16) 49 | binding_of_caller (0.7.2) 50 | debug_inspector (>= 0.0.1) 51 | builder (3.2.2) 52 | byebug (6.0.2) 53 | coffee-rails (4.1.0) 54 | coffee-script (>= 2.2.0) 55 | railties (>= 4.0.0, < 5.0) 56 | coffee-script (2.4.1) 57 | coffee-script-source 58 | execjs 59 | coffee-script-source (1.9.1.1) 60 | debug_inspector (0.0.2) 61 | erubis (2.7.0) 62 | execjs (2.6.0) 63 | globalid (0.3.6) 64 | activesupport (>= 4.1.0) 65 | i18n (0.7.0) 66 | jbuilder (2.3.1) 67 | activesupport (>= 3.0.0, < 5) 68 | multi_json (~> 1.2) 69 | jmespath (1.0.2) 70 | multi_json (~> 1.0) 71 | jquery-rails (4.0.4) 72 | rails-dom-testing (~> 1.0) 73 | railties (>= 4.2.0) 74 | thor (>= 0.14, < 2.0) 75 | json (1.8.3) 76 | loofah (2.0.3) 77 | nokogiri (>= 1.5.9) 78 | mail (2.6.3) 79 | mime-types (>= 1.16, < 3) 80 | mime-types (2.6.1) 81 | mini_portile (0.6.2) 82 | minitest (5.8.0) 83 | multi_json (1.11.2) 84 | nokogiri (1.6.6.2) 85 | mini_portile (~> 0.6.0) 86 | pg (0.18.2) 87 | rack (1.6.4) 88 | rack-test (0.6.3) 89 | rack (>= 1.0) 90 | rails (4.2.3) 91 | actionmailer (= 4.2.3) 92 | actionpack (= 4.2.3) 93 | actionview (= 4.2.3) 94 | activejob (= 4.2.3) 95 | activemodel (= 4.2.3) 96 | activerecord (= 4.2.3) 97 | activesupport (= 4.2.3) 98 | bundler (>= 1.3.0, < 2.0) 99 | railties (= 4.2.3) 100 | sprockets-rails 101 | rails-deprecated_sanitizer (1.0.3) 102 | activesupport (>= 4.2.0.alpha) 103 | rails-dom-testing (1.0.7) 104 | activesupport (>= 4.2.0.beta, < 5.0) 105 | nokogiri (~> 1.6.0) 106 | rails-deprecated_sanitizer (>= 1.0.1) 107 | rails-html-sanitizer (1.0.2) 108 | loofah (~> 2.0) 109 | railties (4.2.3) 110 | actionpack (= 4.2.3) 111 | activesupport (= 4.2.3) 112 | rake (>= 0.8.7) 113 | thor (>= 0.18.1, < 2.0) 114 | rake (10.4.2) 115 | rdoc (4.2.0) 116 | sass (3.4.18) 117 | sass-rails (5.0.3) 118 | railties (>= 4.0.0, < 5.0) 119 | sass (~> 3.1) 120 | sprockets (>= 2.8, < 4.0) 121 | sprockets-rails (>= 2.0, < 4.0) 122 | tilt (~> 1.1) 123 | sdoc (0.4.1) 124 | json (~> 1.7, >= 1.7.7) 125 | rdoc (~> 4.0) 126 | spring (1.3.6) 127 | sprockets (3.3.3) 128 | rack (~> 1.0) 129 | sprockets-rails (2.3.2) 130 | actionpack (>= 3.0) 131 | activesupport (>= 3.0) 132 | sprockets (>= 2.8, < 4.0) 133 | sqlite3 (1.3.10) 134 | thor (0.19.1) 135 | thread_safe (0.3.5) 136 | tilt (1.4.1) 137 | turbolinks (2.5.3) 138 | coffee-rails 139 | tzinfo (1.2.2) 140 | thread_safe (~> 0.1) 141 | uglifier (2.7.2) 142 | execjs (>= 0.3.0) 143 | json (>= 1.8.0) 144 | web-console (2.2.1) 145 | activemodel (>= 4.0) 146 | binding_of_caller (>= 0.7.2) 147 | railties (>= 4.0) 148 | sprockets-rails (>= 2.0, < 4.0) 149 | 150 | PLATFORMS 151 | ruby 152 | 153 | DEPENDENCIES 154 | activerecord4-redshift-adapter (~> 0.2.0) 155 | aws-sdk (~> 2.1.0) 156 | byebug 157 | coffee-rails (~> 4.1.0) 158 | jbuilder (~> 2.0) 159 | jquery-rails 160 | pg 161 | rails (= 4.2.3) 162 | sass-rails (~> 5.0) 163 | sdoc (~> 0.4.0) 164 | spring 165 | sqlite3 166 | turbolinks 167 | uglifier (>= 1.3.0) 168 | web-console (~> 2.0) 169 | 170 | BUNDLED WITH 171 | 1.10.6 172 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Setting up a Data Warehouse with AWS Redshift and Ruby 2 | Published on the [Credible Blog](https://www.credible.com/code/setting-up-a-data-warehouse-with-aws-redshift-and-ruby/) 3 | 4 | ![Screenshot](https://www.credible.com/code/wp-content/uploads/2015/09/AUvn49gey8Y-thumb.png) 5 | 6 | Most startups eventually need a robust solution for storing large amounts of data for analytics. Perhaps you're running a video app trying to understand user drop-off or you're studying user behavior on your website like we do at Credible. 7 | 8 | You might start with a few tables in your primary database. Soon you may create a separate web app with a nightly cron job to sync data. Before you know it, you have more data than you can handle, jobs are taking way too long, and you're being asked to integrate data from more sources. This is where a [data warehouse](https://en.wikipedia.org/wiki/Data_warehouse) comes in handy. It allows your team to store and query terabytes or even petabytes of data from many sources without writing a bunch of custom code. 9 | 10 | In the past, only big companies like Amazon had data warehouses because they were expensive, hard to setup, and time-consuming to maintain. With AWS Redshift and Ruby, we'll show you how to setup your own simple, inexpensive, and scalable data warehouse. We'll provide [sample code](https://github.com/tuesy/redshift-ruby-tutorial) that will show you to how to extract, transform, and load (ETL) data into Redshift as well as how to access the data from a Rails app. 11 | 12 | ## Part I: Setting up AWS Redshift 13 | 14 | ### Creating a Redshift Cluster 15 | 16 | We chose AWS's Redshift offering because it's easy to set up, inexpensive (it's AWS after all), and its interface is pretty similar to that of Postgres so you can manage it using tools like [Postico](https://eggerapps.at/postico/), a Postgres database manager for OSX, and use with Ruby via an [activerecord adapter](https://github.com/aamine/activerecord4-redshift-adapter). Let's begin by logging into your AWS console and creating a new Redshift cluster. Make sure to write down your cluster info as we'll need it later. 17 | 18 | ![Screenshot](https://13217-presscdn-0-50-pagely.netdna-ssl.com/code/wp-content/uploads/2015/09/Redshift_%C2%B7_AWS_Console_4.png) 19 | 20 | We're going with a single node here for development and QA environments but for production, you'll want to create a multi-node cluster so you can get faster importing and querying as well as handle more data. 21 | 22 | ![Screenshot](https://13217-presscdn-0-50-pagely.netdna-ssl.com/code/wp-content/uploads/2015/09/Redshift_%C2%B7_AWS_Console_3.png) 23 | 24 | You can optionally encrypt the data and enable other security settings here. You can go with defaults the rest of the way for the purposes of this tutorial. Note that you'll start incurring charges once you create the cluster ($0.25 an hour for DC1.Large and first 2 months free). 25 | 26 | ![Screenshot](https://13217-presscdn-0-50-pagely.netdna-ssl.com/code/wp-content/uploads/2015/09/Redshift_%C2%B7_AWS_Console_2.png) 27 | 28 | When you're done, you'll see a summary page for the cluster. Please jot down the hostname in the Endpoint. 29 | 30 | ![Screenshot](https://13217-presscdn-0-50-pagely.netdna-ssl.com/code/wp-content/uploads/2015/09/Redshift_%C2%B7_AWS_Console_1.png) 31 | 32 | By default, nothing is allowed to connect to the cluster. You can create one for your computer by going to Security > Add Connection Type > Authorize--AWS will automatically fill in your current IP address for convenience. 33 | 34 | ![Screenshot](https://13217-presscdn-0-50-pagely.netdna-ssl.com/code/wp-content/uploads/2015/09/Redshift_%C2%B7_AWS_Console.png) 35 | 36 | ### Verifying Your Cluster 37 | 38 | Now, let's try connecting to your cluster using [Postico](https://eggerapps.at/postico/). You'll need to create a Favorite and fill in the info you used to create the cluster. Note that the Endpoint url you got from the Redshift cluster contains both the host and port--you'll need to put them in separate fields. 39 | 40 | ![Screenshot](https://13217-presscdn-0-50-pagely.netdna-ssl.com/code/wp-content/uploads/2015/09/Postico_Favorites.png) 41 | 42 | If you're successful, you'll see something like this. 43 | 44 | ![Screenshot](https://13217-presscdn-0-50-pagely.netdna-ssl.com/code/wp-content/uploads/2015/09/redshift-ruby-tutorial_%E2%80%93_analytics_and__bash_profile_%E2%80%94_redshift-ruby-tutorial_and_3__bash.png) 45 | 46 | Congrats, you've created your first data warehouse! For your Production environment, you may want to beef up the security or use a multi-node cluster for redundancy and performance. 47 | 48 | The next step is to configure Redshift so we can load data into it. Redshift acts like Postgres for the most part. For example, you need to create tables ahead of time and you'll need to specify the data types for each column. There are some differences that may trip you up. We ran into issues at first because the default Rails data types don't map correctly. The following are some examples of Rails data types and how they should be mapped to Redshift: 49 | 50 | * integer => int 51 | * string => varchar 52 | * date => date 53 | * datetime => timestamp 54 | * boolean => bool 55 | * text => varchar(65535) 56 | * decimal(precision, scale) => decimal(precision, scale) 57 | 58 | Note that the ID column should be of type "bigint". The [Redshift documentation](https://aws.amazon.com/documentation/redshift/) has more details. Here's how we mapped the "users" table for the sample app. 59 | 60 | ![Screenshot](https://13217-presscdn-0-50-pagely.netdna-ssl.com/code/wp-content/uploads/2015/09/redshift-ruby-tutorial_%E2%80%93_analytics_and_schema_rb_%E2%80%94_redshift-ruby-tutorial_and_3__bash.png) 61 | 62 | You should also note that we didn't map all fields. You'll want to omit sensitive fields like "password" or add fields on an as-needed basis to reduce complexity and costs. 63 | 64 | ## Part 2: Extracting, Transforming, and Loading (ETL) 65 | 66 | ### Create an S3 Bucket 67 | 68 | You'll need to create an S3 bucket either via the AWS Console or through their API. For this sample, we've created one called "redshift-ruby-tutorial". 69 | 70 | ### Setup the Sample App 71 | 72 | We created a [sample Rails app](https://github.com/tuesy/redshift-ruby-tutorial) for this part. It contains a User table, some seed data, and a Loader class that will perform ETL. The high-level approach is to output the User data to CSV files, upload the files to an AWS S3 bucket, and then trigger Redshift to load the CSV files. 73 | 74 | Let's start by cloning the app: 75 | 76 | ```bash 77 | git clone git@github.com:tuesy/redshift-ruby-tutorial.git 78 | cd redshift-ruby-tutorial 79 | ``` 80 | 81 | Next, update your environment variables by editing and sourcing the ~/.bash_profile. You should use the info from when you created your cluster. 82 | 83 | ```bash 84 | # redshift-ruby-tutorial 85 | export REDSHIFT_HOST=redshift-ruby-tutorial.ccmj2nxbsay7.us-east-1.redshift.amazonaws.com 86 | export REDSHIFT_PORT=5439 87 | export REDSHIFT_USER=deploy 88 | export REDSHIFT_PASSWORD= 89 | export REDSHIFT_DATABASE=analytics 90 | export REDSHIFT_BUCKET=redshift-ruby-tutorial 91 | ``` 92 | 93 | We're ready to bundle our gems, create our database, and seed the dummy data: 94 | 95 | ```bash 96 | bundle install 97 | bundle exec rake db:setup 98 | ``` 99 | 100 | Before we run ETL, let's check the connection to Redshift. This should return "0 users" because we haven't loaded any data yet: 101 | 102 | ```bash 103 | bundle exec rails c 104 | RedshiftUser.count 105 | ``` 106 | 107 | Now let's run ETL and then count users again (there should be some users now): 108 | 109 | ```ruby 110 | require 'loader' 111 | Loader.load 112 | RedshiftUser.count 113 | ``` 114 | 115 | Here's an example of the output you should see: 116 | 117 | ```bash 118 | ~/git/redshift-ruby-tutorial(master)$ bundle exec rails c 119 | Loading development environment (Rails 4.2.3) 120 | irb(main):001:0> RedshiftUser.count 121 | unknown OID 16: failed to recognize type of 'attnotnull'. It will be treated as String. 122 | (1055.2ms) SELECT COUNT(*) FROM "users" 123 | => 0 124 | irb(main):002:0> require 'loader' 125 | => true 126 | irb(main):003:0> Loader.load 127 | User Load (0.2ms) SELECT "users".* FROM "users" ORDER BY "users"."id" ASC LIMIT 1000 128 | INFO: Load into table 'users' completed, 6 record(s) loaded successfully. 129 | => # 130 | irb(main):004:0> RedshiftUser.count 131 | (95.7ms) SELECT COUNT(*) FROM "users" 132 | => 6 133 | irb(main):005:0> RedshiftUser.first 134 | RedshiftUser Load (1528.4ms) SELECT "users".* FROM "users" ORDER BY "users"."id" ASC LIMIT 1 135 | => # 136 | ``` 137 | 138 | ### How to Connect to Redshift 139 | 140 | Redshift is based on Postgres so we can use a slightly modified ActiveRecord adapter: 141 | 142 | ```ruby 143 | gem 'activerecord4-redshift-adapter', '~> 0.2.0' 144 | ``` 145 | 146 | We use an initializer to DRY things up a bit: 147 | 148 | ```ruby 149 | Rails.application.secrets.redshift_config = { 150 | host: ENV['REDSHIFT_HOST'], 151 | port: ENV['REDSHIFT_PORT'], 152 | user: ENV['REDSHIFT_USER'], 153 | password: ENV['REDSHIFT_PASSWORD'], 154 | database: ENV['REDSHIFT_DATABASE'], 155 | adapter: 'redshift' 156 | } 157 | ``` 158 | 159 | You can configure each Rails model to connect to a separate database so we created a base class for all the tables we'll use to connect to Redshift: 160 | 161 | ```ruby 162 | class RedshiftBase < ActiveRecord::Base 163 | establish_connection Rails.application.secrets.redshift_config 164 | self.abstract_class = true 165 | end 166 | ``` 167 | 168 | For the RedshiftUser class, we'll just need to specify the name of the table, otherwise Rails would look for a table named "redshift_users". This is also necessary because we have our own User class for the local database. 169 | 170 | ```ruby 171 | class RedshiftUser < RedshiftBase 172 | self.table_name = :users 173 | end 174 | ``` 175 | 176 | With this configured, you can query the table. For associations, you'll have to do some more customizations if you want niceties like "@user.posts". 177 | 178 | ### How to ETL 179 | 180 | This task is performed by the Loader class. We begin by connecting to AWS and Redshift: 181 | 182 | ```ruby 183 | # setup AWS credentials 184 | Aws.config.update({ 185 | region: 'us-east-1', 186 | credentials: Aws::Credentials.new( 187 | ENV['AWS_ACCESS_KEY_ID'], 188 | ENV['AWS_SECRET_ACCESS_KEY']) 189 | }) 190 | 191 | # connect to Redshift 192 | db = PG.connect( 193 | host: ENV['REDSHIFT_HOST'], 194 | port: ENV['REDSHIFT_PORT'], 195 | user: ENV['REDSHIFT_USER'], 196 | password: ENV['REDSHIFT_PASSWORD'], 197 | dbname: ENV['REDSHIFT_DATABASE'], 198 | ) 199 | ``` 200 | 201 | This is the heart of the process. The source data comes from the User table. We're fetching users in fixed-size batches to avoid timeouts. For now, we're querying for all users, but you can modify this to return only active users, for example. 202 | 203 | Don't be alarmed by all the nested blocks--we're just creating temporary files, generating an array with the values for each column, and then compressing the data using gzip so we can save time and money. We're not doing any transformation here, but you could do things like format a column or generate new columns. We upload each CSV file to our S3 bucket after processing each batch but you could upload after everything is generated if desired. 204 | 205 | ```ruby 206 | # extract data to CSV files and upload to S3 207 | User.find_in_batches(batch_size: BATCH_SIZE).with_index do |group, batch| 208 | Tempfile.open(TABLE) do |f| 209 | Zlib::GzipWriter.open(f) do |gz| 210 | csv_string = CSV.generate do |csv| 211 | group.each do |record| 212 | csv << COLUMNS.map{|x| record.send(x)} 213 | end 214 | end 215 | gz.write csv_string 216 | end 217 | # upload to s3 218 | s3 = Aws::S3::Resource.new 219 | key = "#{TABLE}/data-#{batch}.gz" 220 | obj = s3.bucket(BUCKET).object(key) 221 | obj.upload_file(f) 222 | end 223 | end 224 | ``` 225 | 226 | Finally, we clear existing data in this Redshift table and tell Redshift to load the new data from S3. Note that we are specifying the column names for the table so that the right data goes to the right columns in the database. We also specify "GZIP" so that Redshift knows that the files are compressed. Using multiple files also allows Redshift to load data in parallel if you have multiple nodes. 227 | 228 | ```ruby 229 | # clear existing data for this table 230 | db.exec <<-EOS 231 | TRUNCATE #{TABLE} 232 | EOS 233 |   234 | # load the data, specifying the order of the fields 235 | db.exec <<-EOS 236 | COPY #{TABLE} (#{COLUMNS.join(', ')}) 237 | FROM 's3://#{BUCKET}/#{TABLE}/data' 238 | CREDENTIALS 'aws_access_key_id=#{ENV['AWS_ACCESS_KEY_ID']};aws_secret_access_key=#{ENV['AWS_SECRET_ACCESS_KEY']}' 239 | CSV 240 | EMPTYASNULL 241 | GZIP 242 | EOS 243 | ``` 244 | 245 | There are other improvements you can add. For example, using a manifest file, you can have full control over which CSVs are loaded. Also, while the current approach truncates and reloads the table on each run, which can be slow, you can do incremental loads. 246 | 247 | ## Links 248 | 249 | * [Sample Rails app on Github](https://github.com/tuesy/redshift-ruby-tutorial) 250 | * [Postico](https://eggerapps.at/postico/) 251 | * [Redshift documentation](https://aws.amazon.com/documentation/redshift/) 252 | * [activerecord adapter](https://github.com/aamine/activerecord4-redshift-adapter) 253 | * [@mankindforward](https://twitter.com/mankindforward) 254 | 255 | ## We're hiring! 256 | Checkout our [jobs page](https://angel.co/credible/jobs/) 257 | --------------------------------------------------------------------------------