├── .gitignore ├── .rspec ├── .ruby-version ├── .travis.yml ├── Appraisals ├── CHANGELOG.md ├── CODE_OF_CONDUCT.md ├── Gemfile ├── LICENSE.txt ├── README.md ├── Rakefile ├── gemfiles ├── rails32.gemfile ├── rails40.gemfile ├── rails41.gemfile ├── rails42.gemfile ├── rails50.gemfile ├── rails51.gemfile ├── rails52.gemfile ├── rails60.gemfile └── rails70.gemfile ├── lib ├── initializers │ ├── arel.rb │ └── big_decimal.rb ├── polo.rb └── polo │ ├── adapters │ ├── mysql.rb │ └── postgres.rb │ ├── collector.rb │ ├── configuration.rb │ ├── sql_translator.rb │ ├── translator.rb │ └── version.rb ├── polo.gemspec └── spec ├── adapters ├── mysql_spec.rb └── postgres_spec.rb ├── configuration_spec.rb ├── polo_spec.rb ├── spec_helper.rb ├── sql_translator_spec.rb ├── support ├── activerecord_models.rb ├── factories.rb └── schema.rb └── translator_spec.rb /.gitignore: -------------------------------------------------------------------------------- 1 | /.bundle/ 2 | /.yardoc 3 | /Gemfile.lock 4 | /_yardoc/ 5 | /coverage/ 6 | /doc/ 7 | /pkg/ 8 | /spec/reports/ 9 | /tmp/ 10 | /gemfiles/.bundle/ 11 | /gemfiles/*.gemfile.lock 12 | -------------------------------------------------------------------------------- /.rspec: -------------------------------------------------------------------------------- 1 | --format documentation 2 | --color 3 | -------------------------------------------------------------------------------- /.ruby-version: -------------------------------------------------------------------------------- 1 | 2.7.5 2 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: ruby 2 | rvm: 3 | - 2.6.5 4 | gemfile: 5 | - gemfiles/rails32.gemfile 6 | - gemfiles/rails40.gemfile 7 | - gemfiles/rails41.gemfile 8 | - gemfiles/rails42.gemfile 9 | - gemfiles/rails50.gemfile 10 | - gemfiles/rails51.gemfile 11 | - gemfiles/rails52.gemfile 12 | - gemfiles/rails60.gemfile 13 | before_install: gem install bundler -v 2.1.4 14 | -------------------------------------------------------------------------------- /Appraisals: -------------------------------------------------------------------------------- 1 | appraise "rails32" do 2 | gem "activerecord", "3.2.22.5" 3 | gem "sqlite3", "1.3.13" 4 | end 5 | 6 | appraise "rails40" do 7 | gem "activerecord", "4.0.13" 8 | gem "sqlite3", "1.3.13" 9 | end 10 | 11 | appraise "rails41" do 12 | gem "activerecord", "4.1.16" 13 | gem "sqlite3", "1.3.13" 14 | end 15 | 16 | appraise "rails42" do 17 | gem "activerecord", "4.2.11.1" 18 | gem "sqlite3", "1.3.13" 19 | end 20 | 21 | appraise "rails50" do 22 | gem "activerecord", "5.0.7.2" 23 | gem "sqlite3", "1.3.13" 24 | end 25 | 26 | appraise "rails51" do 27 | gem "activerecord", "5.1.7" 28 | gem "sqlite3", "1.4.2" 29 | end 30 | 31 | appraise "rails52" do 32 | gem "activerecord", "5.2.4.2" 33 | gem "sqlite3", "1.4.2" 34 | end 35 | 36 | appraise "rails60" do 37 | gem "activerecord", "6.0.0" 38 | end 39 | 40 | appraise "rails70" do 41 | gem "activerecord", "7.0.0" 42 | end 43 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | ## 0.5 2 | 3 | ### Breaking Changes 4 | 5 | - None 6 | 7 | ### Added 8 | - [#43](https://github.com/IFTTT/polo/pull/43) Rails 5 support 9 | - [#44](https://github.com/IFTTT/polo/pull/44) Remove all Rails 5 deprecation notices 10 | 11 | ## 0.4.1 12 | 13 | ### Breaking Changes 14 | 15 | - None 16 | 17 | ### Fixed 18 | - [#42](https://github.com/IFTTT/polo/pull/42) Escape columns in DUPLICATE KEY statements 19 | 20 | ## 0.4.0 21 | 22 | ### Breaking Changes 23 | 24 | - None 25 | 26 | ### Added 27 | - [#40](https://github.com/IFTTT/polo/pull/40) Support for optional 2nd instance argument 28 | 29 | 30 | ## 0.3.0 31 | 32 | ### Breaking Changes 33 | 34 | - None 35 | 36 | ### Added 37 | 38 | - [#30](https://github.com/IFTTT/polo/pull/30) Advanced obfuscation 39 | - [#37](https://github.com/IFTTT/polo/pull/37) Custom adapters for Postgres and MySQL 40 | 41 | ### Fixed 42 | 43 | - [#26](https://github.com/IFTTT/polo/pull/26) Postgres - Use ActiveRecord methods to generate INSERT SQLs 44 | - [#25](https://github.com/IFTTT/polo/pull/25) Fix custom strategies bug 45 | - [#28](https://github.com/IFTTT/polo/pull/28) Only obfuscate fields when they are present 46 | - [#35](https://github.com/IFTTT/polo/pull/35) Better support for Rails 4.0 47 | - [#31](https://github.com/IFTTT/polo/pull/31) Fix link to Code of Conduct 48 | 49 | ## 0.2.0 50 | 51 | ### Breaking Changes 52 | 53 | - None 54 | 55 | ### Added 56 | 57 | - [#8](https://github.com/IFTTT/polo/pull/8) Global settings 58 | - [#17](https://github.com/IFTTT/polo/pull/17) Using random generator instead of character shuffle for data obfuscation 59 | - [#18](https://github.com/IFTTT/polo/pull/18) Add a CHANGELOG 60 | - [#20](https://github.com/IFTTT/polo/pull/20) Postgres Support 61 | 62 | ### Fixed 63 | 64 | - Typo fixes on the README: [#9](https://github.com/IFTTT/polo/pull/9), [#10](https://github.com/IFTTT/polo/pull/10) 65 | - [#11]() Some ActiveRecord classes do not use id as the primary key 66 | - [#25](https://github.com/IFTTT/polo/pull/25) Fix Custom Strategy 67 | 68 | ## 0.1.0 69 | 70 | ### Breaking Changes 71 | 72 | - None 73 | 74 | ### Added 75 | 76 | - [#2](https://github.com/IFTTT/polo/pull/2) Add :ignore and :override options to deal with data collision 77 | - [#3](https://github.com/IFTTT/polo/pull/3) Add option to obfuscate fields 78 | - [#4](https://github.com/IFTTT/polo/pull/4) Add intro to Update / Ignore section 79 | - [#6](https://github.com/IFTTT/polo/pull/6) Set up Appraisal to run specs across Rails 3.2 through 4.2 80 | 81 | ### Fixed 82 | 83 | - [#7](https://github.com/IFTTT/polo/pull/7) Fix casting of values 84 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Contributor Code of Conduct 2 | 3 | As contributors and maintainers of this project, we pledge to respect all people who contribute through reporting issues, posting feature requests, updating documentation, submitting pull requests or patches, and other activities. 4 | 5 | We are committed to making participation in this project a harassment-free experience for everyone, regardless of level of experience, gender, gender identity and expression, sexual orientation, disability, personal appearance, body size, race, ethnicity, age, or religion. 6 | 7 | Examples of unacceptable behavior by participants include the use of sexual language or imagery, derogatory comments or personal attacks, trolling, public or private harassment, insults, or other unprofessional conduct. 8 | 9 | Project maintainers have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, issues, and other contributions that are not aligned to this Code of Conduct. Project maintainers who do not follow the Code of Conduct may be removed from the project team. 10 | 11 | Instances of abusive, harassing, or otherwise unacceptable behavior may be reported by opening an issue or contacting one or more of the project maintainers. 12 | 13 | This Code of Conduct is adapted from the [Contributor Covenant](http://contributor-covenant.org), version 1.0.0, available at [http://contributor-covenant.org/version/1/0/0/](http://contributor-covenant.org/version/1/0/0/) 14 | -------------------------------------------------------------------------------- /Gemfile: -------------------------------------------------------------------------------- 1 | source 'https://rubygems.org' 2 | 3 | gemspec 4 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2015 IFTTT 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in 13 | all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | THE SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [![Open Source at IFTTT](http://ifttt.github.io/images/open-source-ifttt.svg)](http://ifttt.github.io) 2 | [![Build Status](https://travis-ci.org/IFTTT/polo.svg?branch=master)](https://travis-ci.org/IFTTT/polo) 3 | 4 | ![Polo](https://raw.githubusercontent.com/IFTTT/polo/images/images/polo.png "Polo") 5 | 6 | # Polo 7 | Polo travels through your database and creates sample snapshots so you can work with real world data in any environment. 8 | 9 | Polo takes an `ActiveRecord::Base` seed object and traverses every whitelisted `ActiveRecord::Association` generating SQL `INSERTs` along the way. 10 | 11 | You can then save those SQL `INSERTS` to .sql file and import the data to your favorite environment. 12 | 13 | ## Motivation 14 | Read our [blog post](https://medium.com/engineering-at-ifttt/happier-rails-development-with-polo-9df6819136d3#.f8ll3azeq) or check out this [presentation](https://speakerdeck.com/nettofarah/polo-working-with-real-world-data-in-development). 15 | 16 | ## Usage 17 | Given the following data model: 18 | ```ruby 19 | class Chef < ActiveRecord::Base 20 | has_many :recipes 21 | has_many :ingredients, through: :recipes 22 | end 23 | 24 | class Recipe < ActiveRecord::Base 25 | has_many :recipes_ingredients 26 | has_many :ingredients, through: :recipes_ingredients 27 | end 28 | 29 | class Ingredient < ActiveRecord::Base 30 | end 31 | 32 | class RecipesIngredient < ActiveRecord::Base 33 | belongs_to :recipe 34 | belongs_to :ingredient 35 | end 36 | ``` 37 | 38 | ### Simple ActiveRecord Objects 39 | ```ruby 40 | inserts = Polo.explore(Chef, 1) 41 | ``` 42 | ```sql 43 | INSERT INTO `chefs` (`id`, `name`) VALUES (1, 'Netto') 44 | ``` 45 | 46 | Where `Chef` is the seed object class, and `1` is the seed object id. 47 | 48 | ### Simple Associations 49 | ```ruby 50 | inserts = Polo.explore(Chef, 1, :recipes) 51 | ``` 52 | ```sql 53 | INSERT INTO `chefs` (`id`, `name`) VALUES (1, 'Netto') 54 | INSERT INTO `recipes` (`id`, `title`, `num_steps`, `chef_id`) VALUES (1, 'Turkey Sandwich', NULL, 1) 55 | INSERT INTO `recipes` (`id`, `title`, `num_steps`, `chef_id`) VALUES (2, 'Cheese Burger', NULL, 1) 56 | ``` 57 | 58 | ### Complex nested associations 59 | ```ruby 60 | inserts = Polo.explore(Chef, 1, :recipes => :ingredients) 61 | ``` 62 | 63 | ```sql 64 | INSERT INTO `chefs` (`id`, `name`) VALUES (1, 'Netto') 65 | INSERT INTO `recipes` (`id`, `title`, `num_steps`, `chef_id`) VALUES (1, 'Turkey Sandwich', NULL, 1) 66 | INSERT INTO `recipes` (`id`, `title`, `num_steps`, `chef_id`) VALUES (2, 'Cheese Burger', NULL, 1) 67 | INSERT INTO `recipes_ingredients` (`id`, `recipe_id`, `ingredient_id`) VALUES (1, 1, 1) 68 | INSERT INTO `recipes_ingredients` (`id`, `recipe_id`, `ingredient_id`) VALUES (2, 1, 2) 69 | INSERT INTO `recipes_ingredients` (`id`, `recipe_id`, `ingredient_id`) VALUES (3, 2, 3) 70 | INSERT INTO `recipes_ingredients` (`id`, `recipe_id`, `ingredient_id`) VALUES (4, 2, 4) 71 | INSERT INTO `ingredients` (`id`, `name`, `quantity`) VALUES (1, 'Turkey', 'a lot') 72 | INSERT INTO `ingredients` (`id`, `name`, `quantity`) VALUES (2, 'Cheese', '1 slice') 73 | INSERT INTO `ingredients` (`id`, `name`, `quantity`) VALUES (3, 'Patty', '1') 74 | INSERT INTO `ingredients` (`id`, `name`, `quantity`) VALUES (4, 'Cheese', '2 slices') 75 | ``` 76 | 77 | ## Advanced Usage 78 | 79 | Occasionally, you might have a dataset that you want to refresh. A production database that has data that might be useful on your local copy of the database. Polo doesn't have an opinion about your data; if you try to import data with a key that's already in your local database, Polo doesn't necessarily know how you want to handle that conflict. 80 | 81 | Advanced users will find the `on_duplicate` option to be helpful in this context. It gives Polo instructions on how to handle collisions. 82 | *Note: This feature is currently only supported for MySQL databases. (PRs for other databases are welcome!)* 83 | 84 | There are two possible values for the `on_duplicate` key: `:ignore` and `:override`. Ignore keeps the old data. Override keeps the new data. If there's a collision and the on_duplicate param is not set, Polo will simpy stop importing the data. 85 | 86 | ### Ignore 87 | A.K.A the Ostrich Approach: stick your head in the sand and pretend nothing happened. 88 | 89 | ```ruby 90 | Polo.configure do 91 | on_duplicate :ignore 92 | end 93 | 94 | Polo::Traveler.explore(Chef, 1, :recipes) 95 | ``` 96 | 97 | ```sql 98 | INSERT IGNORE INTO `chefs` (`id`, `name`) VALUES (1, 'Netto') 99 | INSERT IGNORE INTO `recipes` (`id`, `title`, `num_steps`, `chef_id`) VALUES (1, 'Turkey Sandwich', NULL, 1) 100 | INSERT IGNORE INTO `recipes` (`id`, `title`, `num_steps`, `chef_id`) VALUES (2, 'Cheese Burger', NULL, 1) 101 | ``` 102 | 103 | ### Override 104 | Use the option `on_duplicate: :override` to override your local data with new data from your Polo script. 105 | 106 | ```ruby 107 | Polo.configure do 108 | on_duplicate :override 109 | end 110 | 111 | Polo::Traveler.explore(Chef, 1, :recipes) 112 | ``` 113 | 114 | ```sql 115 | INSERT INTO `chefs` (`id`, `name`) VALUES (1, 'Netto') 116 | ON DUPLICATE KEY UPDATE id = VALUES(id), name = VALUES(name) 117 | ... 118 | ``` 119 | 120 | ### Sensitive Fields 121 | You can use the `obfuscate` option to obfuscate sensitive fields like emails or 122 | user logins. 123 | 124 | ```ruby 125 | Polo.configure do 126 | obfuscate :email, :credit_card 127 | end 128 | 129 | Polo::Traveler.explore(AR::Chef, 1) 130 | ``` 131 | 132 | ```sql 133 | INSERT INTO `chefs` (`id`, `name`, `email`) VALUES (1, 'Netto', 'eahorctmaagfo.nitm@l') 134 | ``` 135 | 136 | Warning: This is not a security feature. Fields can still easily be rearranged back to their original format. Polo will simply scramble the order of strings so you don't accidentally end up causing side effects when using production data in development. It is not a good practice to use highly sensitive data in development. 137 | 138 | #### Advanced Obfuscation 139 | 140 | For more advanced obfuscation, you can pass in a custom obfuscation strategy. Polo will take in a lambda that can be used to transform sensitive data. 141 | 142 | Using a `:symbol` as an obfuscate key targets all columns of that name. Passing an SQL selector as a `String` will target columns within the specified table. 143 | 144 | ````ruby 145 | Polo.configure do 146 | 147 | email_strategy = lambda do |email| 148 | first_part = email.split("@")[0] 149 | "#{first_part}@test.com" 150 | end 151 | 152 | credit_card_strategy = lambda do |credit_card| 153 | "4123 4567 8910 1112" 154 | end 155 | 156 | # If you need the context of the record for its fields, it is accessible 157 | # in the second argument of the strategy 158 | social_security_strategy = lambda do |ssn, instance| 159 | sprintf("%09d", instance.id) 160 | end 161 | 162 | obfuscate({ 163 | 'chefs.email' => email_strategy, # This only applies to the "email" column in the "chefs" table 164 | :credit_card => credit_card_strategy, # This applies to any column named "credit_card" across every table 165 | :ssn_strategy => social_security_strategy 166 | }) 167 | end 168 | 169 | Polo::Traveler.explore(AR::Chef, 1) 170 | ```` 171 | 172 | ```sql 173 | INSERT INTO `chefs` (`id`, `name`, `email`) VALUES (1, 'Netto', 'netto_test.@example.com') 174 | ``` 175 | 176 | ## Installation 177 | 178 | Add this line to your application's Gemfile: 179 | 180 | ```ruby 181 | gem 'polo' 182 | ``` 183 | 184 | And then execute: 185 | ```bash 186 | $ bundle 187 | ``` 188 | 189 | Or install it yourself as: 190 | ```bash 191 | $ gem install polo 192 | ``` 193 | 194 | ## Contributing 195 | 196 | Bug reports and pull requests are welcome on GitHub at https://github.com/IFTTT/polo. This project is intended to be a safe, welcoming space for collaboration, and contributors are expected to adhere to the [Code of Conduct](https://github.com/IFTTT/polo/blob/master/CODE_OF_CONDUCT.md). 197 | 198 | To run the specs across all supported version of Rails, check out the repo and 199 | follow these steps: 200 | 201 | ```bash 202 | $ bundle install 203 | $ bundle exec appraisal install 204 | $ bundle exec appraisal rake 205 | ``` 206 | 207 | ## License 208 | 209 | The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT). 210 | -------------------------------------------------------------------------------- /Rakefile: -------------------------------------------------------------------------------- 1 | require "bundler/gem_tasks" 2 | require "rspec/core/rake_task" 3 | 4 | RSpec::Core::RakeTask.new(:spec) 5 | 6 | task :default => :spec 7 | -------------------------------------------------------------------------------- /gemfiles/rails32.gemfile: -------------------------------------------------------------------------------- 1 | # This file was generated by Appraisal 2 | 3 | source "https://rubygems.org" 4 | 5 | gem "activerecord", "3.2.22.5" 6 | gem "sqlite3", "1.3.13" 7 | 8 | gemspec path: "../" 9 | -------------------------------------------------------------------------------- /gemfiles/rails40.gemfile: -------------------------------------------------------------------------------- 1 | # This file was generated by Appraisal 2 | 3 | source "https://rubygems.org" 4 | 5 | gem "activerecord", "4.0.13" 6 | gem "sqlite3", "1.3.13" 7 | 8 | gemspec path: "../" 9 | -------------------------------------------------------------------------------- /gemfiles/rails41.gemfile: -------------------------------------------------------------------------------- 1 | # This file was generated by Appraisal 2 | 3 | source "https://rubygems.org" 4 | 5 | gem "activerecord", "4.1.16" 6 | gem "sqlite3", "1.3.13" 7 | 8 | gemspec path: "../" 9 | -------------------------------------------------------------------------------- /gemfiles/rails42.gemfile: -------------------------------------------------------------------------------- 1 | # This file was generated by Appraisal 2 | 3 | source "https://rubygems.org" 4 | 5 | gem "activerecord", "4.2.11.1" 6 | gem "sqlite3", "1.3.13" 7 | 8 | gemspec path: "../" 9 | -------------------------------------------------------------------------------- /gemfiles/rails50.gemfile: -------------------------------------------------------------------------------- 1 | # This file was generated by Appraisal 2 | 3 | source "https://rubygems.org" 4 | 5 | gem "activerecord", "5.0.7.2" 6 | gem "sqlite3", "1.3.13" 7 | 8 | gemspec path: "../" 9 | -------------------------------------------------------------------------------- /gemfiles/rails51.gemfile: -------------------------------------------------------------------------------- 1 | # This file was generated by Appraisal 2 | 3 | source "https://rubygems.org" 4 | 5 | gem "activerecord", "5.1.7" 6 | gem "sqlite3", "1.4.2" 7 | 8 | gemspec path: "../" 9 | -------------------------------------------------------------------------------- /gemfiles/rails52.gemfile: -------------------------------------------------------------------------------- 1 | # This file was generated by Appraisal 2 | 3 | source "https://rubygems.org" 4 | 5 | gem "activerecord", "5.2.4.2" 6 | gem "sqlite3", "1.4.2" 7 | 8 | gemspec path: "../" 9 | -------------------------------------------------------------------------------- /gemfiles/rails60.gemfile: -------------------------------------------------------------------------------- 1 | # This file was generated by Appraisal 2 | 3 | source "https://rubygems.org" 4 | 5 | gem "activerecord", "6.0.0" 6 | 7 | gemspec path: "../" 8 | -------------------------------------------------------------------------------- /gemfiles/rails70.gemfile: -------------------------------------------------------------------------------- 1 | # This file was generated by Appraisal 2 | 3 | source "https://rubygems.org" 4 | 5 | gem "activerecord", "7.0.0" 6 | 7 | gemspec path: "../" 8 | -------------------------------------------------------------------------------- /lib/initializers/arel.rb: -------------------------------------------------------------------------------- 1 | # https://stackoverflow.com/a/44286212 2 | if ActiveRecord::VERSION::MAJOR < 5 3 | module Arel 4 | module Visitors 5 | class DepthFirst < Arel::Visitors::Visitor 6 | alias :visit_Integer :terminal 7 | end 8 | 9 | class Dot < Arel::Visitors::Visitor 10 | alias :visit_Integer :visit_String 11 | end 12 | 13 | if ActiveRecord::VERSION::MAJOR == 4 && ActiveRecord::VERSION::MINOR >= 2 14 | class ToSql < Arel::Visitors::Reduce 15 | alias :visit_Integer :literal 16 | end 17 | else 18 | class ToSql < Arel::Visitors::Visitor 19 | alias :visit_Integer :literal 20 | end 21 | end 22 | end 23 | end 24 | end 25 | -------------------------------------------------------------------------------- /lib/initializers/big_decimal.rb: -------------------------------------------------------------------------------- 1 | # Support old BigDecimal constructor for sqlite3 (among others?) 2 | class BigDecimal 3 | def self.new(*args) 4 | BigDecimal(*args) 5 | end 6 | end 7 | -------------------------------------------------------------------------------- /lib/polo.rb: -------------------------------------------------------------------------------- 1 | require "polo/version" 2 | require "polo/collector" 3 | require "polo/translator" 4 | require "polo/configuration" 5 | require "initializers/arel" 6 | require "initializers/big_decimal" 7 | 8 | module Polo 9 | 10 | # Public: Traverses a dependency graph based on a seed ActiveRecord object 11 | # and generates all the necessary INSERT queries for each one of the records 12 | # it finds along the way. 13 | # 14 | # base_class - An ActiveRecord::Base class for the seed record. 15 | # id - An ID used to find the desired seed record. 16 | # 17 | # dependency_tree - An ActiveRecord::Associations::Preloader compliant that 18 | # will define the path Polo will traverse. 19 | # 20 | # (from ActiveRecord::Associations::Preloader docs) 21 | # It may be: 22 | # - a Symbol or a String which specifies a single association name. For 23 | # example, specifying +:books+ allows this method to preload all books 24 | # for an Author. 25 | # - an Array which specifies multiple association names. This array 26 | # is processed recursively. For example, specifying [:avatar, :books] 27 | # allows this method to preload an author's avatar as well as all of his 28 | # books. 29 | # - a Hash which specifies multiple association names, as well as 30 | # association names for the to-be-preloaded association objects. For 31 | # example, specifying { author: :avatar } will preload a 32 | # book's author, as well as that author's avatar. 33 | # 34 | # +:associations+ has the same format as the +:include+ option for 35 | # ActiveRecord::Base.find. So +associations+ could look like this: 36 | # 37 | # :books 38 | # [ :books, :author ] 39 | # { author: :avatar } 40 | # [ :books, { author: :avatar } ] 41 | # 42 | def self.explore(base_class, id, dependencies={}) 43 | Traveler.collect(base_class, id, dependencies).translate(defaults) 44 | end 45 | 46 | 47 | # Public: Sets up global settings for Polo 48 | # 49 | # block - Takes a block with the settings you decide to use 50 | # 51 | # obfuscate - Takes a blacklist with sensitive fields you wish to scramble 52 | # on_duplicate - Defines the on_duplicate strategy for your INSERTS 53 | # e.g. :override, :ignore 54 | # 55 | # usage: 56 | # Polo.configure do 57 | # obfuscate(:email, :password, :credit_card) 58 | # on_duplicate(:override) 59 | # end 60 | # 61 | def self.configure(&block) 62 | @configuration = Configuration.new 63 | @configuration.instance_eval(&block) if block_given? 64 | @configuration 65 | end 66 | 67 | # Public: Returns the default settings 68 | # 69 | def self.defaults 70 | @configuration || configure 71 | end 72 | 73 | 74 | class Traveler 75 | 76 | def self.collect(base_class, id, dependencies={}) 77 | selects = Collector.new(base_class, id, dependencies).collect 78 | new(selects) 79 | end 80 | 81 | def initialize(selects) 82 | @selects = selects 83 | end 84 | 85 | def translate(configuration=Configuration.new) 86 | Translator.new(@selects, configuration).translate 87 | end 88 | end 89 | end 90 | -------------------------------------------------------------------------------- /lib/polo/adapters/mysql.rb: -------------------------------------------------------------------------------- 1 | module Polo 2 | module Adapters 3 | class MySQL 4 | def on_duplicate_key_update(inserts, records) 5 | insert_and_record = inserts.zip(records) 6 | insert_and_record.map do |insert, record| 7 | attrs = record.is_a?(Hash) ? record.fetch(:values) : record.attributes 8 | values_syntax = attrs.keys.map do |key| 9 | "`#{key}` = VALUES(`#{key}`)" 10 | end 11 | 12 | on_dup_syntax = "ON DUPLICATE KEY UPDATE #{values_syntax.join(', ')}" 13 | 14 | "#{insert} #{on_dup_syntax}" 15 | end 16 | end 17 | 18 | def ignore_transform(inserts, records) 19 | inserts.map do |insert| 20 | insert.gsub("INSERT", "INSERT IGNORE") 21 | end 22 | end 23 | end 24 | end 25 | end -------------------------------------------------------------------------------- /lib/polo/adapters/postgres.rb: -------------------------------------------------------------------------------- 1 | module Polo 2 | module Adapters 3 | class Postgres 4 | # TODO: Implement UPSERT. This command became available in 9.1. 5 | # 6 | # See: http://www.the-art-of-web.com/sql/upsert/ 7 | def on_duplicate_key_update(inserts, records) 8 | raise 'on_duplicate: :override is not currently supported in the PostgreSQL adapter' 9 | end 10 | 11 | # Internal: Transforms an INSERT with PostgreSQL-specific syntax. Ignores 12 | # records that already exist in the table. To do this, it uses 13 | # a heuristic, i.e. checks if there is a record with the same id 14 | # in the table. 15 | # See: http://stackoverflow.com/a/6527838/32816 16 | # 17 | # inserts - The Array of INSERT statements. 18 | # records - The Array of Arel objects. 19 | # 20 | # Returns the Array of transformed INSERT statements. 21 | def ignore_transform(inserts, records) 22 | insert_and_record = inserts.zip(records) 23 | insert_and_record.map do |insert, record| 24 | if record.is_a?(Hash) 25 | id = record.fetch(:values)[:id] 26 | table_name = record.fetch(:table_name) 27 | else 28 | id = record[:id] 29 | table_name = record.class.arel_table.name 30 | end 31 | insert = insert.gsub(/VALUES \((.+)\)$/m, 'SELECT \\1') 32 | insert << " WHERE NOT EXISTS (SELECT 1 FROM #{table_name} WHERE id=#{id});" 33 | end 34 | end 35 | end 36 | end 37 | end 38 | -------------------------------------------------------------------------------- /lib/polo/collector.rb: -------------------------------------------------------------------------------- 1 | module Polo 2 | class Collector 3 | 4 | def initialize(base_class, id, dependency_tree={}) 5 | @base_class = base_class 6 | @id = id 7 | @dependency_tree = dependency_tree 8 | @selects = [] 9 | end 10 | 11 | # Public: Traverses the dependency tree and collects every SQL query. 12 | # 13 | # This is done by wrapping a top level call to includes(...) with a 14 | # ActiveSupport::Notifications block and collecting every generate SQL query. 15 | # 16 | def collect 17 | unprepared_statement do 18 | ActiveSupport::Notifications.subscribed(collector, 'sql.active_record') do 19 | base_finder = @base_class.includes(@dependency_tree).where(@base_class.primary_key => @id) 20 | collect_sql(klass: @base_class, sql: base_finder.to_sql) 21 | base_finder.to_a 22 | end 23 | end 24 | 25 | @selects.compact.uniq 26 | end 27 | 28 | private 29 | 30 | # Internal: Store ActiveRecord queries in @selects 31 | # 32 | # Collector will intersect every ActiveRecord query performed within the 33 | # ActiveSupport::Notifications.subscribed block defined in #run and store 34 | # the resulting SQL query in @selects 35 | # 36 | def collector 37 | lambda do |name, start, finish, id, payload| 38 | sql = payload[:sql] 39 | if payload[:name] =~ /^HABTM_.* Load$/ 40 | collect_sql(connection: @base_class.connection, sql: sql) 41 | elsif payload[:name] =~ /^(.*) Load$/ 42 | begin 43 | class_name = $1.constantize 44 | collect_sql(klass: class_name, sql: sql) 45 | rescue ActiveRecord::StatementInvalid, NameError 46 | # invalid table name (common when prefetching schemas) 47 | end 48 | end 49 | end 50 | end 51 | 52 | def collect_sql(select) 53 | @selects << select 54 | end 55 | 56 | def unprepared_statement 57 | if ActiveRecord::Base.connection.respond_to?(:unprepared_statement) 58 | ActiveRecord::Base.connection.unprepared_statement do 59 | yield 60 | end 61 | else 62 | yield 63 | end 64 | end 65 | end 66 | end 67 | -------------------------------------------------------------------------------- /lib/polo/configuration.rb: -------------------------------------------------------------------------------- 1 | module Polo 2 | 3 | class Configuration 4 | attr_reader :on_duplicate_strategy, :blacklist, :adapter 5 | 6 | def initialize(options={}) 7 | options = { on_duplicate: nil, obfuscate: {}, adapter: :mysql }.merge(options) 8 | @adapter = options[:adapter] 9 | @on_duplicate_strategy = options[:on_duplicate] 10 | obfuscate(options[:obfuscate]) 11 | end 12 | 13 | # TODO: document this 14 | # This normalizes an array or hash of fields to a hash of 15 | # { field_name => strategy } 16 | def obfuscate(*fields) 17 | if fields.is_a?(Array) 18 | fields = fields.flatten 19 | end 20 | 21 | fields_and_strategies = {} 22 | 23 | fields.each do |field| 24 | if field.is_a?(Symbol) || field.is_a?(String) 25 | fields_and_strategies[field] = nil 26 | elsif field.is_a?(Hash) 27 | fields_and_strategies = fields_and_strategies.merge(field) 28 | end 29 | end 30 | 31 | @blacklist = fields_and_strategies 32 | end 33 | 34 | def on_duplicate(strategy) 35 | @on_duplicate_strategy = strategy 36 | end 37 | 38 | def set_adapter(db) 39 | @adapter = db 40 | end 41 | end 42 | end 43 | -------------------------------------------------------------------------------- /lib/polo/sql_translator.rb: -------------------------------------------------------------------------------- 1 | require 'active_record' 2 | require 'polo/configuration' 3 | require 'polo/adapters/mysql' 4 | require 'polo/adapters/postgres' 5 | 6 | module Polo 7 | class SqlTranslator 8 | 9 | def initialize(object, configuration = Configuration.new) 10 | @record = object 11 | @configuration = configuration 12 | 13 | case @configuration.adapter 14 | when :mysql 15 | @adapter = Polo::Adapters::MySQL.new 16 | when :postgres 17 | @adapter = Polo::Adapters::Postgres.new 18 | else 19 | raise "Unknown SQL adapter: #{@configuration.adapter}" 20 | end 21 | end 22 | 23 | def to_sql 24 | case @configuration.on_duplicate_strategy 25 | when :ignore 26 | @adapter.ignore_transform(inserts, records) 27 | when :override 28 | @adapter.on_duplicate_key_update(inserts, records) 29 | else inserts 30 | end 31 | end 32 | 33 | def records 34 | Array.wrap(@record) 35 | end 36 | 37 | def inserts 38 | records.map do |record| 39 | if record.is_a?(Hash) 40 | raw_sql_from_hash(record) 41 | else 42 | raw_sql_from_record(record) 43 | end 44 | end 45 | end 46 | 47 | private 48 | 49 | # Internal: Generates an insert SQL statement for a given record 50 | # 51 | # It will make use of the InsertManager class from the Arel gem to generate 52 | # insert statements 53 | # 54 | def raw_sql_from_record(record) 55 | record.class.arel_table.create_insert.tap do |insert_manager| 56 | insert_manager.insert(insert_values(record)) 57 | end.to_sql 58 | end 59 | 60 | # Internal: Generates an insert SQL statement from a hash of values 61 | def raw_sql_from_hash(hash) 62 | connection = ActiveRecord::Base.connection 63 | attributes = hash.fetch(:values) 64 | table_name = connection.quote_table_name(hash.fetch(:table_name)) 65 | columns = attributes.keys.map{|k| connection.quote_column_name(k)}.join(", ") 66 | value_placeholders = attributes.values.map{|v| "?" }.join(", ") 67 | ActiveRecord::Base.send(:sanitize_sql_array, ["INSERT INTO #{table_name} (#{columns}) VALUES (#{value_placeholders})", *attributes.values]) 68 | end 69 | 70 | # Internal: Returns an object's attribute definitions along with 71 | # their set values (for Rails 3.x). 72 | # 73 | module ActiveRecordLessThanFour 74 | def insert_values(record) 75 | record.send(:arel_attributes_values) 76 | end 77 | end 78 | 79 | # Internal: Returns an object's attribute definitions along with 80 | # their set values (for Rails >= 4.x). 81 | # 82 | # From Rails 4.2 onwards, for some reason attributes with custom serializers 83 | # wouldn't be properly serialized automatically. That's why explict 84 | # 'type_cast' call are necessary. 85 | # 86 | module ActiveRecordFour 87 | def insert_values(record) 88 | connection = ActiveRecord::Base.connection 89 | values = record.send(:arel_attributes_with_values_for_create, connection.schema_cache.columns(record.class.table_name).map(&:name)) 90 | values.each do |attribute, value| 91 | column = record.send(:column_for_attribute, attribute.name) 92 | values[attribute] = connection.type_cast(value, column) 93 | end 94 | end 95 | end 96 | 97 | # Internal: Returns an object's attribute definitions along with 98 | # their set values (for Rails 5.0 & 5.1). 99 | # 100 | # Serializers have changed again in rails 5. 101 | # We now use the type_caster from the arel_table. 102 | # 103 | module ActiveRecordFivePointZeroOrOne 104 | # Based on the codepath used in Rails 5 105 | def raw_sql_from_record(record) 106 | values = record.send(:arel_attributes_with_values_for_create, record.class.column_names) 107 | model = record.class 108 | substitutes, binds = model.unscoped.substitute_values(values) 109 | 110 | insert_manager = model.arel_table.create_insert 111 | insert_manager.insert substitutes 112 | 113 | model.connection.unprepared_statement do 114 | model.connection.to_sql(insert_manager, binds) 115 | end 116 | end 117 | end 118 | 119 | # Internal: Returns an object's attribute definitions along with 120 | # their set values (for Rails >= 5.2). 121 | module ActiveRecordFive 122 | def raw_sql_from_record(record) 123 | values = record.send(:attributes_with_values_for_create, record.class.column_names) 124 | model = record.class 125 | substitutes_and_binds = model.send(:_substitute_values, values) 126 | 127 | insert_manager = model.arel_table.create_insert 128 | insert_manager.insert substitutes_and_binds 129 | 130 | model.connection.unprepared_statement do 131 | model.connection.to_sql(insert_manager) 132 | end 133 | end 134 | end 135 | 136 | # Internal: Returns an object's attribute definitions along with 137 | # their set values (for Rails 6.0). 138 | module ActiveRecordSix 139 | def raw_sql_from_record(record) 140 | model = record.class 141 | values = record.send(:attributes_with_values, record.send(:attributes_for_create, model.column_names)) 142 | substitutes_and_binds = model.send(:_substitute_values, values) 143 | 144 | insert_manager = model.arel_table.create_insert 145 | insert_manager.insert substitutes_and_binds 146 | 147 | model.connection.unprepared_statement do 148 | model.connection.to_sql(insert_manager) 149 | end 150 | end 151 | end 152 | 153 | # Internal: Returns an object's attribute definitions along with 154 | # their set values (for Rails 7.0). 155 | module ActiveRecordSeven 156 | def raw_sql_from_record(record) 157 | values = record.send(:attributes_with_values, record.class.column_names) 158 | model = record.class 159 | substitutes_and_binds = values.transform_keys { |name| model.arel_table[name] } 160 | 161 | insert_manager = Arel::InsertManager.new(model.arel_table) 162 | insert_manager.insert substitutes_and_binds 163 | 164 | model.connection.unprepared_statement do 165 | model.connection.to_sql(insert_manager) 166 | end 167 | end 168 | end 169 | 170 | if ActiveRecord::VERSION::MAJOR < 4 171 | include ActiveRecordLessThanFour 172 | elsif ActiveRecord::VERSION::MAJOR == 4 173 | include ActiveRecordFour 174 | elsif ActiveRecord::VERSION::MAJOR == 5 && ActiveRecord::VERSION::MINOR < 2 175 | prepend ActiveRecordFivePointZeroOrOne 176 | elsif ActiveRecord::VERSION::MAJOR == 5 177 | prepend ActiveRecordFive 178 | elsif ActiveRecord::VERSION::MAJOR == 6 179 | prepend ActiveRecordSix 180 | else 181 | prepend ActiveRecordSeven 182 | end 183 | end 184 | end 185 | -------------------------------------------------------------------------------- /lib/polo/translator.rb: -------------------------------------------------------------------------------- 1 | require "polo/sql_translator" 2 | require "polo/configuration" 3 | 4 | module Polo 5 | class Translator 6 | 7 | # Public: Creates a new Polo::Collector 8 | # 9 | # selects - An array of SELECT queries 10 | # 11 | def initialize(selects, configuration=Configuration.new) 12 | @selects = selects 13 | @configuration = configuration 14 | end 15 | 16 | # Public: Translates SELECT queries into INSERTS. 17 | # 18 | def translate 19 | SqlTranslator.new(instances, @configuration).to_sql.uniq 20 | end 21 | 22 | def instances 23 | active_record_selects, raw_selects = @selects.partition{|s| s[:klass]} 24 | 25 | active_record_instances = active_record_selects.flat_map do |select| 26 | select[:klass].find_by_sql(select[:sql]).to_a 27 | end 28 | 29 | if fields = @configuration.blacklist 30 | obfuscate!(active_record_instances, fields) 31 | end 32 | 33 | raw_instance_values = raw_selects.flat_map do |select| 34 | table_name = select[:sql][/^SELECT .* FROM (?:"|`)([^"`]+)(?:"|`)/, 1] 35 | select[:connection].select_all(select[:sql]).map { |values| {table_name: table_name, values: values} } 36 | end 37 | 38 | active_record_instances + raw_instance_values 39 | end 40 | 41 | private 42 | 43 | def obfuscate!(instances, fields) 44 | instances.each do |instance| 45 | next if intersection(instance.attributes.keys, fields).empty? 46 | 47 | fields.each do |field, strategy| 48 | field = field.to_s 49 | 50 | if table = table_name(field) 51 | field = field_name(field) 52 | end 53 | 54 | correct_table = table.nil? || instance.class.table_name == table 55 | 56 | if correct_table && instance.attributes[field] 57 | instance.send("#{field}=", new_field_value(field, strategy, instance)) 58 | end 59 | end 60 | end 61 | end 62 | 63 | def field_name(field) 64 | field.to_s.include?('.') ? field.split('.').last : field.to_s 65 | end 66 | 67 | def table_name(field) 68 | field.to_s.include?('.') ? field.split('.').first : nil 69 | end 70 | 71 | def intersection(attrs, fields) 72 | attrs & fields.map { |pair| field_name(pair.first) } 73 | end 74 | 75 | def new_field_value(field, strategy, instance) 76 | value = instance.attributes[field] 77 | if strategy.nil? 78 | value.split("").shuffle.join 79 | else 80 | strategy.arity == 1 ? strategy.call(value) : strategy.call(value, instance) 81 | end 82 | end 83 | end 84 | end 85 | -------------------------------------------------------------------------------- /lib/polo/version.rb: -------------------------------------------------------------------------------- 1 | module Polo 2 | VERSION = "0.6.0" 3 | end 4 | -------------------------------------------------------------------------------- /polo.gemspec: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | lib = File.expand_path('../lib', __FILE__) 3 | $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib) 4 | require 'polo/version' 5 | 6 | Gem::Specification.new do |spec| 7 | spec.name = "polo" 8 | spec.version = Polo::VERSION 9 | spec.authors = ["Netto Farah"] 10 | spec.email = ["nettofarah@gmail.com"] 11 | 12 | spec.summary = %q{Bring life back to your development environment with samples from production data.} 13 | spec.description = %q{Polo travels through your database and creates sample snapshots so you can work with real world data in development.} 14 | spec.homepage = "http://ifttt.github.io" 15 | spec.license = "MIT" 16 | 17 | spec.files = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) } 18 | spec.bindir = "exe" 19 | spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) } 20 | spec.require_paths = ["lib"] 21 | 22 | spec.add_dependency "activerecord", ">= 3.2" 23 | 24 | spec.add_development_dependency "appraisal" 25 | spec.add_development_dependency "bundler", "~> 2.3" 26 | spec.add_development_dependency "rake", "~> 13.0.1" 27 | spec.add_development_dependency "rspec" 28 | 29 | spec.add_development_dependency "sqlite3", ">= 1.3.13" 30 | end 31 | -------------------------------------------------------------------------------- /spec/adapters/mysql_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | 3 | describe Polo::Adapters::MySQL do 4 | 5 | let(:adapter) { Polo::Adapters::MySQL.new } 6 | 7 | let(:netto) do 8 | AR::Chef.where(name: 'Netto').first 9 | end 10 | 11 | before(:all) do 12 | TestData.create_netto 13 | end 14 | 15 | let(:translator) { Polo::SqlTranslator.new(netto, Polo::Configuration.new(adapter: :mysql)) } 16 | 17 | describe '#ignore_transform' do 18 | it 'appends the IGNORE command after INSERTs' do 19 | insert_netto = [%q{INSERT IGNORE INTO "chefs" ("id", "name", "email") VALUES (1, 'Netto', 'nettofarah@gmail.com')}] 20 | 21 | records = translator.records 22 | inserts = translator.inserts 23 | translated_sql = adapter.ignore_transform(inserts, records) 24 | expect(translated_sql).to eq(insert_netto) 25 | end 26 | end 27 | 28 | 29 | describe '#on_duplicate_key_update' do 30 | it 'appends ON DUPLICATE KEY UPDATE with all values to the current INSERT statement' do 31 | insert_netto = [ 32 | %q{INSERT INTO "chefs" ("id", "name", "email") VALUES (1, 'Netto', 'nettofarah@gmail.com') ON DUPLICATE KEY UPDATE `id` = VALUES(`id`), `name` = VALUES(`name`), `email` = VALUES(`email`)} 33 | ] 34 | 35 | inserts = translator.inserts 36 | records = translator.records 37 | translated_sql = adapter.on_duplicate_key_update(inserts, records) 38 | expect(translated_sql).to eq(insert_netto) 39 | end 40 | it 'works for hash-values instead of ActiveRecord instances' do 41 | insert_netto = [ 42 | %q{INSERT INTO "chefs" ("id", "name", "email") VALUES (1, 'Netto', 'nettofarah@gmail.com') ON DUPLICATE KEY UPDATE `id` = VALUES(`id`), `name` = VALUES(`name`), `email` = VALUES(`email`)} 43 | ] 44 | 45 | inserts = translator.inserts 46 | records = [{table_name: "chefs", values: { id: 1, name: "Netto", email: "nettofarah@gmail.com"}}] 47 | translated_sql = adapter.on_duplicate_key_update(inserts, records) 48 | expect(translated_sql).to eq(insert_netto) 49 | end 50 | end 51 | end -------------------------------------------------------------------------------- /spec/adapters/postgres_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | 3 | describe Polo::Adapters::Postgres do 4 | 5 | let(:adapter) { Polo::Adapters::Postgres.new } 6 | 7 | let(:netto) do 8 | AR::Chef.where(name: 'Netto').first 9 | end 10 | 11 | before(:all) do 12 | TestData.create_netto 13 | end 14 | 15 | let(:translator) { Polo::SqlTranslator.new(netto, Polo::Configuration.new(adapter: :postgres)) } 16 | 17 | describe '#on_duplicate_key_update' do 18 | it 'should raise an error' do 19 | expect { adapter.on_duplicate_key_update(double(), double()) }.to raise_error('on_duplicate: :override is not currently supported in the PostgreSQL adapter') 20 | end 21 | end 22 | 23 | describe '#ignore_transform' do 24 | it 'transforms INSERT by appending WHERE NOT EXISTS clause' do 25 | 26 | insert_netto = [%q{INSERT INTO "chefs" ("id", "name", "email") SELECT 1, 'Netto', 'nettofarah@gmail.com' WHERE NOT EXISTS (SELECT 1 FROM chefs WHERE id=1);}] 27 | 28 | records = translator.records 29 | inserts = translator.inserts 30 | translated_sql = adapter.ignore_transform(inserts, records) 31 | expect(translated_sql).to eq(insert_netto) 32 | end 33 | 34 | it 'works for hash-values instead of ActiveRecord instances' do 35 | insert_netto = [%q{INSERT INTO "chefs" ("id", "name", "email") SELECT 1, 'Netto', 'nettofarah@gmail.com' WHERE NOT EXISTS (SELECT 1 FROM chefs WHERE id=1);}] 36 | 37 | inserts = translator.inserts 38 | records = [{table_name: "chefs", values: { id: 1, name: "Netto", email: "nettofarah@gmail.com"}}] 39 | translated_sql = adapter.ignore_transform(inserts, records) 40 | expect(translated_sql).to eq(insert_netto) 41 | end 42 | end 43 | end -------------------------------------------------------------------------------- /spec/configuration_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | 3 | describe Polo::Configuration do 4 | 5 | describe 'adapter' do 6 | it 'defaults to mysql' do 7 | expect(Polo.defaults.adapter).to be :mysql 8 | end 9 | end 10 | 11 | describe 'on_duplicate' do 12 | it 'defaults to nothing' do 13 | expect(Polo.defaults.on_duplicate_strategy).to be nil 14 | end 15 | 16 | it 'accepts custom strategies' do 17 | Polo.configure do 18 | on_duplicate(:ignore) 19 | end 20 | 21 | defaults = Polo.defaults 22 | expect(defaults.on_duplicate_strategy).to eq(:ignore) 23 | end 24 | end 25 | 26 | describe 'obfuscate' do 27 | it 'defaults to an empty list' do 28 | expect(Polo.defaults.blacklist).to be_empty 29 | end 30 | 31 | it 'allows for the user to define fields to blacklist' do 32 | Polo.configure do 33 | obfuscate(:email, :password) 34 | end 35 | 36 | defaults = Polo.defaults 37 | expect(defaults.blacklist).to eq({ :email => nil, :password => nil }) 38 | end 39 | 40 | it 'allows for the user to define fields with strategies in blacklist' do 41 | Polo.configure do 42 | 43 | email_strategy = lambda {|e| "#{e.split("@")[0]}_test@example.com" } 44 | credit_card_strategy = lambda {|_| "4111111111111111"} 45 | 46 | obfuscate({email: email_strategy, credit_card: credit_card_strategy}) 47 | end 48 | 49 | defaults = Polo.defaults 50 | expect(defaults.blacklist[:email].call("a@b.com")).to eq("a_test@example.com") 51 | end 52 | end 53 | end 54 | -------------------------------------------------------------------------------- /spec/polo_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | 3 | describe Polo do 4 | 5 | before(:all) do 6 | TestData.create_netto 7 | end 8 | 9 | it 'generates an insert query for the base object' do 10 | exp = Polo.explore(AR::Chef, 1) 11 | insert = %q{INSERT INTO "chefs" ("id", "name", "email") VALUES (1, 'Netto', 'nettofarah@gmail.com')} 12 | expect(exp).to include(insert) 13 | end 14 | 15 | it 'generates an insert query for the objects with non-standard primary keys' do 16 | exp = Polo.explore(AR::Person, 1) 17 | insert = %q{INSERT INTO "people" ("ssn", "name") VALUES (1, 'John Doe')} 18 | expect(exp).to include(insert) 19 | end 20 | 21 | it 'generates insert queries for dependencies' do 22 | if ActiveRecord::VERSION::STRING >= "4.2" 23 | serialized_nil = "NULL" 24 | else 25 | serialized_nil = "'null'" 26 | end 27 | 28 | turkey_insert = %Q{INSERT INTO "recipes" ("id", "title", "num_steps", "chef_id", "metadata") VALUES (1, 'Turkey Sandwich', NULL, 1, #{serialized_nil})} 29 | cheese_burger_insert = %Q{INSERT INTO "recipes" ("id", "title", "num_steps", "chef_id", "metadata") VALUES (2, 'Cheese Burger', NULL, 1, #{serialized_nil})} 30 | 31 | inserts = Polo.explore(AR::Chef, 1, [:recipes]) 32 | 33 | expect(inserts).to include(turkey_insert) 34 | expect(inserts).to include(cheese_burger_insert) 35 | end 36 | 37 | it 'generates queries for nested dependencies' do 38 | patty = %q{INSERT INTO "ingredients" ("id", "name", "quantity") VALUES (3, 'Patty', '1')} 39 | turkey = %q{INSERT INTO "ingredients" ("id", "name", "quantity") VALUES (1, 'Turkey', 'a lot')} 40 | one_cheese = %q{INSERT INTO "ingredients" ("id", "name", "quantity") VALUES (2, 'Cheese', '1 slice')} 41 | two_cheeses = %q{INSERT INTO "ingredients" ("id", "name", "quantity") VALUES (4, 'Cheese', '2 slices')} 42 | 43 | inserts = Polo.explore(AR::Chef, 1, :recipes => :ingredients) 44 | 45 | expect(inserts).to include(patty) 46 | expect(inserts).to include(turkey) 47 | expect(inserts).to include(one_cheese) 48 | expect(inserts).to include(two_cheeses) 49 | end 50 | 51 | it 'generates inserts for HABTM relationships' do 52 | ar_version = ActiveRecord::VERSION::STRING 53 | skip("Not supported on ActiveRecord #{ar_version}") if ar_version < "4.1.0" 54 | habtm_inserts = [ 55 | %q{INSERT INTO "recipes_tags" ("recipe_id", "tag_id") VALUES (2, 2)}, 56 | %q{INSERT INTO "tags" ("id", "name") VALUES (2, 'burgers')} 57 | ] 58 | 59 | inserts = Polo.explore(AR::Chef, 1, :recipes => :tags) 60 | 61 | habtm_inserts.each do |habtm_insert| 62 | expect(inserts).to include(habtm_insert) 63 | end 64 | end 65 | 66 | it 'generates inserts for many to many relationships' do 67 | many_to_many_inserts = [ 68 | %q{INSERT INTO "recipes_ingredients" ("id", "recipe_id", "ingredient_id") VALUES (1, 1, 1)}, 69 | %q{INSERT INTO "recipes_ingredients" ("id", "recipe_id", "ingredient_id") VALUES (2, 1, 2)}, 70 | %q{INSERT INTO "recipes_ingredients" ("id", "recipe_id", "ingredient_id") VALUES (3, 2, 3)}, 71 | %q{INSERT INTO "recipes_ingredients" ("id", "recipe_id", "ingredient_id") VALUES (4, 2, 4)}, 72 | ] 73 | 74 | inserts = Polo.explore(AR::Chef, 1, :recipes => :ingredients) 75 | 76 | many_to_many_inserts.each do |many_to_many_insert| 77 | expect(inserts).to include(many_to_many_insert) 78 | end 79 | end 80 | 81 | describe "Advanced Options" do 82 | describe 'obfuscate: [fields]' do 83 | 84 | it 'scrambles a predefined field' do 85 | Polo.configure do 86 | obfuscate(:email) 87 | end 88 | 89 | exp = Polo.explore(AR::Chef, 1) 90 | insert = /INSERT INTO "chefs" \("id", "name", "email"\) VALUES \(1, 'Netto', (.+)\)/ 91 | scrambled_email = insert.match(exp.first)[1] 92 | 93 | expect(scrambled_email).to_not eq('nettofarah@gmail.com') 94 | expect(insert).to match(exp.first) 95 | end 96 | 97 | it 'can apply custom strategies' do 98 | Polo.configure do 99 | obfuscate(email: lambda { |_| 'changeme' }) 100 | end 101 | 102 | inserts = Polo.explore(AR::Chef, 1) 103 | 104 | expect(inserts).to eq [ %q{INSERT INTO "chefs" ("id", "name", "email") VALUES (1, 'Netto', 'changeme')} ] 105 | end 106 | 107 | it 'only scrambles instances with the obfuscate field defined' do 108 | Polo.configure do 109 | obfuscate email: ->(e) { "#{e.split("@")[0]}_test@example.com" }, 110 | title: ->(t) { t.chars.reverse!.join } 111 | end 112 | 113 | exp = Polo.explore(AR::Chef, 1, :recipes) 114 | 115 | explore_statement = exp.join(';') 116 | expect(explore_statement).to_not match('nettofarah@gmail.com') 117 | expect(explore_statement).to match('Netto') 118 | end 119 | 120 | it 'can target a specific field in a table' do 121 | Polo.configure do 122 | obfuscate 'ingredients.name' => -> (i) { "Secret" } 123 | end 124 | 125 | exp = Polo.explore(AR::Chef, 1, recipes: :ingredients) 126 | 127 | explore_statement = exp.join(';') 128 | expect(explore_statement).to match('Netto') 129 | expect(explore_statement).to match('Secret') 130 | end 131 | end 132 | 133 | describe 'on_duplicate' do 134 | it 'applies the on_duplicate strategy' do 135 | Polo.configure do 136 | on_duplicate(:ignore) 137 | end 138 | 139 | exp = Polo.explore(AR::Chef, 1) 140 | insert = /INSERT IGNORE INTO "chefs" \("id", "name", "email"\) VALUES \(1, 'Netto', (.+)\)/ 141 | expect(insert).to match(exp.first) 142 | end 143 | end 144 | end 145 | end 146 | -------------------------------------------------------------------------------- /spec/spec_helper.rb: -------------------------------------------------------------------------------- 1 | $LOAD_PATH.unshift File.expand_path('../../lib', __FILE__) 2 | require 'polo' 3 | require 'active_record' 4 | 5 | ActiveRecord::Base.establish_connection(adapter: 'sqlite3', database: ':memory:') 6 | 7 | require 'support/schema' 8 | require 'support/activerecord_models' 9 | require 'support/factories' 10 | 11 | module Polo 12 | def self.reset! 13 | @configuration = Configuration.new 14 | end 15 | end 16 | 17 | RSpec.configure do |c| 18 | c.around(:example) do |example| 19 | Polo.reset! 20 | 21 | ActiveRecord::Base.transaction do 22 | example.run 23 | raise ActiveRecord::Rollback 24 | end 25 | end 26 | end 27 | 28 | def track_queries 29 | selects = [] 30 | queries_collector = lambda do |name, start, finish, id, payload| 31 | selects << payload 32 | end 33 | 34 | ActiveRecord::Base.connection.clear_query_cache 35 | ActiveSupport::Notifications.subscribed(queries_collector, 'sql.active_record') do 36 | yield 37 | end 38 | 39 | selects.map { |sel| sel[:sql] } 40 | end 41 | -------------------------------------------------------------------------------- /spec/sql_translator_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | 3 | describe Polo::SqlTranslator do 4 | 5 | let(:netto) do 6 | AR::Chef.where(name: 'Netto').first 7 | end 8 | 9 | before(:all) do 10 | TestData.create_netto 11 | end 12 | 13 | it 'translates records to inserts' do 14 | insert_netto = [%q{INSERT INTO "chefs" ("id", "name", "email") VALUES (1, 'Netto', 'nettofarah@gmail.com')}] 15 | netto_to_sql = Polo::SqlTranslator.new(netto).to_sql 16 | expect(netto_to_sql).to eq(insert_netto) 17 | end 18 | 19 | it 'encodes serialized fields correctly' do 20 | recipe = AR::Recipe.create(title: 'Polenta', metadata: { quality: 'ok' }) 21 | recipe_to_sql = Polo::SqlTranslator.new(recipe).to_sql.first 22 | expect(recipe_to_sql).to include(%q{'{"quality":"ok"}'}) # JSON, not YAML 23 | end 24 | 25 | it 'encodes attributes not backed by a database column correctly' do 26 | if Gem.loaded_specs["activerecord"].version < Gem::Version.new("4.2.1") 27 | skip "the attributes API was included in rails starting in 4.2.1" 28 | elsif Gem.loaded_specs["activerecord"].version >= Gem::Version.new("4.2.1") && 29 | Gem.loaded_specs["activerecord"].version < Gem::Version.new("5.0.0") 30 | class Employee < ActiveRecord::Base 31 | attribute :on_vacation, Type::Boolean.new 32 | end 33 | else 34 | class Employee < ActiveRecord::Base 35 | attribute :on_vacation, :boolean 36 | end 37 | end 38 | 39 | employee = Employee.create(name: 'John Doe', on_vacation: true) 40 | employee_to_sql = Polo::SqlTranslator.new(employee).to_sql.first 41 | expect(employee_to_sql).to_not include('on_vacation') 42 | end 43 | end 44 | -------------------------------------------------------------------------------- /spec/support/activerecord_models.rb: -------------------------------------------------------------------------------- 1 | module AR 2 | class Recipe < ActiveRecord::Base 3 | belongs_to :chef 4 | has_many :recipes_ingredients 5 | has_many :ingredients, through: :recipes_ingredients 6 | has_and_belongs_to_many :tags 7 | 8 | serialize :metadata, JSON 9 | end 10 | 11 | class Ingredient < ActiveRecord::Base 12 | end 13 | 14 | class Tag < ActiveRecord::Base 15 | has_and_belongs_to_many :recipes 16 | end 17 | 18 | class RecipesIngredient < ActiveRecord::Base 19 | belongs_to :recipe 20 | belongs_to :ingredient 21 | end 22 | 23 | class Restaurant < ActiveRecord::Base 24 | belongs_to :owner, class_name: 'Chef' 25 | has_one :rating 26 | end 27 | 28 | class Rating < ActiveRecord::Base 29 | belongs_to :restaurant 30 | end 31 | 32 | class Chef < ActiveRecord::Base 33 | has_many :recipes 34 | has_many :ingredients, through: :recipes 35 | has_one :restaurant, foreign_key: 'owner_id' 36 | end 37 | 38 | class Person < ActiveRecord::Base 39 | self.primary_key = :ssn 40 | end 41 | end 42 | -------------------------------------------------------------------------------- /spec/support/factories.rb: -------------------------------------------------------------------------------- 1 | module TestData 2 | 3 | def self.create_netto 4 | AR::Chef.create(name: 'Netto', email: 'nettofarah@gmail.com').tap do |netto| 5 | AR::Recipe.create(title: 'Turkey Sandwich', chef: netto).tap do |r| 6 | r.ingredients.create(name: 'Turkey', quantity: 'a lot') 7 | r.ingredients.create(name: 'Cheese', quantity: '1 slice') 8 | r.tags.create(name: 'sandwiches') 9 | end 10 | 11 | AR::Recipe.create(title: 'Cheese Burger', chef: netto).tap do |r| 12 | r.ingredients.create(name: 'Patty', quantity: '1') 13 | r.ingredients.create(name: 'Cheese', quantity: '2 slices') 14 | r.tags.create(name: 'burgers') 15 | end 16 | end 17 | 18 | AR::Person.create(name: 'John Doe') 19 | end 20 | end 21 | -------------------------------------------------------------------------------- /spec/support/schema.rb: -------------------------------------------------------------------------------- 1 | ActiveRecord::Schema.define do 2 | create_table :recipes, force: true do |t| 3 | t.column :title, :string 4 | t.column :num_steps, :integer 5 | t.column :chef_id, :integer 6 | t.column :metadata, :text 7 | end 8 | 9 | create_table :ingredients, force: true do |t| 10 | t.column :name, :string 11 | t.column :quantity, :string 12 | end 13 | 14 | create_table :recipes_ingredients, force: true do |t| 15 | t.column :recipe_id, :integer 16 | t.column :ingredient_id, :integer 17 | end 18 | 19 | create_table :chefs, force: true do |t| 20 | t.column :name, :string 21 | t.column :email, :string 22 | end 23 | 24 | create_table :restaurants, force: true do |t| 25 | t.column :name, :string 26 | t.column :owner_id, :integer 27 | t.column :current_customer_count, :integer 28 | end 29 | 30 | create_table :ratings, force: true do |t| 31 | t.column :value, :string 32 | t.column :restaurant_id, :integer 33 | end 34 | 35 | create_table :people, primary_key: :ssn, force: true do |t| 36 | t.column :name, :string 37 | end 38 | 39 | create_table :employees, force: true do |t| 40 | t.column :name, :string 41 | end 42 | 43 | create_table :tags, force: true do |t| 44 | t.column :name, :string 45 | end 46 | 47 | create_table :recipes_tags, id: false, force: true do |t| 48 | t.column :recipe_id, :integer 49 | t.column :tag_id, :integer 50 | end 51 | end 52 | -------------------------------------------------------------------------------- /spec/translator_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | 3 | describe Polo::Translator do 4 | 5 | let(:email) { 'nettofarah@gmail.com' } 6 | let(:finder) do 7 | { 8 | klass: AR::Chef, 9 | sql: AR::Chef.where(email: email).to_sql 10 | } 11 | end 12 | 13 | before(:all) do 14 | TestData.create_netto 15 | end 16 | 17 | describe "options" do 18 | describe "obfuscate: [fields]" do 19 | let(:translator) { Polo::Translator.new([finder], Polo::Configuration.new(obfuscate: obfuscated_fields)) } 20 | let(:netto) { netto = translator.instances.first } 21 | 22 | context "obfuscated field with no specified strategy" do 23 | let(:obfuscated_fields) {{email: nil }} 24 | 25 | it "shuffles characters in field" do 26 | expect(netto.email).to_not be_nil 27 | expect(netto.email.length).to eq email.length 28 | expect(sorted_characters(netto.email)).to eq sorted_characters(email) 29 | end 30 | end 31 | 32 | context "obfuscated field with obscuration strategy applied which will result in 42" do 33 | let(:obfuscated_fields) {{email: lambda { |_| 42 } } } 34 | 35 | it "replaces contents of field according to the supplied lambda" do 36 | expect(netto.email.to_s).to eq "42" 37 | end 38 | end 39 | 40 | context "complex custom obfuscation strategy" do 41 | let(:obfuscated_fields) do 42 | { email: lambda { |field| "temp_#{field}" } } 43 | end 44 | 45 | it "replaces contents of field according to the supplied lambda" do 46 | expect(netto.email.to_s).to eq "temp_nettofarah@gmail.com" 47 | end 48 | end 49 | 50 | context "custom obfuscation strategy using instance context" do 51 | let(:obfuscated_fields) do 52 | { email: lambda { |field, instance| "#{instance.name}@example.com" } } 53 | end 54 | 55 | it "replaces contents of field according to the supplied lambda" do 56 | expect(netto.email.to_s).to eq "Netto@example.com" 57 | end 58 | end 59 | 60 | context "no strategy passed in" do 61 | let(:obfuscated_fields) { [:email] } 62 | 63 | it "shuffles contents" do 64 | expect(netto.email).to_not eq email 65 | expect(sorted_characters(netto.email)).to eq sorted_characters(email) 66 | end 67 | end 68 | 69 | def sorted_characters(str) 70 | str.split("").sort.join 71 | end 72 | end 73 | end 74 | end 75 | --------------------------------------------------------------------------------