├── .gitignore
├── .rspec
├── .ruby-version
├── .travis.yml
├── Appraisals
├── CHANGELOG.md
├── CODE_OF_CONDUCT.md
├── Gemfile
├── LICENSE.txt
├── README.md
├── Rakefile
├── gemfiles
├── rails32.gemfile
├── rails40.gemfile
├── rails41.gemfile
├── rails42.gemfile
├── rails50.gemfile
├── rails51.gemfile
├── rails52.gemfile
├── rails60.gemfile
└── rails70.gemfile
├── lib
├── initializers
│ ├── arel.rb
│ └── big_decimal.rb
├── polo.rb
└── polo
│ ├── adapters
│ ├── mysql.rb
│ └── postgres.rb
│ ├── collector.rb
│ ├── configuration.rb
│ ├── sql_translator.rb
│ ├── translator.rb
│ └── version.rb
├── polo.gemspec
└── spec
├── adapters
├── mysql_spec.rb
└── postgres_spec.rb
├── configuration_spec.rb
├── polo_spec.rb
├── spec_helper.rb
├── sql_translator_spec.rb
├── support
├── activerecord_models.rb
├── factories.rb
└── schema.rb
└── translator_spec.rb
/.gitignore:
--------------------------------------------------------------------------------
1 | /.bundle/
2 | /.yardoc
3 | /Gemfile.lock
4 | /_yardoc/
5 | /coverage/
6 | /doc/
7 | /pkg/
8 | /spec/reports/
9 | /tmp/
10 | /gemfiles/.bundle/
11 | /gemfiles/*.gemfile.lock
12 |
--------------------------------------------------------------------------------
/.rspec:
--------------------------------------------------------------------------------
1 | --format documentation
2 | --color
3 |
--------------------------------------------------------------------------------
/.ruby-version:
--------------------------------------------------------------------------------
1 | 2.7.5
2 |
--------------------------------------------------------------------------------
/.travis.yml:
--------------------------------------------------------------------------------
1 | language: ruby
2 | rvm:
3 | - 2.6.5
4 | gemfile:
5 | - gemfiles/rails32.gemfile
6 | - gemfiles/rails40.gemfile
7 | - gemfiles/rails41.gemfile
8 | - gemfiles/rails42.gemfile
9 | - gemfiles/rails50.gemfile
10 | - gemfiles/rails51.gemfile
11 | - gemfiles/rails52.gemfile
12 | - gemfiles/rails60.gemfile
13 | before_install: gem install bundler -v 2.1.4
14 |
--------------------------------------------------------------------------------
/Appraisals:
--------------------------------------------------------------------------------
1 | appraise "rails32" do
2 | gem "activerecord", "3.2.22.5"
3 | gem "sqlite3", "1.3.13"
4 | end
5 |
6 | appraise "rails40" do
7 | gem "activerecord", "4.0.13"
8 | gem "sqlite3", "1.3.13"
9 | end
10 |
11 | appraise "rails41" do
12 | gem "activerecord", "4.1.16"
13 | gem "sqlite3", "1.3.13"
14 | end
15 |
16 | appraise "rails42" do
17 | gem "activerecord", "4.2.11.1"
18 | gem "sqlite3", "1.3.13"
19 | end
20 |
21 | appraise "rails50" do
22 | gem "activerecord", "5.0.7.2"
23 | gem "sqlite3", "1.3.13"
24 | end
25 |
26 | appraise "rails51" do
27 | gem "activerecord", "5.1.7"
28 | gem "sqlite3", "1.4.2"
29 | end
30 |
31 | appraise "rails52" do
32 | gem "activerecord", "5.2.4.2"
33 | gem "sqlite3", "1.4.2"
34 | end
35 |
36 | appraise "rails60" do
37 | gem "activerecord", "6.0.0"
38 | end
39 |
40 | appraise "rails70" do
41 | gem "activerecord", "7.0.0"
42 | end
43 |
--------------------------------------------------------------------------------
/CHANGELOG.md:
--------------------------------------------------------------------------------
1 | ## 0.5
2 |
3 | ### Breaking Changes
4 |
5 | - None
6 |
7 | ### Added
8 | - [#43](https://github.com/IFTTT/polo/pull/43) Rails 5 support
9 | - [#44](https://github.com/IFTTT/polo/pull/44) Remove all Rails 5 deprecation notices
10 |
11 | ## 0.4.1
12 |
13 | ### Breaking Changes
14 |
15 | - None
16 |
17 | ### Fixed
18 | - [#42](https://github.com/IFTTT/polo/pull/42) Escape columns in DUPLICATE KEY statements
19 |
20 | ## 0.4.0
21 |
22 | ### Breaking Changes
23 |
24 | - None
25 |
26 | ### Added
27 | - [#40](https://github.com/IFTTT/polo/pull/40) Support for optional 2nd instance argument
28 |
29 |
30 | ## 0.3.0
31 |
32 | ### Breaking Changes
33 |
34 | - None
35 |
36 | ### Added
37 |
38 | - [#30](https://github.com/IFTTT/polo/pull/30) Advanced obfuscation
39 | - [#37](https://github.com/IFTTT/polo/pull/37) Custom adapters for Postgres and MySQL
40 |
41 | ### Fixed
42 |
43 | - [#26](https://github.com/IFTTT/polo/pull/26) Postgres - Use ActiveRecord methods to generate INSERT SQLs
44 | - [#25](https://github.com/IFTTT/polo/pull/25) Fix custom strategies bug
45 | - [#28](https://github.com/IFTTT/polo/pull/28) Only obfuscate fields when they are present
46 | - [#35](https://github.com/IFTTT/polo/pull/35) Better support for Rails 4.0
47 | - [#31](https://github.com/IFTTT/polo/pull/31) Fix link to Code of Conduct
48 |
49 | ## 0.2.0
50 |
51 | ### Breaking Changes
52 |
53 | - None
54 |
55 | ### Added
56 |
57 | - [#8](https://github.com/IFTTT/polo/pull/8) Global settings
58 | - [#17](https://github.com/IFTTT/polo/pull/17) Using random generator instead of character shuffle for data obfuscation
59 | - [#18](https://github.com/IFTTT/polo/pull/18) Add a CHANGELOG
60 | - [#20](https://github.com/IFTTT/polo/pull/20) Postgres Support
61 |
62 | ### Fixed
63 |
64 | - Typo fixes on the README: [#9](https://github.com/IFTTT/polo/pull/9), [#10](https://github.com/IFTTT/polo/pull/10)
65 | - [#11]() Some ActiveRecord classes do not use id as the primary key
66 | - [#25](https://github.com/IFTTT/polo/pull/25) Fix Custom Strategy
67 |
68 | ## 0.1.0
69 |
70 | ### Breaking Changes
71 |
72 | - None
73 |
74 | ### Added
75 |
76 | - [#2](https://github.com/IFTTT/polo/pull/2) Add :ignore and :override options to deal with data collision
77 | - [#3](https://github.com/IFTTT/polo/pull/3) Add option to obfuscate fields
78 | - [#4](https://github.com/IFTTT/polo/pull/4) Add intro to Update / Ignore section
79 | - [#6](https://github.com/IFTTT/polo/pull/6) Set up Appraisal to run specs across Rails 3.2 through 4.2
80 |
81 | ### Fixed
82 |
83 | - [#7](https://github.com/IFTTT/polo/pull/7) Fix casting of values
84 |
--------------------------------------------------------------------------------
/CODE_OF_CONDUCT.md:
--------------------------------------------------------------------------------
1 | # Contributor Code of Conduct
2 |
3 | As contributors and maintainers of this project, we pledge to respect all people who contribute through reporting issues, posting feature requests, updating documentation, submitting pull requests or patches, and other activities.
4 |
5 | We are committed to making participation in this project a harassment-free experience for everyone, regardless of level of experience, gender, gender identity and expression, sexual orientation, disability, personal appearance, body size, race, ethnicity, age, or religion.
6 |
7 | Examples of unacceptable behavior by participants include the use of sexual language or imagery, derogatory comments or personal attacks, trolling, public or private harassment, insults, or other unprofessional conduct.
8 |
9 | Project maintainers have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, issues, and other contributions that are not aligned to this Code of Conduct. Project maintainers who do not follow the Code of Conduct may be removed from the project team.
10 |
11 | Instances of abusive, harassing, or otherwise unacceptable behavior may be reported by opening an issue or contacting one or more of the project maintainers.
12 |
13 | This Code of Conduct is adapted from the [Contributor Covenant](http://contributor-covenant.org), version 1.0.0, available at [http://contributor-covenant.org/version/1/0/0/](http://contributor-covenant.org/version/1/0/0/)
14 |
--------------------------------------------------------------------------------
/Gemfile:
--------------------------------------------------------------------------------
1 | source 'https://rubygems.org'
2 |
3 | gemspec
4 |
--------------------------------------------------------------------------------
/LICENSE.txt:
--------------------------------------------------------------------------------
1 | The MIT License (MIT)
2 |
3 | Copyright (c) 2015 IFTTT
4 |
5 | Permission is hereby granted, free of charge, to any person obtaining a copy
6 | of this software and associated documentation files (the "Software"), to deal
7 | in the Software without restriction, including without limitation the rights
8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9 | copies of the Software, and to permit persons to whom the Software is
10 | furnished to do so, subject to the following conditions:
11 |
12 | The above copyright notice and this permission notice shall be included in
13 | all copies or substantial portions of the Software.
14 |
15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
21 | THE SOFTWARE.
22 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | [](http://ifttt.github.io)
2 | [](https://travis-ci.org/IFTTT/polo)
3 |
4 | 
5 |
6 | # Polo
7 | Polo travels through your database and creates sample snapshots so you can work with real world data in any environment.
8 |
9 | Polo takes an `ActiveRecord::Base` seed object and traverses every whitelisted `ActiveRecord::Association` generating SQL `INSERTs` along the way.
10 |
11 | You can then save those SQL `INSERTS` to .sql file and import the data to your favorite environment.
12 |
13 | ## Motivation
14 | Read our [blog post](https://medium.com/engineering-at-ifttt/happier-rails-development-with-polo-9df6819136d3#.f8ll3azeq) or check out this [presentation](https://speakerdeck.com/nettofarah/polo-working-with-real-world-data-in-development).
15 |
16 | ## Usage
17 | Given the following data model:
18 | ```ruby
19 | class Chef < ActiveRecord::Base
20 | has_many :recipes
21 | has_many :ingredients, through: :recipes
22 | end
23 |
24 | class Recipe < ActiveRecord::Base
25 | has_many :recipes_ingredients
26 | has_many :ingredients, through: :recipes_ingredients
27 | end
28 |
29 | class Ingredient < ActiveRecord::Base
30 | end
31 |
32 | class RecipesIngredient < ActiveRecord::Base
33 | belongs_to :recipe
34 | belongs_to :ingredient
35 | end
36 | ```
37 |
38 | ### Simple ActiveRecord Objects
39 | ```ruby
40 | inserts = Polo.explore(Chef, 1)
41 | ```
42 | ```sql
43 | INSERT INTO `chefs` (`id`, `name`) VALUES (1, 'Netto')
44 | ```
45 |
46 | Where `Chef` is the seed object class, and `1` is the seed object id.
47 |
48 | ### Simple Associations
49 | ```ruby
50 | inserts = Polo.explore(Chef, 1, :recipes)
51 | ```
52 | ```sql
53 | INSERT INTO `chefs` (`id`, `name`) VALUES (1, 'Netto')
54 | INSERT INTO `recipes` (`id`, `title`, `num_steps`, `chef_id`) VALUES (1, 'Turkey Sandwich', NULL, 1)
55 | INSERT INTO `recipes` (`id`, `title`, `num_steps`, `chef_id`) VALUES (2, 'Cheese Burger', NULL, 1)
56 | ```
57 |
58 | ### Complex nested associations
59 | ```ruby
60 | inserts = Polo.explore(Chef, 1, :recipes => :ingredients)
61 | ```
62 |
63 | ```sql
64 | INSERT INTO `chefs` (`id`, `name`) VALUES (1, 'Netto')
65 | INSERT INTO `recipes` (`id`, `title`, `num_steps`, `chef_id`) VALUES (1, 'Turkey Sandwich', NULL, 1)
66 | INSERT INTO `recipes` (`id`, `title`, `num_steps`, `chef_id`) VALUES (2, 'Cheese Burger', NULL, 1)
67 | INSERT INTO `recipes_ingredients` (`id`, `recipe_id`, `ingredient_id`) VALUES (1, 1, 1)
68 | INSERT INTO `recipes_ingredients` (`id`, `recipe_id`, `ingredient_id`) VALUES (2, 1, 2)
69 | INSERT INTO `recipes_ingredients` (`id`, `recipe_id`, `ingredient_id`) VALUES (3, 2, 3)
70 | INSERT INTO `recipes_ingredients` (`id`, `recipe_id`, `ingredient_id`) VALUES (4, 2, 4)
71 | INSERT INTO `ingredients` (`id`, `name`, `quantity`) VALUES (1, 'Turkey', 'a lot')
72 | INSERT INTO `ingredients` (`id`, `name`, `quantity`) VALUES (2, 'Cheese', '1 slice')
73 | INSERT INTO `ingredients` (`id`, `name`, `quantity`) VALUES (3, 'Patty', '1')
74 | INSERT INTO `ingredients` (`id`, `name`, `quantity`) VALUES (4, 'Cheese', '2 slices')
75 | ```
76 |
77 | ## Advanced Usage
78 |
79 | Occasionally, you might have a dataset that you want to refresh. A production database that has data that might be useful on your local copy of the database. Polo doesn't have an opinion about your data; if you try to import data with a key that's already in your local database, Polo doesn't necessarily know how you want to handle that conflict.
80 |
81 | Advanced users will find the `on_duplicate` option to be helpful in this context. It gives Polo instructions on how to handle collisions.
82 | *Note: This feature is currently only supported for MySQL databases. (PRs for other databases are welcome!)*
83 |
84 | There are two possible values for the `on_duplicate` key: `:ignore` and `:override`. Ignore keeps the old data. Override keeps the new data. If there's a collision and the on_duplicate param is not set, Polo will simpy stop importing the data.
85 |
86 | ### Ignore
87 | A.K.A the Ostrich Approach: stick your head in the sand and pretend nothing happened.
88 |
89 | ```ruby
90 | Polo.configure do
91 | on_duplicate :ignore
92 | end
93 |
94 | Polo::Traveler.explore(Chef, 1, :recipes)
95 | ```
96 |
97 | ```sql
98 | INSERT IGNORE INTO `chefs` (`id`, `name`) VALUES (1, 'Netto')
99 | INSERT IGNORE INTO `recipes` (`id`, `title`, `num_steps`, `chef_id`) VALUES (1, 'Turkey Sandwich', NULL, 1)
100 | INSERT IGNORE INTO `recipes` (`id`, `title`, `num_steps`, `chef_id`) VALUES (2, 'Cheese Burger', NULL, 1)
101 | ```
102 |
103 | ### Override
104 | Use the option `on_duplicate: :override` to override your local data with new data from your Polo script.
105 |
106 | ```ruby
107 | Polo.configure do
108 | on_duplicate :override
109 | end
110 |
111 | Polo::Traveler.explore(Chef, 1, :recipes)
112 | ```
113 |
114 | ```sql
115 | INSERT INTO `chefs` (`id`, `name`) VALUES (1, 'Netto')
116 | ON DUPLICATE KEY UPDATE id = VALUES(id), name = VALUES(name)
117 | ...
118 | ```
119 |
120 | ### Sensitive Fields
121 | You can use the `obfuscate` option to obfuscate sensitive fields like emails or
122 | user logins.
123 |
124 | ```ruby
125 | Polo.configure do
126 | obfuscate :email, :credit_card
127 | end
128 |
129 | Polo::Traveler.explore(AR::Chef, 1)
130 | ```
131 |
132 | ```sql
133 | INSERT INTO `chefs` (`id`, `name`, `email`) VALUES (1, 'Netto', 'eahorctmaagfo.nitm@l')
134 | ```
135 |
136 | Warning: This is not a security feature. Fields can still easily be rearranged back to their original format. Polo will simply scramble the order of strings so you don't accidentally end up causing side effects when using production data in development. It is not a good practice to use highly sensitive data in development.
137 |
138 | #### Advanced Obfuscation
139 |
140 | For more advanced obfuscation, you can pass in a custom obfuscation strategy. Polo will take in a lambda that can be used to transform sensitive data.
141 |
142 | Using a `:symbol` as an obfuscate key targets all columns of that name. Passing an SQL selector as a `String` will target columns within the specified table.
143 |
144 | ````ruby
145 | Polo.configure do
146 |
147 | email_strategy = lambda do |email|
148 | first_part = email.split("@")[0]
149 | "#{first_part}@test.com"
150 | end
151 |
152 | credit_card_strategy = lambda do |credit_card|
153 | "4123 4567 8910 1112"
154 | end
155 |
156 | # If you need the context of the record for its fields, it is accessible
157 | # in the second argument of the strategy
158 | social_security_strategy = lambda do |ssn, instance|
159 | sprintf("%09d", instance.id)
160 | end
161 |
162 | obfuscate({
163 | 'chefs.email' => email_strategy, # This only applies to the "email" column in the "chefs" table
164 | :credit_card => credit_card_strategy, # This applies to any column named "credit_card" across every table
165 | :ssn_strategy => social_security_strategy
166 | })
167 | end
168 |
169 | Polo::Traveler.explore(AR::Chef, 1)
170 | ````
171 |
172 | ```sql
173 | INSERT INTO `chefs` (`id`, `name`, `email`) VALUES (1, 'Netto', 'netto_test.@example.com')
174 | ```
175 |
176 | ## Installation
177 |
178 | Add this line to your application's Gemfile:
179 |
180 | ```ruby
181 | gem 'polo'
182 | ```
183 |
184 | And then execute:
185 | ```bash
186 | $ bundle
187 | ```
188 |
189 | Or install it yourself as:
190 | ```bash
191 | $ gem install polo
192 | ```
193 |
194 | ## Contributing
195 |
196 | Bug reports and pull requests are welcome on GitHub at https://github.com/IFTTT/polo. This project is intended to be a safe, welcoming space for collaboration, and contributors are expected to adhere to the [Code of Conduct](https://github.com/IFTTT/polo/blob/master/CODE_OF_CONDUCT.md).
197 |
198 | To run the specs across all supported version of Rails, check out the repo and
199 | follow these steps:
200 |
201 | ```bash
202 | $ bundle install
203 | $ bundle exec appraisal install
204 | $ bundle exec appraisal rake
205 | ```
206 |
207 | ## License
208 |
209 | The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT).
210 |
--------------------------------------------------------------------------------
/Rakefile:
--------------------------------------------------------------------------------
1 | require "bundler/gem_tasks"
2 | require "rspec/core/rake_task"
3 |
4 | RSpec::Core::RakeTask.new(:spec)
5 |
6 | task :default => :spec
7 |
--------------------------------------------------------------------------------
/gemfiles/rails32.gemfile:
--------------------------------------------------------------------------------
1 | # This file was generated by Appraisal
2 |
3 | source "https://rubygems.org"
4 |
5 | gem "activerecord", "3.2.22.5"
6 | gem "sqlite3", "1.3.13"
7 |
8 | gemspec path: "../"
9 |
--------------------------------------------------------------------------------
/gemfiles/rails40.gemfile:
--------------------------------------------------------------------------------
1 | # This file was generated by Appraisal
2 |
3 | source "https://rubygems.org"
4 |
5 | gem "activerecord", "4.0.13"
6 | gem "sqlite3", "1.3.13"
7 |
8 | gemspec path: "../"
9 |
--------------------------------------------------------------------------------
/gemfiles/rails41.gemfile:
--------------------------------------------------------------------------------
1 | # This file was generated by Appraisal
2 |
3 | source "https://rubygems.org"
4 |
5 | gem "activerecord", "4.1.16"
6 | gem "sqlite3", "1.3.13"
7 |
8 | gemspec path: "../"
9 |
--------------------------------------------------------------------------------
/gemfiles/rails42.gemfile:
--------------------------------------------------------------------------------
1 | # This file was generated by Appraisal
2 |
3 | source "https://rubygems.org"
4 |
5 | gem "activerecord", "4.2.11.1"
6 | gem "sqlite3", "1.3.13"
7 |
8 | gemspec path: "../"
9 |
--------------------------------------------------------------------------------
/gemfiles/rails50.gemfile:
--------------------------------------------------------------------------------
1 | # This file was generated by Appraisal
2 |
3 | source "https://rubygems.org"
4 |
5 | gem "activerecord", "5.0.7.2"
6 | gem "sqlite3", "1.3.13"
7 |
8 | gemspec path: "../"
9 |
--------------------------------------------------------------------------------
/gemfiles/rails51.gemfile:
--------------------------------------------------------------------------------
1 | # This file was generated by Appraisal
2 |
3 | source "https://rubygems.org"
4 |
5 | gem "activerecord", "5.1.7"
6 | gem "sqlite3", "1.4.2"
7 |
8 | gemspec path: "../"
9 |
--------------------------------------------------------------------------------
/gemfiles/rails52.gemfile:
--------------------------------------------------------------------------------
1 | # This file was generated by Appraisal
2 |
3 | source "https://rubygems.org"
4 |
5 | gem "activerecord", "5.2.4.2"
6 | gem "sqlite3", "1.4.2"
7 |
8 | gemspec path: "../"
9 |
--------------------------------------------------------------------------------
/gemfiles/rails60.gemfile:
--------------------------------------------------------------------------------
1 | # This file was generated by Appraisal
2 |
3 | source "https://rubygems.org"
4 |
5 | gem "activerecord", "6.0.0"
6 |
7 | gemspec path: "../"
8 |
--------------------------------------------------------------------------------
/gemfiles/rails70.gemfile:
--------------------------------------------------------------------------------
1 | # This file was generated by Appraisal
2 |
3 | source "https://rubygems.org"
4 |
5 | gem "activerecord", "7.0.0"
6 |
7 | gemspec path: "../"
8 |
--------------------------------------------------------------------------------
/lib/initializers/arel.rb:
--------------------------------------------------------------------------------
1 | # https://stackoverflow.com/a/44286212
2 | if ActiveRecord::VERSION::MAJOR < 5
3 | module Arel
4 | module Visitors
5 | class DepthFirst < Arel::Visitors::Visitor
6 | alias :visit_Integer :terminal
7 | end
8 |
9 | class Dot < Arel::Visitors::Visitor
10 | alias :visit_Integer :visit_String
11 | end
12 |
13 | if ActiveRecord::VERSION::MAJOR == 4 && ActiveRecord::VERSION::MINOR >= 2
14 | class ToSql < Arel::Visitors::Reduce
15 | alias :visit_Integer :literal
16 | end
17 | else
18 | class ToSql < Arel::Visitors::Visitor
19 | alias :visit_Integer :literal
20 | end
21 | end
22 | end
23 | end
24 | end
25 |
--------------------------------------------------------------------------------
/lib/initializers/big_decimal.rb:
--------------------------------------------------------------------------------
1 | # Support old BigDecimal constructor for sqlite3 (among others?)
2 | class BigDecimal
3 | def self.new(*args)
4 | BigDecimal(*args)
5 | end
6 | end
7 |
--------------------------------------------------------------------------------
/lib/polo.rb:
--------------------------------------------------------------------------------
1 | require "polo/version"
2 | require "polo/collector"
3 | require "polo/translator"
4 | require "polo/configuration"
5 | require "initializers/arel"
6 | require "initializers/big_decimal"
7 |
8 | module Polo
9 |
10 | # Public: Traverses a dependency graph based on a seed ActiveRecord object
11 | # and generates all the necessary INSERT queries for each one of the records
12 | # it finds along the way.
13 | #
14 | # base_class - An ActiveRecord::Base class for the seed record.
15 | # id - An ID used to find the desired seed record.
16 | #
17 | # dependency_tree - An ActiveRecord::Associations::Preloader compliant that
18 | # will define the path Polo will traverse.
19 | #
20 | # (from ActiveRecord::Associations::Preloader docs)
21 | # It may be:
22 | # - a Symbol or a String which specifies a single association name. For
23 | # example, specifying +:books+ allows this method to preload all books
24 | # for an Author.
25 | # - an Array which specifies multiple association names. This array
26 | # is processed recursively. For example, specifying [:avatar, :books]
27 | # allows this method to preload an author's avatar as well as all of his
28 | # books.
29 | # - a Hash which specifies multiple association names, as well as
30 | # association names for the to-be-preloaded association objects. For
31 | # example, specifying { author: :avatar } will preload a
32 | # book's author, as well as that author's avatar.
33 | #
34 | # +:associations+ has the same format as the +:include+ option for
35 | # ActiveRecord::Base.find. So +associations+ could look like this:
36 | #
37 | # :books
38 | # [ :books, :author ]
39 | # { author: :avatar }
40 | # [ :books, { author: :avatar } ]
41 | #
42 | def self.explore(base_class, id, dependencies={})
43 | Traveler.collect(base_class, id, dependencies).translate(defaults)
44 | end
45 |
46 |
47 | # Public: Sets up global settings for Polo
48 | #
49 | # block - Takes a block with the settings you decide to use
50 | #
51 | # obfuscate - Takes a blacklist with sensitive fields you wish to scramble
52 | # on_duplicate - Defines the on_duplicate strategy for your INSERTS
53 | # e.g. :override, :ignore
54 | #
55 | # usage:
56 | # Polo.configure do
57 | # obfuscate(:email, :password, :credit_card)
58 | # on_duplicate(:override)
59 | # end
60 | #
61 | def self.configure(&block)
62 | @configuration = Configuration.new
63 | @configuration.instance_eval(&block) if block_given?
64 | @configuration
65 | end
66 |
67 | # Public: Returns the default settings
68 | #
69 | def self.defaults
70 | @configuration || configure
71 | end
72 |
73 |
74 | class Traveler
75 |
76 | def self.collect(base_class, id, dependencies={})
77 | selects = Collector.new(base_class, id, dependencies).collect
78 | new(selects)
79 | end
80 |
81 | def initialize(selects)
82 | @selects = selects
83 | end
84 |
85 | def translate(configuration=Configuration.new)
86 | Translator.new(@selects, configuration).translate
87 | end
88 | end
89 | end
90 |
--------------------------------------------------------------------------------
/lib/polo/adapters/mysql.rb:
--------------------------------------------------------------------------------
1 | module Polo
2 | module Adapters
3 | class MySQL
4 | def on_duplicate_key_update(inserts, records)
5 | insert_and_record = inserts.zip(records)
6 | insert_and_record.map do |insert, record|
7 | attrs = record.is_a?(Hash) ? record.fetch(:values) : record.attributes
8 | values_syntax = attrs.keys.map do |key|
9 | "`#{key}` = VALUES(`#{key}`)"
10 | end
11 |
12 | on_dup_syntax = "ON DUPLICATE KEY UPDATE #{values_syntax.join(', ')}"
13 |
14 | "#{insert} #{on_dup_syntax}"
15 | end
16 | end
17 |
18 | def ignore_transform(inserts, records)
19 | inserts.map do |insert|
20 | insert.gsub("INSERT", "INSERT IGNORE")
21 | end
22 | end
23 | end
24 | end
25 | end
--------------------------------------------------------------------------------
/lib/polo/adapters/postgres.rb:
--------------------------------------------------------------------------------
1 | module Polo
2 | module Adapters
3 | class Postgres
4 | # TODO: Implement UPSERT. This command became available in 9.1.
5 | #
6 | # See: http://www.the-art-of-web.com/sql/upsert/
7 | def on_duplicate_key_update(inserts, records)
8 | raise 'on_duplicate: :override is not currently supported in the PostgreSQL adapter'
9 | end
10 |
11 | # Internal: Transforms an INSERT with PostgreSQL-specific syntax. Ignores
12 | # records that already exist in the table. To do this, it uses
13 | # a heuristic, i.e. checks if there is a record with the same id
14 | # in the table.
15 | # See: http://stackoverflow.com/a/6527838/32816
16 | #
17 | # inserts - The Array of INSERT statements.
18 | # records - The Array of Arel objects.
19 | #
20 | # Returns the Array of transformed INSERT statements.
21 | def ignore_transform(inserts, records)
22 | insert_and_record = inserts.zip(records)
23 | insert_and_record.map do |insert, record|
24 | if record.is_a?(Hash)
25 | id = record.fetch(:values)[:id]
26 | table_name = record.fetch(:table_name)
27 | else
28 | id = record[:id]
29 | table_name = record.class.arel_table.name
30 | end
31 | insert = insert.gsub(/VALUES \((.+)\)$/m, 'SELECT \\1')
32 | insert << " WHERE NOT EXISTS (SELECT 1 FROM #{table_name} WHERE id=#{id});"
33 | end
34 | end
35 | end
36 | end
37 | end
38 |
--------------------------------------------------------------------------------
/lib/polo/collector.rb:
--------------------------------------------------------------------------------
1 | module Polo
2 | class Collector
3 |
4 | def initialize(base_class, id, dependency_tree={})
5 | @base_class = base_class
6 | @id = id
7 | @dependency_tree = dependency_tree
8 | @selects = []
9 | end
10 |
11 | # Public: Traverses the dependency tree and collects every SQL query.
12 | #
13 | # This is done by wrapping a top level call to includes(...) with a
14 | # ActiveSupport::Notifications block and collecting every generate SQL query.
15 | #
16 | def collect
17 | unprepared_statement do
18 | ActiveSupport::Notifications.subscribed(collector, 'sql.active_record') do
19 | base_finder = @base_class.includes(@dependency_tree).where(@base_class.primary_key => @id)
20 | collect_sql(klass: @base_class, sql: base_finder.to_sql)
21 | base_finder.to_a
22 | end
23 | end
24 |
25 | @selects.compact.uniq
26 | end
27 |
28 | private
29 |
30 | # Internal: Store ActiveRecord queries in @selects
31 | #
32 | # Collector will intersect every ActiveRecord query performed within the
33 | # ActiveSupport::Notifications.subscribed block defined in #run and store
34 | # the resulting SQL query in @selects
35 | #
36 | def collector
37 | lambda do |name, start, finish, id, payload|
38 | sql = payload[:sql]
39 | if payload[:name] =~ /^HABTM_.* Load$/
40 | collect_sql(connection: @base_class.connection, sql: sql)
41 | elsif payload[:name] =~ /^(.*) Load$/
42 | begin
43 | class_name = $1.constantize
44 | collect_sql(klass: class_name, sql: sql)
45 | rescue ActiveRecord::StatementInvalid, NameError
46 | # invalid table name (common when prefetching schemas)
47 | end
48 | end
49 | end
50 | end
51 |
52 | def collect_sql(select)
53 | @selects << select
54 | end
55 |
56 | def unprepared_statement
57 | if ActiveRecord::Base.connection.respond_to?(:unprepared_statement)
58 | ActiveRecord::Base.connection.unprepared_statement do
59 | yield
60 | end
61 | else
62 | yield
63 | end
64 | end
65 | end
66 | end
67 |
--------------------------------------------------------------------------------
/lib/polo/configuration.rb:
--------------------------------------------------------------------------------
1 | module Polo
2 |
3 | class Configuration
4 | attr_reader :on_duplicate_strategy, :blacklist, :adapter
5 |
6 | def initialize(options={})
7 | options = { on_duplicate: nil, obfuscate: {}, adapter: :mysql }.merge(options)
8 | @adapter = options[:adapter]
9 | @on_duplicate_strategy = options[:on_duplicate]
10 | obfuscate(options[:obfuscate])
11 | end
12 |
13 | # TODO: document this
14 | # This normalizes an array or hash of fields to a hash of
15 | # { field_name => strategy }
16 | def obfuscate(*fields)
17 | if fields.is_a?(Array)
18 | fields = fields.flatten
19 | end
20 |
21 | fields_and_strategies = {}
22 |
23 | fields.each do |field|
24 | if field.is_a?(Symbol) || field.is_a?(String)
25 | fields_and_strategies[field] = nil
26 | elsif field.is_a?(Hash)
27 | fields_and_strategies = fields_and_strategies.merge(field)
28 | end
29 | end
30 |
31 | @blacklist = fields_and_strategies
32 | end
33 |
34 | def on_duplicate(strategy)
35 | @on_duplicate_strategy = strategy
36 | end
37 |
38 | def set_adapter(db)
39 | @adapter = db
40 | end
41 | end
42 | end
43 |
--------------------------------------------------------------------------------
/lib/polo/sql_translator.rb:
--------------------------------------------------------------------------------
1 | require 'active_record'
2 | require 'polo/configuration'
3 | require 'polo/adapters/mysql'
4 | require 'polo/adapters/postgres'
5 |
6 | module Polo
7 | class SqlTranslator
8 |
9 | def initialize(object, configuration = Configuration.new)
10 | @record = object
11 | @configuration = configuration
12 |
13 | case @configuration.adapter
14 | when :mysql
15 | @adapter = Polo::Adapters::MySQL.new
16 | when :postgres
17 | @adapter = Polo::Adapters::Postgres.new
18 | else
19 | raise "Unknown SQL adapter: #{@configuration.adapter}"
20 | end
21 | end
22 |
23 | def to_sql
24 | case @configuration.on_duplicate_strategy
25 | when :ignore
26 | @adapter.ignore_transform(inserts, records)
27 | when :override
28 | @adapter.on_duplicate_key_update(inserts, records)
29 | else inserts
30 | end
31 | end
32 |
33 | def records
34 | Array.wrap(@record)
35 | end
36 |
37 | def inserts
38 | records.map do |record|
39 | if record.is_a?(Hash)
40 | raw_sql_from_hash(record)
41 | else
42 | raw_sql_from_record(record)
43 | end
44 | end
45 | end
46 |
47 | private
48 |
49 | # Internal: Generates an insert SQL statement for a given record
50 | #
51 | # It will make use of the InsertManager class from the Arel gem to generate
52 | # insert statements
53 | #
54 | def raw_sql_from_record(record)
55 | record.class.arel_table.create_insert.tap do |insert_manager|
56 | insert_manager.insert(insert_values(record))
57 | end.to_sql
58 | end
59 |
60 | # Internal: Generates an insert SQL statement from a hash of values
61 | def raw_sql_from_hash(hash)
62 | connection = ActiveRecord::Base.connection
63 | attributes = hash.fetch(:values)
64 | table_name = connection.quote_table_name(hash.fetch(:table_name))
65 | columns = attributes.keys.map{|k| connection.quote_column_name(k)}.join(", ")
66 | value_placeholders = attributes.values.map{|v| "?" }.join(", ")
67 | ActiveRecord::Base.send(:sanitize_sql_array, ["INSERT INTO #{table_name} (#{columns}) VALUES (#{value_placeholders})", *attributes.values])
68 | end
69 |
70 | # Internal: Returns an object's attribute definitions along with
71 | # their set values (for Rails 3.x).
72 | #
73 | module ActiveRecordLessThanFour
74 | def insert_values(record)
75 | record.send(:arel_attributes_values)
76 | end
77 | end
78 |
79 | # Internal: Returns an object's attribute definitions along with
80 | # their set values (for Rails >= 4.x).
81 | #
82 | # From Rails 4.2 onwards, for some reason attributes with custom serializers
83 | # wouldn't be properly serialized automatically. That's why explict
84 | # 'type_cast' call are necessary.
85 | #
86 | module ActiveRecordFour
87 | def insert_values(record)
88 | connection = ActiveRecord::Base.connection
89 | values = record.send(:arel_attributes_with_values_for_create, connection.schema_cache.columns(record.class.table_name).map(&:name))
90 | values.each do |attribute, value|
91 | column = record.send(:column_for_attribute, attribute.name)
92 | values[attribute] = connection.type_cast(value, column)
93 | end
94 | end
95 | end
96 |
97 | # Internal: Returns an object's attribute definitions along with
98 | # their set values (for Rails 5.0 & 5.1).
99 | #
100 | # Serializers have changed again in rails 5.
101 | # We now use the type_caster from the arel_table.
102 | #
103 | module ActiveRecordFivePointZeroOrOne
104 | # Based on the codepath used in Rails 5
105 | def raw_sql_from_record(record)
106 | values = record.send(:arel_attributes_with_values_for_create, record.class.column_names)
107 | model = record.class
108 | substitutes, binds = model.unscoped.substitute_values(values)
109 |
110 | insert_manager = model.arel_table.create_insert
111 | insert_manager.insert substitutes
112 |
113 | model.connection.unprepared_statement do
114 | model.connection.to_sql(insert_manager, binds)
115 | end
116 | end
117 | end
118 |
119 | # Internal: Returns an object's attribute definitions along with
120 | # their set values (for Rails >= 5.2).
121 | module ActiveRecordFive
122 | def raw_sql_from_record(record)
123 | values = record.send(:attributes_with_values_for_create, record.class.column_names)
124 | model = record.class
125 | substitutes_and_binds = model.send(:_substitute_values, values)
126 |
127 | insert_manager = model.arel_table.create_insert
128 | insert_manager.insert substitutes_and_binds
129 |
130 | model.connection.unprepared_statement do
131 | model.connection.to_sql(insert_manager)
132 | end
133 | end
134 | end
135 |
136 | # Internal: Returns an object's attribute definitions along with
137 | # their set values (for Rails 6.0).
138 | module ActiveRecordSix
139 | def raw_sql_from_record(record)
140 | model = record.class
141 | values = record.send(:attributes_with_values, record.send(:attributes_for_create, model.column_names))
142 | substitutes_and_binds = model.send(:_substitute_values, values)
143 |
144 | insert_manager = model.arel_table.create_insert
145 | insert_manager.insert substitutes_and_binds
146 |
147 | model.connection.unprepared_statement do
148 | model.connection.to_sql(insert_manager)
149 | end
150 | end
151 | end
152 |
153 | # Internal: Returns an object's attribute definitions along with
154 | # their set values (for Rails 7.0).
155 | module ActiveRecordSeven
156 | def raw_sql_from_record(record)
157 | values = record.send(:attributes_with_values, record.class.column_names)
158 | model = record.class
159 | substitutes_and_binds = values.transform_keys { |name| model.arel_table[name] }
160 |
161 | insert_manager = Arel::InsertManager.new(model.arel_table)
162 | insert_manager.insert substitutes_and_binds
163 |
164 | model.connection.unprepared_statement do
165 | model.connection.to_sql(insert_manager)
166 | end
167 | end
168 | end
169 |
170 | if ActiveRecord::VERSION::MAJOR < 4
171 | include ActiveRecordLessThanFour
172 | elsif ActiveRecord::VERSION::MAJOR == 4
173 | include ActiveRecordFour
174 | elsif ActiveRecord::VERSION::MAJOR == 5 && ActiveRecord::VERSION::MINOR < 2
175 | prepend ActiveRecordFivePointZeroOrOne
176 | elsif ActiveRecord::VERSION::MAJOR == 5
177 | prepend ActiveRecordFive
178 | elsif ActiveRecord::VERSION::MAJOR == 6
179 | prepend ActiveRecordSix
180 | else
181 | prepend ActiveRecordSeven
182 | end
183 | end
184 | end
185 |
--------------------------------------------------------------------------------
/lib/polo/translator.rb:
--------------------------------------------------------------------------------
1 | require "polo/sql_translator"
2 | require "polo/configuration"
3 |
4 | module Polo
5 | class Translator
6 |
7 | # Public: Creates a new Polo::Collector
8 | #
9 | # selects - An array of SELECT queries
10 | #
11 | def initialize(selects, configuration=Configuration.new)
12 | @selects = selects
13 | @configuration = configuration
14 | end
15 |
16 | # Public: Translates SELECT queries into INSERTS.
17 | #
18 | def translate
19 | SqlTranslator.new(instances, @configuration).to_sql.uniq
20 | end
21 |
22 | def instances
23 | active_record_selects, raw_selects = @selects.partition{|s| s[:klass]}
24 |
25 | active_record_instances = active_record_selects.flat_map do |select|
26 | select[:klass].find_by_sql(select[:sql]).to_a
27 | end
28 |
29 | if fields = @configuration.blacklist
30 | obfuscate!(active_record_instances, fields)
31 | end
32 |
33 | raw_instance_values = raw_selects.flat_map do |select|
34 | table_name = select[:sql][/^SELECT .* FROM (?:"|`)([^"`]+)(?:"|`)/, 1]
35 | select[:connection].select_all(select[:sql]).map { |values| {table_name: table_name, values: values} }
36 | end
37 |
38 | active_record_instances + raw_instance_values
39 | end
40 |
41 | private
42 |
43 | def obfuscate!(instances, fields)
44 | instances.each do |instance|
45 | next if intersection(instance.attributes.keys, fields).empty?
46 |
47 | fields.each do |field, strategy|
48 | field = field.to_s
49 |
50 | if table = table_name(field)
51 | field = field_name(field)
52 | end
53 |
54 | correct_table = table.nil? || instance.class.table_name == table
55 |
56 | if correct_table && instance.attributes[field]
57 | instance.send("#{field}=", new_field_value(field, strategy, instance))
58 | end
59 | end
60 | end
61 | end
62 |
63 | def field_name(field)
64 | field.to_s.include?('.') ? field.split('.').last : field.to_s
65 | end
66 |
67 | def table_name(field)
68 | field.to_s.include?('.') ? field.split('.').first : nil
69 | end
70 |
71 | def intersection(attrs, fields)
72 | attrs & fields.map { |pair| field_name(pair.first) }
73 | end
74 |
75 | def new_field_value(field, strategy, instance)
76 | value = instance.attributes[field]
77 | if strategy.nil?
78 | value.split("").shuffle.join
79 | else
80 | strategy.arity == 1 ? strategy.call(value) : strategy.call(value, instance)
81 | end
82 | end
83 | end
84 | end
85 |
--------------------------------------------------------------------------------
/lib/polo/version.rb:
--------------------------------------------------------------------------------
1 | module Polo
2 | VERSION = "0.6.0"
3 | end
4 |
--------------------------------------------------------------------------------
/polo.gemspec:
--------------------------------------------------------------------------------
1 | # coding: utf-8
2 | lib = File.expand_path('../lib', __FILE__)
3 | $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
4 | require 'polo/version'
5 |
6 | Gem::Specification.new do |spec|
7 | spec.name = "polo"
8 | spec.version = Polo::VERSION
9 | spec.authors = ["Netto Farah"]
10 | spec.email = ["nettofarah@gmail.com"]
11 |
12 | spec.summary = %q{Bring life back to your development environment with samples from production data.}
13 | spec.description = %q{Polo travels through your database and creates sample snapshots so you can work with real world data in development.}
14 | spec.homepage = "http://ifttt.github.io"
15 | spec.license = "MIT"
16 |
17 | spec.files = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
18 | spec.bindir = "exe"
19 | spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
20 | spec.require_paths = ["lib"]
21 |
22 | spec.add_dependency "activerecord", ">= 3.2"
23 |
24 | spec.add_development_dependency "appraisal"
25 | spec.add_development_dependency "bundler", "~> 2.3"
26 | spec.add_development_dependency "rake", "~> 13.0.1"
27 | spec.add_development_dependency "rspec"
28 |
29 | spec.add_development_dependency "sqlite3", ">= 1.3.13"
30 | end
31 |
--------------------------------------------------------------------------------
/spec/adapters/mysql_spec.rb:
--------------------------------------------------------------------------------
1 | require 'spec_helper'
2 |
3 | describe Polo::Adapters::MySQL do
4 |
5 | let(:adapter) { Polo::Adapters::MySQL.new }
6 |
7 | let(:netto) do
8 | AR::Chef.where(name: 'Netto').first
9 | end
10 |
11 | before(:all) do
12 | TestData.create_netto
13 | end
14 |
15 | let(:translator) { Polo::SqlTranslator.new(netto, Polo::Configuration.new(adapter: :mysql)) }
16 |
17 | describe '#ignore_transform' do
18 | it 'appends the IGNORE command after INSERTs' do
19 | insert_netto = [%q{INSERT IGNORE INTO "chefs" ("id", "name", "email") VALUES (1, 'Netto', 'nettofarah@gmail.com')}]
20 |
21 | records = translator.records
22 | inserts = translator.inserts
23 | translated_sql = adapter.ignore_transform(inserts, records)
24 | expect(translated_sql).to eq(insert_netto)
25 | end
26 | end
27 |
28 |
29 | describe '#on_duplicate_key_update' do
30 | it 'appends ON DUPLICATE KEY UPDATE with all values to the current INSERT statement' do
31 | insert_netto = [
32 | %q{INSERT INTO "chefs" ("id", "name", "email") VALUES (1, 'Netto', 'nettofarah@gmail.com') ON DUPLICATE KEY UPDATE `id` = VALUES(`id`), `name` = VALUES(`name`), `email` = VALUES(`email`)}
33 | ]
34 |
35 | inserts = translator.inserts
36 | records = translator.records
37 | translated_sql = adapter.on_duplicate_key_update(inserts, records)
38 | expect(translated_sql).to eq(insert_netto)
39 | end
40 | it 'works for hash-values instead of ActiveRecord instances' do
41 | insert_netto = [
42 | %q{INSERT INTO "chefs" ("id", "name", "email") VALUES (1, 'Netto', 'nettofarah@gmail.com') ON DUPLICATE KEY UPDATE `id` = VALUES(`id`), `name` = VALUES(`name`), `email` = VALUES(`email`)}
43 | ]
44 |
45 | inserts = translator.inserts
46 | records = [{table_name: "chefs", values: { id: 1, name: "Netto", email: "nettofarah@gmail.com"}}]
47 | translated_sql = adapter.on_duplicate_key_update(inserts, records)
48 | expect(translated_sql).to eq(insert_netto)
49 | end
50 | end
51 | end
--------------------------------------------------------------------------------
/spec/adapters/postgres_spec.rb:
--------------------------------------------------------------------------------
1 | require 'spec_helper'
2 |
3 | describe Polo::Adapters::Postgres do
4 |
5 | let(:adapter) { Polo::Adapters::Postgres.new }
6 |
7 | let(:netto) do
8 | AR::Chef.where(name: 'Netto').first
9 | end
10 |
11 | before(:all) do
12 | TestData.create_netto
13 | end
14 |
15 | let(:translator) { Polo::SqlTranslator.new(netto, Polo::Configuration.new(adapter: :postgres)) }
16 |
17 | describe '#on_duplicate_key_update' do
18 | it 'should raise an error' do
19 | expect { adapter.on_duplicate_key_update(double(), double()) }.to raise_error('on_duplicate: :override is not currently supported in the PostgreSQL adapter')
20 | end
21 | end
22 |
23 | describe '#ignore_transform' do
24 | it 'transforms INSERT by appending WHERE NOT EXISTS clause' do
25 |
26 | insert_netto = [%q{INSERT INTO "chefs" ("id", "name", "email") SELECT 1, 'Netto', 'nettofarah@gmail.com' WHERE NOT EXISTS (SELECT 1 FROM chefs WHERE id=1);}]
27 |
28 | records = translator.records
29 | inserts = translator.inserts
30 | translated_sql = adapter.ignore_transform(inserts, records)
31 | expect(translated_sql).to eq(insert_netto)
32 | end
33 |
34 | it 'works for hash-values instead of ActiveRecord instances' do
35 | insert_netto = [%q{INSERT INTO "chefs" ("id", "name", "email") SELECT 1, 'Netto', 'nettofarah@gmail.com' WHERE NOT EXISTS (SELECT 1 FROM chefs WHERE id=1);}]
36 |
37 | inserts = translator.inserts
38 | records = [{table_name: "chefs", values: { id: 1, name: "Netto", email: "nettofarah@gmail.com"}}]
39 | translated_sql = adapter.ignore_transform(inserts, records)
40 | expect(translated_sql).to eq(insert_netto)
41 | end
42 | end
43 | end
--------------------------------------------------------------------------------
/spec/configuration_spec.rb:
--------------------------------------------------------------------------------
1 | require 'spec_helper'
2 |
3 | describe Polo::Configuration do
4 |
5 | describe 'adapter' do
6 | it 'defaults to mysql' do
7 | expect(Polo.defaults.adapter).to be :mysql
8 | end
9 | end
10 |
11 | describe 'on_duplicate' do
12 | it 'defaults to nothing' do
13 | expect(Polo.defaults.on_duplicate_strategy).to be nil
14 | end
15 |
16 | it 'accepts custom strategies' do
17 | Polo.configure do
18 | on_duplicate(:ignore)
19 | end
20 |
21 | defaults = Polo.defaults
22 | expect(defaults.on_duplicate_strategy).to eq(:ignore)
23 | end
24 | end
25 |
26 | describe 'obfuscate' do
27 | it 'defaults to an empty list' do
28 | expect(Polo.defaults.blacklist).to be_empty
29 | end
30 |
31 | it 'allows for the user to define fields to blacklist' do
32 | Polo.configure do
33 | obfuscate(:email, :password)
34 | end
35 |
36 | defaults = Polo.defaults
37 | expect(defaults.blacklist).to eq({ :email => nil, :password => nil })
38 | end
39 |
40 | it 'allows for the user to define fields with strategies in blacklist' do
41 | Polo.configure do
42 |
43 | email_strategy = lambda {|e| "#{e.split("@")[0]}_test@example.com" }
44 | credit_card_strategy = lambda {|_| "4111111111111111"}
45 |
46 | obfuscate({email: email_strategy, credit_card: credit_card_strategy})
47 | end
48 |
49 | defaults = Polo.defaults
50 | expect(defaults.blacklist[:email].call("a@b.com")).to eq("a_test@example.com")
51 | end
52 | end
53 | end
54 |
--------------------------------------------------------------------------------
/spec/polo_spec.rb:
--------------------------------------------------------------------------------
1 | require 'spec_helper'
2 |
3 | describe Polo do
4 |
5 | before(:all) do
6 | TestData.create_netto
7 | end
8 |
9 | it 'generates an insert query for the base object' do
10 | exp = Polo.explore(AR::Chef, 1)
11 | insert = %q{INSERT INTO "chefs" ("id", "name", "email") VALUES (1, 'Netto', 'nettofarah@gmail.com')}
12 | expect(exp).to include(insert)
13 | end
14 |
15 | it 'generates an insert query for the objects with non-standard primary keys' do
16 | exp = Polo.explore(AR::Person, 1)
17 | insert = %q{INSERT INTO "people" ("ssn", "name") VALUES (1, 'John Doe')}
18 | expect(exp).to include(insert)
19 | end
20 |
21 | it 'generates insert queries for dependencies' do
22 | if ActiveRecord::VERSION::STRING >= "4.2"
23 | serialized_nil = "NULL"
24 | else
25 | serialized_nil = "'null'"
26 | end
27 |
28 | turkey_insert = %Q{INSERT INTO "recipes" ("id", "title", "num_steps", "chef_id", "metadata") VALUES (1, 'Turkey Sandwich', NULL, 1, #{serialized_nil})}
29 | cheese_burger_insert = %Q{INSERT INTO "recipes" ("id", "title", "num_steps", "chef_id", "metadata") VALUES (2, 'Cheese Burger', NULL, 1, #{serialized_nil})}
30 |
31 | inserts = Polo.explore(AR::Chef, 1, [:recipes])
32 |
33 | expect(inserts).to include(turkey_insert)
34 | expect(inserts).to include(cheese_burger_insert)
35 | end
36 |
37 | it 'generates queries for nested dependencies' do
38 | patty = %q{INSERT INTO "ingredients" ("id", "name", "quantity") VALUES (3, 'Patty', '1')}
39 | turkey = %q{INSERT INTO "ingredients" ("id", "name", "quantity") VALUES (1, 'Turkey', 'a lot')}
40 | one_cheese = %q{INSERT INTO "ingredients" ("id", "name", "quantity") VALUES (2, 'Cheese', '1 slice')}
41 | two_cheeses = %q{INSERT INTO "ingredients" ("id", "name", "quantity") VALUES (4, 'Cheese', '2 slices')}
42 |
43 | inserts = Polo.explore(AR::Chef, 1, :recipes => :ingredients)
44 |
45 | expect(inserts).to include(patty)
46 | expect(inserts).to include(turkey)
47 | expect(inserts).to include(one_cheese)
48 | expect(inserts).to include(two_cheeses)
49 | end
50 |
51 | it 'generates inserts for HABTM relationships' do
52 | ar_version = ActiveRecord::VERSION::STRING
53 | skip("Not supported on ActiveRecord #{ar_version}") if ar_version < "4.1.0"
54 | habtm_inserts = [
55 | %q{INSERT INTO "recipes_tags" ("recipe_id", "tag_id") VALUES (2, 2)},
56 | %q{INSERT INTO "tags" ("id", "name") VALUES (2, 'burgers')}
57 | ]
58 |
59 | inserts = Polo.explore(AR::Chef, 1, :recipes => :tags)
60 |
61 | habtm_inserts.each do |habtm_insert|
62 | expect(inserts).to include(habtm_insert)
63 | end
64 | end
65 |
66 | it 'generates inserts for many to many relationships' do
67 | many_to_many_inserts = [
68 | %q{INSERT INTO "recipes_ingredients" ("id", "recipe_id", "ingredient_id") VALUES (1, 1, 1)},
69 | %q{INSERT INTO "recipes_ingredients" ("id", "recipe_id", "ingredient_id") VALUES (2, 1, 2)},
70 | %q{INSERT INTO "recipes_ingredients" ("id", "recipe_id", "ingredient_id") VALUES (3, 2, 3)},
71 | %q{INSERT INTO "recipes_ingredients" ("id", "recipe_id", "ingredient_id") VALUES (4, 2, 4)},
72 | ]
73 |
74 | inserts = Polo.explore(AR::Chef, 1, :recipes => :ingredients)
75 |
76 | many_to_many_inserts.each do |many_to_many_insert|
77 | expect(inserts).to include(many_to_many_insert)
78 | end
79 | end
80 |
81 | describe "Advanced Options" do
82 | describe 'obfuscate: [fields]' do
83 |
84 | it 'scrambles a predefined field' do
85 | Polo.configure do
86 | obfuscate(:email)
87 | end
88 |
89 | exp = Polo.explore(AR::Chef, 1)
90 | insert = /INSERT INTO "chefs" \("id", "name", "email"\) VALUES \(1, 'Netto', (.+)\)/
91 | scrambled_email = insert.match(exp.first)[1]
92 |
93 | expect(scrambled_email).to_not eq('nettofarah@gmail.com')
94 | expect(insert).to match(exp.first)
95 | end
96 |
97 | it 'can apply custom strategies' do
98 | Polo.configure do
99 | obfuscate(email: lambda { |_| 'changeme' })
100 | end
101 |
102 | inserts = Polo.explore(AR::Chef, 1)
103 |
104 | expect(inserts).to eq [ %q{INSERT INTO "chefs" ("id", "name", "email") VALUES (1, 'Netto', 'changeme')} ]
105 | end
106 |
107 | it 'only scrambles instances with the obfuscate field defined' do
108 | Polo.configure do
109 | obfuscate email: ->(e) { "#{e.split("@")[0]}_test@example.com" },
110 | title: ->(t) { t.chars.reverse!.join }
111 | end
112 |
113 | exp = Polo.explore(AR::Chef, 1, :recipes)
114 |
115 | explore_statement = exp.join(';')
116 | expect(explore_statement).to_not match('nettofarah@gmail.com')
117 | expect(explore_statement).to match('Netto')
118 | end
119 |
120 | it 'can target a specific field in a table' do
121 | Polo.configure do
122 | obfuscate 'ingredients.name' => -> (i) { "Secret" }
123 | end
124 |
125 | exp = Polo.explore(AR::Chef, 1, recipes: :ingredients)
126 |
127 | explore_statement = exp.join(';')
128 | expect(explore_statement).to match('Netto')
129 | expect(explore_statement).to match('Secret')
130 | end
131 | end
132 |
133 | describe 'on_duplicate' do
134 | it 'applies the on_duplicate strategy' do
135 | Polo.configure do
136 | on_duplicate(:ignore)
137 | end
138 |
139 | exp = Polo.explore(AR::Chef, 1)
140 | insert = /INSERT IGNORE INTO "chefs" \("id", "name", "email"\) VALUES \(1, 'Netto', (.+)\)/
141 | expect(insert).to match(exp.first)
142 | end
143 | end
144 | end
145 | end
146 |
--------------------------------------------------------------------------------
/spec/spec_helper.rb:
--------------------------------------------------------------------------------
1 | $LOAD_PATH.unshift File.expand_path('../../lib', __FILE__)
2 | require 'polo'
3 | require 'active_record'
4 |
5 | ActiveRecord::Base.establish_connection(adapter: 'sqlite3', database: ':memory:')
6 |
7 | require 'support/schema'
8 | require 'support/activerecord_models'
9 | require 'support/factories'
10 |
11 | module Polo
12 | def self.reset!
13 | @configuration = Configuration.new
14 | end
15 | end
16 |
17 | RSpec.configure do |c|
18 | c.around(:example) do |example|
19 | Polo.reset!
20 |
21 | ActiveRecord::Base.transaction do
22 | example.run
23 | raise ActiveRecord::Rollback
24 | end
25 | end
26 | end
27 |
28 | def track_queries
29 | selects = []
30 | queries_collector = lambda do |name, start, finish, id, payload|
31 | selects << payload
32 | end
33 |
34 | ActiveRecord::Base.connection.clear_query_cache
35 | ActiveSupport::Notifications.subscribed(queries_collector, 'sql.active_record') do
36 | yield
37 | end
38 |
39 | selects.map { |sel| sel[:sql] }
40 | end
41 |
--------------------------------------------------------------------------------
/spec/sql_translator_spec.rb:
--------------------------------------------------------------------------------
1 | require 'spec_helper'
2 |
3 | describe Polo::SqlTranslator do
4 |
5 | let(:netto) do
6 | AR::Chef.where(name: 'Netto').first
7 | end
8 |
9 | before(:all) do
10 | TestData.create_netto
11 | end
12 |
13 | it 'translates records to inserts' do
14 | insert_netto = [%q{INSERT INTO "chefs" ("id", "name", "email") VALUES (1, 'Netto', 'nettofarah@gmail.com')}]
15 | netto_to_sql = Polo::SqlTranslator.new(netto).to_sql
16 | expect(netto_to_sql).to eq(insert_netto)
17 | end
18 |
19 | it 'encodes serialized fields correctly' do
20 | recipe = AR::Recipe.create(title: 'Polenta', metadata: { quality: 'ok' })
21 | recipe_to_sql = Polo::SqlTranslator.new(recipe).to_sql.first
22 | expect(recipe_to_sql).to include(%q{'{"quality":"ok"}'}) # JSON, not YAML
23 | end
24 |
25 | it 'encodes attributes not backed by a database column correctly' do
26 | if Gem.loaded_specs["activerecord"].version < Gem::Version.new("4.2.1")
27 | skip "the attributes API was included in rails starting in 4.2.1"
28 | elsif Gem.loaded_specs["activerecord"].version >= Gem::Version.new("4.2.1") &&
29 | Gem.loaded_specs["activerecord"].version < Gem::Version.new("5.0.0")
30 | class Employee < ActiveRecord::Base
31 | attribute :on_vacation, Type::Boolean.new
32 | end
33 | else
34 | class Employee < ActiveRecord::Base
35 | attribute :on_vacation, :boolean
36 | end
37 | end
38 |
39 | employee = Employee.create(name: 'John Doe', on_vacation: true)
40 | employee_to_sql = Polo::SqlTranslator.new(employee).to_sql.first
41 | expect(employee_to_sql).to_not include('on_vacation')
42 | end
43 | end
44 |
--------------------------------------------------------------------------------
/spec/support/activerecord_models.rb:
--------------------------------------------------------------------------------
1 | module AR
2 | class Recipe < ActiveRecord::Base
3 | belongs_to :chef
4 | has_many :recipes_ingredients
5 | has_many :ingredients, through: :recipes_ingredients
6 | has_and_belongs_to_many :tags
7 |
8 | serialize :metadata, JSON
9 | end
10 |
11 | class Ingredient < ActiveRecord::Base
12 | end
13 |
14 | class Tag < ActiveRecord::Base
15 | has_and_belongs_to_many :recipes
16 | end
17 |
18 | class RecipesIngredient < ActiveRecord::Base
19 | belongs_to :recipe
20 | belongs_to :ingredient
21 | end
22 |
23 | class Restaurant < ActiveRecord::Base
24 | belongs_to :owner, class_name: 'Chef'
25 | has_one :rating
26 | end
27 |
28 | class Rating < ActiveRecord::Base
29 | belongs_to :restaurant
30 | end
31 |
32 | class Chef < ActiveRecord::Base
33 | has_many :recipes
34 | has_many :ingredients, through: :recipes
35 | has_one :restaurant, foreign_key: 'owner_id'
36 | end
37 |
38 | class Person < ActiveRecord::Base
39 | self.primary_key = :ssn
40 | end
41 | end
42 |
--------------------------------------------------------------------------------
/spec/support/factories.rb:
--------------------------------------------------------------------------------
1 | module TestData
2 |
3 | def self.create_netto
4 | AR::Chef.create(name: 'Netto', email: 'nettofarah@gmail.com').tap do |netto|
5 | AR::Recipe.create(title: 'Turkey Sandwich', chef: netto).tap do |r|
6 | r.ingredients.create(name: 'Turkey', quantity: 'a lot')
7 | r.ingredients.create(name: 'Cheese', quantity: '1 slice')
8 | r.tags.create(name: 'sandwiches')
9 | end
10 |
11 | AR::Recipe.create(title: 'Cheese Burger', chef: netto).tap do |r|
12 | r.ingredients.create(name: 'Patty', quantity: '1')
13 | r.ingredients.create(name: 'Cheese', quantity: '2 slices')
14 | r.tags.create(name: 'burgers')
15 | end
16 | end
17 |
18 | AR::Person.create(name: 'John Doe')
19 | end
20 | end
21 |
--------------------------------------------------------------------------------
/spec/support/schema.rb:
--------------------------------------------------------------------------------
1 | ActiveRecord::Schema.define do
2 | create_table :recipes, force: true do |t|
3 | t.column :title, :string
4 | t.column :num_steps, :integer
5 | t.column :chef_id, :integer
6 | t.column :metadata, :text
7 | end
8 |
9 | create_table :ingredients, force: true do |t|
10 | t.column :name, :string
11 | t.column :quantity, :string
12 | end
13 |
14 | create_table :recipes_ingredients, force: true do |t|
15 | t.column :recipe_id, :integer
16 | t.column :ingredient_id, :integer
17 | end
18 |
19 | create_table :chefs, force: true do |t|
20 | t.column :name, :string
21 | t.column :email, :string
22 | end
23 |
24 | create_table :restaurants, force: true do |t|
25 | t.column :name, :string
26 | t.column :owner_id, :integer
27 | t.column :current_customer_count, :integer
28 | end
29 |
30 | create_table :ratings, force: true do |t|
31 | t.column :value, :string
32 | t.column :restaurant_id, :integer
33 | end
34 |
35 | create_table :people, primary_key: :ssn, force: true do |t|
36 | t.column :name, :string
37 | end
38 |
39 | create_table :employees, force: true do |t|
40 | t.column :name, :string
41 | end
42 |
43 | create_table :tags, force: true do |t|
44 | t.column :name, :string
45 | end
46 |
47 | create_table :recipes_tags, id: false, force: true do |t|
48 | t.column :recipe_id, :integer
49 | t.column :tag_id, :integer
50 | end
51 | end
52 |
--------------------------------------------------------------------------------
/spec/translator_spec.rb:
--------------------------------------------------------------------------------
1 | require 'spec_helper'
2 |
3 | describe Polo::Translator do
4 |
5 | let(:email) { 'nettofarah@gmail.com' }
6 | let(:finder) do
7 | {
8 | klass: AR::Chef,
9 | sql: AR::Chef.where(email: email).to_sql
10 | }
11 | end
12 |
13 | before(:all) do
14 | TestData.create_netto
15 | end
16 |
17 | describe "options" do
18 | describe "obfuscate: [fields]" do
19 | let(:translator) { Polo::Translator.new([finder], Polo::Configuration.new(obfuscate: obfuscated_fields)) }
20 | let(:netto) { netto = translator.instances.first }
21 |
22 | context "obfuscated field with no specified strategy" do
23 | let(:obfuscated_fields) {{email: nil }}
24 |
25 | it "shuffles characters in field" do
26 | expect(netto.email).to_not be_nil
27 | expect(netto.email.length).to eq email.length
28 | expect(sorted_characters(netto.email)).to eq sorted_characters(email)
29 | end
30 | end
31 |
32 | context "obfuscated field with obscuration strategy applied which will result in 42" do
33 | let(:obfuscated_fields) {{email: lambda { |_| 42 } } }
34 |
35 | it "replaces contents of field according to the supplied lambda" do
36 | expect(netto.email.to_s).to eq "42"
37 | end
38 | end
39 |
40 | context "complex custom obfuscation strategy" do
41 | let(:obfuscated_fields) do
42 | { email: lambda { |field| "temp_#{field}" } }
43 | end
44 |
45 | it "replaces contents of field according to the supplied lambda" do
46 | expect(netto.email.to_s).to eq "temp_nettofarah@gmail.com"
47 | end
48 | end
49 |
50 | context "custom obfuscation strategy using instance context" do
51 | let(:obfuscated_fields) do
52 | { email: lambda { |field, instance| "#{instance.name}@example.com" } }
53 | end
54 |
55 | it "replaces contents of field according to the supplied lambda" do
56 | expect(netto.email.to_s).to eq "Netto@example.com"
57 | end
58 | end
59 |
60 | context "no strategy passed in" do
61 | let(:obfuscated_fields) { [:email] }
62 |
63 | it "shuffles contents" do
64 | expect(netto.email).to_not eq email
65 | expect(sorted_characters(netto.email)).to eq sorted_characters(email)
66 | end
67 | end
68 |
69 | def sorted_characters(str)
70 | str.split("").sort.join
71 | end
72 | end
73 | end
74 | end
75 |
--------------------------------------------------------------------------------