├── .gitignore ├── lib ├── conformist │ ├── version.rb │ ├── builder.rb │ ├── hash_struct.rb │ ├── column.rb │ └── schema.rb └── conformist.rb ├── test ├── fixtures │ ├── states.xls │ ├── citizens.csv │ ├── acma.csv │ └── fcc.txt ├── helper.rb ├── schemas │ ├── citizens.rb │ ├── acma.rb │ └── fcc.rb └── unit │ ├── conformist_test.rb │ ├── conformist │ ├── builder_test.rb │ ├── hash_struct_test.rb │ ├── column_test.rb │ └── schema_test.rb │ └── integration_test.rb ├── Gemfile ├── .travis.yml ├── Rakefile ├── LICENSE ├── conformist.gemspec ├── CHANGELOG.md └── README.md /.gitignore: -------------------------------------------------------------------------------- 1 | *.gem 2 | .bundle 3 | Gemfile.lock 4 | pkg/* 5 | -------------------------------------------------------------------------------- /lib/conformist/version.rb: -------------------------------------------------------------------------------- 1 | module Conformist 2 | VERSION = '0.2.5' 3 | end 4 | -------------------------------------------------------------------------------- /test/fixtures/states.xls: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tatey/conformist/HEAD/test/fixtures/states.xls -------------------------------------------------------------------------------- /Gemfile: -------------------------------------------------------------------------------- 1 | source "http://rubygems.org" 2 | 3 | # Specify your gem's dependencies in conformist.gemspec 4 | gemspec 5 | -------------------------------------------------------------------------------- /test/fixtures/citizens.csv: -------------------------------------------------------------------------------- 1 | Name,Age,Gender 2 | Aaron,21,Male 3 | Bec,30,Female 4 | Charlie,50,Male 5 | Debbie,26,Female -------------------------------------------------------------------------------- /test/helper.rb: -------------------------------------------------------------------------------- 1 | require 'conformist' 2 | require 'minitest/autorun' 3 | require 'pry' 4 | 5 | include Conformist 6 | -------------------------------------------------------------------------------- /test/schemas/citizens.rb: -------------------------------------------------------------------------------- 1 | class Citizens 2 | extend Conformist 3 | 4 | column :name, 0 5 | column :age, 1 6 | column :gender, 2 7 | end -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: ruby 2 | rvm: 3 | - 1.9.3 4 | - 2.0.0 5 | - 2.1.0 6 | - 2.2.0 7 | - 2.3.1 8 | - 2.4.0 9 | - jruby 10 | 11 | before_install: 12 | - gem install bundler -v ">= 1.8.9" 13 | -------------------------------------------------------------------------------- /Rakefile: -------------------------------------------------------------------------------- 1 | require 'rubygems' 2 | require 'bundler/gem_tasks' 3 | require 'bundler/setup' 4 | require 'rake/testtask' 5 | 6 | Rake::TestTask.new :test do |test| 7 | test.libs << 'test' 8 | test.pattern = 'test/**/*_test.rb' 9 | end 10 | 11 | task :default => :test 12 | -------------------------------------------------------------------------------- /test/schemas/acma.rb: -------------------------------------------------------------------------------- 1 | class ACMA 2 | extend Conformist 3 | 4 | column :name, 11 do |value| 5 | value.match(/[A-Z]+$/)[0].upcase 6 | end 7 | column :callsign, 1 8 | column :latitude, 15 9 | end 10 | 11 | class ACMA::Digital < ACMA 12 | column :signal_type do 13 | 'digital' 14 | end 15 | end 16 | -------------------------------------------------------------------------------- /test/schemas/fcc.rb: -------------------------------------------------------------------------------- 1 | class FCC 2 | extend Conformist 3 | 4 | column :name, 10, 11 do |values| 5 | "#{values[0].upcase}, #{values[-1]}" 6 | end 7 | column :callsign, 1 8 | column :latitude, 20, 21, 22, 19 do |values| 9 | values.join(' ') 10 | end 11 | column :signtal_type do 12 | 'digital' 13 | end 14 | end 15 | -------------------------------------------------------------------------------- /lib/conformist/builder.rb: -------------------------------------------------------------------------------- 1 | module Conformist 2 | class Builder 3 | def self.call schema, enumerable, context = nil 4 | columns = schema.columns 5 | hash = columns.each_with_object({}) do |column, hash| 6 | hash[column.name] = column.values_in(enumerable, context) 7 | end 8 | HashStruct.new hash 9 | end 10 | end 11 | end 12 | -------------------------------------------------------------------------------- /lib/conformist.rb: -------------------------------------------------------------------------------- 1 | require 'forwardable' 2 | 3 | require 'conformist/builder' 4 | require 'conformist/column' 5 | require 'conformist/hash_struct' 6 | require 'conformist/schema' 7 | 8 | module Conformist 9 | def self.extended base 10 | base.extend Schema 11 | end 12 | 13 | def self.new *args, &block 14 | Class.new { include Schema }.new *args, &block 15 | end 16 | end 17 | -------------------------------------------------------------------------------- /test/unit/conformist_test.rb: -------------------------------------------------------------------------------- 1 | require 'helper' 2 | 3 | class ConformistTest < Minitest::Test 4 | def test_extended 5 | definition = Class.new { extend Conformist } 6 | assert definition.respond_to?(:builder) 7 | assert definition.respond_to?(:columns) 8 | assert definition.respond_to?(:conform) 9 | end 10 | 11 | def test_new 12 | assert Conformist.new.class.include?(Schema) 13 | end 14 | end 15 | -------------------------------------------------------------------------------- /test/unit/conformist/builder_test.rb: -------------------------------------------------------------------------------- 1 | require 'helper' 2 | 3 | class Conformist::BuilderTest < Minitest::Test 4 | def test_call 5 | column = Minitest::Mock.new 6 | column.expect :name, :a 7 | column.expect :values_in, [1], [Array, Hash] 8 | definition = Minitest::Mock.new 9 | definition.expect :columns, [column] 10 | assert_equal HashStruct.new({:a => [1]}), Builder.call(definition, [], {}) 11 | end 12 | end 13 | -------------------------------------------------------------------------------- /lib/conformist/hash_struct.rb: -------------------------------------------------------------------------------- 1 | module Conformist 2 | class HashStruct 3 | extend Forwardable 4 | 5 | attr_accessor :attributes 6 | 7 | def_delegators :attributes, :[], :[]=, :fetch, :key? 8 | 9 | def initialize attributes = {} 10 | self.attributes = attributes 11 | end 12 | 13 | def == other 14 | other.class == self.class && attributes == other.attributes 15 | end 16 | 17 | protected 18 | 19 | def respond_to_missing? method, include_private 20 | key?(method) || super 21 | end 22 | 23 | def method_missing method, *args, &block 24 | fetch(method) { super } 25 | end 26 | end 27 | end 28 | -------------------------------------------------------------------------------- /test/fixtures/acma.csv: -------------------------------------------------------------------------------- 1 | Adelaide,ABS2,64.26,National,H,124,OD,200000,57001,1198821,23139,ntl Tower Summit Road CRAFERS,54,290707,6126723,34 58 49S,138 42 25E,SA,2,,,,Issued 2 | Adelaide,SAS7,182.26,Commercial,H,122,OD,200000,122001,1139429,23181,Channel 7/10 Site 115 Summit Road CRAFERS,54,290670,6126481,34 58 57S,138 42 24E,SA,7,122,ADELAIDE TV1,162,Issued 3 | Adelaide,NWS9,196.26,Commercial,H,144,OD,200000,121001,1384392,23132,NWS 9 site 109 Summit Road CRAFERS,54,290698,6126331,34 59 02S,138 42 25E,SA,9,121,ADELAIDE TV1,162,Issued 4 | Adelaide,ADS10,209.25,Commercial,H,144,OD,200000,120001,1904367,23132,NWS 9 site 109 Summit Road CRAFERS,54,290698,6126331,34 59 02S,138 42 25E,SA,10,120,ADELAIDE TV1,162,Issued 5 | Adelaide,ADS10,209.25,Commercial,H,122,OD,200000,120001,1384005,23181,Channel 7/10 Site 115 Summit Road CRAFERS,54,290670,6126481,34 58 57S,138 42 24E,SA,10,120,ADELAIDE TV1,162,Issued 6 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License 2 | 3 | Copyright (c) 2011 Tate Johnson 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in 13 | all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | THE SOFTWARE. 22 | -------------------------------------------------------------------------------- /conformist.gemspec: -------------------------------------------------------------------------------- 1 | # -*- encoding: utf-8 -*- 2 | $:.push File.expand_path("../lib", __FILE__) 3 | require "conformist/version" 4 | 5 | Gem::Specification.new do |s| 6 | s.name = "conformist" 7 | s.version = Conformist::VERSION 8 | s.platform = Gem::Platform::RUBY 9 | s.authors = ["Tate Johnson"] 10 | s.email = ["tate@tatey.com"] 11 | s.homepage = "https://github.com/tatey/conformist" 12 | s.summary = %q{Bend CSVs to your will with declarative schemas.} 13 | s.description = %q{Bend CSVs to your will with declarative schemas.} 14 | 15 | s.rubyforge_project = "conformist" 16 | 17 | s.required_ruby_version = '>= 1.9.3' 18 | 19 | s.add_development_dependency 'minitest', '>= 5.4.0' 20 | s.add_development_dependency 'pry' 21 | s.add_development_dependency 'rake' 22 | s.add_development_dependency 'spreadsheet', '>= 1.1.4' 23 | 24 | s.files = `git ls-files`.split("\n") 25 | s.test_files = `git ls-files -- {test,spec,features}/*`.split("\n") 26 | s.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) } 27 | s.require_paths = ["lib"] 28 | end 29 | -------------------------------------------------------------------------------- /test/unit/conformist/hash_struct_test.rb: -------------------------------------------------------------------------------- 1 | require 'helper' 2 | 3 | class Conformist::HashStructTest < Minitest::Test 4 | def test_initialize 5 | assert_equal({:a => 1}, HashStruct.new({:a => 1}).attributes) 6 | assert_empty HashStruct.new.attributes 7 | end 8 | 9 | def test_delegates 10 | hash = HashStruct.new 11 | assert hash.respond_to?(:[]) 12 | assert hash.respond_to?(:[]=) 13 | assert hash.respond_to?(:fetch) 14 | assert hash.respond_to?(:key?) 15 | end 16 | 17 | def test_equality 18 | hash1 = HashStruct.new :a => 1 19 | hash2 = HashStruct.new :a => 1 20 | hash3 = Minitest::Mock.new 21 | hash3.expect :attributes, {:a => 1} 22 | hash3.expect :class, Minitest::Mock 23 | assert_equal hash1, hash2 24 | refute_equal hash1, hash3 25 | end 26 | 27 | def test_readers_with_method_missing 28 | hash = HashStruct.new :a => 1, :c_d => 1 29 | assert_equal 1, hash.a 30 | assert_equal 1, hash.c_d 31 | end 32 | 33 | if respond_to? :respond_to_missing? # Compatible with 1.9 34 | def test_readers_with_respond_to_missing 35 | hash = HashStruct.new :a => 1, :c_d => 1 36 | assert hash.respond_to?(:a) 37 | assert hash.respond_to?(:c_d) 38 | end 39 | end 40 | end 41 | -------------------------------------------------------------------------------- /lib/conformist/column.rb: -------------------------------------------------------------------------------- 1 | module Conformist 2 | class Column 3 | attr_accessor :name, :sources, :indexes, :preprocessor 4 | 5 | def initialize name, *sources, &preprocessor 6 | self.name = name 7 | self.sources = sources 8 | self.indexes = sources 9 | self.preprocessor = preprocessor 10 | end 11 | 12 | def calculate_indices!(headers) 13 | headers = Array(headers).collect {|header| header.to_s.downcase.squeeze(' ').strip } 14 | 15 | self.indexes = sources.collect do |source| 16 | if source.is_a?(String) 17 | headers.index(source.downcase) 18 | else 19 | source 20 | end 21 | end 22 | end 23 | 24 | def values_in enumerable, context = nil 25 | enumerable = Array(enumerable) 26 | 27 | values = Array(indexes).map do |index| 28 | value = enumerable.at(index) if index 29 | 30 | if value.respond_to? :strip 31 | value.strip 32 | else 33 | value 34 | end 35 | end 36 | values = values.first if values.size == 1 37 | if preprocessor 38 | if preprocessor.arity == 1 39 | preprocessor.call values 40 | else 41 | preprocessor.call values, context 42 | end 43 | else 44 | values 45 | end 46 | end 47 | end 48 | end 49 | -------------------------------------------------------------------------------- /test/unit/conformist/column_test.rb: -------------------------------------------------------------------------------- 1 | require 'helper' 2 | 3 | class Conformist::ColumnTest < Minitest::Test 4 | def stub_row 5 | ('a'..'d').to_a 6 | end 7 | 8 | def test_name 9 | column = Column.new :foo 10 | assert_equal :foo, column.name 11 | end 12 | 13 | def test_one_index 14 | column = Column.new :foo, 0 15 | assert_equal 'a', column.values_in(stub_row) 16 | end 17 | 18 | def test_preprocess_with_one_index 19 | column = Column.new(:foo, 0) { |value| value.upcase } 20 | assert_equal 'A', column.values_in(stub_row) 21 | end 22 | 23 | def test_many_indexes 24 | column = Column.new :foo, 1, 2, 3 25 | assert_equal ['b', 'c', 'd'], column.values_in(stub_row) 26 | end 27 | 28 | def test_preprocess_with_many_indexes 29 | column = Column.new(:foo, 1, 2, 3) { |values| values.reverse } 30 | assert_equal ['d', 'c', 'b'], column.values_in(stub_row) 31 | end 32 | 33 | def test_virtual 34 | column = Column.new(:foo) { 'a' } 35 | assert_equal 'a', column.values_in(stub_row) 36 | end 37 | 38 | def test_strip_whitespace 39 | column = Column.new :foo, 0 40 | assert_equal 'a', column.values_in([' a ']) 41 | end 42 | 43 | def test_array 44 | column = Column.new :foo, 0 45 | assert_equal 'a', column.values_in(['a']) 46 | assert_equal 'a', column.values_in('a') 47 | end 48 | 49 | def test_nil 50 | column = Column.new :foo, 0 51 | assert_nil column.values_in([]) 52 | end 53 | 54 | def test_passes_context 55 | context = {'a' => 'b'} 56 | column = Column.new(:foo, 0) { |_, ctx| ctx } 57 | assert_equal context, column.values_in([], context) 58 | end 59 | 60 | def test_respects_preprocessor_arity 61 | context = {'a' => 'b'} 62 | column = Column.new(:foo, 0, &lambda { |values| }) 63 | column.values_in([], context) 64 | end 65 | end 66 | -------------------------------------------------------------------------------- /lib/conformist/schema.rb: -------------------------------------------------------------------------------- 1 | module Conformist 2 | module Schema 3 | def self.included base 4 | base.send :include, InstanceExtensions 5 | base.send :include, Methods 6 | end 7 | 8 | def self.extended base 9 | base.extend ClassExtensions 10 | base.extend Methods 11 | end 12 | 13 | module ClassExtensions 14 | def inherited base 15 | base.builder = builder.dup 16 | base.columns = columns.dup 17 | end 18 | end 19 | 20 | module InstanceExtensions 21 | def initialize super_schema = nil, &block 22 | if super_schema 23 | self.builder = super_schema.builder.dup 24 | self.columns = super_schema.columns.dup 25 | end 26 | if block 27 | instance_eval &block 28 | end 29 | end 30 | end 31 | 32 | module Methods 33 | def builder 34 | @builder ||= Builder 35 | end 36 | 37 | def builder= value 38 | @builder = value 39 | end 40 | 41 | def column *args, &block 42 | args << columns.length if args.length == 1 43 | columns << Column.new(*args, &block) 44 | end 45 | 46 | def columns 47 | @columns ||= [] 48 | end 49 | 50 | def columns= value 51 | @columns = value 52 | end 53 | 54 | def conform enumerables, options = {} 55 | options = options.dup 56 | context = options.delete(:context) 57 | 58 | Enumerator.new do |yielder| 59 | enumerables.each do |enumerable| 60 | if options.delete :skip_first 61 | columns.each {|column| column.calculate_indices!(enumerable) } 62 | next 63 | end 64 | 65 | yielder.yield builder.call(self, enumerable, context) 66 | end 67 | end 68 | end 69 | end 70 | end 71 | end 72 | -------------------------------------------------------------------------------- /test/fixtures/fcc.txt: -------------------------------------------------------------------------------- 1 | |KVTU-LP |- |LD |3 |DA | |- |- |APP |LOS ANGELES |CA |US |BDFCDVL-20100309ABS |0.25 kW |- |0.0 |- |130176 |N |34 |11 |13.90 |W |117 |42 |1.00 |MARK C. ALLEN | 0.00 km | 0.00 mi | 0.00 deg |1702. m|- |20780 |240. |1015693 |9. |1359585 | 2 | |NEW |- |LD |3 |ND | |- |- |APP |BAKERSFIELD |CA |US |BMJADVL-20100520ABV |0.3 kW |- |0.0 |- |130049 |N |35 |12 |6.00 |W |119 |5 |26.10 |JOHN R. POWLEY | 0.00 km | 0.00 mi | 0.00 deg |143.2 m|- |100298 |- |1015627 |50. |1368486 | 3 | |NEW |- |LD |3 |ND | |- |- |APP |YUCA VALLEY |CA |US |BMJADVL-20100520ADI |0.3 kW |- |0.0 |- |129831 |N |34 |9 |10.10 |W |116 |23 |19.10 |JOHN R. POWLEY | 0.00 km | 0.00 mi | 0.00 deg |1179. m|- |100305 |- |1028564 |20. |1370146 | 4 | |KCSO-LD |- |LD |3 |DA | |- |- |LIC |SACRAMENTO |CA |US |BLDVL -20100825AAQ |0.3 kW |- |0.0 |- |18998 |N |38 |7 |10.00 |W |120 |43 |27.00 |SAINTE 51, L.P. | 0.00 km | 0.00 mi | 0.00 deg |948.5 m|- |96076 |270. |1050375 |114. |1396970 | 5 | |KVTU-LP |- |TX |3 |DA |+ |- |- |CP MOD |LOS ANGELES |CA |US |BMPTVL -20100826AFT |0.498 kW |- |0.0 |- |130176 |N |34 |13 |38.00 |W |118 |4 |3.00 |MARK C. ALLEN | 0.00 km | 0.00 mi | 0.00 deg |1784. m|- |101585 |0. |1007719 |43. |1397199 | 6 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # CHANGELOG 2 | 3 | ## 0.2.5 / 2017-01-31 4 | 5 | * Add support for context. (@jcmfernandes) 6 | * Official support for Ruby 2.4.0. 7 | 8 | ## 0.2.4 / 2016-10-14 9 | 10 | * Speed up processing large amounts of columns (@splattael) 11 | * Official support for Ruby 2.3.1 (@splattael) 12 | 13 | ## 0.2.3 / 2015-03-04 14 | 15 | * Always return nil when named column isn't found. (@rywall) 16 | 17 | ## 0.2.2 / 2015-02-12 18 | 19 | * Don't munge headers. (@rywall) 20 | 21 | ## 0.2.1 / 2015-01-31 22 | 23 | * Add support for named columns. (@rywall) 24 | * Dropped support for Ruby 1.8.7, 1.9.2. 25 | 26 | ## 0.2.0 / 2012-08-18 27 | 28 | * `Conformist::Schema#confrom` takes option to skip first row. 29 | * Removed deprecated classes and warnings. 30 | 31 | ## 0.1.3 / 2012-02-09 32 | 33 | * Column indexes are implicitly incremented when the index argument is omitted. Implicit indexing is all or nothing. (@coop) 34 | 35 | ## 0.1.2 / 2012-01-19 36 | 37 | * `Conformist::Builder` coerces enumerables into an Array. Works with Spreadsheet for conforming Microsoft Excel spreadsheets. 38 | 39 | ## 0.1.1 / 2012-01-17 40 | 41 | * Explicitly required `Forwardable`. 42 | 43 | ## 0.1.0 / 2012-01-05 44 | 45 | * Added anonymous schemas. 46 | * Added `Conformist::Schema::Methods#conform` for lazily applying schema to input. 47 | * Added capability to access columns with methods. 48 | * FasterCSV is no longer included, use `require 'fastercsv'` instead. 49 | * `include Conformist::Base` has been removed. 50 | * `Conformist.foreach` has been removed. 51 | * `Conformist::Base::ClassMethods#load` has been removed. 52 | 53 | ## 0.0.3 / 2011-05-07 54 | 55 | * Inheriting from a class which mixes in Conformist::Base gives you access to all of the superclasses' columns. 56 | 57 | ## 0.0.2 / 2011-04-28 58 | 59 | * Column#values_in will be nil if the index is out of range rather than an empty string. This is consistent with CSV 60 | * Fixed 1.8.7 and JRuby dependencies. Gemspec will not let you specify dependencies based on the version of Ruby. Everyone gets it 61 | 62 | ## 0.0.1 / 2011-04-27 63 | 64 | * Initial release 65 | -------------------------------------------------------------------------------- /test/unit/conformist/schema_test.rb: -------------------------------------------------------------------------------- 1 | require 'helper' 2 | 3 | class Conformist::SchemaTest < Minitest::Test 4 | def test_initialize_with_instance 5 | parent = Class.new { include Schema }.new.tap { |d| d.columns = [0] } 6 | child1 = Class.new { include Schema }.new(parent).tap { |d| d.columns << 1 } 7 | child2 = Class.new { include Schema }.new(parent).tap { |d| d.columns << 2 } 8 | assert_equal [0], parent.columns 9 | assert_equal [0, 1], child1.columns 10 | assert_equal [0, 2], child2.columns 11 | end 12 | 13 | def test_initialize_with_block 14 | anonymous = Class.new { include Schema }.new do 15 | column :a, 0 16 | column :b, 1 17 | end 18 | assert_equal 2, anonymous.columns.size 19 | end 20 | 21 | def test_builder_reader 22 | assert_equal Builder, Class.new { extend Schema }.builder 23 | end 24 | 25 | def test_builder_writer 26 | definition = Class.new { extend Schema } 27 | definition.builder = Object 28 | assert_equal Object, definition.builder 29 | end 30 | 31 | def test_columns_reader 32 | assert_empty Class.new { extend Schema }.columns 33 | end 34 | 35 | def test_columns_writer 36 | definition = Class.new { extend Schema } 37 | definition.columns = [1] 38 | assert_equal [1], definition.columns 39 | end 40 | 41 | def test_column 42 | definition = Class.new { extend Schema } 43 | definition.column :a, 0 44 | definition.column :b, 1 45 | assert_equal 2, definition.columns.size 46 | end 47 | 48 | def test_column_with_implict_index 49 | definition = Class.new { extend Schema } 50 | definition.column :a 51 | definition.column :b 52 | assert_equal [0, 1], definition.columns.map { |column| column.indexes }.flatten 53 | end 54 | 55 | def test_conform_returns_enumerable 56 | definition = Class.new { extend Schema } 57 | assert definition.conform([]).respond_to?(:each) 58 | assert definition.conform([]).respond_to?(:map) 59 | end 60 | 61 | def test_conform_skip_first 62 | definition = Class.new { extend Schema } 63 | definition.column :a, 0 64 | assert_equal HashStruct.new({:a => 'value'}), definition.conform(['header', 'value'], :skip_first => true).first 65 | end 66 | 67 | def test_conform_calls_builders_call_method 68 | definition = Class.new { extend Schema } 69 | definition.builder = lambda { |definition, value, context| value } 70 | assert_equal [2, 4], definition.conform([1, 2]).map { |value| value * 2 } 71 | end 72 | 73 | def test_inheritance 74 | parent = Class.new { extend Schema }.tap { |d| d.columns = [0] } 75 | child1 = Class.new(parent).tap { |d| d.columns << 1 } 76 | child2 = Class.new(parent).tap { |d| d.columns << 2 } 77 | assert_equal [0], parent.columns 78 | assert_equal [0, 1], child1.columns 79 | assert_equal [0, 2], child2.columns 80 | end 81 | end 82 | -------------------------------------------------------------------------------- /test/unit/integration_test.rb: -------------------------------------------------------------------------------- 1 | require 'csv' 2 | require 'helper' 3 | require 'schemas/acma' 4 | require 'schemas/citizens' 5 | require 'schemas/fcc' 6 | require 'spreadsheet' 7 | 8 | class IntegrationTest < Minitest::Test 9 | def fixture filename 10 | File.expand_path "../../fixtures/#{filename}", __FILE__ 11 | end 12 | 13 | def open_csv filename, options = {} 14 | if CSV.method(:open).arity == -3 # 1.8 CSV 15 | CSV.open fixture(filename), 'r', options[:col_sep] 16 | else 17 | CSV.open fixture(filename), options 18 | end 19 | end 20 | 21 | def test_class_with_csv 22 | enumerable = ACMA.conform open_csv('acma.csv') 23 | last = enumerable.to_a.last 24 | assert_equal HashStruct.new(:name=>'CRAFERS', :callsign=>'ADS10', :latitude=>'34 58 57S'), last 25 | end 26 | 27 | def test_class_with_csv_including_headers 28 | enumerable = Citizens.conform open_csv('citizens.csv'), :skip_first => true 29 | first = enumerable.to_a.first 30 | assert_equal HashStruct.new(:name => 'Aaron', :age => '21', :gender => 'Male'), first 31 | end 32 | 33 | def test_inherited_class_with_csv 34 | enumerable = ACMA::Digital.conform open_csv('acma.csv') 35 | last = enumerable.to_a.last 36 | assert_equal HashStruct.new(:name=>'CRAFERS', :callsign=>'ADS10', :latitude=>'34 58 57S', :signal_type => 'digital'), last 37 | end 38 | 39 | def test_class_with_psv 40 | enumerable = FCC.conform open_csv('fcc.txt', :col_sep => '|') 41 | last = enumerable.to_a.last 42 | assert_equal HashStruct.new(:name => 'LOS ANGELES, CA', :callsign => 'KVTU-LP', :latitude => '34 13 38.00 N', :signtal_type => 'digital'), last 43 | end 44 | 45 | def test_instance_with_spreadsheet 46 | book = Spreadsheet.open fixture('states.xls') 47 | sheet = book.worksheet 0 48 | schema = Conformist.new { column :state, 0 } 49 | enumerable = schema.conform sheet 50 | last = enumerable.to_a.last 51 | assert_equal HashStruct.new(:state => 'QLD'), last 52 | end 53 | 54 | def test_instance_with_array_of_arrays 55 | data = Array.new.tap do |d| 56 | d << ['NSW', 'New South Wales', 'Sydney'] 57 | d << ['VIC', 'Victoria', 'Melbourne'] 58 | d << ['QLD', 'Queensland', 'Brisbane'] 59 | end 60 | schema = Conformist.new do 61 | column :state, 0, 1 do |values| 62 | "#{values.first}, #{values.last}" 63 | end 64 | column :capital, 2 65 | end 66 | enumerable = schema.conform data 67 | last = enumerable.to_a.last 68 | assert_equal HashStruct.new(:state => 'QLD, Queensland', :capital => 'Brisbane'), last 69 | end 70 | 71 | def test_inherited_instance_with_array_of_arrays 72 | data = Array.new.tap do |d| 73 | d << ['NSW', 'New South Wales', 'Sydney'] 74 | d << ['VIC', 'Victoria', 'Melbourne'] 75 | d << ['QLD', 'Queensland', 'Brisbane'] 76 | end 77 | parent = Conformist.new do 78 | column :state, 0, 1 do |values| 79 | "#{values.first}, #{values.last}" 80 | end 81 | column :capital, 2 82 | end 83 | child = Conformist.new parent do 84 | column :country do 85 | 'Australia' 86 | end 87 | end 88 | enumerable = child.conform data 89 | last = enumerable.to_a.last 90 | assert_equal HashStruct.new(:state => 'QLD, Queensland', :capital => 'Brisbane', :country => 'Australia'), last 91 | end 92 | 93 | def test_named_columns 94 | schema = Conformist.new do 95 | column :name, 'Name' 96 | column :age, 'Age' 97 | column :gender, 'Gender' 98 | end 99 | enumerable = schema.conform open_csv('citizens.csv'), :skip_first => true 100 | first = enumerable.to_a.first 101 | assert_equal HashStruct.new(:name => 'Aaron', :age => '21', :gender => 'Male'), first 102 | end 103 | 104 | def test_missing_named_column 105 | schema = Conformist.new do 106 | column :state, 'State' 107 | end 108 | enumerable = schema.conform open_csv('citizens.csv'), :skip_first => true 109 | first = enumerable.to_a.first 110 | assert_equal HashStruct.new(:state => nil), first 111 | end 112 | end 113 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Conformist 2 | 3 | [![Build Status](https://secure.travis-ci.org/tatey/conformist.png)](http://travis-ci.org/tatey/conformist) 4 | [![Code Climate](https://codeclimate.com/github/tatey/conformist.png)](https://codeclimate.com/github/tatey/conformist) 5 | 6 | Bend CSVs to your will with declarative schemas. Map one or many columns, preprocess cells and lazily enumerate. Declarative schemas are easier to understand, quicker to setup and independent of I/O. Use [CSV](http://www.ruby-doc.org/stdlib-1.9.3/libdoc/csv/rdoc/CSV.html) (Formally [FasterCSV](https://rubygems.org/gems/fastercsv)), [Spreadsheet](https://rubygems.org/gems/spreadsheet) or any array of array-like data structure. 7 | 8 | ![](http://f.cl.ly/items/00191n3O1J2E1a342F1L/conformist.jpg) 9 | 10 | ## Quick and Dirty Examples 11 | 12 | Open a CSV file and declare a schema. A schema compromises of columns. A column takes an arbitrary name followed by its position in the input. A column may be derived from multiple positions. 13 | 14 | ``` ruby 15 | require 'conformist' 16 | require 'csv' 17 | 18 | csv = CSV.open '~/transmitters.csv' 19 | schema = Conformist.new do 20 | column :callsign, 1 21 | column :latitude, 1, 2, 3 22 | column :longitude, 3, 4, 5 23 | column :name, 0 do |value| 24 | value.upcase 25 | end 26 | end 27 | ``` 28 | 29 | Insert the transmitters into a SQLite database. 30 | 31 | ``` ruby 32 | require 'sqlite3' 33 | 34 | db = SQLite3::Database.new 'transmitters.db' 35 | schema.conform(csv).each do |transmitter| 36 | db.execute "INSERT INTO transmitters (callsign, ...) VALUES ('#{transmitter.callsign}', ...);" 37 | end 38 | ``` 39 | 40 | Only insert the transmitters with the name "Mount Cooth-tha" using ActiveRecord or DataMapper. 41 | 42 | ``` ruby 43 | transmitters = schema.conform(csv).select do |transmitter| 44 | transmitter.name == 'Mount Coot-tha' 45 | end 46 | transmitters.each do |transmitter| 47 | Transmitter.create! transmitter.attributes 48 | end 49 | ``` 50 | 51 | Source from multiple, different input files and insert transmitters together into a single database. 52 | 53 | ``` ruby 54 | require 'conformist' 55 | require 'csv' 56 | require 'sqlite3' 57 | 58 | au_schema = Conformist.new do 59 | column :callsign, 8 60 | column :latitude, 10 61 | end 62 | us_schema = Conformist.new do 63 | column :callsign, 1 64 | column :latitude, 1, 2, 3 65 | end 66 | 67 | au_csv = CSV.open '~/au/transmitters.csv' 68 | us_csv = CSV.open '~/us/transmitters.csv' 69 | 70 | db = SQLite3::Database.new 'transmitters.db' 71 | 72 | [au_schema.conform(au_csv), us_schema.conform(us_csv)].each do |schema| 73 | schema.each do |transmitter| 74 | db.execute "INSERT INTO transmitters (callsign, ...) VALUES ('#{transmitter.callsign}', ...);" 75 | end 76 | end 77 | ``` 78 | 79 | Open a Microsoft Excel spreadsheet and declare a schema. 80 | 81 | ``` ruby 82 | require 'conformist' 83 | require 'spreadsheet' 84 | 85 | book = Spreadsheet.open '~/states.xls' 86 | sheet = book.worksheet 0 87 | schema = Conformist.new do 88 | column :state, 0, 1 do |values| 89 | "#{values.first}, #{values.last}" 90 | end 91 | column :capital, 2 92 | end 93 | ``` 94 | 95 | Print each state's attributes to standard out. 96 | 97 | ``` ruby 98 | schema.conform(sheet).each do |state| 99 | $stdout.puts state.attributes 100 | end 101 | ``` 102 | 103 | For more examples see [test/fixtures](https://github.com/tatey/conformist/tree/master/test/fixtures), [test/schemas](https://github.com/tatey/conformist/tree/master/test/schemas) and [test/unit/integration_test.rb](https://github.com/tatey/conformist/blob/master/test/unit/integration_test.rb). 104 | 105 | ## Installation 106 | 107 | Conformist is available as a gem. Install it at the command line. 108 | 109 | ``` sh 110 | $ [sudo] gem install conformist 111 | ``` 112 | 113 | Or add it to your Gemfile and run `$ bundle install`. 114 | 115 | ``` ruby 116 | gem 'conformist' 117 | ``` 118 | 119 | ## Usage 120 | 121 | ### Anonymous Schema 122 | 123 | Anonymous schemas are quick to declare and don't have the overhead of creating an explicit class. 124 | 125 | ``` ruby 126 | citizen = Conformist.new do 127 | column :name, 0, 1 128 | column :email, 2 129 | end 130 | 131 | citizen.conform [['Tate', 'Johnson', 'tate@tatey.com']] 132 | ``` 133 | 134 | ### Class Schema 135 | 136 | Class schemas are explicit. Class schemas were the only type available in earlier versions of Conformist. 137 | 138 | ``` ruby 139 | class Citizen 140 | extend Conformist 141 | 142 | column :name, 0, 1 143 | column :email, 2 144 | end 145 | 146 | Citizen.conform [['Tate', 'Johnson', 'tate@tatey.com']] 147 | ``` 148 | 149 | ### Implicit Indexing 150 | 151 | Column indexes are implicitly incremented when the index argument is omitted. Implicit indexing is all or nothing. 152 | 153 | ``` ruby 154 | column :account_number # => 0 155 | column :date { |v| Time.new *v.split('/').reverse } # => 1 156 | column :description # => 2 157 | column :debit # => 3 158 | column :credit # => 4 159 | ``` 160 | 161 | ### Conform 162 | 163 | Conform is the principle method for lazily applying a schema to the given input. 164 | 165 | ``` ruby 166 | enumerator = schema.conform CSV.open('~/file.csv') 167 | enumerator.each do |row| 168 | puts row.attributes 169 | end 170 | ``` 171 | 172 | #### Input 173 | 174 | `#conform` expects any object that responds to `#each` to return an array-like object. 175 | 176 | ``` ruby 177 | CSV.open('~/file.csv').responds_to? :each # => true 178 | [[], [], []].responds_to? :each # => true 179 | ``` 180 | 181 | #### Header Row 182 | 183 | `#conform` takes an option to skip the first row of input. Given a typical CSV document, 184 | the first row is the header row and irrelevant for enumeration. 185 | 186 | ``` ruby 187 | schema.conform CSV.open('~/file_with_headers.csv'), :skip_first => true 188 | ``` 189 | 190 | #### Named Columns 191 | 192 | Strings can be used as column indexes instead of integers. These strings will be matched 193 | against the first row to determine the appropriate numerical index. 194 | 195 | ``` ruby 196 | citizen = Conformist.new do 197 | column :email, 'EM' 198 | column :name, 'FN', 'LN' 199 | end 200 | 201 | citizen.conform [['FN', 'LN', 'EM'], ['Tate', 'Johnson', 'tate@tatey.com']], :skip_first => true 202 | ``` 203 | 204 | #### Enumerator 205 | 206 | `#conform` is lazy, returning an [Enumerator](http://www.ruby-doc.org/core-1.9.3/Enumerator.html). Input is not parsed until you call `#each`, `#map` or any method defined in [Enumerable](http://www.ruby-doc.org/core-1.9.3/Enumerable.html). That means schemas can be assigned now and evaluated later. `#each` has the lowest memory footprint because it does not build a collection. 207 | 208 | #### Struct 209 | 210 | The argument passed into the block is a struct-like object. You can access columns as methods or keys. Columns were only accessible as keys in earlier versions of Conformist. Methods are now the preferred syntax. 211 | 212 | ``` ruby 213 | citizen[:name] # => "Tate Johnson" 214 | citizen.name # => "Tate Johnson" 215 | ``` 216 | 217 | For convenience the `#attributes` method returns a hash of key-value pairs suitable for creating ActiveRecord or DataMapper records. 218 | 219 | ``` ruby 220 | citizen.attributes # => {:name => "Tate Johnson", :email => "tate@tatey.com"} 221 | ``` 222 | 223 | ### One Column 224 | 225 | Maps the first column in the input file to `:first_name`. Column indexing starts at zero. 226 | 227 | ``` ruby 228 | column :first_name, 0 229 | ``` 230 | 231 | ### Many Columns 232 | 233 | Maps the first and second columns in the input file to `:name`. 234 | 235 | ``` ruby 236 | column :name, 0, 1 237 | ``` 238 | 239 | Indexing is completely arbitrary and you can map any combination. 240 | 241 | ``` ruby 242 | column :name_and_city 0, 1, 2 243 | ``` 244 | 245 | Many columns are implicitly concatenated. Behaviour can be changed by passing a block. See *preprocessing*. 246 | 247 | ### Preprocessing 248 | 249 | Sometimes values need to be manipulated before they're conformed. Passing a block gets access to values. The return value of the block becomes the conformed output. 250 | 251 | ``` ruby 252 | column :name, 0, 1 do |values| 253 | values.map(&:upcase) * ' ' 254 | end 255 | ``` 256 | 257 | Works with one column too. Instead of getting a collection of objects, one object is passed to the block. 258 | 259 | ``` ruby 260 | column :first_name, 0 do |value| 261 | value.upcase 262 | end 263 | ``` 264 | 265 | It's also possible to provide a context object that is made available during preprocessing. 266 | 267 | ``` ruby 268 | citizen = Conformist.new do 269 | column :name, 0, 1 do |values, context| 270 | (context[:upcase?] ? values.map(&:upcase) : values) * ' ' 271 | end 272 | end 273 | 274 | citizen.conform [['tate', 'johnson']], context: {upcase?: true} 275 | ``` 276 | 277 | ### Virtual Columns 278 | 279 | Virtual columns are not sourced from input. Omit the index to create a virtual column. Like real columns, virtual columns are included in the conformed output. 280 | 281 | ``` ruby 282 | column :day do 283 | 1 284 | end 285 | ``` 286 | 287 | ### Inheritance 288 | 289 | Inheriting from a schema gives access to all of the parent schema's columns. 290 | 291 | #### Anonymous Schema 292 | 293 | Anonymous inheritance takes inspiration from Ruby's syntax for [instantiating new classes](http://ruby-doc.org/core-1.9.3/Class.html#method-c-new). 294 | 295 | ``` ruby 296 | parent = Conformist.new do 297 | column :name, 0, 1 298 | end 299 | 300 | child = Conformist.new parent do 301 | column :category do 302 | 'Child' 303 | end 304 | end 305 | ``` 306 | 307 | #### Class Schema 308 | 309 | Classical inheritance works as expected. 310 | 311 | ``` ruby 312 | class Parent 313 | extend Conformist 314 | 315 | column :name, 0, 1 316 | end 317 | 318 | class Child < Parent 319 | column :category do 320 | 'Child' 321 | end 322 | end 323 | ``` 324 | 325 | ## Upgrading from <= 0.0.3 to >= 0.1.0 326 | 327 | Where previously you had 328 | 329 | ``` ruby 330 | class Citizen 331 | include Conformist::Base 332 | 333 | column :name, 0, 1 334 | end 335 | 336 | Citizen.load('~/file.csv').foreach do |citizen| 337 | # ... 338 | end 339 | ``` 340 | 341 | You should now do 342 | 343 | ``` ruby 344 | require 'fastercsv' 345 | 346 | class Citizen 347 | extend Conformist 348 | 349 | column :name, 0, 1 350 | end 351 | 352 | Citizen.conform(FasterCSV.open('~/file.csv')).each do |citizen| 353 | # ... 354 | end 355 | ``` 356 | 357 | See CHANGELOG.md for a full list of changes. 358 | 359 | ## Compatibility 360 | 361 | * MRI 2.4.0, 2.3.1, 2.2.0, 2.1.0, 2.0.0, 1.9.3 362 | * JRuby 363 | 364 | ## Dependencies 365 | 366 | No explicit dependencies, although `CSV` and `Spreadsheet` are commonly used. 367 | 368 | ## Contributing 369 | 370 | 1. Fork 371 | 2. Install dependancies by running `$ bundle install` 372 | 3. Write tests and code 373 | 4. Make sure the tests pass locally by running `$ bundle exec rake` 374 | 5. Push to GitHub and make sure continuous integration tests pass at 375 | https://travis-ci.org/tatey/conformist/pull_requests 376 | 5. Send a pull request on GitHub 377 | 378 | Please do not increment the version number in `lib/conformist/version.rb`. 379 | The version number will be incremented by the maintainer after the patch 380 | is accepted. 381 | 382 | ## Motivation 383 | 384 | Motivation for this project came from the desire to simplify importing data from various government organisations into [Antenna Mate](http://antennamate.com). The data from each government was similar, but had completely different formatting. Some pieces of data needed preprocessing while others simply needed to be concatenated together. Not wanting to write a parser for each new government organisation, I created Conformist. 385 | 386 | ## Copyright 387 | 388 | Copyright © 2016 Tate Johnson. Conformist is released under the MIT license. See LICENSE for details. 389 | --------------------------------------------------------------------------------