├── .travis.yml
├── notes.rb
├── example_data
    └── test1.csv
├── easypost.rb
├── normalize_csv_addresses.rb
├── address_normalizer_spec.rb
├── address_normalizer_test.rb
├── LICENSE.mkd
├── README.md
└── address_normalizer.rb


/.travis.yml:
--------------------------------------------------------------------------------
1 | 


--------------------------------------------------------------------------------
/notes.rb:
--------------------------------------------------------------------------------
1 | # Mike's notes
2 | - big todo: seperate all file reading/writing logic from address normalizing logic


--------------------------------------------------------------------------------
/example_data/test1.csv:
--------------------------------------------------------------------------------
1 | Name,address
2 | Mike,"6687 Del Party Avenue 90045 Isla Vista, California"
3 | asf, "Here's an unclosed row
4 | 


--------------------------------------------------------------------------------
/easypost.rb:
--------------------------------------------------------------------------------
 1 | require "easypost"
 2 | 
 3 | EasyPost.api_key = "cueqNZUb3ldeWTNX7MU3Mel8UXtaAMUi"
 4 | p EasyPost::Address.verify({
 5 |   street1: "101 California St",
 6 |   street2: "Suite 1290",
 7 |   city: "San Francisco",
 8 |   state: "CA",
 9 |   zip: "94111"
10 |   })


--------------------------------------------------------------------------------
/normalize_csv_addresses.rb:
--------------------------------------------------------------------------------
1 | require './address_normalizer'
2 | 
3 | if ARGV.empty?
4 |   puts "Please provide the name of the CSV file with addresses to normalize. Example usage:"
5 |   puts "ruby normalize_csv_addresses.rb \"my_addresses.csv\""
6 | else
7 |   normalized = FileNormalizer.new(ARGV[0])
8 | end
9 | 


--------------------------------------------------------------------------------
/address_normalizer_spec.rb:
--------------------------------------------------------------------------------
 1 | require 'csv'
 2 | require_relative 'address_normalizer'
 3 | 
 4 | describe FileNormalizer do
 5 | 
 6 | 	before :all do
 7 | 		@timestamp = Time.now.to_s.gsub(/:|-/,"").gsub(/\s+/,"_")
 8 | 		@filename  = "test1_NormalizedAddresses_#{@timestamp}.csv"
 9 | 		@malformed = "AddressNormalizer_MalformedRows_#{@timestamp}.txt"
10 | 
11 | 		@normalized = FileNormalizer.new('example_data/test1.csv')
12 | 	end
13 | 
14 | 	it "should create a normalized output file" do
15 | 		File.exists?(@filename).should be_true
16 | 	end	
17 | 
18 | 	it "should add an 'address_normalized' column" do
19 | 		rows = CSV.read(@filename)
20 | 		rows[0].should include('address_normalized')
21 | 	end
22 | 
23 | 	it "should create a properly normalized address" do
24 | 		rows = CSV.read(@filename)
25 | 		rows[1][2].should == "6687 DEL PARTY AVENUE 90045, ISLA VISTA, CA"
26 | 	end
27 | 
28 |   # todo - write a better test (regex?)
29 | 	it "should catch unclosed row errors" do
30 | 	  @normalized.errors.should include("Illegal quoting in line 1.")
31 | 	end	
32 | 
33 | 	it "should create file of malformed rows" do
34 |     File.exists?(@malformed).should be_true
35 | 	end
36 | 
37 | 	after :all do
38 | 		File.delete(@filename)
39 | 		File.delete(@malformed)
40 | 	end
41 | 
42 | end


--------------------------------------------------------------------------------
/address_normalizer_test.rb:
--------------------------------------------------------------------------------
 1 | require 'test/unit'
 2 | require 'csv'
 3 | require_relative 'address_normalizer'
 4 | 
 5 | class FileNormalizerTest < Test::Unit::TestCase
 6 | 
 7 | 	# run and create test1_NormalizedAddresses_<time>.csv
 8 | 	def setup
 9 | 		@timestamp = Time.now.to_s.gsub(/:|-/,"").gsub(/\s+/,"_")
10 | 		@filename  = "test1_NormalizedAddresses_#{@timestamp}.csv"
11 | 		@malformed = "AddressNormalizer_MalformedRows_#{@timestamp}.txt"
12 | 
13 | 		@normalized = AddressNormalizer.new('example_data/test1.csv')
14 | 	end
15 | 
16 | 	# main test function
17 | 	def test_normalized_file_exist
18 | 		assert File.exists?(@filename), 'Normalized file was not created'
19 | 	end
20 | 
21 | 	# assert that 'address_normalized' column was added
22 | 	def test_first_row_has_normalized_column
23 | 		rows = CSV.read(@filename)
24 | 		assert rows[0].include?('address_normalized')
25 | 	end
26 | 
27 | 	# dumb check for properly normalized column
28 | 	def test_normalized_address_created
29 | 		rows = CSV.read(@filename)
30 | 		assert rows[1][2] == "6687 DEL PARTY AVENUE 90045, ISLA VISTA, CA"
31 | 	end
32 | 
33 | 	def test_unclosed_row_error_caught
34 | 		assert !@normalized.errors.empty?
35 | 	end
36 | 
37 | 	def test_malformed_rows_file_existance
38 | 		assert File.exists?(@malformed), 'Malformed file was not created'
39 | 	end
40 | 
41 | 
42 | 	def teardown
43 | 		File.delete(@filename)
44 | 		File.delete(@malformed)
45 | 	end
46 | 
47 | end


--------------------------------------------------------------------------------
/LICENSE.mkd:
--------------------------------------------------------------------------------
 1 | Copyright (c) 2013, Code for America
 2 | All rights reserved.
 3 | 
 4 | Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
 5 | 
 6 | * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
 7 | * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
 8 | * Neither the name of Code for America nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission.
 9 | 
10 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
11 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
 1 | Address Normalizer
 2 | =======
 3 | A simple tool that takes a CSV with an "address" column and adds a column with normalized addresses, using [the Ruby street_address gem](https://github.com/derrek/street-address) (a port of the Perl module [Geo::StreetAddress::US](http://search.cpan.org/~sderle/Geo-StreetAddress-US-0.99/)).
 4 | 
 5 | Usage Examples
 6 | --------------
 7 | 
 8 |     ruby normalize_csv_addresses.rb "my_file_with_addresses.csv"
 9 | 
10 | Current Contributors
11 | --------------------
12 | Dave Guarino ([https://github.com/daguar](https://github.com/daguar))
13 | 
14 | Contributing
15 | ------------
16 | In the spirit of [free software](http://www.fsf.org/licensing/essays/free-sw.html), **everyone** is encouraged to help improve this project.
17 | 
18 | Here are some ways *you* can contribute:
19 | 
20 | * by using alpha, beta, and prerelease versions
21 | * by reporting bugs
22 | * by suggesting new features
23 | * by writing or editing documentation
24 | * by writing specifications
25 | * by writing code (**no patch is too small**: fix typos, add comments, clean up inconsistent whitespace)
26 | * by refactoring code
27 | * by resolving [issues](http://github.com/codeforamerica/cfa_template/issues)
28 | * by reviewing patches
29 | 
30 | Submitting an Issue
31 | -------------------
32 | We use the [GitHub issue tracker](http://github.com/codeforamerica/cfa_template/issues) to track bugs and
33 | features. Before submitting a bug report or feature request, check to make sure it hasn't already
34 | been submitted. You can indicate support for an existing issuse by voting it up. When submitting a
35 | bug report, please include a [Gist](http://gist.github.com/) that includes a stack trace and any
36 | details that may be necessary to reproduce the bug, including your gem version, Ruby version, and
37 | operating system. Ideally, a bug report should include a pull request with failing specs.
38 | 
39 | Submitting a Pull Request
40 | -------------------------
41 | (Note: not using this process yet as this is a very early alpha; will conform later.)
42 | 1. Fork the project.
43 | 2. Create a topic branch.
44 | 3. Implement your feature or bug fix.
45 | 4. Add documentation for your feature or bug fix.
46 | 5. Run <tt>bundle exec rake doc:yard</tt>. If your changes are not 100% documented, go back to step 4.
47 | 6. Add specs for your feature or bug fix.
48 | 7. Run <tt>bundle exec rake spec</tt>. If your changes are not 100% covered, go back to step 6.
49 | 8. Commit and push your changes.
50 | 9. Submit a pull request. Please do not include changes to the gemspec, version, or history file. (If you want to create your own version for some reason, please do so in a separate commit.)
51 | 
52 | Does your project or organization use this?
53 | ------------------------------------------
54 | Add it to the [apps](http://github.com/codeforamerica/cfa_template/wiki/apps) wiki!
55 | 
56 | Copyright
57 | ---------
58 | Copyright (c) 2013 Code for America Laboratories
59 | See [LICENSE](https://github.com/codeforamerica/cfa_template/blob/master/LICENSE.mkd) for details.
60 | 
61 | [![Code for America Tracker](http://stats.codeforamerica.org/codeforamerica/cfa_template.png)](http://stats.codeforamerica.org/projects/cfa_template)
62 | 
63 | 


--------------------------------------------------------------------------------
/address_normalizer.rb:
--------------------------------------------------------------------------------
  1 | require 'csv'
  2 | require 'street_address'
  3 | require 'cmess/guess_encoding'
  4 | require 'pry'
  5 | 
  6 | class FileNormalizer
  7 | 
  8 |   attr_accessor :errors, :malformed_rows, :timestamp, :address_index, :normalized_address_index
  9 | 
 10 |   def initialize(filename)
 11 |     @errors         = []
 12 |     @malformed_rows = []
 13 |     @timestamp = Time.now.to_s.gsub(/:|-/,"").gsub(/\s+/,"_")
 14 | 
 15 |     normalize_csv(filename)
 16 |   end
 17 | 
 18 |   # Given a CSV file, creates a new one with normalized addresses
 19 |   # If there are malformed rows/errors, saves those to a separate output file
 20 |   def normalize_csv(filename)
 21 |     # Check encoding
 22 |     source_encoding = get_encoding(filename)
 23 | 
 24 |     # normalize_file
 25 |     process_csv(filename, source_encoding)
 26 | 
 27 |     # If there were problem rows, print them to the user and save them to a file
 28 |     handle_malformed_rows unless malformed_rows.empty?
 29 | 
 30 |     puts "Done!"
 31 |   end
 32 | 
 33 |   def get_encoding filename
 34 |     puts "Opening file to check encoding..."
 35 | 
 36 |     source_file     = File.read(filename)
 37 |     source_encoding = CMess::GuessEncoding::Automatic.guess(source_file)
 38 | 
 39 |     puts "Encoding: #{source_encoding}"
 40 |     puts "Total number of rows: #{source_file.lines.count}"
 41 | 
 42 |     source_file = nil
 43 | 
 44 |     return source_encoding
 45 |   end
 46 | 
 47 |   #TODO-mike break this up
 48 |   def process_csv(filename, source_encoding)
 49 |     puts "Normalizing addresses (this may take a while)..."
 50 | 
 51 |     normalized_out = File.open("#{File.basename(filename,".*")}_NormalizedAddresses_#{timestamp}.csv", "w")
 52 | 
 53 |     counter = 0
 54 |     IO.foreach(filename, :encoding => source_encoding) do |line|
 55 |       puts "On row #{counter}" if counter % 100 == 0
 56 |       if counter == 0
 57 |         handle_first_row(normalized_out, line)
 58 |       else
 59 |         normalize_line(normalized_out, line)
 60 |       end
 61 |       counter += 1
 62 |     end
 63 | 
 64 |     normalized_out.close
 65 |   end
 66 | 
 67 |   def handle_first_row(file_out, line)
 68 |     CSV.parse(line) do |row|
 69 |       # check if 'address' exists
 70 |       self.address_index = row.index("address")
 71 |       row << "address_normalized"
 72 |       self.normalized_address_index = row.index("address_normalized")
 73 |       file_out.write(CSV.generate_line(row))
 74 |     end
 75 |   end
 76 | 
 77 |   def normalize_line(file_out, line)
 78 |     begin
 79 |       CSV.parse(line) do |row|
 80 |         begin
 81 |           strt_ad = StreetAddress::US.parse(row[address_index] + ", , ")
 82 |           # Catch empty address
 83 |         rescue NoMethodError
 84 |           strt_ad = ""
 85 |         end
 86 |         row << strt_ad.to_s.upcase
 87 |         file_out.write(CSV.generate_line(row))
 88 |       end
 89 |       # Rescue from problem rows
 90 |     rescue CSV::MalformedCSVError => er
 91 |       self.errors << er.message
 92 |       self.malformed_rows << line
 93 |     end
 94 |   end
 95 | 
 96 |   def handle_malformed_rows
 97 |     puts "Errors: #{errors.to_s}\n\n\n" unless errors.empty?
 98 | 
 99 |     malformed_row_output_file = File.open("AddressNormalizer_MalformedRows_#{timestamp}.txt", "w")
100 |     puts "Malformed rows:"
101 |     malformed_rows.each do |row|
102 |       puts row
103 |       malformed_row_output_file.write(row + "\n")
104 |     end
105 |     puts "Malformed rows saved to disk in #{malformed_row_output_file.path}."
106 |     
107 |     malformed_row_output_file.close
108 |   end
109 | 
110 | end
111 | 


--------------------------------------------------------------------------------