├── .gitignore ├── .ruby-version ├── Gemfile ├── Gemfile.lock ├── LICENSE ├── README.md ├── Rakefile ├── config └── css_paths.yml ├── db └── .gitkeep └── lib └── amazon ├── order.rb ├── order_importer.rb ├── shipment.rb └── shipment_order.rb /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | config/account.yml 3 | db/orders.sqlite3 4 | test/data 5 | -------------------------------------------------------------------------------- /.ruby-version: -------------------------------------------------------------------------------- 1 | 2.5.0 2 | -------------------------------------------------------------------------------- /Gemfile: -------------------------------------------------------------------------------- 1 | source 'https://rubygems.org' 2 | 3 | gem 'actionview', require: 'action_view' 4 | gem 'activerecord', '~> 5', require: 'active_record' 5 | gem 'activesupport', require: 'active_support/all' 6 | gem 'awesome_print' 7 | gem 'byebug' 8 | gem 'colored' 9 | gem 'hashdiff' 10 | gem 'highline', require: 'highline/import' 11 | gem 'hirb' 12 | gem 'map_by_method' 13 | gem 'mechanize' 14 | gem 'nokogiri' 15 | gem 'pry' 16 | gem 'rake' 17 | gem 'sqlite3' 18 | gem 'terminal-table' 19 | -------------------------------------------------------------------------------- /Gemfile.lock: -------------------------------------------------------------------------------- 1 | GEM 2 | remote: https://rubygems.org/ 3 | specs: 4 | actionview (5.1.5) 5 | activesupport (= 5.1.5) 6 | builder (~> 3.1) 7 | erubi (~> 1.4) 8 | rails-dom-testing (~> 2.0) 9 | rails-html-sanitizer (~> 1.0, >= 1.0.3) 10 | activemodel (5.1.5) 11 | activesupport (= 5.1.5) 12 | activerecord (5.1.5) 13 | activemodel (= 5.1.5) 14 | activesupport (= 5.1.5) 15 | arel (~> 8.0) 16 | activesupport (5.1.5) 17 | concurrent-ruby (~> 1.0, >= 1.0.2) 18 | i18n (~> 0.7) 19 | minitest (~> 5.1) 20 | tzinfo (~> 1.1) 21 | arel (8.0.0) 22 | awesome_print (1.8.0) 23 | builder (3.2.3) 24 | byebug (10.0.0) 25 | coderay (1.1.2) 26 | colored (1.2) 27 | concurrent-ruby (1.0.5) 28 | crass (1.0.3) 29 | domain_name (0.5.20170404) 30 | unf (>= 0.0.5, < 1.0.0) 31 | erubi (1.7.1) 32 | hashdiff (0.3.7) 33 | highline (1.7.10) 34 | hirb (0.7.3) 35 | http-cookie (1.0.3) 36 | domain_name (~> 0.5) 37 | i18n (0.9.5) 38 | concurrent-ruby (~> 1.0) 39 | loofah (2.2.0) 40 | crass (~> 1.0.2) 41 | nokogiri (>= 1.5.9) 42 | map_by_method (0.8.3) 43 | mechanize (2.7.5) 44 | domain_name (~> 0.5, >= 0.5.1) 45 | http-cookie (~> 1.0) 46 | mime-types (>= 1.17.2) 47 | net-http-digest_auth (~> 1.1, >= 1.1.1) 48 | net-http-persistent (~> 2.5, >= 2.5.2) 49 | nokogiri (~> 1.6) 50 | ntlm-http (~> 0.1, >= 0.1.1) 51 | webrobots (>= 0.0.9, < 0.2) 52 | method_source (0.9.0) 53 | mime-types (3.1) 54 | mime-types-data (~> 3.2015) 55 | mime-types-data (3.2016.0521) 56 | mini_portile2 (2.3.0) 57 | minitest (5.11.3) 58 | net-http-digest_auth (1.4.1) 59 | net-http-persistent (2.9.4) 60 | nokogiri (1.8.2) 61 | mini_portile2 (~> 2.3.0) 62 | ntlm-http (0.1.1) 63 | pry (0.11.3) 64 | coderay (~> 1.1.0) 65 | method_source (~> 0.9.0) 66 | rails-dom-testing (2.0.3) 67 | activesupport (>= 4.2.0) 68 | nokogiri (>= 1.6) 69 | rails-html-sanitizer (1.0.3) 70 | loofah (~> 2.0) 71 | rake (12.3.0) 72 | sqlite3 (1.3.13) 73 | terminal-table (1.8.0) 74 | unicode-display_width (~> 1.1, >= 1.1.1) 75 | thread_safe (0.3.6) 76 | tzinfo (1.2.5) 77 | thread_safe (~> 0.1) 78 | unf (0.1.4) 79 | unf_ext 80 | unf_ext (0.0.7.5) 81 | unicode-display_width (1.3.0) 82 | webrobots (0.1.2) 83 | 84 | PLATFORMS 85 | ruby 86 | 87 | DEPENDENCIES 88 | actionview 89 | activerecord (~> 5) 90 | activesupport 91 | awesome_print 92 | byebug 93 | colored 94 | hashdiff 95 | highline 96 | hirb 97 | map_by_method 98 | mechanize 99 | nokogiri 100 | pry 101 | rake 102 | sqlite3 103 | terminal-table 104 | 105 | BUNDLED WITH 106 | 1.16.1 107 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright 2015 Chris Bielinski. 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining 4 | a copy of this software and associated documentation files (the 5 | "Software"), to deal in the Software without restriction, including 6 | without limitation the rights to use, copy, modify, merge, publish, 7 | distribute, sublicense, and/or sell copies of the Software, and to 8 | permit persons to whom the Software is furnished to do so, subject to 9 | the following conditions: 10 | 11 | The above copyright notice and this permission notice shall be 12 | included in all copies or substantial portions of the Software. 13 | 14 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 15 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 16 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 17 | NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE 18 | LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 19 | OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 20 | WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 21 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Amazon Orders 2 | 3 | A collection of Rake tasks for scraping _Amazon.com_ and fetching info about your order history. Useful for determining stats about your purchase habits or automatic fetching of open order status like delivery dates and tracking numbers. 4 | 5 | Data is stored in a SQLite database and ActiveRecord adapters are provided. 6 | 7 | ## Install 8 | 9 | ```bash 10 | git clone https://github.com/chrisb/amazon-orders.git 11 | cd amazon-orders 12 | bundle 13 | bundle exec rake orders:fetch 14 | ``` 15 | 16 | ## Usage 17 | 18 | Once you've imported your orders (it can take a while!), you can display a nicely-formatted table of some interesting stats with `rake stats`. 19 | 20 | You'll get output that looks something like this: 21 | 22 | ```bash 23 | $ rake stats 24 | 25 | +-----------------------------------+----------------------------+ 26 | | Your Amazon.com Stats | 27 | +-----------------------------------+----------------------------+ 28 | | Customer Since | April 2007 (about 8 years) | 29 | | Total Orders | 123 | 30 | | Amount Spent | $1,234.56 | 31 | | Average Amount Spent per Order | $12.34 | 32 | +-----------------------------------+----------------------------+ 33 | | Cumulative Month with Most Orders | January (12 orders) | 34 | | Cumulative Month with Most Spent | January ($123.45) | 35 | +-----------------------------------+----------------------------+ 36 | | Calendar Month with Most Orders | January 2014 (12 orders) | 37 | | Calendar Month with Most Spent | January 2011 ($1,234.56) | 38 | +-----------------------------------+----------------------------+ 39 | ``` 40 | 41 | To update your database, just run `rake orders:fetch` again. By default the task will only update orders that are more than 18 hours old. 42 | 43 | ## Advanced Usage 44 | 45 | The following Rake tasks are available: 46 | 47 | | Task Name | Description | 48 | | ------------- | ----------- | 49 | | `rake stats` | Display some interesting stats about your order history. 50 | | `rake orders:fetch` | Log in to Amazon.com and fetch all orders in your history. 51 | | `rake config:generate` | Generate a `config/account.yml` file with your Amazon.com credentials. 52 | | `rake db:reset` | Wipe all local order data. 53 | | `rake db:migrate` | Ensure the DB schema is up-to-date. 54 | | `rake console` | Open an interactive console with the environment loaded (helpful if you want run your own queries or poke around your data). 55 | 56 | ## Contributing 57 | 58 | This project may not capture every order type and probably doesn't work on international Amazon sites. 59 | 60 | If you spot a bug, feel free to send me a pull request; currently there are no tests, sorry. 61 | 62 | There are a whole bunch of things I'd like to do with this, but I don't have much time presently, so who knows how this project will shape up. 63 | 64 | ## Authors 65 | 66 | [Several people](https://github.com/chrisb/amazon-orders/graphs/contributors) have contributed to this project. 67 | 68 | ## License 69 | 70 | [MIT License](https://github.com/chrisb/amazon-orders/blob/master/LICENSE). Copyright 2015 Chris Bielinski. 71 | -------------------------------------------------------------------------------- /Rakefile: -------------------------------------------------------------------------------- 1 | desc 'load all required gems and dependencies' 2 | task :dependencies do 3 | require 'rubygems' 4 | require 'yaml' 5 | require 'bundler' 6 | Bundler.require 7 | end 8 | 9 | task establish_connection: :dependencies do 10 | ActiveRecord::Base.establish_connection adapter: 'sqlite3', database: 'db/orders.sqlite3' 11 | end 12 | 13 | desc 'load the environment' 14 | task environment: :dependencies do 15 | Rake::Task['establish_connection'].execute 16 | require './lib/amazon/order' 17 | require './lib/amazon/shipment' 18 | require './lib/amazon/shipment_order' 19 | require './lib/amazon/order_importer' 20 | end 21 | 22 | desc 'load an interactive console' 23 | task console: :environment do 24 | require 'pry' 25 | ARGV.clear 26 | Pry.start 27 | end 28 | 29 | desc 'display some interesting stats about your order history' 30 | task stats: :environment do 31 | include ActiveSupport::NumberHelper 32 | include ActionView::Helpers::DateHelper 33 | include Amazon 34 | 35 | if Order.count == 0 36 | puts "ERROR: Import some orders first with #{'rake orders:fetch'.yellow}!\n\n" 37 | next 38 | end 39 | 40 | def format_row(row_or_first, second = nil) 41 | arr = second ? [row_or_first, second] : row_or_first 42 | title = arr.first.to_s.blue 43 | value = arr.second.yellow 44 | parenthetical_content = value[/\(.*?\)/] 45 | value = value.gsub(parenthetical_content, parenthetical_content.green) if parenthetical_content 46 | [title, value] 47 | end 48 | 49 | orders_by_calendar_month = Order.all.each_with_object({}) do |order, hsh| 50 | hsh[order.date.beginning_of_month] ||= [] 51 | hsh[order.date.beginning_of_month] << order 52 | end 53 | 54 | orders_by_calendar_month_sorted = orders_by_calendar_month.keys.each_with_object({}) do |calendar_month, hsh| 55 | hsh[calendar_month] = orders_by_calendar_month[calendar_month].count 56 | end.sort_by { |date, orders| orders }.reverse 57 | 58 | amount_by_calendar_month = orders_by_calendar_month.each_with_object({}) do |month_and_orders, hsh| 59 | hsh[month_and_orders.first] = month_and_orders.last.map(&:grand_total).sum 60 | end.sort_by { |date, amount| amount }.reverse 61 | 62 | orders_by_month_number = Order.all.each_with_object({}) do |order, hsh| 63 | hsh[order.date.month] ||= [] 64 | hsh[order.date.month] << order 65 | end 66 | 67 | order_totals_by_month = orders_by_month_number.keys.each_with_object({}) do |month_number, hsh| 68 | hsh[Date::MONTHNAMES[month_number]] ||= 0 69 | hsh[Date::MONTHNAMES[month_number]] += orders_by_month_number[month_number].map(&:grand_total).sum 70 | end.sort_by { |month, orders| orders }.reverse 71 | 72 | order_counts_by_month = orders_by_month_number.keys.each_with_object({}) do |month_number, hsh| 73 | hsh[Date::MONTHNAMES[month_number]] = orders_by_month_number[month_number].count 74 | end.sort_by { |month, amount| amount }.reverse 75 | 76 | first_order = Order.order('date ASC').first 77 | 78 | table = Terminal::Table.new title: 'Your Amazon.com Stats'.red do |t| 79 | t << format_row('Customer Since', "#{first_order.date.strftime('%B %Y')} (#{distance_of_time_in_words_to_now first_order.date})") 80 | t << format_row('Total Orders', number_to_human(Order.count)) 81 | t << format_row('Amount Spent', number_to_currency(Order.sum :grand_total)) 82 | t << format_row('Average Amount Spent per Order', number_to_currency(Order.average :grand_total)) 83 | t << :separator 84 | t << format_row('Cumulative Month with Most Orders', "#{order_counts_by_month.first.first} (#{order_counts_by_month.first.last.to_s.green} #{'orders'.green})") 85 | t << format_row('Cumulative Month with Most Spent', "#{order_totals_by_month.first.first} (#{number_to_currency(order_totals_by_month.first.last).green})") 86 | t << :separator 87 | t << format_row('Calendar Month with Most Orders', "#{orders_by_calendar_month_sorted.first.first.strftime '%B %Y'} (#{orders_by_calendar_month_sorted.first.last.to_s.green} #{'orders'.green})") 88 | t << format_row('Calendar Month with Most Spent', "#{amount_by_calendar_month.first.first.strftime '%B %Y'} (#{number_to_currency(amount_by_calendar_month.first.last).green})") 89 | end 90 | 91 | puts "\n" 92 | puts table 93 | puts "\n\n" 94 | end 95 | 96 | namespace :config do 97 | desc 'generate a config/account.yml file with your Amazon.com credentials' 98 | task generate: :dependencies do 99 | puts "\nNOTE: The credentials you enter\nhere are stored in #{'PLAIN TEXT'.red}.\n\n" 100 | puts "The credentials will be stored in:\n#{File.expand_path('./config/account.yml').to_s.red}\n\n" 101 | 102 | email = ask 'Amazon.com Account Email: ' 103 | password = ask('Amazon.com Account Password (hidden): ') { |q| q.echo = '*' } 104 | 105 | # confirmation; may or may not want ... 106 | password_confirm = ask('Confirm Amazon.com Account Password (hidden): ') { |q| q.echo = '*' } 107 | password = nil unless password == password_confirm 108 | 109 | puts "\n" 110 | 111 | if password.blank? || email.blank? 112 | puts "No luck with that; please try again.\n\n" 113 | next 114 | end 115 | 116 | File.open('./config/account.yml', 'w') { |f| f.puts({ email: email, password: password }.to_yaml) } 117 | puts "Wrote values to #{File.expand_path('./config/account.yml').to_s}\n\n" 118 | end 119 | end 120 | 121 | namespace :db do 122 | desc 'wipe all local order data' 123 | task reset: :establish_connection do 124 | File.unlink './db/orders.sqlite3' rescue Errno::ENOENT 125 | Rake::Task['db:migrate'].execute 126 | end 127 | 128 | desc 'ensure the DB schema is up-to-date' 129 | task migrate: :establish_connection do 130 | CURRENT_SCHEMA_VERSION = 1 131 | if ActiveRecord::Migrator.current_version != CURRENT_SCHEMA_VERSION 132 | ActiveRecord::Schema.define(version: CURRENT_SCHEMA_VERSION) do 133 | create_table :orders, id: false, force: true do |t| 134 | t.string :order_id, null: false, index: true, unique: true 135 | t.date :date, null: false 136 | 137 | t.float :grand_total, default: 0, null: false 138 | t.float :items_subtotal, default: 0, null: false 139 | t.float :estimated_tax_to_be_collected, default: 0, null: false 140 | 141 | t.boolean :gift, default: false, null: false 142 | t.boolean :completed, default: false, null: false 143 | t.text :line_items 144 | t.timestamps null: false 145 | end 146 | create_table :shipments, force: true, id: false do |t| 147 | t.string :shipment_id, null: false, index: true, unique: true 148 | t.string :carrier 149 | t.string :tracking_number, index: true 150 | t.string :ship_to 151 | t.string :shipment_status, null: false, default: 'Unknown' 152 | t.boolean :delivered, default: false, null: false, index: true 153 | t.timestamps null: false 154 | end 155 | create_table :shipment_orders, force: true, id: false do |t| 156 | t.string :shipment_id, null: false, index: true 157 | t.string :order_id, null: false, index: true 158 | end 159 | end 160 | end 161 | end 162 | end 163 | 164 | namespace :orders do 165 | 166 | desc 'log in to amazon and fetch all orders' 167 | task fetch: :environment do 168 | begin 169 | puts "Loading account settings ..." 170 | account = YAML.load_file('./config/account.yml').with_indifferent_access 171 | rescue Errno::ENOENT 172 | puts "\nHmm... no settings found.\n".red 173 | puts "Let's create a configuration file!".green 174 | Rake::Task['config:generate'].execute 175 | retry 176 | end 177 | 178 | agent = Mechanize.new { |a| a.user_agent_alias = 'Mac Safari' } 179 | puts "Loading #{'Amazon.com'.yellow}..." 180 | agent.get('https://www.amazon.com/') do |page| 181 | puts "Loading the login page.." 182 | login_page = agent.click(page.search('a[data-nav-role="signin"]')[0]) 183 | puts "Filling out the form and logging in..." 184 | begin 185 | post_login_page = login_page.form_with(action: %r{ap/signin}) do |f| 186 | account.each_pair { |k,v| f.send "#{k}=", v } 187 | end.click_button 188 | orders_page = agent.click(post_login_page.link_with(text: 'Your Orders')) 189 | rescue NoMethodError 190 | puts "\nLogging in to Amazon.com failed.\n".red 191 | puts "Try running #{'rake config:generate'.yellow} and updating your credentials." 192 | next 193 | end 194 | 195 | puts "Logged in successfully -- loading up your orders!\n" 196 | 197 | begin 198 | Amazon::Order.count 199 | rescue ActiveRecord::StatementInvalid 200 | Rake::Task['db:migrate'].execute 201 | end 202 | 203 | years = orders_page.search('form#timePeriodForm select[name=orderFilter] option[value^="year-"]').map do |option_tag| 204 | option_tag.content.strip.to_i 205 | end 206 | 207 | years.each do |year| 208 | puts "Looking up orders from #{year.to_s.blue}..." 209 | orders_page = orders_page.form_with(id: 'timePeriodForm') do |f| 210 | f.orderFilter = "year-#{year}" 211 | end.submit 212 | page = 0 213 | catch :no_more_orders do 214 | loop do 215 | puts " Parsing orders from #{year.to_s.blue} (page #{(page+1).to_s.red})" 216 | orders_page.search('#ordersContainer > .order').each do |node| 217 | order = Amazon::OrderImporter.import(node, agent) 218 | action = order.skipped? ? 'skipped' : nil 219 | ( action = order.new_order? ? 'imported' : 'updated' ) if action.nil? 220 | puts " #{action.titleize.yellow} Order ##{order.order_id.green} (#{order.reload.shipments.count} shipments)" 221 | end 222 | begin 223 | last_pagination_element = orders_page.search('ul.a-pagination li.a-last')[0] 224 | link = last_pagination_element.css('a')[0] 225 | rescue NoMethodError 226 | throw :no_more_orders 227 | end 228 | throw :no_more_orders unless link 229 | orders_page = agent.click(link) 230 | page += 1 231 | end 232 | end 233 | 234 | end 235 | 236 | puts "\nYippee! All done." 237 | end 238 | end 239 | end 240 | -------------------------------------------------------------------------------- /config/css_paths.yml: -------------------------------------------------------------------------------- 1 | date: .order-info > div > div > div > div.a-fixed-right-grid-col.a-col-left > div > div.a-column.a-span3 > div.a-row.a-size-base > span 2 | amount_paid: .order-info > div > div > div > div.a-fixed-right-grid-col.a-col-left > div > div.a-column.a-span2 > div.a-row.a-size-base > span 3 | order_id: .order-info > div > div > div > .actions > div > span.value 4 | tracking_number: '#a-page > div.a-section.a-spacing-none.ship-track-page-container > div:nth-child(6) > div.a-column.a-span6.a-span-last > div.a-row.a-expander-container.a-spacing-base.a-expander-extend-container > div.a-box-group.ship-track-latest-event-wrapper > div:nth-child(2) > div > div > div.a-column.a-span9.ship-track-grid-responsive-column.a-span-last > div' 5 | order_line_item: '#od-subtotals > div' 6 | status_node: 'div > div.a-row.shipment-top-row > div:nth-child(1) > div:nth-child(1) > span.a-size-medium.a-text-bold' 7 | sub_status: div > div.a-row.shipment-top-row > div:nth-child(1) > div:nth-child(2) > div 8 | shipment_node: '#orderDetails .shipment' 9 | -------------------------------------------------------------------------------- /db/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chrisb/amazon-orders/cfb2163a975017f067b74dabf47d4932c1926867/db/.gitkeep -------------------------------------------------------------------------------- /lib/amazon/order.rb: -------------------------------------------------------------------------------- 1 | module Amazon 2 | class Order < ActiveRecord::Base 3 | self.primary_key = :order_id 4 | 5 | attr_accessor :skipped 6 | attr_accessor :new_order 7 | 8 | has_many :shipment_orders, class_name: 'Amazon::ShipmentOrder' 9 | has_many :shipments, through: :shipment_orders, class_name: 'Amazon::Shipment' 10 | 11 | with_options numericality: { greater_than_or_equal_to: 0 } do |m| 12 | m.validates :grand_total 13 | m.validates :items_subtotal 14 | m.validates :estimated_tax_to_be_collected 15 | end 16 | 17 | before_save :set_completion 18 | before_save :set_new_order, on: :create 19 | 20 | before_validation :set_amounts_from_line_items, on: :create 21 | 22 | serialize :line_items, JSON 23 | 24 | after_initialize :set_default_values 25 | 26 | def set_default_values 27 | self.skipped = !new_record? 28 | self.new_order = new_record? 29 | self.line_items = {} 30 | end 31 | 32 | def skipped? ; @skipped ; end 33 | 34 | def new_order? ; @new_order ; end 35 | 36 | def set_amounts_from_line_items 37 | line_items.each_pair { |key, amount| self.send "#{key}=", amount if self.respond_to?("#{key}=") } 38 | end 39 | 40 | def set_new_order 41 | self.new_order = true 42 | end 43 | 44 | def set_completion 45 | self.skipped = false 46 | self.completed = shipments.count == shipments.delivered.count 47 | true 48 | end 49 | end 50 | end 51 | -------------------------------------------------------------------------------- /lib/amazon/order_importer.rb: -------------------------------------------------------------------------------- 1 | module Amazon 2 | class OrderImporter 3 | EXPIRATION_THRESHOLD = 18.hours 4 | 5 | @@css_paths = YAML.load_file('./config/css_paths.yml').with_indifferent_access 6 | 7 | class << self 8 | def order_id_from_node(node) 9 | node.css(@@css_paths['order_id']).first.content.strip 10 | end 11 | 12 | def shipment_from_feedback_link(shipment_node) 13 | link = shipment_node.css('a').find { |l| l['href'].include? 'od_aui_pack_feedback' } 14 | return nil unless link 15 | 16 | shipment_id = CGI.parse(link['href'].split('?').last)['specificShipmentId'].first 17 | shipment = Amazon::Shipment.find_or_create_by(shipment_id: shipment_id) 18 | shipment.update_attributes delivered: true 19 | shipment 20 | end 21 | 22 | def ad_hoc_shipment(order, shipment_index) 23 | fake_id = "ORDER-#{order.order_id}-SHIPMENT-#{shipment_index}" 24 | shipment = Amazon::Shipment.find_or_create_by shipment_id: fake_id 25 | shipment.update_attributes delivered: true 26 | shipment 27 | end 28 | 29 | def shipment_from_tracking_page(shipment_id, tracking_page) 30 | shipment = Amazon::Shipment.find_or_initialize_by(shipment_id: shipment_id) 31 | tracking_number_string = tracking_page.search(@@css_paths['tracking_number']).first.content rescue nil 32 | 33 | if tracking_number_string 34 | carrier, tracking_number = tracking_number_string.split(',').map { |str| str.split(':').last }.map(&:strip) 35 | shipment.update_attributes carrier: carrier, tracking_number: tracking_number 36 | end 37 | 38 | shipment 39 | end 40 | 41 | def parse_link_for_shipment_id 42 | CGI.parse(link['href'].split('?').last)['shipmentId'].first 43 | end 44 | 45 | def shipment_id_from_node(shipment_node) 46 | link = shipment_node.css('a').find { |l| l['href'].include?('ship-track') } 47 | return nil unless link 48 | 49 | tracking_page = agent.get(link['href']) 50 | 51 | return nil if tracking_page.body.match(/No tracking details/) 52 | 53 | shipment_from_tracking_page parse_link_for_shipment_id(link), tracking_page 54 | end 55 | 56 | def parse_shipments_for_order(agent, order, shipment_nodes) 57 | shipment_nodes.each_with_index do |shipment_node, shipment_index| 58 | status_node = shipment_node.css(@@css_paths['status_node']).first 59 | shipment_status = status_node.content.strip rescue 'Uknown' 60 | sub_status = shipment_node.search(@@css_paths['sub_status']).first 61 | shipment_status = "#{shipment_status}: #{sub_status}" if sub_status 62 | shipment = nil 63 | 64 | shipment_node.css('a').each do |link| # find the 'track package' link if possible 65 | next unless link['href'].include?('ship-track') 66 | 67 | shipment_id = CGI.parse(link['href'].split('?').last)['shipmentId'].first 68 | tracking_page = agent.get(link['href']) 69 | 70 | next if tracking_page.body.match(/No tracking details/) 71 | 72 | shipment = shipment_from_tracking_page(shipment_id, tracking_page) 73 | break 74 | end 75 | 76 | shipment = shipment_from_feedback_link shipment_node unless shipment # no package tracking link, find shipmentId from feedback link 77 | shipment = ad_hoc_shipment order, shipment_index unless shipment 78 | shipment.update_attributes shipment_status: shipment_status 79 | shipment.save! 80 | 81 | order.shipments << shipment 82 | end 83 | 84 | begin 85 | order.save! 86 | rescue ActiveRecord::RecordInvalid, ActiveRecord::RecordNotSaved => e 87 | handle_error_saving_order e, order 88 | end 89 | 90 | order 91 | end 92 | 93 | def should_skip_order?(order) 94 | return false if order.new_record? || order.shipments.empty? 95 | order.updated_at < Time.now + Amazon::OrderImporter::EXPIRATION_THRESHOLD 96 | end 97 | 98 | def will_skip_order?(order) 99 | order.skipped = true 100 | order 101 | end 102 | 103 | def formatted_line_item_name(name) 104 | name.gsub('(', '').gsub(')', '').parameterize.gsub('-', '_').to_sym 105 | end 106 | 107 | def parse_line_items_for_order(order, line_item_nodes) 108 | order.line_items = line_item_nodes.each_with_object({}) do |line_item_node, hsh| 109 | next unless line_item_node.css('div > span').size == 2 110 | name, amount = line_item_node.css('div > span').map(&:content).map(&:strip) 111 | hsh[formatted_line_item_name name] = currency_to_number(amount) 112 | end 113 | end 114 | 115 | def parse_date_for_order(order, node) 116 | order.date = Date.parse(node.css(@@css_paths['date']).first.content) 117 | order.save! 118 | end 119 | 120 | def import(node, agent) 121 | order = Amazon::Order.find_or_initialize_by(order_id: order_id_from_node(node)) 122 | 123 | if should_skip_order?(order) 124 | order.skipped = true 125 | return order 126 | end 127 | 128 | parse_date_for_order order, node 129 | 130 | url = "https://www.amazon.com/gp/your-account/order-details?orderID=#{order.order_id}" 131 | order_details_page = agent.get(url) 132 | 133 | parse_line_items_for_order order, order_details_page.search(@@css_paths['order_line_item']) 134 | parse_shipments_for_order agent, order, order_details_page.search(@@css_paths['shipment_node']) 135 | end 136 | 137 | def handle_error_saving_order(exception, order) 138 | puts "Unable to save order: #{order.inspect}" 139 | ap order 140 | puts 'Shipments:' 141 | ap order.shipments 142 | fail exception 143 | end 144 | 145 | def currency_to_number(currency) 146 | currency.to_s.gsub(/[$,]/, '').to_f 147 | end 148 | end 149 | end 150 | end 151 | -------------------------------------------------------------------------------- /lib/amazon/shipment.rb: -------------------------------------------------------------------------------- 1 | module Amazon 2 | class Shipment < ActiveRecord::Base 3 | self.primary_key = :shipment_id 4 | 5 | has_many :shipment_orders, class_name: 'Amazon::ShipmentOrder' 6 | has_many :orders, through: :shipment_orders, class_name: 'Amazon::Order' 7 | 8 | validates :delivered, inclusion: { in: [true, false] } 9 | validates :shipment_status, presence: true 10 | # validates :ship_to, presence: true 11 | # validates :shipping_address, presence: true 12 | 13 | scope :delivered, -> { where delivered: true } 14 | 15 | before_validation :set_delivered_status 16 | 17 | def set_delivered_status 18 | self.delivered = shipment_status.downcase.include?('delivered') 19 | true 20 | end 21 | end 22 | end 23 | -------------------------------------------------------------------------------- /lib/amazon/shipment_order.rb: -------------------------------------------------------------------------------- 1 | module Amazon 2 | class ShipmentOrder < ActiveRecord::Base 3 | belongs_to :shipment, class_name: 'Amazon::Shipment' 4 | belongs_to :order, class_name: 'Amazon::Order' 5 | validates :shipment_id, presence: true 6 | validates :order_id, presence: true 7 | end 8 | end 9 | --------------------------------------------------------------------------------