├── .bundle └── install.log ├── .ruby-version ├── Gemfile ├── Gemfile.lock ├── Procfile ├── README.md ├── Rakefile ├── app.rb ├── company-scraper.rb ├── config.ru ├── config └── newrelic.yml ├── data_mapper_custom.rb ├── data_mapper_setup.rb ├── job-scraper.rb └── lib ├── benefit.rb ├── company.rb ├── job.rb └── tag.rb /.bundle/install.log: -------------------------------------------------------------------------------- 1 | # Logfile created on 2014-02-05 18:34:50 +0000 by logger.rb/41954 2 | I, [2014-02-05T18:34:50.017876 #34437] INFO -- : 0: rake (10.1.1) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/rake-10.1.1.gemspec 3 | I, [2014-02-05T18:34:50.018345 #34437] INFO -- : 0: i18n (0.6.9) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/i18n-0.6.9.gemspec 4 | I, [2014-02-05T18:34:50.018780 #34437] INFO -- : 0: minitest (4.7.5) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/minitest-4.7.5.gemspec 5 | I, [2014-02-05T18:34:50.019187 #34437] INFO -- : 0: multi_json (1.8.4) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/multi_json-1.8.4.gemspec 6 | I, [2014-02-05T18:34:50.019572 #34437] INFO -- : 0: atomic (1.1.14) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/atomic-1.1.14.gemspec 7 | I, [2014-02-05T18:34:50.019957 #34437] INFO -- : 0: thread_safe (0.1.3) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/thread_safe-0.1.3.gemspec 8 | I, [2014-02-05T18:34:50.020346 #34437] INFO -- : 0: tzinfo (0.3.38) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/tzinfo-0.3.38.gemspec 9 | I, [2014-02-05T18:34:50.020654 #34437] INFO -- : 0: activesupport (4.0.2) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/activesupport-4.0.2.gemspec 10 | I, [2014-02-05T18:34:50.021043 #34437] INFO -- : 0: addressable (2.3.5) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/addressable-2.3.5.gemspec 11 | I, [2014-02-05T18:34:50.021383 #34437] INFO -- : 0: descendants_tracker (0.0.3) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/descendants_tracker-0.0.3.gemspec 12 | I, [2014-02-05T18:34:50.021749 #34437] INFO -- : 0: ice_nine (0.11.0) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/ice_nine-0.11.0.gemspec 13 | I, [2014-02-05T18:34:50.022085 #34437] INFO -- : 0: axiom-types (0.0.5) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/axiom-types-0.0.5.gemspec 14 | I, [2014-02-05T18:34:50.022408 #34437] INFO -- : 0: bcrypt-ruby (3.1.2) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/bcrypt-ruby-3.1.2.gemspec 15 | I, [2014-02-05T18:34:50.022728 #34437] INFO -- : 0: builder (3.2.2) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/builder-3.2.2.gemspec 16 | I, [2014-02-05T18:34:50.023054 #34437] INFO -- : 0: mime-types (2.1) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/mime-types-2.1.gemspec 17 | I, [2014-02-05T18:34:50.023365 #34437] INFO -- : 0: mini_portile (0.5.2) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/mini_portile-0.5.2.gemspec 18 | I, [2014-02-05T18:34:50.023685 #34437] INFO -- : 0: nokogiri (1.6.1) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/nokogiri-1.6.1.gemspec 19 | I, [2014-02-05T18:34:50.023994 #34437] INFO -- : 0: rack (1.5.2) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/rack-1.5.2.gemspec 20 | I, [2014-02-05T18:34:50.024313 #34437] INFO -- : 0: rack-test (0.6.2) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/rack-test-0.6.2.gemspec 21 | I, [2014-02-05T18:34:50.024605 #34437] INFO -- : 0: xpath (2.0.0) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/xpath-2.0.0.gemspec 22 | I, [2014-02-05T18:34:50.024938 #34437] INFO -- : 0: capybara (2.2.1) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/capybara-2.2.1.gemspec 23 | I, [2014-02-05T18:34:50.025259 #34437] INFO -- : 0: coercible (1.0.0) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/coercible-1.0.0.gemspec 24 | I, [2014-02-05T18:34:50.025568 #34437] INFO -- : 0: dm-core (1.2.1) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/dm-core-1.2.1.gemspec 25 | I, [2014-02-05T18:34:50.025878 #34437] INFO -- : 0: dm-aggregates (1.2.0) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/dm-aggregates-1.2.0.gemspec 26 | I, [2014-02-05T18:34:50.026192 #34437] INFO -- : 0: dm-constraints (1.2.0) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/dm-constraints-1.2.0.gemspec 27 | I, [2014-02-05T18:34:50.026498 #34437] INFO -- : 0: dm-migrations (1.2.0) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/dm-migrations-1.2.0.gemspec 28 | I, [2014-02-05T18:34:50.026808 #34437] INFO -- : 0: fastercsv (1.5.5) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/fastercsv-1.5.5.gemspec 29 | I, [2014-02-05T18:34:50.027110 #34437] INFO -- : 0: json (1.8.1) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/json-1.8.1.gemspec 30 | I, [2014-02-05T18:34:50.027546 #34437] INFO -- : 0: json_pure (1.8.1) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/json_pure-1.8.1.gemspec 31 | I, [2014-02-05T18:34:50.028017 #34437] INFO -- : 0: dm-serializer (1.2.2) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/dm-serializer-1.2.2.gemspec 32 | I, [2014-02-05T18:34:50.028440 #34437] INFO -- : 0: dm-timestamps (1.2.0) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/dm-timestamps-1.2.0.gemspec 33 | I, [2014-02-05T18:34:50.028739 #34437] INFO -- : 0: dm-transactions (1.2.0) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/dm-transactions-1.2.0.gemspec 34 | I, [2014-02-05T18:34:50.029111 #34437] INFO -- : 0: stringex (1.5.1) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/stringex-1.5.1.gemspec 35 | I, [2014-02-05T18:34:50.029399 #34437] INFO -- : 0: uuidtools (2.1.4) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/uuidtools-2.1.4.gemspec 36 | I, [2014-02-05T18:34:50.029695 #34437] INFO -- : 0: dm-types (1.2.2) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/dm-types-1.2.2.gemspec 37 | I, [2014-02-05T18:34:50.029973 #34437] INFO -- : 0: dm-validations (1.2.0) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/dm-validations-1.2.0.gemspec 38 | I, [2014-02-05T18:34:50.030267 #34437] INFO -- : 0: data_mapper (1.2.0) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/data_mapper-1.2.0.gemspec 39 | I, [2014-02-05T18:34:50.030536 #34437] INFO -- : 0: data_objects (0.10.13) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/data_objects-0.10.13.gemspec 40 | I, [2014-02-05T18:34:50.030820 #34437] INFO -- : 0: database_cleaner (1.2.0) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/database_cleaner-1.2.0.gemspec 41 | I, [2014-02-05T18:34:50.031107 #34437] INFO -- : 0: diff-lcs (1.2.5) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/diff-lcs-1.2.5.gemspec 42 | I, [2014-02-05T18:34:51.774191 #34437] INFO -- : 0: dm-ar-finders (1.2.0) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/dm-ar-finders-1.2.0.gemspec 43 | I, [2014-02-05T18:34:51.774583 #34437] INFO -- : 0: dm-do-adapter (1.2.0) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/dm-do-adapter-1.2.0.gemspec 44 | I, [2014-02-05T18:34:51.774851 #34437] INFO -- : 0: do_postgres (0.10.13) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/do_postgres-0.10.13.gemspec 45 | I, [2014-02-05T18:34:51.775106 #34437] INFO -- : 0: dm-postgres-adapter (1.2.0) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/dm-postgres-adapter-1.2.0.gemspec 46 | I, [2014-02-05T18:34:51.775345 #34437] INFO -- : 0: equalizer (0.0.9) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/equalizer-0.0.9.gemspec 47 | I, [2014-02-05T18:34:51.775582 #34437] INFO -- : 0: hashie (2.0.5) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/hashie-2.0.5.gemspec 48 | I, [2014-02-05T18:34:51.775819 #34437] INFO -- : 0: multi_xml (0.5.5) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/multi_xml-0.5.5.gemspec 49 | I, [2014-02-05T18:34:51.776054 #34437] INFO -- : 0: rack-accept (0.4.5) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/rack-accept-0.4.5.gemspec 50 | I, [2014-02-05T18:34:51.776286 #34437] INFO -- : 0: rack-mount (0.8.3) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/rack-mount-0.8.3.gemspec 51 | I, [2014-02-05T18:34:51.776520 #34437] INFO -- : 0: virtus (1.0.1) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/virtus-1.0.1.gemspec 52 | I, [2014-02-05T18:34:51.776752 #34437] INFO -- : 0: grape (0.6.1) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/grape-0.6.1.gemspec 53 | I, [2014-02-05T18:34:51.777019 #34437] INFO -- : 0: rspec-core (2.14.7) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/rspec-core-2.14.7.gemspec 54 | I, [2014-02-05T18:34:57.784453 #34437] INFO -- : 0: rspec-expectations (2.14.5) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/rspec-expectations-2.14.5.gemspec 55 | I, [2014-02-05T18:35:02.337219 #34437] INFO -- : 0: rspec-mocks (2.14.5) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/rspec-mocks-2.14.5.gemspec 56 | I, [2014-02-05T18:35:02.337638 #34437] INFO -- : 0: rspec (2.14.1) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/rspec-2.14.1.gemspec 57 | I, [2014-02-05T18:35:02.338031 #34437] INFO -- : 0: bundler (1.5.1) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/bundler-1.5.1.gemspec 58 | I, [2014-02-05T18:40:08.301174 #34515] INFO -- : 0: rake (10.1.1) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/rake-10.1.1.gemspec 59 | I, [2014-02-05T18:40:08.301753 #34515] INFO -- : 0: i18n (0.6.9) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/i18n-0.6.9.gemspec 60 | I, [2014-02-05T18:40:08.302226 #34515] INFO -- : 0: minitest (4.7.5) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/minitest-4.7.5.gemspec 61 | I, [2014-02-05T18:40:08.302650 #34515] INFO -- : 0: multi_json (1.8.4) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/multi_json-1.8.4.gemspec 62 | I, [2014-02-05T18:40:08.303044 #34515] INFO -- : 0: atomic (1.1.14) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/atomic-1.1.14.gemspec 63 | I, [2014-02-05T18:40:08.303408 #34515] INFO -- : 0: thread_safe (0.1.3) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/thread_safe-0.1.3.gemspec 64 | I, [2014-02-05T18:40:08.303786 #34515] INFO -- : 0: tzinfo (0.3.38) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/tzinfo-0.3.38.gemspec 65 | I, [2014-02-05T18:40:08.304163 #34515] INFO -- : 0: activesupport (4.0.2) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/activesupport-4.0.2.gemspec 66 | I, [2014-02-05T18:40:08.304532 #34515] INFO -- : 0: addressable (2.3.5) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/addressable-2.3.5.gemspec 67 | I, [2014-02-05T18:40:08.304899 #34515] INFO -- : 0: descendants_tracker (0.0.3) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/descendants_tracker-0.0.3.gemspec 68 | I, [2014-02-05T18:40:08.305309 #34515] INFO -- : 0: ice_nine (0.11.0) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/ice_nine-0.11.0.gemspec 69 | I, [2014-02-05T18:40:08.305702 #34515] INFO -- : 0: axiom-types (0.0.5) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/axiom-types-0.0.5.gemspec 70 | I, [2014-02-05T18:40:08.306109 #34515] INFO -- : 0: bcrypt-ruby (3.1.2) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/bcrypt-ruby-3.1.2.gemspec 71 | I, [2014-02-05T18:40:08.306507 #34515] INFO -- : 0: builder (3.2.2) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/builder-3.2.2.gemspec 72 | I, [2014-02-05T18:40:08.306807 #34515] INFO -- : 0: mime-types (2.1) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/mime-types-2.1.gemspec 73 | I, [2014-02-05T18:40:08.307093 #34515] INFO -- : 0: mini_portile (0.5.2) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/mini_portile-0.5.2.gemspec 74 | I, [2014-02-05T18:40:08.307360 #34515] INFO -- : 0: nokogiri (1.6.1) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/nokogiri-1.6.1.gemspec 75 | I, [2014-02-05T18:40:08.307636 #34515] INFO -- : 0: rack (1.5.2) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/rack-1.5.2.gemspec 76 | I, [2014-02-05T18:40:08.307907 #34515] INFO -- : 0: rack-test (0.6.2) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/rack-test-0.6.2.gemspec 77 | I, [2014-02-05T18:40:08.308179 #34515] INFO -- : 0: xpath (2.0.0) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/xpath-2.0.0.gemspec 78 | I, [2014-02-05T18:40:08.308457 #34515] INFO -- : 0: capybara (2.2.1) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/capybara-2.2.1.gemspec 79 | I, [2014-02-05T18:40:08.308731 #34515] INFO -- : 0: coercible (1.0.0) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/coercible-1.0.0.gemspec 80 | I, [2014-02-05T18:40:08.308988 #34515] INFO -- : 0: daemons (1.1.9) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/daemons-1.1.9.gemspec 81 | I, [2014-02-05T18:40:08.309261 #34515] INFO -- : 0: dm-core (1.2.1) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/dm-core-1.2.1.gemspec 82 | I, [2014-02-05T18:40:08.309532 #34515] INFO -- : 0: dm-aggregates (1.2.0) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/dm-aggregates-1.2.0.gemspec 83 | I, [2014-02-05T18:40:08.309820 #34515] INFO -- : 0: dm-constraints (1.2.0) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/dm-constraints-1.2.0.gemspec 84 | I, [2014-02-05T18:40:08.310095 #34515] INFO -- : 0: dm-migrations (1.2.0) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/dm-migrations-1.2.0.gemspec 85 | I, [2014-02-05T18:40:08.310352 #34515] INFO -- : 0: fastercsv (1.5.5) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/fastercsv-1.5.5.gemspec 86 | I, [2014-02-05T18:40:08.310626 #34515] INFO -- : 0: json (1.8.1) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/json-1.8.1.gemspec 87 | I, [2014-02-05T18:40:08.310896 #34515] INFO -- : 0: json_pure (1.8.1) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/json_pure-1.8.1.gemspec 88 | I, [2014-02-05T18:40:08.311173 #34515] INFO -- : 0: dm-serializer (1.2.2) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/dm-serializer-1.2.2.gemspec 89 | I, [2014-02-05T18:40:08.311456 #34515] INFO -- : 0: dm-timestamps (1.2.0) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/dm-timestamps-1.2.0.gemspec 90 | I, [2014-02-05T18:40:08.311757 #34515] INFO -- : 0: dm-transactions (1.2.0) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/dm-transactions-1.2.0.gemspec 91 | I, [2014-02-05T18:40:08.312014 #34515] INFO -- : 0: stringex (1.5.1) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/stringex-1.5.1.gemspec 92 | I, [2014-02-05T18:40:08.312282 #34515] INFO -- : 0: uuidtools (2.1.4) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/uuidtools-2.1.4.gemspec 93 | I, [2014-02-05T18:40:08.312554 #34515] INFO -- : 0: dm-types (1.2.2) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/dm-types-1.2.2.gemspec 94 | I, [2014-02-05T18:40:08.312832 #34515] INFO -- : 0: dm-validations (1.2.0) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/dm-validations-1.2.0.gemspec 95 | I, [2014-02-05T18:40:08.313105 #34515] INFO -- : 0: data_mapper (1.2.0) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/data_mapper-1.2.0.gemspec 96 | I, [2014-02-05T18:40:08.313378 #34515] INFO -- : 0: data_objects (0.10.13) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/data_objects-0.10.13.gemspec 97 | I, [2014-02-05T18:40:08.313633 #34515] INFO -- : 0: database_cleaner (1.2.0) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/database_cleaner-1.2.0.gemspec 98 | I, [2014-02-05T18:40:08.313901 #34515] INFO -- : 0: diff-lcs (1.2.5) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/diff-lcs-1.2.5.gemspec 99 | I, [2014-02-05T18:40:08.314172 #34515] INFO -- : 0: dm-ar-finders (1.2.0) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/dm-ar-finders-1.2.0.gemspec 100 | I, [2014-02-05T18:40:08.314442 #34515] INFO -- : 0: dm-do-adapter (1.2.0) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/dm-do-adapter-1.2.0.gemspec 101 | I, [2014-02-05T18:40:08.314714 #34515] INFO -- : 0: do_postgres (0.10.13) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/do_postgres-0.10.13.gemspec 102 | I, [2014-02-05T18:40:08.314967 #34515] INFO -- : 0: dm-postgres-adapter (1.2.0) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/dm-postgres-adapter-1.2.0.gemspec 103 | I, [2014-02-05T18:40:08.315240 #34515] INFO -- : 0: equalizer (0.0.9) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/equalizer-0.0.9.gemspec 104 | I, [2014-02-05T18:40:08.315533 #34515] INFO -- : 0: eventmachine (1.0.3) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/eventmachine-1.0.3.gemspec 105 | I, [2014-02-05T18:40:08.315974 #34515] INFO -- : 0: hashie (2.0.5) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/hashie-2.0.5.gemspec 106 | I, [2014-02-05T18:40:08.316397 #34515] INFO -- : 0: multi_xml (0.5.5) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/multi_xml-0.5.5.gemspec 107 | I, [2014-02-05T18:40:08.316784 #34515] INFO -- : 0: rack-accept (0.4.5) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/rack-accept-0.4.5.gemspec 108 | I, [2014-02-05T18:40:08.317175 #34515] INFO -- : 0: rack-mount (0.8.3) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/rack-mount-0.8.3.gemspec 109 | I, [2014-02-05T18:40:08.317571 #34515] INFO -- : 0: virtus (1.0.1) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/virtus-1.0.1.gemspec 110 | I, [2014-02-05T18:40:08.317939 #34515] INFO -- : 0: grape (0.6.1) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/grape-0.6.1.gemspec 111 | I, [2014-02-05T18:40:08.318310 #34515] INFO -- : 0: rspec-core (2.14.7) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/rspec-core-2.14.7.gemspec 112 | I, [2014-02-05T18:40:08.318690 #34515] INFO -- : 0: rspec-expectations (2.14.5) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/rspec-expectations-2.14.5.gemspec 113 | I, [2014-02-05T18:40:08.319062 #34515] INFO -- : 0: rspec-mocks (2.14.5) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/rspec-mocks-2.14.5.gemspec 114 | I, [2014-02-05T18:40:08.319413 #34515] INFO -- : 0: rspec (2.14.1) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/rspec-2.14.1.gemspec 115 | I, [2014-02-05T18:40:08.319778 #34515] INFO -- : 0: thin (1.6.1) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/thin-1.6.1.gemspec 116 | I, [2014-02-05T18:40:08.320134 #34515] INFO -- : 0: bundler (1.5.1) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/bundler-1.5.1.gemspec 117 | I, [2014-02-06T15:15:38.932527 #39074] INFO -- : 0: rake (10.1.1) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/rake-10.1.1.gemspec 118 | I, [2014-02-06T15:15:38.935025 #39074] INFO -- : 0: i18n (0.6.9) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/i18n-0.6.9.gemspec 119 | I, [2014-02-06T15:15:38.935345 #39074] INFO -- : 0: minitest (4.7.5) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/minitest-4.7.5.gemspec 120 | I, [2014-02-06T15:15:38.935613 #39074] INFO -- : 0: multi_json (1.8.4) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/multi_json-1.8.4.gemspec 121 | I, [2014-02-06T15:15:38.935882 #39074] INFO -- : 0: atomic (1.1.14) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/atomic-1.1.14.gemspec 122 | I, [2014-02-06T15:15:38.936129 #39074] INFO -- : 0: thread_safe (0.1.3) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/thread_safe-0.1.3.gemspec 123 | I, [2014-02-06T15:15:38.936379 #39074] INFO -- : 0: tzinfo (0.3.38) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/tzinfo-0.3.38.gemspec 124 | I, [2014-02-06T15:15:38.936643 #39074] INFO -- : 0: activesupport (4.0.2) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/activesupport-4.0.2.gemspec 125 | I, [2014-02-06T15:15:38.936887 #39074] INFO -- : 0: addressable (2.3.5) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/addressable-2.3.5.gemspec 126 | I, [2014-02-06T15:15:38.937131 #39074] INFO -- : 0: descendants_tracker (0.0.3) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/descendants_tracker-0.0.3.gemspec 127 | I, [2014-02-06T15:15:38.937374 #39074] INFO -- : 0: ice_nine (0.11.0) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/ice_nine-0.11.0.gemspec 128 | I, [2014-02-06T15:15:38.937624 #39074] INFO -- : 0: axiom-types (0.0.5) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/axiom-types-0.0.5.gemspec 129 | I, [2014-02-06T15:15:38.937862 #39074] INFO -- : 0: bcrypt-ruby (3.1.2) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/bcrypt-ruby-3.1.2.gemspec 130 | I, [2014-02-06T15:15:38.938102 #39074] INFO -- : 0: builder (3.2.2) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/builder-3.2.2.gemspec 131 | I, [2014-02-06T15:15:38.938361 #39074] INFO -- : 0: mime-types (2.1) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/mime-types-2.1.gemspec 132 | I, [2014-02-06T15:15:38.938682 #39074] INFO -- : 0: mini_portile (0.5.2) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/mini_portile-0.5.2.gemspec 133 | I, [2014-02-06T15:15:38.938931 #39074] INFO -- : 0: nokogiri (1.6.1) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/nokogiri-1.6.1.gemspec 134 | I, [2014-02-06T15:15:38.939167 #39074] INFO -- : 0: rack (1.5.2) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/rack-1.5.2.gemspec 135 | I, [2014-02-06T15:15:38.939414 #39074] INFO -- : 0: rack-test (0.6.2) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/rack-test-0.6.2.gemspec 136 | I, [2014-02-06T15:15:38.939666 #39074] INFO -- : 0: xpath (2.0.0) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/xpath-2.0.0.gemspec 137 | I, [2014-02-06T15:15:38.939922 #39074] INFO -- : 0: capybara (2.2.1) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/capybara-2.2.1.gemspec 138 | I, [2014-02-06T15:15:38.940158 #39074] INFO -- : 0: coercible (1.0.0) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/coercible-1.0.0.gemspec 139 | I, [2014-02-06T15:15:38.940388 #39074] INFO -- : 0: daemons (1.1.9) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/daemons-1.1.9.gemspec 140 | I, [2014-02-06T15:15:38.940639 #39074] INFO -- : 0: dm-core (1.2.1) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/dm-core-1.2.1.gemspec 141 | I, [2014-02-06T15:15:38.940881 #39074] INFO -- : 0: dm-aggregates (1.2.0) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/dm-aggregates-1.2.0.gemspec 142 | I, [2014-02-06T15:15:38.941113 #39074] INFO -- : 0: dm-constraints (1.2.0) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/dm-constraints-1.2.0.gemspec 143 | I, [2014-02-06T15:15:38.941347 #39074] INFO -- : 0: dm-migrations (1.2.0) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/dm-migrations-1.2.0.gemspec 144 | I, [2014-02-06T15:15:38.941991 #39074] INFO -- : 0: fastercsv (1.5.5) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/fastercsv-1.5.5.gemspec 145 | I, [2014-02-06T15:15:38.942337 #39074] INFO -- : 0: json (1.8.1) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/json-1.8.1.gemspec 146 | I, [2014-02-06T15:15:38.942682 #39074] INFO -- : 0: json_pure (1.8.1) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/json_pure-1.8.1.gemspec 147 | I, [2014-02-06T15:15:38.942994 #39074] INFO -- : 0: dm-serializer (1.2.2) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/dm-serializer-1.2.2.gemspec 148 | I, [2014-02-06T15:15:38.943331 #39074] INFO -- : 0: dm-timestamps (1.2.0) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/dm-timestamps-1.2.0.gemspec 149 | I, [2014-02-06T15:15:38.943615 #39074] INFO -- : 0: dm-transactions (1.2.0) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/dm-transactions-1.2.0.gemspec 150 | I, [2014-02-06T15:15:38.943888 #39074] INFO -- : 0: stringex (1.5.1) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/stringex-1.5.1.gemspec 151 | I, [2014-02-06T15:15:38.944165 #39074] INFO -- : 0: uuidtools (2.1.4) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/uuidtools-2.1.4.gemspec 152 | I, [2014-02-06T15:15:38.944461 #39074] INFO -- : 0: dm-types (1.2.2) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/dm-types-1.2.2.gemspec 153 | I, [2014-02-06T15:15:38.944764 #39074] INFO -- : 0: dm-validations (1.2.0) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/dm-validations-1.2.0.gemspec 154 | I, [2014-02-06T15:15:38.945077 #39074] INFO -- : 0: data_mapper (1.2.0) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/data_mapper-1.2.0.gemspec 155 | I, [2014-02-06T15:15:38.945397 #39074] INFO -- : 0: data_objects (0.10.13) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/data_objects-0.10.13.gemspec 156 | I, [2014-02-06T15:15:38.945803 #39074] INFO -- : 0: database_cleaner (1.2.0) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/database_cleaner-1.2.0.gemspec 157 | I, [2014-02-06T15:15:38.946387 #39074] INFO -- : 0: diff-lcs (1.2.5) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/diff-lcs-1.2.5.gemspec 158 | I, [2014-02-06T15:15:38.947044 #39074] INFO -- : 0: dm-ar-finders (1.2.0) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/dm-ar-finders-1.2.0.gemspec 159 | I, [2014-02-06T15:15:38.947394 #39074] INFO -- : 0: dm-do-adapter (1.2.0) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/dm-do-adapter-1.2.0.gemspec 160 | I, [2014-02-06T15:15:38.947724 #39074] INFO -- : 0: do_postgres (0.10.13) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/do_postgres-0.10.13.gemspec 161 | I, [2014-02-06T15:15:38.948048 #39074] INFO -- : 0: dm-postgres-adapter (1.2.0) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/dm-postgres-adapter-1.2.0.gemspec 162 | I, [2014-02-06T15:15:38.948422 #39074] INFO -- : 0: equalizer (0.0.9) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/equalizer-0.0.9.gemspec 163 | I, [2014-02-06T15:15:38.948811 #39074] INFO -- : 0: eventmachine (1.0.3) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/eventmachine-1.0.3.gemspec 164 | I, [2014-02-06T15:15:38.949148 #39074] INFO -- : 0: hashie (2.0.5) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/hashie-2.0.5.gemspec 165 | I, [2014-02-06T15:15:38.949479 #39074] INFO -- : 0: multi_xml (0.5.5) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/multi_xml-0.5.5.gemspec 166 | I, [2014-02-06T15:15:38.949811 #39074] INFO -- : 0: rack-accept (0.4.5) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/rack-accept-0.4.5.gemspec 167 | I, [2014-02-06T15:15:38.950140 #39074] INFO -- : 0: rack-mount (0.8.3) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/rack-mount-0.8.3.gemspec 168 | I, [2014-02-06T15:15:38.950472 #39074] INFO -- : 0: virtus (1.0.1) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/virtus-1.0.1.gemspec 169 | I, [2014-02-06T15:15:38.950781 #39074] INFO -- : 0: grape (0.6.1) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/grape-0.6.1.gemspec 170 | I, [2014-02-06T15:15:40.908668 #39074] INFO -- : 0: grape-entity (0.4.0) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/grape-entity-0.4.0.gemspec 171 | I, [2014-02-06T15:15:40.909177 #39074] INFO -- : 0: rspec-core (2.14.7) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/rspec-core-2.14.7.gemspec 172 | I, [2014-02-06T15:15:40.909513 #39074] INFO -- : 0: rspec-expectations (2.14.5) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/rspec-expectations-2.14.5.gemspec 173 | I, [2014-02-06T15:15:40.909892 #39074] INFO -- : 0: rspec-mocks (2.14.5) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/rspec-mocks-2.14.5.gemspec 174 | I, [2014-02-06T15:15:40.910317 #39074] INFO -- : 0: rspec (2.14.1) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/rspec-2.14.1.gemspec 175 | I, [2014-02-06T15:15:40.910605 #39074] INFO -- : 0: thin (1.6.1) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/thin-1.6.1.gemspec 176 | I, [2014-02-06T15:15:40.910872 #39074] INFO -- : 0: bundler (1.5.1) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/bundler-1.5.1.gemspec 177 | I, [2014-02-07T15:33:26.472389 #47044] INFO -- : 0: rake (10.1.1) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/rake-10.1.1.gemspec 178 | I, [2014-02-07T15:33:26.473543 #47044] INFO -- : 0: i18n (0.6.9) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/i18n-0.6.9.gemspec 179 | I, [2014-02-07T15:33:26.473837 #47044] INFO -- : 0: minitest (4.7.5) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/minitest-4.7.5.gemspec 180 | I, [2014-02-07T15:33:26.474121 #47044] INFO -- : 0: multi_json (1.8.4) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/multi_json-1.8.4.gemspec 181 | I, [2014-02-07T15:33:26.474527 #47044] INFO -- : 0: atomic (1.1.14) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/atomic-1.1.14.gemspec 182 | I, [2014-02-07T15:33:26.474802 #47044] INFO -- : 0: thread_safe (0.1.3) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/thread_safe-0.1.3.gemspec 183 | I, [2014-02-07T15:33:26.475061 #47044] INFO -- : 0: tzinfo (0.3.38) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/tzinfo-0.3.38.gemspec 184 | I, [2014-02-07T15:33:26.475319 #47044] INFO -- : 0: activesupport (4.0.2) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/activesupport-4.0.2.gemspec 185 | I, [2014-02-07T15:33:26.475577 #47044] INFO -- : 0: addressable (2.3.5) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/addressable-2.3.5.gemspec 186 | I, [2014-02-07T15:33:26.475829 #47044] INFO -- : 0: descendants_tracker (0.0.3) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/descendants_tracker-0.0.3.gemspec 187 | I, [2014-02-07T15:33:26.476079 #47044] INFO -- : 0: ice_nine (0.11.0) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/ice_nine-0.11.0.gemspec 188 | I, [2014-02-07T15:33:26.476309 #47044] INFO -- : 0: axiom-types (0.0.5) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/axiom-types-0.0.5.gemspec 189 | I, [2014-02-07T15:33:26.476626 #47044] INFO -- : 0: bcrypt-ruby (3.1.2) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/bcrypt-ruby-3.1.2.gemspec 190 | I, [2014-02-07T15:33:26.476964 #47044] INFO -- : 0: builder (3.2.2) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/builder-3.2.2.gemspec 191 | I, [2014-02-07T15:33:26.477225 #47044] INFO -- : 0: mime-types (2.1) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/mime-types-2.1.gemspec 192 | I, [2014-02-07T15:33:26.477470 #47044] INFO -- : 0: mini_portile (0.5.2) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/mini_portile-0.5.2.gemspec 193 | I, [2014-02-07T15:33:26.477701 #47044] INFO -- : 0: nokogiri (1.6.1) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/nokogiri-1.6.1.gemspec 194 | I, [2014-02-07T15:33:26.477947 #47044] INFO -- : 0: rack (1.5.2) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/rack-1.5.2.gemspec 195 | I, [2014-02-07T15:33:26.478193 #47044] INFO -- : 0: rack-test (0.6.2) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/rack-test-0.6.2.gemspec 196 | I, [2014-02-07T15:33:26.478436 #47044] INFO -- : 0: xpath (2.0.0) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/xpath-2.0.0.gemspec 197 | I, [2014-02-07T15:33:26.478684 #47044] INFO -- : 0: capybara (2.2.1) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/capybara-2.2.1.gemspec 198 | I, [2014-02-07T15:33:26.478946 #47044] INFO -- : 0: coercible (1.0.0) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/coercible-1.0.0.gemspec 199 | I, [2014-02-07T15:33:26.479303 #47044] INFO -- : 0: daemons (1.1.9) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/daemons-1.1.9.gemspec 200 | I, [2014-02-07T15:33:26.479558 #47044] INFO -- : 0: dm-core (1.2.1) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/dm-core-1.2.1.gemspec 201 | I, [2014-02-07T15:33:26.479807 #47044] INFO -- : 0: dm-aggregates (1.2.0) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/dm-aggregates-1.2.0.gemspec 202 | I, [2014-02-07T15:33:26.480075 #47044] INFO -- : 0: dm-constraints (1.2.0) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/dm-constraints-1.2.0.gemspec 203 | I, [2014-02-07T15:33:26.480321 #47044] INFO -- : 0: dm-migrations (1.2.0) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/dm-migrations-1.2.0.gemspec 204 | I, [2014-02-07T15:33:26.480568 #47044] INFO -- : 0: fastercsv (1.5.5) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/fastercsv-1.5.5.gemspec 205 | I, [2014-02-07T15:33:26.480944 #47044] INFO -- : 0: json (1.8.1) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/json-1.8.1.gemspec 206 | I, [2014-02-07T15:33:26.481354 #47044] INFO -- : 0: json_pure (1.8.1) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/json_pure-1.8.1.gemspec 207 | I, [2014-02-07T15:33:26.481718 #47044] INFO -- : 0: dm-serializer (1.2.2) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/dm-serializer-1.2.2.gemspec 208 | I, [2014-02-07T15:33:26.482408 #47044] INFO -- : 0: dm-timestamps (1.2.0) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/dm-timestamps-1.2.0.gemspec 209 | I, [2014-02-07T15:33:26.482823 #47044] INFO -- : 0: dm-transactions (1.2.0) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/dm-transactions-1.2.0.gemspec 210 | I, [2014-02-07T15:33:26.483162 #47044] INFO -- : 0: stringex (1.5.1) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/stringex-1.5.1.gemspec 211 | I, [2014-02-07T15:33:26.483469 #47044] INFO -- : 0: uuidtools (2.1.4) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/uuidtools-2.1.4.gemspec 212 | I, [2014-02-07T15:33:26.483788 #47044] INFO -- : 0: dm-types (1.2.2) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/dm-types-1.2.2.gemspec 213 | I, [2014-02-07T15:33:26.484099 #47044] INFO -- : 0: dm-validations (1.2.0) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/dm-validations-1.2.0.gemspec 214 | I, [2014-02-07T15:33:26.484413 #47044] INFO -- : 0: data_mapper (1.2.0) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/data_mapper-1.2.0.gemspec 215 | I, [2014-02-07T15:33:26.484726 #47044] INFO -- : 0: data_objects (0.10.13) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/data_objects-0.10.13.gemspec 216 | I, [2014-02-07T15:33:26.485037 #47044] INFO -- : 0: database_cleaner (1.2.0) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/database_cleaner-1.2.0.gemspec 217 | I, [2014-02-07T15:33:26.485351 #47044] INFO -- : 0: diff-lcs (1.2.5) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/diff-lcs-1.2.5.gemspec 218 | I, [2014-02-07T15:33:26.485668 #47044] INFO -- : 0: dm-ar-finders (1.2.0) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/dm-ar-finders-1.2.0.gemspec 219 | I, [2014-02-07T15:33:26.485978 #47044] INFO -- : 0: dm-do-adapter (1.2.0) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/dm-do-adapter-1.2.0.gemspec 220 | I, [2014-02-07T15:33:26.486287 #47044] INFO -- : 0: do_postgres (0.10.13) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/do_postgres-0.10.13.gemspec 221 | I, [2014-02-07T15:33:26.486596 #47044] INFO -- : 0: dm-postgres-adapter (1.2.0) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/dm-postgres-adapter-1.2.0.gemspec 222 | I, [2014-02-07T15:33:26.486886 #47044] INFO -- : 0: equalizer (0.0.9) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/equalizer-0.0.9.gemspec 223 | I, [2014-02-07T15:33:26.487188 #47044] INFO -- : 0: eventmachine (1.0.3) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/eventmachine-1.0.3.gemspec 224 | I, [2014-02-07T15:33:26.487490 #47044] INFO -- : 0: hashie (2.0.5) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/hashie-2.0.5.gemspec 225 | I, [2014-02-07T15:33:26.487803 #47044] INFO -- : 0: multi_xml (0.5.5) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/multi_xml-0.5.5.gemspec 226 | I, [2014-02-07T15:33:26.488109 #47044] INFO -- : 0: rack-accept (0.4.5) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/rack-accept-0.4.5.gemspec 227 | I, [2014-02-07T15:33:26.488414 #47044] INFO -- : 0: rack-mount (0.8.3) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/rack-mount-0.8.3.gemspec 228 | I, [2014-02-07T15:33:26.488703 #47044] INFO -- : 0: virtus (1.0.1) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/virtus-1.0.1.gemspec 229 | I, [2014-02-07T15:33:26.489023 #47044] INFO -- : 0: grape (0.6.1) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/grape-0.6.1.gemspec 230 | I, [2014-02-07T15:33:26.489328 #47044] INFO -- : 0: grape-entity (0.4.0) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/grape-entity-0.4.0.gemspec 231 | I, [2014-02-07T15:33:26.489634 #47044] INFO -- : 0: rspec-core (2.14.7) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/rspec-core-2.14.7.gemspec 232 | I, [2014-02-07T15:33:26.489951 #47044] INFO -- : 0: rspec-expectations (2.14.5) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/rspec-expectations-2.14.5.gemspec 233 | I, [2014-02-07T15:33:26.490245 #47044] INFO -- : 0: rspec-mocks (2.14.5) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/rspec-mocks-2.14.5.gemspec 234 | I, [2014-02-07T15:33:26.490548 #47044] INFO -- : 0: rspec (2.14.1) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/rspec-2.14.1.gemspec 235 | I, [2014-02-07T15:33:26.490847 #47044] INFO -- : 0: thin (1.6.1) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/thin-1.6.1.gemspec 236 | I, [2014-02-07T15:33:26.491156 #47044] INFO -- : 0: bundler (1.5.1) from /Users/m/.rvm/gems/ruby-2.0.0-p353/specifications/bundler-1.5.1.gemspec 237 | -------------------------------------------------------------------------------- /.ruby-version: -------------------------------------------------------------------------------- 1 | 2.0.0-p353 2 | -------------------------------------------------------------------------------- /Gemfile: -------------------------------------------------------------------------------- 1 | source 'https://rubygems.org' 2 | 3 | # ruby '2.0.0' 4 | 5 | gem 'data_mapper' 6 | gem 'dm-postgres-adapter' 7 | gem 'rake' 8 | gem 'grape' 9 | gem 'grape-entity' 10 | gem 'dm-ar-finders' 11 | gem 'dm-constraints' 12 | gem 'thin' 13 | gem 'nokogiri' 14 | gem 'dm-core' 15 | gem 'dm-do-adapter' 16 | gem 'newrelic_rpm' 17 | gem 'grape-pagination' 18 | gem 'will_paginate', '~> 3.0' 19 | 20 | group :test, :development do 21 | gem 'rspec' 22 | gem 'database_cleaner' 23 | gem 'capybara' 24 | end -------------------------------------------------------------------------------- /Gemfile.lock: -------------------------------------------------------------------------------- 1 | GEM 2 | remote: https://rubygems.org/ 3 | specs: 4 | activesupport (4.0.2) 5 | i18n (~> 0.6, >= 0.6.4) 6 | minitest (~> 4.2) 7 | multi_json (~> 1.3) 8 | thread_safe (~> 0.1) 9 | tzinfo (~> 0.3.37) 10 | addressable (2.3.5) 11 | atomic (1.1.14) 12 | axiom-types (0.0.5) 13 | descendants_tracker (~> 0.0.1) 14 | ice_nine (~> 0.9) 15 | bcrypt-ruby (3.1.2) 16 | builder (3.2.2) 17 | capybara (2.2.1) 18 | mime-types (>= 1.16) 19 | nokogiri (>= 1.3.3) 20 | rack (>= 1.0.0) 21 | rack-test (>= 0.5.4) 22 | xpath (~> 2.0) 23 | coercible (1.0.0) 24 | descendants_tracker (~> 0.0.1) 25 | daemons (1.1.9) 26 | data_mapper (1.2.0) 27 | dm-aggregates (~> 1.2.0) 28 | dm-constraints (~> 1.2.0) 29 | dm-core (~> 1.2.0) 30 | dm-migrations (~> 1.2.0) 31 | dm-serializer (~> 1.2.0) 32 | dm-timestamps (~> 1.2.0) 33 | dm-transactions (~> 1.2.0) 34 | dm-types (~> 1.2.0) 35 | dm-validations (~> 1.2.0) 36 | data_objects (0.10.14) 37 | addressable (~> 2.1) 38 | database_cleaner (1.2.0) 39 | descendants_tracker (0.0.3) 40 | diff-lcs (1.2.5) 41 | dm-aggregates (1.2.0) 42 | dm-core (~> 1.2.0) 43 | dm-ar-finders (1.2.0) 44 | dm-core (~> 1.2.0) 45 | dm-constraints (1.2.0) 46 | dm-core (~> 1.2.0) 47 | dm-core (1.2.1) 48 | addressable (~> 2.3) 49 | dm-do-adapter (1.2.0) 50 | data_objects (~> 0.10.6) 51 | dm-core (~> 1.2.0) 52 | dm-migrations (1.2.0) 53 | dm-core (~> 1.2.0) 54 | dm-postgres-adapter (1.2.0) 55 | dm-do-adapter (~> 1.2.0) 56 | do_postgres (~> 0.10.6) 57 | dm-serializer (1.2.2) 58 | dm-core (~> 1.2.0) 59 | fastercsv (~> 1.5) 60 | json (~> 1.6) 61 | json_pure (~> 1.6) 62 | multi_json (~> 1.0) 63 | dm-timestamps (1.2.0) 64 | dm-core (~> 1.2.0) 65 | dm-transactions (1.2.0) 66 | dm-core (~> 1.2.0) 67 | dm-types (1.2.2) 68 | bcrypt-ruby (~> 3.0) 69 | dm-core (~> 1.2.0) 70 | fastercsv (~> 1.5) 71 | json (~> 1.6) 72 | multi_json (~> 1.0) 73 | stringex (~> 1.4) 74 | uuidtools (~> 2.1) 75 | dm-validations (1.2.0) 76 | dm-core (~> 1.2.0) 77 | do_postgres (0.10.14) 78 | data_objects (= 0.10.14) 79 | equalizer (0.0.9) 80 | eventmachine (1.0.3) 81 | fastercsv (1.5.5) 82 | grape (0.6.1) 83 | activesupport 84 | builder 85 | hashie (>= 1.2.0) 86 | multi_json (>= 1.3.2) 87 | multi_xml (>= 0.5.2) 88 | rack (>= 1.3.0) 89 | rack-accept 90 | rack-mount 91 | virtus (>= 1.0.0) 92 | grape-entity (0.4.1) 93 | activesupport 94 | multi_json (>= 1.3.2) 95 | grape-pagination (0.1.0) 96 | addressable 97 | grape (~> 0.3) 98 | hashie (2.0.5) 99 | i18n (0.6.9) 100 | ice_nine (0.11.0) 101 | json (1.8.1) 102 | json_pure (1.8.1) 103 | mime-types (2.1) 104 | mini_portile (0.5.2) 105 | minitest (4.7.5) 106 | multi_json (1.8.4) 107 | multi_xml (0.5.5) 108 | newrelic_rpm (3.7.2.192) 109 | nokogiri (1.6.1) 110 | mini_portile (~> 0.5.0) 111 | rack (1.5.2) 112 | rack-accept (0.4.5) 113 | rack (>= 0.4) 114 | rack-mount (0.8.3) 115 | rack (>= 1.0.0) 116 | rack-test (0.6.2) 117 | rack (>= 1.0) 118 | rake (10.1.1) 119 | rspec (2.14.1) 120 | rspec-core (~> 2.14.0) 121 | rspec-expectations (~> 2.14.0) 122 | rspec-mocks (~> 2.14.0) 123 | rspec-core (2.14.7) 124 | rspec-expectations (2.14.5) 125 | diff-lcs (>= 1.1.3, < 2.0) 126 | rspec-mocks (2.14.5) 127 | stringex (1.5.1) 128 | thin (1.6.1) 129 | daemons (>= 1.0.9) 130 | eventmachine (>= 1.0.0) 131 | rack (>= 1.0.0) 132 | thread_safe (0.1.3) 133 | atomic 134 | tzinfo (0.3.38) 135 | uuidtools (2.1.4) 136 | virtus (1.0.1) 137 | axiom-types (~> 0.0.5) 138 | coercible (~> 1.0) 139 | descendants_tracker (~> 0.0.1) 140 | equalizer (~> 0.0.7) 141 | will_paginate (3.0.5) 142 | xpath (2.0.0) 143 | nokogiri (~> 1.3) 144 | 145 | PLATFORMS 146 | ruby 147 | 148 | DEPENDENCIES 149 | capybara 150 | data_mapper 151 | database_cleaner 152 | dm-ar-finders 153 | dm-constraints 154 | dm-core 155 | dm-do-adapter 156 | dm-postgres-adapter 157 | grape 158 | grape-entity 159 | grape-pagination 160 | newrelic_rpm 161 | nokogiri 162 | rake 163 | rspec 164 | thin 165 | will_paginate (~> 3.0) 166 | -------------------------------------------------------------------------------- /Procfile: -------------------------------------------------------------------------------- 1 | web: bundle exec rackup config.ru -p $PORT -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Unofficial Stack Overflow API 2 | ============================= 3 | ==IMPORTANT=== This repo is no longer maintained. 4 | 5 | Taking inspiration from [this post] on Stackoverflow, we decided to make an API for the Stackoverflow Careers website as a final project for [Makers Academy]. 6 | 7 | We wrote two very polite scrapers which gather new jobs / companies posted on Stackoverflow on a daily basis. The data is availabe in JSON format on Heroku at various URLs on http://stackable.herokuapp.com, for example: 8 | 9 | - **/api/companies** (*all companies*) 10 | - **/api/jobs** (*all jobs*) 11 | - **/api/relocation** (*relocation offered*) 12 | - **/api/remote** (*remote allowed*) 13 | - **/api/senior** (*senior position*) 14 | - **/api/jobs/tags/ruby&java** (*jobs by tags - separated by ampersand*) 15 | - **/api/companies/benefits/gym** (*companies by benefits - keywords separated by ampersand*) 16 | - **/api/jobs/location/london** (*jobs by location*) 17 | 18 | (The prefix /api/ was added with Grape. If you simply open the root it will return a 404 Not Found!) 19 | 20 | Usage 21 | ----- 22 | It is possible to query the API by making a GET request to the following URLs: 23 | 24 | | URL | Paginated? | 25 | | ---- | :------: | 26 | | http://stackable.herokuapp.com/api/jobs | Yes | 27 | | http://stackable.herokuapp.com/api/companies | Yes | 28 | | http://stackable.herokuapp.com/api/relocation | No | 29 | | http://stackable.herokuapp.com/api/senior | No | 30 | | http://stackable.herokuapp.com/api/remote | No | 31 | | http://stackable.herokuapp.com/api/jobs/tags/ruby | No | 32 | | http://stackable.herokuapp.com/api/jobs/tags/ruby&java | No | 33 | | http://stackable.herokuapp.com/api/jobs/location/london | No | 34 | | http://stackable.herokuapp.com/api/companies/benefits/gym | No | 35 | | http://stackable.herokuapp.com/api/companies/benefits/gym | No | 36 | | http://stackable.herokuapp.com/api/companies/benefits/gym&dental | No | 37 | 38 | We strongly recommend using a Chrome plugin to 'prettify' the JSON output, such as **JSON Formatter**. 39 | 40 | Example 41 | ------ 42 | The query below will return a list of the first 30 jobs available. The command -i will show you the headers, where you can see the second page available in the series: http://stackable.herokuapp.com/api/companies?page=2&per_page=30 43 | 44 | From the command line, type: 45 | 46 | ``` 47 | curl -i http://stackable.herokuapp.com/api/jobs 48 | ``` 49 | As you may have gathered, results are paginated for two queries: /api/jobs and /api/companies. URLs are built by appending the following string to the main URL: 50 | 51 | ``` 52 | ?page=2&per_page=30 53 | ``` 54 | To make a GET request you can either use curl or net-http straight from your Ruby code. There are other solutions, one of which is our [Gem Stackable]. This makes it easier to pull jobs from your Ruby code as it handles GET requests for you (similarly to what Octokit does for Github API). 55 | 56 | However, the Stackable Gem still cannot handle pagination (it will be able to soon). For this reason, all queries will work fine except get_all_companies and all_jobs, which will only return the first 30 jobs / companies. For these two queries, please query the API directly for now. 57 | 58 | Important 59 | --------- 60 | While [all Jobs] and [all Companies] listings are paginated, all other queries aren't yet. This may slow down the API. We will launch pagination for all results as soon as possible. 61 | 62 | Tech Stack 63 | ---------- 64 | 65 | This API was built with the [Grape Gem]. Full tech stack: 66 | 67 | - Grape; 68 | - Postgres; 69 | - DataMapper; 70 | - Heroku; 71 | - DataMapper plugins 72 | 73 | Although it's very quick to get things going with DataMapper, we soon came to know its limits and had to pass raw SQL queries to get what we wanted. We also had to integrate with a few other plugins - among which the custom 'ilike' function for case-insensitive queries. 74 | 75 | Grape::Entity::DSL module made it possible to avoid exposing user-unfriendly database columns such as the 'scraping_round'. 76 | 77 | :small_red_triangle_down: Gems: Grape, Grape Entity, Grape-pagination, Nokogiri. 78 | 79 | [this post]: http://meta.stackoverflow.com/questions/158005/stackoverflow-careers-api 80 | [all Jobs]: http://stackable.herokuapp.com/api/jobs 81 | [all Companies]: http://stackable.herokuapp.com/api/companies 82 | [Gem Stackable]: https://github.com/mfisher90/stackable 83 | [Grape]: https://github.com/intridea/grape 84 | [Makers Academy]: http://www.makersacademy.com/ 85 | -------------------------------------------------------------------------------- /Rakefile: -------------------------------------------------------------------------------- 1 | require 'data_mapper' 2 | require 'open-uri' 3 | require 'grape-entity' 4 | require 'nokogiri' 5 | require './lib/benefit' 6 | require './lib/company' 7 | require './lib/job' 8 | require './lib/tag' 9 | require_relative 'data_mapper_setup' 10 | require_relative 'job-scraper' 11 | require_relative 'company-scraper' 12 | 13 | def parse_jobs(pages) 14 | scraping_round = Job.last ? Job.last.scraping_round + 1 : 1 15 | 16 | pages.compact.each_with_index do |jobs, index| 17 | puts "Parsing page #{index + 1}" 18 | jobs.each do |job_info| 19 | tag_names = job_info.delete(:tags) 20 | job_info[:scraping_round] = scraping_round 21 | # Job.raise_on_save_failure = true 22 | job = Job.create(job_info) 23 | raise job.errors.inspect if job.errors.any? 24 | if tag_names 25 | tag_names.each do |tag_name| 26 | job.tags << Tag.first_or_create(name: tag_name) 27 | job.save 28 | end 29 | end 30 | end 31 | end 32 | end 33 | 34 | def parse_companies(pages) 35 | scraping_round = Company.last ? (Company.last.scraping_round || 0) + 1 : 1 36 | 37 | @jobs_links = [] 38 | pages.each do |company_info| 39 | company_info[:scraping_round] = scraping_round 40 | tag_names = company_info.delete(:tags) 41 | benefits = company_info.delete(:benefits) 42 | jobs = company_info.delete(:jobs) 43 | 44 | company = Company.create(company_info) 45 | raise company.errors.inspect if company.errors.any? 46 | 47 | if tag_names 48 | tag_names.each do |tag_name| 49 | company.tags << Tag.first_or_create(name: tag_name) 50 | end 51 | end 52 | 53 | if benefits 54 | benefits.each do |benefit| 55 | company.benefits << Benefit.first_or_create(name: benefit) 56 | company.save 57 | end 58 | end 59 | 60 | if jobs 61 | jobs.each do |job_id| 62 | job = Job.first(job_id: job_id) 63 | if job 64 | puts 'saving job' 65 | job.company = company 66 | job.save 67 | else 68 | puts "WARN: job not in DB: #{job_id}" 69 | Job.create(job_id: job_id, title: "Job Missing", company: company) 70 | end 71 | end 72 | end 73 | end 74 | end 75 | 76 | 77 | desc "Upgrade database" 78 | task :auto_upgrade do 79 | DataMapper.auto_upgrade! 80 | puts "Auto-upgrade complete (no data loss)" 81 | end 82 | 83 | desc "Migrate database" 84 | task :auto_migrate do 85 | DataMapper.auto_migrate! 86 | puts "Auto-migrate complete" 87 | end 88 | 89 | desc "Get most recent jobs and save to database" 90 | task :scrape_jobs do 91 | last_job_id = Job.first(:order => [:scraping_round.desc]).job_id.to_s rescue nil 92 | pages = JobScraper.new('http://careers.stackoverflow.com/jobs', last_job_id).scrape 93 | parse_jobs(pages) 94 | end 95 | 96 | desc "Get most recent companies and save to database" 97 | task :scrape_companies do 98 | last_company_id = Company.first(:order => [:scraping_round.desc]).company_id.to_s rescue nil 99 | pages = CompanyScraper.new('http://careers.stackoverflow.com/jobs/companies', last_company_id).scrape 100 | parse_companies(pages) 101 | end 102 | 103 | desc "Scrape jobs not in db" 104 | task :jobs_not_in_db do 105 | job_ids = Job.all(:title => "Job Missing").map(&:job_id) 106 | jobs = CompanyScraper.scrape_jobs(job_ids) 107 | jobs.compact.each do |job_info| 108 | tag_names = job_info.delete(:tags) 109 | job = Job.first(job_id: job_info[:job_id]) 110 | job.update(job_info) 111 | raise job.errors.inspect if job.errors.any? 112 | 113 | if tag_names 114 | tag_names.each do |tag_name| 115 | job.tags << Tag.first_or_create(name: tag_name) 116 | job.save 117 | end 118 | end 119 | end 120 | end 121 | 122 | desc "Cleanup jobs not found" 123 | task :cleanup_jobs do 124 | Job.all(:title => "Job Missing").delete 125 | end 126 | 127 | desc "Refresh jobs database" 128 | task :refresh_jobs do 129 | JobTag.destroy 130 | Job.destroy 131 | pages = JobScraper.new('http://careers.stackoverflow.com/jobs').scrape 132 | parse_jobs(pages) 133 | end 134 | 135 | desc "Refresh companies database" 136 | task :refresh_companies do 137 | CompanyTag.destroy 138 | Company.destroy 139 | pages = CompanyScraper.new('http://careers.stackoverflow.com/jobs/companies').scrape 140 | parse_companies(pages) 141 | end 142 | 143 | 144 | 145 | 146 | 147 | 148 | 149 | 150 | 151 | 152 | 153 | 154 | 155 | 156 | 157 | 158 | 159 | -------------------------------------------------------------------------------- /app.rb: -------------------------------------------------------------------------------- 1 | require 'grape' 2 | require 'grape-entity' 3 | require 'data_mapper' 4 | require 'dm-ar-finders' 5 | require './lib/benefit' 6 | require './lib/company' 7 | require './lib/job' 8 | require './lib/tag' 9 | require 'grape-pagination' 10 | require 'will_paginate/array' 11 | require_relative 'data_mapper_setup' 12 | require_relative 'data_mapper_custom' 13 | 14 | def show_jobs(jobs) 15 | present jobs, with: Job::Entity 16 | end 17 | 18 | def show_companies(companies) 19 | present companies, with: Company::Entity 20 | end 21 | 22 | class StackAPI < Grape::API 23 | 24 | format :json 25 | default_format :json 26 | prefix 'api' 27 | 28 | 29 | desc "Returns a list of jobs." 30 | paginate 31 | get :jobs do 32 | jobs = paginate(Job.all) 33 | show_jobs(jobs) 34 | end 35 | 36 | desc "Return a single job." 37 | resource :jobs do 38 | params do 39 | requires :id, type: String, desc: "Job id." 40 | end 41 | route_param :id do 42 | get do 43 | show_jobs(Job.first(job_id: params[:id])) 44 | end 45 | end 46 | 47 | desc "Return a list of jobs by a tag from user query." 48 | resource :tags do 49 | params do 50 | requires :id, type: String, desc: "Tag." 51 | end 52 | route_param :id do 53 | get do 54 | tags = params[:id].split('&').map(&:downcase) 55 | show_jobs(Tag.all(name: tags).map(&:jobs).flatten) 56 | end 57 | end 58 | end 59 | desc "Returns a list of jobs by location" 60 | resource :location do 61 | params do 62 | requires :id, type: String, desc: "Location" 63 | end 64 | route_param :id do 65 | get do 66 | location = params[:id] 67 | show_jobs(Job.all(:location.ilike => "%#{location}%")) 68 | end 69 | end 70 | end 71 | end 72 | 73 | 74 | desc "Returns a list of companies." 75 | paginate 76 | get :companies do 77 | companies = paginate(Company.all) 78 | show_companies(companies) 79 | end 80 | 81 | desc "Returns a single company." 82 | resource :companies do 83 | params do 84 | requires :id, type: String, desc: "Company id." 85 | end 86 | route_param :id do 87 | get do 88 | show_companies(Company.first(company_id: params[:id])) 89 | end 90 | end 91 | desc "Returns a list of companies by a tag" 92 | resource :tags do 93 | params do 94 | requires :id, type: String, desc: "Tag." 95 | end 96 | route_param :id do 97 | get do 98 | tags = params[:id].split('&') 99 | show_companies(Tag.all(name: tags).map(&:companies).flatten) 100 | end 101 | end 102 | end 103 | 104 | desc "Returns a list of companies by benefits keywords" 105 | resource :benefits do 106 | params do 107 | requires :id, type: String, desc: "Benefits" 108 | end 109 | route_param :id do 110 | get do 111 | benefit_names = params[:id].split('&') 112 | benefits = benefit_names.map { |benefit| Benefit.first(:name.ilike => "%#{benefit}%") } 113 | show_companies(Company.all(benefits: benefits.flatten)) 114 | end 115 | end 116 | end 117 | end 118 | 119 | desc "Returns a list of jobs where relocation is offered." 120 | get :relocation do 121 | show_jobs(Job.find_by_sql("SELECT * FROM jobs WHERE location ILIKE '%(relocation offered)%'")) 122 | end 123 | 124 | desc "Returns a list of jobs where remote working is allowed." 125 | get :remote do 126 | show_jobs(Job.find_by_sql("SELECT * FROM jobs WHERE location ILIKE '%(allows remote)%'")) 127 | end 128 | 129 | desc "Returns a list of senior level jobs." 130 | get :senior do 131 | show_jobs(Job.find_by_sql("SELECT * FROM jobs WHERE title ILIKE '%senior%'")) 132 | end 133 | 134 | desc "Return a list of full stack jobs" 135 | get :full_stack do 136 | show_jobs(Job.find_by_sql("SELECT * FROM jobs WHERE title SIMILAR TO '%((F|f)ull(\s)(S|s)tack)%'")) 137 | end 138 | 139 | end 140 | 141 | -------------------------------------------------------------------------------- /company-scraper.rb: -------------------------------------------------------------------------------- 1 | require 'rubygems' 2 | require 'nokogiri' 3 | require 'open-uri' 4 | 5 | class CompanyScraper 6 | 7 | def initialize(url, last_company_id='nonsense') 8 | @page = Nokogiri::HTML(open(url)) 9 | @last_page = @page.css('div.pagination a.job-link')[-2].text.to_i 10 | @last_company_id = last_company_id 11 | end 12 | 13 | def build_urls 14 | # ["http://careers.stackoverflow.com/jobs/companies?pg=1"] 15 | urls = [] 16 | (1..@last_page).each {|n| urls << "http://careers.stackoverflow.com/jobs/companies?pg=#{n}"} 17 | urls 18 | end 19 | 20 | def company_urls 21 | urls = build_urls 22 | company_urls = [] 23 | urls.map do |url| 24 | page = Nokogiri::HTML(open(url)) 25 | sleep 0.5 26 | 27 | break if page.css('.list.companies a.title').first['href'].split('/').last == @last_company_id 28 | 29 | company_urls << page.css('.list.companies a.title').map {|x| "http://careers.stackoverflow.com/uk" + x['href']} 30 | company_urls.flatten! 31 | end 32 | company_urls 33 | end 34 | 35 | def posted_at 36 | Time.now.strftime("%Y-%m-%d %H:%M:%S") 37 | end 38 | 39 | def get_name 40 | @page.css('h1').text 41 | end 42 | 43 | def get_avatar 44 | "http:" + @page.css('div.logo-container img').first['src'] rescue nil 45 | end 46 | 47 | def get_size 48 | @page.css('table.basics tr').first.children[1].text rescue nil 49 | end 50 | 51 | def get_status 52 | @page.css('table.basics tr')[1].children[1].text rescue nil 53 | end 54 | 55 | def get_founded 56 | @page.css('table.basics tr')[2].children[1].text.to_i rescue nil 57 | end 58 | 59 | def get_url 60 | @page.css('a.cp-links-url').text rescue nil 61 | end 62 | 63 | def get_tags 64 | @page.css('div.tags span.post-tag').map(&:text) rescue [] 65 | end 66 | 67 | def get_benefits 68 | @page.css('div.benefits-list span.benefit').map(&:text) rescue [] 69 | end 70 | 71 | def get_open_jobs 72 | @page.css('div.job a').map {|link| link[:href][/\d+/]} rescue [] 73 | end 74 | 75 | def scrape 76 | @company_urls = company_urls 77 | 78 | index = 1 79 | @company_urls.map do |company_url| 80 | puts "Parsing page #{index}" 81 | index += 1 82 | sleep 0.5 83 | @page = Nokogiri::HTML(open(company_url)) 84 | 85 | { name: get_name, 86 | avatar: get_avatar, 87 | size: get_size, 88 | status: get_status, 89 | founded: get_founded, 90 | url: get_url, 91 | company_id: company_url.split('/').last, 92 | tags: get_tags, 93 | benefits: get_benefits, 94 | jobs: get_open_jobs, 95 | created_at: posted_at, 96 | stack_url: company_url 97 | } 98 | end 99 | end 100 | 101 | def self.get_meta_desc 102 | desc = @page.css("meta[name='Description']").first 103 | desc['content'] 104 | end 105 | 106 | def self.scrape_jobs(job_ids) 107 | index = 1 108 | job_ids.map do |job_id| 109 | url = 'http://careers.stackoverflow.com/jobs/' + job_id 110 | puts "Parsing page #{index}" 111 | index += 1 112 | sleep 0.6 113 | puts "Current job is #{url}" 114 | begin 115 | open(url) do |resp| 116 | url = resp.base_uri.to_s 117 | 118 | @page = Nokogiri::HTML(resp.read) 119 | { 120 | job_id: job_id, 121 | title: @page.css('h1 .title').text, 122 | description: get_meta_desc, 123 | url: url, 124 | jscore: (@page.css("h3").text.match(/(\d+) out of/)[1].to_i rescue nil), 125 | location: @page.css('span.location').text, 126 | company_name: @page.css('a.employer').text, 127 | tags: [@page.css('a.post-tag.job-link').map(&:text)].flatten, 128 | created_at: Time.now.strftime("%Y-%m-%d %H:%M:%S") 129 | } 130 | end 131 | rescue OpenURI::HTTPError => ex 132 | next 133 | end 134 | end 135 | end 136 | 137 | end 138 | 139 | 140 | -------------------------------------------------------------------------------- /config.ru: -------------------------------------------------------------------------------- 1 | require './app' 2 | run StackAPI -------------------------------------------------------------------------------- /config/newrelic.yml: -------------------------------------------------------------------------------- 1 | # 2 | # This file configures the New Relic Agent. New Relic monitors 3 | # Ruby, Java, .NET, PHP, and Python applications with deep visibility and low overhead. 4 | # For more information, visit www.newrelic.com. 5 | # 6 | # Generated February 16, 2014 7 | # 8 | # This configuration file is custom generated for Makers Academy_1 9 | 10 | 11 | # Here are the settings that are common to all environments 12 | common: &default_settings 13 | # ============================== LICENSE KEY =============================== 14 | 15 | # You must specify the license key associated with your New Relic 16 | # account. This key binds your Agent's data to your account in the 17 | # New Relic service. 18 | license_key: ENV['NEW_RELIC_TOKEN'] 19 | 20 | # Agent Enabled (Ruby/Rails Only) 21 | # Use this setting to force the agent to run or not run. 22 | # Default is 'auto' which means the agent will install and run only 23 | # if a valid dispatcher such as Mongrel is running. This prevents 24 | # it from running with Rake or the console. Set to false to 25 | # completely turn the agent off regardless of the other settings. 26 | # Valid values are true, false and auto. 27 | # 28 | # agent_enabled: auto 29 | 30 | # Application Name Set this to be the name of your application as 31 | # you'd like it show up in New Relic. The service will then auto-map 32 | # instances of your application into an "application" on your 33 | # dashboard page. If you want to map this instance into multiple 34 | # apps, like "AJAX Requests" and "All UI" then specify a semicolon 35 | # separated list of up to three distinct names, or a yaml list. 36 | # Defaults to the capitalized RAILS_ENV or RACK_ENV (i.e., 37 | # Production, Staging, etc) 38 | # 39 | # Example: 40 | # 41 | # app_name: 42 | # - Ajax Service 43 | # - All Services 44 | # 45 | app_name: Stackable 46 | 47 | # When "true", the agent collects performance data about your 48 | # application and reports this data to the New Relic service at 49 | # newrelic.com. This global switch is normally overridden for each 50 | # environment below. (formerly called 'enabled') 51 | monitor_mode: true 52 | 53 | # Developer mode should be off in every environment but 54 | # development as it has very high overhead in memory. 55 | developer_mode: false 56 | 57 | # The newrelic agent generates its own log file to keep its logging 58 | # information separate from that of your application. Specify its 59 | # log level here. 60 | log_level: info 61 | 62 | # Optionally set the path to the log file This is expanded from the 63 | # root directory (may be relative or absolute, e.g. 'log/' or 64 | # '/var/log/') The agent will attempt to create this directory if it 65 | # does not exist. 66 | # log_file_path: 'log' 67 | 68 | # Optionally set the name of the log file, defaults to 'newrelic_agent.log' 69 | # log_file_name: 'newrelic_agent.log' 70 | 71 | # The newrelic agent communicates with the service via https by default. This 72 | # prevents eavesdropping on the performance metrics transmitted by the agent. 73 | # The encryption required by SSL introduces a nominal amount of CPU overhead, 74 | # which is performed asynchronously in a background thread. If you'd prefer 75 | # to send your metrics over http uncomment the following line. 76 | # ssl: false 77 | 78 | #============================== Browser Monitoring =============================== 79 | # New Relic Real User Monitoring gives you insight into the performance real users are 80 | # experiencing with your website. This is accomplished by measuring the time it takes for 81 | # your users' browsers to download and render your web pages by injecting a small amount 82 | # of JavaScript code into the header and footer of each page. 83 | browser_monitoring: 84 | # By default the agent automatically injects the monitoring JavaScript 85 | # into web pages. Set this attribute to false to turn off this behavior. 86 | auto_instrument: true 87 | 88 | # Proxy settings for connecting to the New Relic server. 89 | # 90 | # If a proxy is used, the host setting is required. Other settings 91 | # are optional. Default port is 8080. 92 | # 93 | # proxy_host: hostname 94 | # proxy_port: 8080 95 | # proxy_user: 96 | # proxy_pass: 97 | 98 | # The agent can optionally log all data it sends to New Relic servers to a 99 | # separate log file for human inspection and auditing purposes. To enable this 100 | # feature, change 'enabled' below to true. 101 | # See: https://newrelic.com/docs/ruby/audit-log 102 | audit_log: 103 | enabled: false 104 | 105 | # Tells transaction tracer and error collector (when enabled) 106 | # whether or not to capture HTTP params. When true, frameworks can 107 | # exclude HTTP parameters from being captured. 108 | # Rails: the RoR filter_parameter_logging excludes parameters 109 | # Java: create a config setting called "ignored_params" and set it to 110 | # a comma separated list of HTTP parameter names. 111 | # ex: ignored_params: credit_card, ssn, password 112 | capture_params: false 113 | 114 | # Transaction tracer captures deep information about slow 115 | # transactions and sends this to the New Relic service once a 116 | # minute. Included in the transaction is the exact call sequence of 117 | # the transactions including any SQL statements issued. 118 | transaction_tracer: 119 | 120 | # Transaction tracer is enabled by default. Set this to false to 121 | # turn it off. This feature is only available at the Professional 122 | # and above product levels. 123 | enabled: true 124 | 125 | # Threshold in seconds for when to collect a transaction 126 | # trace. When the response time of a controller action exceeds 127 | # this threshold, a transaction trace will be recorded and sent to 128 | # New Relic. Valid values are any float value, or (default) "apdex_f", 129 | # which will use the threshold for an dissatisfying Apdex 130 | # controller action - four times the Apdex T value. 131 | transaction_threshold: apdex_f 132 | 133 | # When transaction tracer is on, SQL statements can optionally be 134 | # recorded. The recorder has three modes, "off" which sends no 135 | # SQL, "raw" which sends the SQL statement in its original form, 136 | # and "obfuscated", which strips out numeric and string literals. 137 | record_sql: obfuscated 138 | 139 | # Threshold in seconds for when to collect stack trace for a SQL 140 | # call. In other words, when SQL statements exceed this threshold, 141 | # then capture and send to New Relic the current stack trace. This is 142 | # helpful for pinpointing where long SQL calls originate from. 143 | stack_trace_threshold: 0.500 144 | 145 | # Determines whether the agent will capture query plans for slow 146 | # SQL queries. Only supported in mysql and postgres. Should be 147 | # set to false when using other adapters. 148 | # explain_enabled: true 149 | 150 | # Threshold for query execution time below which query plans will 151 | # not be captured. Relevant only when `explain_enabled` is true. 152 | # explain_threshold: 0.5 153 | 154 | # Error collector captures information about uncaught exceptions and 155 | # sends them to New Relic for viewing 156 | error_collector: 157 | 158 | # Error collector is enabled by default. Set this to false to turn 159 | # it off. This feature is only available at the Professional and above 160 | # product levels. 161 | enabled: true 162 | 163 | # Rails Only - tells error collector whether or not to capture a 164 | # source snippet around the place of the error when errors are View 165 | # related. 166 | capture_source: true 167 | 168 | # To stop specific errors from reporting to New Relic, set this property 169 | # to comma-separated values. Default is to ignore routing errors, 170 | # which are how 404's get triggered. 171 | ignore_errors: "ActionController::RoutingError,Sinatra::NotFound" 172 | 173 | # If you're interested in capturing memcache keys as though they 174 | # were SQL uncomment this flag. Note that this does increase 175 | # overhead slightly on every memcached call, and can have security 176 | # implications if your memcached keys are sensitive 177 | # capture_memcache_keys: true 178 | 179 | # Application Environments 180 | # ------------------------------------------ 181 | # Environment-specific settings are in this section. 182 | # For Rails applications, RAILS_ENV is used to determine the environment. 183 | # For Java applications, pass -Dnewrelic.environment to set 184 | # the environment. 185 | 186 | # NOTE if your application has other named environments, you should 187 | # provide newrelic configuration settings for these environments here. 188 | 189 | development: 190 | <<: *default_settings 191 | # Turn off communication to New Relic service in development mode (also 192 | # 'enabled'). 193 | # NOTE: for initial evaluation purposes, you may want to temporarily 194 | # turn the agent on in development mode. 195 | monitor_mode: false 196 | 197 | # Rails Only - when running in Developer Mode, the New Relic Agent will 198 | # present performance information on the last 100 transactions you have 199 | # executed since starting the mongrel. 200 | # NOTE: There is substantial overhead when running in developer mode. 201 | # Do not use for production or load testing. 202 | developer_mode: true 203 | 204 | # Enable textmate links 205 | # textmate: true 206 | 207 | test: 208 | <<: *default_settings 209 | # It almost never makes sense to turn on the agent when running 210 | # unit, functional or integration tests or the like. 211 | monitor_mode: false 212 | 213 | # Turn on the agent in production for 24x7 monitoring. NewRelic 214 | # testing shows an average performance impact of < 5 ms per 215 | # transaction, you can leave this on all the time without 216 | # incurring any user-visible performance degradation. 217 | production: 218 | <<: *default_settings 219 | monitor_mode: true 220 | 221 | # Many applications have a staging environment which behaves 222 | # identically to production. Support for that environment is provided 223 | # here. By default, the staging environment has the agent turned on. 224 | staging: 225 | <<: *default_settings 226 | monitor_mode: true 227 | # app_name: My Application (Staging) 228 | -------------------------------------------------------------------------------- /data_mapper_custom.rb: -------------------------------------------------------------------------------- 1 | require 'dm-core' 2 | require 'dm-do-adapter' 3 | 4 | # Credits to kabakiyo for the ILIKE Comparator. 5 | 6 | module DataMapper 7 | class Query 8 | module Conditions 9 | class SimilarToComparison < AbstractComparison 10 | slug :similar 11 | 12 | private 13 | 14 | # Overloads the +expected+ method in AbstractComparison 15 | # 16 | # Expects the input value to be a valid Regular expression. 17 | # 18 | # @return [Regexp] 19 | # 20 | # @see AbtractComparison#expected 21 | # 22 | # @api semipublic 23 | def expected 24 | loaded_value.kind_of?(Regexp) 25 | end 26 | 27 | # @return [String] 28 | # 29 | # @see AbstractComparison#to_s 30 | # 31 | # @api private 32 | def comparator_string 33 | 'SIMILAR TO' 34 | end 35 | end # class LikeComparison 36 | class ILikeComparison < AbstractComparison 37 | slug :ilike 38 | 39 | private 40 | 41 | # Overloads the +expected+ method in AbstractComparison 42 | # 43 | # Return a regular expression suitable for matching against the 44 | # records value. 45 | # 46 | # @return [Regexp] 47 | # 48 | # @see AbtractComparison#expected 49 | # 50 | # @api semipublic 51 | def expected 52 | Regexp.new('\A' << super.gsub('%', '.*').tr('_', '.') << '\z') 53 | end 54 | 55 | # @return [String] 56 | # 57 | # @see AbstractComparison#to_s 58 | # 59 | # @api private 60 | def comparator_string 61 | 'ILIKE' 62 | end 63 | end # class LikeComparison 64 | end 65 | end 66 | 67 | module Adapters 68 | class DataObjectsAdapter < AbstractAdapter 69 | module SQL #:nodoc: 70 | 71 | alias :old_comparison_operator :comparison_operator 72 | 73 | def comparison_operator(comparison) 74 | case comparison.slug 75 | when :similar then 'SIMILAR TO' 76 | when :ilike then 'ILIKE' 77 | else old_comparison_operator(comparison) 78 | end 79 | end 80 | end 81 | end 82 | end 83 | end 84 | 85 | class Symbol 86 | def ilike 87 | DataMapper::Query::Operator.new(self, :ilike) 88 | end 89 | 90 | def similar 91 | DataMapper::Query::Operator.new(self, :similar) 92 | end 93 | end # class Symbol -------------------------------------------------------------------------------- /data_mapper_setup.rb: -------------------------------------------------------------------------------- 1 | env = ENV["RACK_ENV"] || "development" 2 | 3 | DataMapper.setup(:default, ENV["DATABASE_URL"] || "postgres://localhost/stackoverflow-api-#{env}") 4 | 5 | DataMapper.finalize 6 | -------------------------------------------------------------------------------- /job-scraper.rb: -------------------------------------------------------------------------------- 1 | require 'rubygems' 2 | require 'nokogiri' 3 | require 'open-uri' 4 | require 'active_support/core_ext/string' 5 | 6 | class JobScraper 7 | 8 | def initialize(url, last_job_id='nonsense') 9 | @last_job_id = last_job_id 10 | @page = Nokogiri::HTML(open(url)) 11 | @last_page = @page.css('div.pagination a.job-link')[-2].text.to_i 12 | end 13 | 14 | def build_urls 15 | # ["http://careers.stackoverflow.com/jobs?pg=1"] 16 | urls = [] 17 | (1..@last_page).each {|n| urls << "http://careers.stackoverflow.com/jobs?pg=#{n}"} 18 | urls 19 | end 20 | 21 | def posted_at 22 | Time.now.strftime("%Y-%m-%d %H:%M:%S") 23 | end 24 | 25 | def get_title(row) 26 | row.css('a.job-link').first.text 27 | end 28 | 29 | def get_desc(row) 30 | row.css('p.description').text 31 | end 32 | 33 | def get_url(row) 34 | "http://careers.stackoverflow.com" + row.css('a.job-link').first['href'].split(/\?/)[0] 35 | end 36 | 37 | def get_jscore(row) 38 | row.css('.joeltestscore').text.to_i 39 | end 40 | 41 | def get_name(row) 42 | row.css('p.location span.employer').text.squish 43 | end 44 | 45 | def get_location(row) 46 | row.css('p.location').text.split(' - ')[1].squish 47 | end 48 | 49 | def get_tags(row) 50 | [row.css('a.post-tag.job-link').map(&:text)].flatten 51 | end 52 | 53 | def scrape 54 | puts "Scraping..." 55 | @urls = build_urls 56 | index = 0 57 | job_already_scraped = false 58 | 59 | @urls.map do |url| 60 | unless job_already_scraped 61 | puts "Scraping page #{index + 1}" 62 | index += 1 63 | 64 | @page = Nokogiri::HTML(open(url)) 65 | @rows = @page.css('.list.jobs div[data-jobid]') 66 | job_info = [] 67 | sleep 0.6 68 | @rows.each do |row| 69 | job_already_scraped = (row['data-jobid'] == @last_job_id) 70 | break if job_already_scraped 71 | 72 | job_info << { job_id: row['data-jobid'], 73 | title: get_title(row), 74 | description: get_desc(row), 75 | url: get_url(row), 76 | jscore: get_jscore(row), 77 | company_name: get_name(row), 78 | location: get_location(row), 79 | tags: get_tags(row), 80 | created_at: posted_at 81 | } 82 | end 83 | job_info 84 | end 85 | end 86 | end 87 | 88 | end -------------------------------------------------------------------------------- /lib/benefit.rb: -------------------------------------------------------------------------------- 1 | class Benefit 2 | 3 | include DataMapper::Resource 4 | 5 | property :id, Serial 6 | property :name, Text 7 | 8 | belongs_to :company, required: true 9 | 10 | end -------------------------------------------------------------------------------- /lib/company.rb: -------------------------------------------------------------------------------- 1 | class Company 2 | 3 | include DataMapper::Resource 4 | include Grape::Entity::DSL 5 | 6 | entity do 7 | expose :id 8 | expose :name 9 | expose :avatar 10 | expose :size 11 | expose :status 12 | expose :founded 13 | expose :url 14 | expose :company_id 15 | expose :stack_url 16 | end 17 | 18 | has n, :tags, :through => Resource 19 | has n, :benefits 20 | has n, :jobs 21 | 22 | property :id, Serial, :required => true 23 | property :name, Text, :required => true 24 | property :avatar, Text 25 | property :size, String 26 | property :status, Text 27 | property :founded, Integer 28 | property :url, Text 29 | property :company_id, Text 30 | property :created_at, Time 31 | property :scraping_round, Integer 32 | property :stack_url, Text 33 | 34 | 35 | def tag_names 36 | tags.map(&:name) 37 | end 38 | end 39 | 40 | -------------------------------------------------------------------------------- /lib/job.rb: -------------------------------------------------------------------------------- 1 | class Job 2 | 3 | include DataMapper::Resource 4 | include Grape::Entity::DSL 5 | 6 | entity do 7 | expose :id 8 | expose :job_id 9 | expose :title 10 | expose :description 11 | expose :url 12 | expose :jscore 13 | expose :location 14 | expose :company_name 15 | expose :tag_names 16 | end 17 | 18 | has n, :tags, :through => Resource 19 | belongs_to :company, required: false 20 | 21 | property :id, Serial 22 | property :job_id, String, :required => true 23 | property :title, Text, :required => true 24 | property :description, Text 25 | property :url, Text 26 | property :jscore, Integer 27 | property :location, Text 28 | property :company_name, Text 29 | property :scraping_round, Integer 30 | property :created_at, Time 31 | 32 | def tag_names 33 | tags.map(&:name) 34 | end 35 | 36 | end -------------------------------------------------------------------------------- /lib/tag.rb: -------------------------------------------------------------------------------- 1 | class Tag 2 | 3 | include DataMapper::Resource 4 | 5 | has n, :jobs, :through => Resource 6 | has n, :companies, :through => Resource 7 | 8 | property :id, Serial 9 | property :name, Text 10 | 11 | end --------------------------------------------------------------------------------