├── .gitignore ├── COPYING ├── Gemfile ├── Gemfile.lock ├── README.md ├── README.rdoc ├── Rakefile ├── app ├── assets │ ├── images │ │ └── rails.png │ ├── javascripts │ │ ├── application.js │ │ ├── data_sources.js.coffee │ │ ├── home.js.coffee │ │ ├── oauth.js.coffee │ │ ├── profiles.js.coffee │ │ ├── status_updates.js.coffee │ │ └── unknown_subjects.js.coffee │ └── stylesheets │ │ ├── application.css │ │ ├── data_sources.css.scss │ │ ├── home.css.scss │ │ ├── oauth.css.scss │ │ ├── profiles.css.scss │ │ ├── status_updates.css.scss │ │ └── unknown_subjects.css.scss ├── controllers │ ├── application_controller.rb │ ├── data_sources_controller.rb │ ├── home_controller.rb │ ├── oauth_controller.rb │ ├── profiles_controller.rb │ ├── status_updates_controller.rb │ └── unknown_subjects_controller.rb ├── helpers │ ├── application_helper.rb │ ├── data_sources_helper.rb │ ├── home_helper.rb │ ├── oauth_helper.rb │ ├── profiles_helper.rb │ ├── status_updates_helper.rb │ └── unknown_subjects_helper.rb ├── mailers │ └── .gitkeep ├── models │ ├── .gitkeep │ ├── data_source.rb │ ├── facebook_data_source.rb │ ├── profile.rb │ ├── status_update.rb │ ├── twitter_data_source.rb │ ├── unknown_subject.rb │ └── user.rb └── views │ ├── data_sources │ ├── _data_source_form.html.erb │ ├── edit.html.erb │ ├── new.html.erb │ └── show.html.erb │ ├── home │ ├── .DS_Store │ └── index.html.erb │ ├── kaminari │ ├── _first_page.html.erb │ ├── _gap.html.erb │ ├── _last_page.html.erb │ ├── _next_page.html.erb │ ├── _page.html.erb │ ├── _paginator.html.erb │ └── _prev_page.html.erb │ ├── layouts │ └── application.html.erb │ ├── profiles │ ├── _profile_criteria.html.erb │ ├── input.html.erb │ ├── new.html.erb │ ├── nlp_tree.html.erb │ ├── search.html.erb │ └── show.html.erb │ ├── status_updates │ └── show.html.erb │ └── unknown_subjects │ ├── _data_source.html.erb │ ├── _profile.html.erb │ ├── _unknown_subject.html.erb │ ├── _unknown_subject_form.html.erb │ ├── edit.html.erb │ ├── export_profile.kml.builder │ ├── index.html.erb │ ├── new.html.erb │ └── show.html.erb ├── config.ru ├── config ├── application.rb ├── boot.rb ├── environment.rb ├── environments │ ├── development.rb │ ├── production.rb │ └── test.rb ├── initializers │ ├── backtrace_silencers.rb │ ├── inflections.rb │ ├── kaminari_config.rb │ ├── mime_types.rb │ ├── secret_token.rb │ ├── session_store.rb │ └── wrap_parameters.rb ├── locales │ └── en.yml ├── mongoid.yml ├── oauth.yml └── routes.rb ├── db └── seeds.rb ├── doc └── README_FOR_APP ├── lib ├── assets │ └── .gitkeep ├── tasks │ └── .gitkeep └── twitter.rb ├── log └── .gitkeep ├── public ├── 404.html ├── 422.html ├── 500.html ├── bootstrap │ ├── css │ │ ├── bootstrap-responsive.css │ │ ├── bootstrap-responsive.min.css │ │ ├── bootstrap.css │ │ └── bootstrap.min.css │ ├── img │ │ ├── glyphicons-halflings-white.png │ │ └── glyphicons-halflings.png │ └── js │ │ ├── bootstrap.js │ │ └── bootstrap.min.js ├── default_profile_image.png ├── favicon.ico └── robots.txt ├── script └── rails ├── test ├── .DS_Store ├── fixtures │ ├── .gitkeep │ ├── data_sources.yml │ ├── profiles.yml │ ├── status_updates.yml │ ├── unknown_subjects.yml │ └── users.yml ├── functional │ ├── .gitkeep │ ├── data_sources_controller_test.rb │ ├── home_controller_test.rb │ ├── oauth_controller_test.rb │ ├── profiles_controller_test.rb │ ├── status_updates_controller_test.rb │ └── unknown_subjects_controller_test.rb ├── integration │ └── .gitkeep ├── performance │ └── browsing_test.rb ├── test_helper.rb └── unit │ ├── .gitkeep │ ├── data_source_test.rb │ ├── helpers │ ├── data_sources_helper_test.rb │ ├── home_helper_test.rb │ ├── oauth_helper_test.rb │ ├── profiles_helper_test.rb │ ├── status_updates_helper_test.rb │ └── unknown_subjects_helper_test.rb │ ├── profile_test.rb │ ├── status_update_test.rb │ ├── unknown_subject_test.rb │ └── user_test.rb └── vendor ├── assets ├── javascripts │ └── .gitkeep └── stylesheets │ └── .gitkeep └── plugins └── .gitkeep /.gitignore: -------------------------------------------------------------------------------- 1 | # See http://help.github.com/ignore-files/ for more about ignoring files. 2 | # 3 | # If you find yourself ignoring temporary files generated by your text editor 4 | # or operating system, you probably want to add a global ignore instead: 5 | # git config --global core.excludesfile ~/.gitignore_global 6 | 7 | # Ignore bundler config 8 | /.bundle 9 | 10 | # Ignore the default SQLite database. 11 | /db/*.sqlite3 12 | 13 | # Ignore all logfiles and tempfiles. 14 | /log/*.log 15 | /tmp 16 | -------------------------------------------------------------------------------- /Gemfile: -------------------------------------------------------------------------------- 1 | source 'https://rubygems.org' 2 | 3 | gem 'rails', '3.2.13' 4 | 5 | # Bundle edge Rails instead: 6 | # gem 'rails', :git => 'git://github.com/rails/rails.git' 7 | 8 | # Use Mongoid instead of ActiveRecord for ORM 9 | gem 'mongoid', '~> 3.1.0' 10 | 11 | # Use OAuth 12 | gem 'oauth' 13 | 14 | # Use DelayedJob for background job scheduling 15 | gem 'daemons' 16 | gem 'delayed_job_mongoid' 17 | 18 | # Use Kaminari for pagination 19 | gem 'kaminari' 20 | 21 | # Enforce rubyzip version to guarantee 22 | # compatibility with treat 23 | gem 'rubyzip', '~> 0.9.9' 24 | 25 | # Use treat for NLP 26 | gem 'engtagger' 27 | gem 'scalpel' 28 | gem 'stanford-core-nlp' 29 | gem 'treat' 30 | gem 'nokogiri' 31 | 32 | # Use lazy-high-charts for graphing/charts 33 | gem 'lazy_high_charts' 34 | 35 | # Gems used only for assets and not required 36 | # in production environments by default. 37 | group :assets do 38 | gem 'sass-rails', '~> 3.2.3' 39 | gem 'coffee-rails', '~> 3.2.1' 40 | 41 | # See https://github.com/sstephenson/execjs#readme for more supported runtimes 42 | # gem 'therubyracer', :platforms => :ruby 43 | 44 | gem 'uglifier', '>= 1.0.3' 45 | end 46 | 47 | gem 'jquery-rails' 48 | 49 | # To use ActiveModel has_secure_password 50 | # gem 'bcrypt-ruby', '~> 3.0.0' 51 | 52 | # To use Jbuilder templates for JSON 53 | # gem 'jbuilder' 54 | 55 | # Use unicorn as the app server 56 | # gem 'unicorn' 57 | 58 | # Deploy with Capistrano 59 | # gem 'capistrano' 60 | 61 | # To use debugger 62 | # gem 'debugger' 63 | -------------------------------------------------------------------------------- /Gemfile.lock: -------------------------------------------------------------------------------- 1 | GEM 2 | remote: https://rubygems.org/ 3 | specs: 4 | actionmailer (3.2.13) 5 | actionpack (= 3.2.13) 6 | mail (~> 2.5.3) 7 | actionpack (3.2.13) 8 | activemodel (= 3.2.13) 9 | activesupport (= 3.2.13) 10 | builder (~> 3.0.0) 11 | erubis (~> 2.7.0) 12 | journey (~> 1.0.4) 13 | rack (~> 1.4.5) 14 | rack-cache (~> 1.2) 15 | rack-test (~> 0.6.1) 16 | sprockets (~> 2.2.1) 17 | activemodel (3.2.13) 18 | activesupport (= 3.2.13) 19 | builder (~> 3.0.0) 20 | activerecord (3.2.13) 21 | activemodel (= 3.2.13) 22 | activesupport (= 3.2.13) 23 | arel (~> 3.0.2) 24 | tzinfo (~> 0.3.29) 25 | activeresource (3.2.13) 26 | activemodel (= 3.2.13) 27 | activesupport (= 3.2.13) 28 | activesupport (3.2.13) 29 | i18n (= 0.6.1) 30 | multi_json (~> 1.0) 31 | arel (3.0.3) 32 | bind-it (0.2.7) 33 | rjb (~> 1.4.3) 34 | birch (0.0.8) 35 | builder (3.0.4) 36 | coffee-rails (3.2.2) 37 | coffee-script (>= 2.2.0) 38 | railties (~> 3.2.0) 39 | coffee-script (2.2.0) 40 | coffee-script-source 41 | execjs 42 | coffee-script-source (1.6.3) 43 | daemons (1.1.9) 44 | delayed_job (3.0.5) 45 | activesupport (~> 3.0) 46 | delayed_job_mongoid (2.0.0) 47 | delayed_job (~> 3.0) 48 | mongoid (~> 3.0) 49 | engtagger (0.1.2) 50 | erubis (2.7.0) 51 | execjs (2.0.2) 52 | hash-deep-merge (0.1.1) 53 | hike (1.2.3) 54 | i18n (0.6.1) 55 | journey (1.0.4) 56 | jquery-rails (3.0.4) 57 | railties (>= 3.0, < 5.0) 58 | thor (>= 0.14, < 2.0) 59 | json (1.8.1) 60 | kaminari (0.15.0) 61 | actionpack (>= 3.0.0) 62 | activesupport (>= 3.0.0) 63 | lazy_high_charts (1.5.0) 64 | bundler (>= 1.0) 65 | hash-deep-merge 66 | mail (2.5.4) 67 | mime-types (~> 1.16) 68 | treetop (~> 1.4.8) 69 | mime-types (1.25.1) 70 | mini_portile (0.5.2) 71 | mongoid (3.1.5) 72 | activemodel (~> 3.2) 73 | moped (~> 1.4) 74 | origin (~> 1.0) 75 | tzinfo (~> 0.3.29) 76 | moped (1.5.1) 77 | multi_json (1.8.2) 78 | nokogiri (1.6.0) 79 | mini_portile (~> 0.5.0) 80 | oauth (0.4.7) 81 | origin (1.1.0) 82 | polyglot (0.3.3) 83 | progressbar (0.21.0) 84 | rack (1.4.5) 85 | rack-cache (1.2) 86 | rack (>= 0.4) 87 | rack-ssl (1.3.3) 88 | rack 89 | rack-test (0.6.2) 90 | rack (>= 1.0) 91 | rails (3.2.13) 92 | actionmailer (= 3.2.13) 93 | actionpack (= 3.2.13) 94 | activerecord (= 3.2.13) 95 | activeresource (= 3.2.13) 96 | activesupport (= 3.2.13) 97 | bundler (~> 1.0) 98 | railties (= 3.2.13) 99 | railties (3.2.13) 100 | actionpack (= 3.2.13) 101 | activesupport (= 3.2.13) 102 | rack-ssl (~> 1.3.2) 103 | rake (>= 0.8.7) 104 | rdoc (~> 3.4) 105 | thor (>= 0.14.6, < 2.0) 106 | rake (10.1.0) 107 | rdoc (3.12.2) 108 | json (~> 1.4) 109 | rjb (1.4.8) 110 | rubyzip (0.9.9) 111 | sass (3.2.12) 112 | sass-rails (3.2.6) 113 | railties (~> 3.2.0) 114 | sass (>= 3.1.10) 115 | tilt (~> 1.3) 116 | scalpel (0.2.1) 117 | schiphol (0.9.5) 118 | progressbar (>= 0.10.0) 119 | rubyzip (~> 0.9.9) 120 | sprockets (2.2.2) 121 | hike (~> 1.2) 122 | multi_json (~> 1.0) 123 | rack (~> 1.0) 124 | tilt (~> 1.1, != 1.3.0) 125 | stanford-core-nlp (0.5.1) 126 | bind-it (~> 0.2.5) 127 | thor (0.18.1) 128 | tilt (1.4.1) 129 | treat (2.0.7) 130 | birch 131 | schiphol 132 | yomu 133 | treetop (1.4.15) 134 | polyglot 135 | polyglot (>= 0.3.1) 136 | tzinfo (0.3.38) 137 | uglifier (2.3.1) 138 | execjs (>= 0.3.0) 139 | json (>= 1.8.0) 140 | yomu (0.1.9) 141 | mime-types (~> 1.23) 142 | 143 | PLATFORMS 144 | ruby 145 | 146 | DEPENDENCIES 147 | coffee-rails (~> 3.2.1) 148 | daemons 149 | delayed_job_mongoid 150 | engtagger 151 | jquery-rails 152 | kaminari 153 | lazy_high_charts 154 | mongoid (~> 3.1.0) 155 | nokogiri 156 | oauth 157 | rails (= 3.2.13) 158 | rubyzip (~> 0.9.9) 159 | sass-rails (~> 3.2.3) 160 | scalpel 161 | stanford-core-nlp 162 | treat 163 | uglifier (>= 1.0.3) 164 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # µphisher 2 | 3 | :warning: *NOTE: This tool is no longer under active maintenance.* 4 | 5 | ## Introduction 6 | 7 | µphisher is a tool designed to support social engineering activities. It 8 | automates the collecting and analysis of public social network data (the 9 | current version only handles [Twitter](https://www.twitter.com "Twitter")), and 10 | provides an interactive assisted text input interface which provides users 11 | real time metrics on word usage, hashtag and user references, as well as word 12 | suggestion based on usage frequency on the social network source. 13 | 14 | µphisher can also be used to validate suspect input against a known user 15 | profile using the same metrics. 16 | 17 | ## Installation 18 | 19 | µphisher was developed using Ruby 1.9.3 and 2.0.0, both installed 20 | through [RVM](https://rvm.io/ "Ruby Version Manager"), under OSX 10.6 and 21 | 10.7, as well as Ubuntu 13.04 and 13.10. Getting the correct Ruby version 22 | running on your particular environment is beyond the scope of this document, 23 | but the authors strongly advise the use of RVM installed as a regular user, 24 | as it provides the easiest way to handle external dependencies and gems 25 | without requiring administrative privileges on the system. 26 | 27 | ### rubygems 28 | 29 | µphisher is a Ruby on Rails application, and the installation of most 30 | dependencies is handled by the `bundler` gem. but the application also relies 31 | on `treat`, which requires a separate installation step to fetch additional 32 | resources. 33 | 34 | ``` 35 | $ git clone https://github.com/urma/microphisher 36 | $ gem install bundler 37 | $ bundle update && bundle install 38 | $ irb 39 | irb> require 'treat' 40 | irb> Treat::Core::Installer.install 'english' 41 | ``` 42 | 43 | Some users have reported issues while installing `rjb`, a gem used to allow 44 | communication between `treat` and the Java code in 45 | [OpenNLP](https://opennlp.apache.org/ "OpenNLP"). `rjb` requires a Java 6 JDK 46 | or later, and the `JAVA_HOME` variable must be set appropriately *before* 47 | running bundler. The `JAVA_HOME` must point to the root of the JDK installation 48 | directory, under which a `include` directory should exist. 49 | 50 | ### External Dependencies 51 | 52 | µphisher uses [MongoDB](http://www.mongodb.org/ "MongoDB") for persistence, 53 | which must be properly installed and configured before running the application 54 | for the first time. There are no specific configuration requirements for it, 55 | so standard distribution packages (for Linux systems), 56 | [Macports](https://www.macports.org/ "Macports") or 57 | [Homebrew](http://brew.sh/ "Homebrew") default installations should work out of 58 | the box. 59 | 60 | ## Configuration 61 | 62 | ### Database 63 | 64 | Database configuration resides in `config/mongoid.yml`, which is documented in 65 | the [Mongoid site] 66 | (http://mongoid.org/en/mongoid/docs/installation.html#configuration 67 | "Mongoid documentation"). Unless the user tweaked the MongoDB configuration, 68 | the default configuration file should work without any changes. 69 | 70 | Since we are not using a relational database, there is no need to execute 71 | a separate step to create tables and columns -- Mongoid will create those 72 | as needed. 73 | 74 | ### OAuth 75 | 76 | The OAuth configuration file, `config/oauth.yml`, requires the user to 77 | register a new Twitter application at https://twitter.com/oauth_clients/new. 78 | The parameter values will be provided by Twitter after the application is 79 | registered. Please notice that the application *must have* a callback 80 | URL configured (although its values is not used by µphisher) -- this 81 | is a restriction of the Twitter OAuth implementation when used as an 82 | authentication service. 83 | 84 | The `config/oauth.yml` also provides separate entries for development, 85 | test and production environments. This allows the user to use separate 86 | OAuth tokens for each scenario, as well as provide stub/mock REST API 87 | implementations. 88 | 89 | ### delayed_job 90 | 91 | [delayed_job](https://github.com/collectiveidea/delayed_job 92 | "delayed_job") is a batch/background job scheduler used by 93 | µphisher to handle long-running activities, such as the NLP 94 | parsing and fetching of status updates. Since it runs asynchronously 95 | from the main web interface of the tool, it requires a separate 96 | Ruby instance. This can be started by executing (copied directly 97 | from the `delayed_job` documentation): 98 | 99 | ``` 100 | RAILS_ENV=production script/delayed_job start 101 | RAILS_ENV=production script/delayed_job stop 102 | 103 | # Runs two workers in separate processes. 104 | RAILS_ENV=production script/delayed_job -n 2 start 105 | RAILS_ENV=production script/delayed_job stop 106 | 107 | # Set the --queue or --queues option to work from a particular queue. 108 | RAILS_ENV=production script/delayed_job --queue=tracking start 109 | RAILS_ENV=production script/delayed_job --queues=mailers,tasks start 110 | 111 | # Runs all available jobs and then exits 112 | RAILS_ENV=production script/delayed_job start --exit-on-complete 113 | # or to run in the foreground 114 | RAILS_ENV=production script/delayed_job run --exit-on-complete 115 | ``` 116 | 117 | *Important:* If `delayed_job` is not running then any attempts 118 | to add new data sources or create user profiles will silently be 119 | held in a suspended state until the job scheduler is started. 120 | 121 | ## Usage 122 | 123 | If you arrived at this project then you probably saw our presentation at Black Hat 2013 and knows how it works 124 | already, right? =) 125 | 126 | ## Copyright 127 | 128 | microphisher - a spear phishing support tool 129 | 130 | Created by [urma](https://github.com/urma) and [jespinhara](https://github.com/jespinhara) 131 | Copyright (C) 2013 [Trustwave Holdings, Inc.](https://www.trustwave.com/ "Trustwave") 132 | 133 | This program is free software: you can redistribute it and/or modify 134 | it under the terms of the GNU General Public License as published by 135 | the Free Software Foundation, either version 3 of the License, or 136 | (at your option) any later version. 137 | 138 | This program is distributed in the hope that it will be useful, 139 | but WITHOUT ANY WARRANTY; without even the implied warranty of 140 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 141 | GNU General Public License for more details. 142 | 143 | You should have received a copy of the GNU General Public License 144 | along with this program. If not, see . 145 | -------------------------------------------------------------------------------- /README.rdoc: -------------------------------------------------------------------------------- 1 | == Welcome to Rails 2 | 3 | Rails is a web-application framework that includes everything needed to create 4 | database-backed web applications according to the Model-View-Control pattern. 5 | 6 | This pattern splits the view (also called the presentation) into "dumb" 7 | templates that are primarily responsible for inserting pre-built data in between 8 | HTML tags. The model contains the "smart" domain objects (such as Account, 9 | Product, Person, Post) that holds all the business logic and knows how to 10 | persist themselves to a database. The controller handles the incoming requests 11 | (such as Save New Account, Update Product, Show Post) by manipulating the model 12 | and directing data to the view. 13 | 14 | In Rails, the model is handled by what's called an object-relational mapping 15 | layer entitled Active Record. This layer allows you to present the data from 16 | database rows as objects and embellish these data objects with business logic 17 | methods. You can read more about Active Record in 18 | link:files/vendor/rails/activerecord/README.html. 19 | 20 | The controller and view are handled by the Action Pack, which handles both 21 | layers by its two parts: Action View and Action Controller. These two layers 22 | are bundled in a single package due to their heavy interdependence. This is 23 | unlike the relationship between the Active Record and Action Pack that is much 24 | more separate. Each of these packages can be used independently outside of 25 | Rails. You can read more about Action Pack in 26 | link:files/vendor/rails/actionpack/README.html. 27 | 28 | 29 | == Getting Started 30 | 31 | 1. At the command prompt, create a new Rails application: 32 | rails new myapp (where myapp is the application name) 33 | 34 | 2. Change directory to myapp and start the web server: 35 | cd myapp; rails server (run with --help for options) 36 | 37 | 3. Go to http://localhost:3000/ and you'll see: 38 | "Welcome aboard: You're riding Ruby on Rails!" 39 | 40 | 4. Follow the guidelines to start developing your application. You can find 41 | the following resources handy: 42 | 43 | * The Getting Started Guide: http://guides.rubyonrails.org/getting_started.html 44 | * Ruby on Rails Tutorial Book: http://www.railstutorial.org/ 45 | 46 | 47 | == Debugging Rails 48 | 49 | Sometimes your application goes wrong. Fortunately there are a lot of tools that 50 | will help you debug it and get it back on the rails. 51 | 52 | First area to check is the application log files. Have "tail -f" commands 53 | running on the server.log and development.log. Rails will automatically display 54 | debugging and runtime information to these files. Debugging info will also be 55 | shown in the browser on requests from 127.0.0.1. 56 | 57 | You can also log your own messages directly into the log file from your code 58 | using the Ruby logger class from inside your controllers. Example: 59 | 60 | class WeblogController < ActionController::Base 61 | def destroy 62 | @weblog = Weblog.find(params[:id]) 63 | @weblog.destroy 64 | logger.info("#{Time.now} Destroyed Weblog ID ##{@weblog.id}!") 65 | end 66 | end 67 | 68 | The result will be a message in your log file along the lines of: 69 | 70 | Mon Oct 08 14:22:29 +1000 2007 Destroyed Weblog ID #1! 71 | 72 | More information on how to use the logger is at http://www.ruby-doc.org/core/ 73 | 74 | Also, Ruby documentation can be found at http://www.ruby-lang.org/. There are 75 | several books available online as well: 76 | 77 | * Programming Ruby: http://www.ruby-doc.org/docs/ProgrammingRuby/ (Pickaxe) 78 | * Learn to Program: http://pine.fm/LearnToProgram/ (a beginners guide) 79 | 80 | These two books will bring you up to speed on the Ruby language and also on 81 | programming in general. 82 | 83 | 84 | == Debugger 85 | 86 | Debugger support is available through the debugger command when you start your 87 | Mongrel or WEBrick server with --debugger. This means that you can break out of 88 | execution at any point in the code, investigate and change the model, and then, 89 | resume execution! You need to install ruby-debug to run the server in debugging 90 | mode. With gems, use sudo gem install ruby-debug. Example: 91 | 92 | class WeblogController < ActionController::Base 93 | def index 94 | @posts = Post.all 95 | debugger 96 | end 97 | end 98 | 99 | So the controller will accept the action, run the first line, then present you 100 | with a IRB prompt in the server window. Here you can do things like: 101 | 102 | >> @posts.inspect 103 | => "[#nil, "body"=>nil, "id"=>"1"}>, 105 | #"Rails", "body"=>"Only ten..", "id"=>"2"}>]" 107 | >> @posts.first.title = "hello from a debugger" 108 | => "hello from a debugger" 109 | 110 | ...and even better, you can examine how your runtime objects actually work: 111 | 112 | >> f = @posts.first 113 | => #nil, "body"=>nil, "id"=>"1"}> 114 | >> f. 115 | Display all 152 possibilities? (y or n) 116 | 117 | Finally, when you're ready to resume execution, you can enter "cont". 118 | 119 | 120 | == Console 121 | 122 | The console is a Ruby shell, which allows you to interact with your 123 | application's domain model. Here you'll have all parts of the application 124 | configured, just like it is when the application is running. You can inspect 125 | domain models, change values, and save to the database. Starting the script 126 | without arguments will launch it in the development environment. 127 | 128 | To start the console, run rails console from the application 129 | directory. 130 | 131 | Options: 132 | 133 | * Passing the -s, --sandbox argument will rollback any modifications 134 | made to the database. 135 | * Passing an environment name as an argument will load the corresponding 136 | environment. Example: rails console production. 137 | 138 | To reload your controllers and models after launching the console run 139 | reload! 140 | 141 | More information about irb can be found at: 142 | link:http://www.rubycentral.org/pickaxe/irb.html 143 | 144 | 145 | == dbconsole 146 | 147 | You can go to the command line of your database directly through rails 148 | dbconsole. You would be connected to the database with the credentials 149 | defined in database.yml. Starting the script without arguments will connect you 150 | to the development database. Passing an argument will connect you to a different 151 | database, like rails dbconsole production. Currently works for MySQL, 152 | PostgreSQL and SQLite 3. 153 | 154 | == Description of Contents 155 | 156 | The default directory structure of a generated Ruby on Rails application: 157 | 158 | |-- app 159 | | |-- assets 160 | | | |-- images 161 | | | |-- javascripts 162 | | | `-- stylesheets 163 | | |-- controllers 164 | | |-- helpers 165 | | |-- mailers 166 | | |-- models 167 | | `-- views 168 | | `-- layouts 169 | |-- config 170 | | |-- environments 171 | | |-- initializers 172 | | `-- locales 173 | |-- db 174 | |-- doc 175 | |-- lib 176 | | |-- assets 177 | | `-- tasks 178 | |-- log 179 | |-- public 180 | |-- script 181 | |-- test 182 | | |-- fixtures 183 | | |-- functional 184 | | |-- integration 185 | | |-- performance 186 | | `-- unit 187 | |-- tmp 188 | | `-- cache 189 | | `-- assets 190 | `-- vendor 191 | |-- assets 192 | | |-- javascripts 193 | | `-- stylesheets 194 | `-- plugins 195 | 196 | app 197 | Holds all the code that's specific to this particular application. 198 | 199 | app/assets 200 | Contains subdirectories for images, stylesheets, and JavaScript files. 201 | 202 | app/controllers 203 | Holds controllers that should be named like weblogs_controller.rb for 204 | automated URL mapping. All controllers should descend from 205 | ApplicationController which itself descends from ActionController::Base. 206 | 207 | app/models 208 | Holds models that should be named like post.rb. Models descend from 209 | ActiveRecord::Base by default. 210 | 211 | app/views 212 | Holds the template files for the view that should be named like 213 | weblogs/index.html.erb for the WeblogsController#index action. All views use 214 | eRuby syntax by default. 215 | 216 | app/views/layouts 217 | Holds the template files for layouts to be used with views. This models the 218 | common header/footer method of wrapping views. In your views, define a layout 219 | using the layout :default and create a file named default.html.erb. 220 | Inside default.html.erb, call <% yield %> to render the view using this 221 | layout. 222 | 223 | app/helpers 224 | Holds view helpers that should be named like weblogs_helper.rb. These are 225 | generated for you automatically when using generators for controllers. 226 | Helpers can be used to wrap functionality for your views into methods. 227 | 228 | config 229 | Configuration files for the Rails environment, the routing map, the database, 230 | and other dependencies. 231 | 232 | db 233 | Contains the database schema in schema.rb. db/migrate contains all the 234 | sequence of Migrations for your schema. 235 | 236 | doc 237 | This directory is where your application documentation will be stored when 238 | generated using rake doc:app 239 | 240 | lib 241 | Application specific libraries. Basically, any kind of custom code that 242 | doesn't belong under controllers, models, or helpers. This directory is in 243 | the load path. 244 | 245 | public 246 | The directory available for the web server. Also contains the dispatchers and the 247 | default HTML files. This should be set as the DOCUMENT_ROOT of your web 248 | server. 249 | 250 | script 251 | Helper scripts for automation and generation. 252 | 253 | test 254 | Unit and functional tests along with fixtures. When using the rails generate 255 | command, template test files will be generated for you and placed in this 256 | directory. 257 | 258 | vendor 259 | External libraries that the application depends on. Also includes the plugins 260 | subdirectory. If the app has frozen rails, those gems also go here, under 261 | vendor/rails/. This directory is in the load path. 262 | -------------------------------------------------------------------------------- /Rakefile: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env rake 2 | # Add your own tasks in files placed in lib/tasks ending in .rake, 3 | # for example lib/tasks/capistrano.rake, and they will automatically be available to Rake. 4 | 5 | require File.expand_path('../config/application', __FILE__) 6 | 7 | Mongophisher::Application.load_tasks 8 | -------------------------------------------------------------------------------- /app/assets/images/rails.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SpiderLabs/microphisher/56fefc5d30fa2ef69023266629d77af66144b7cc/app/assets/images/rails.png -------------------------------------------------------------------------------- /app/assets/javascripts/application.js: -------------------------------------------------------------------------------- 1 | // This is a manifest file that'll be compiled into application.js, which will include all the files 2 | // listed below. 3 | // 4 | // Any JavaScript/Coffee file within this directory, lib/assets/javascripts, vendor/assets/javascripts, 5 | // or vendor/assets/javascripts of plugins, if any, can be referenced here using a relative path. 6 | // 7 | // It's not advisable to add code directly here, but if you do, it'll appear at the bottom of the 8 | // the compiled file. 9 | // 10 | // WARNING: THE FIRST BLANK LINE MARKS THE END OF WHAT'S TO BE PROCESSED, ANY BLANK LINE SHOULD 11 | // GO AFTER THE REQUIRES BELOW. 12 | // 13 | //= require jquery 14 | //= require jquery_ujs 15 | //= require_tree . 16 | // 17 | // Add lazy-high-charts for graphing/charts 18 | // 19 | //= require highcharts/highcharts 20 | //= require highcharts/highcharts-more 21 | //= require highcharts/highstock 22 | -------------------------------------------------------------------------------- /app/assets/javascripts/data_sources.js.coffee: -------------------------------------------------------------------------------- 1 | # Place all the behaviors and hooks related to the matching controller here. 2 | # All this logic will automatically be available in application.js. 3 | # You can use CoffeeScript in this file: http://jashkenas.github.com/coffee-script/ 4 | -------------------------------------------------------------------------------- /app/assets/javascripts/home.js.coffee: -------------------------------------------------------------------------------- 1 | # Place all the behaviors and hooks related to the matching controller here. 2 | # All this logic will automatically be available in application.js. 3 | # You can use CoffeeScript in this file: http://jashkenas.github.com/coffee-script/ 4 | -------------------------------------------------------------------------------- /app/assets/javascripts/oauth.js.coffee: -------------------------------------------------------------------------------- 1 | # Place all the behaviors and hooks related to the matching controller here. 2 | # All this logic will automatically be available in application.js. 3 | # You can use CoffeeScript in this file: http://jashkenas.github.com/coffee-script/ 4 | -------------------------------------------------------------------------------- /app/assets/javascripts/profiles.js.coffee: -------------------------------------------------------------------------------- 1 | # Place all the behaviors and hooks related to the matching controller here. 2 | # All this logic will automatically be available in application.js. 3 | # You can use CoffeeScript in this file: http://jashkenas.github.com/coffee-script/ 4 | -------------------------------------------------------------------------------- /app/assets/javascripts/status_updates.js.coffee: -------------------------------------------------------------------------------- 1 | # Place all the behaviors and hooks related to the matching controller here. 2 | # All this logic will automatically be available in application.js. 3 | # You can use CoffeeScript in this file: http://jashkenas.github.com/coffee-script/ 4 | -------------------------------------------------------------------------------- /app/assets/javascripts/unknown_subjects.js.coffee: -------------------------------------------------------------------------------- 1 | # Place all the behaviors and hooks related to the matching controller here. 2 | # All this logic will automatically be available in application.js. 3 | # You can use CoffeeScript in this file: http://jashkenas.github.com/coffee-script/ 4 | -------------------------------------------------------------------------------- /app/assets/stylesheets/application.css: -------------------------------------------------------------------------------- 1 | /* 2 | * This is a manifest file that'll be compiled into application.css, which will include all the files 3 | * listed below. 4 | * 5 | * Any CSS and SCSS file within this directory, lib/assets/stylesheets, vendor/assets/stylesheets, 6 | * or vendor/assets/stylesheets of plugins, if any, can be referenced here using a relative path. 7 | * 8 | * You're free to add application-wide styles to this file and they'll appear at the top of the 9 | * compiled file, but it's generally better to create a new file per style scope. 10 | * 11 | *= require_self 12 | *= require_tree . 13 | */ 14 | -------------------------------------------------------------------------------- /app/assets/stylesheets/data_sources.css.scss: -------------------------------------------------------------------------------- 1 | // Place all the styles related to the DataSources controller here. 2 | // They will automatically be included in application.css. 3 | // You can use Sass (SCSS) here: http://sass-lang.com/ 4 | -------------------------------------------------------------------------------- /app/assets/stylesheets/home.css.scss: -------------------------------------------------------------------------------- 1 | // Place all the styles related to the Home controller here. 2 | // They will automatically be included in application.css. 3 | // You can use Sass (SCSS) here: http://sass-lang.com/ 4 | -------------------------------------------------------------------------------- /app/assets/stylesheets/oauth.css.scss: -------------------------------------------------------------------------------- 1 | // Place all the styles related to the oauth controller here. 2 | // They will automatically be included in application.css. 3 | // You can use Sass (SCSS) here: http://sass-lang.com/ 4 | -------------------------------------------------------------------------------- /app/assets/stylesheets/profiles.css.scss: -------------------------------------------------------------------------------- 1 | // Place all the styles related to the Profiles controller here. 2 | // They will automatically be included in application.css. 3 | // You can use Sass (SCSS) here: http://sass-lang.com/ 4 | -------------------------------------------------------------------------------- /app/assets/stylesheets/status_updates.css.scss: -------------------------------------------------------------------------------- 1 | // Place all the styles related to the StatusUpdates controller here. 2 | // They will automatically be included in application.css. 3 | // You can use Sass (SCSS) here: http://sass-lang.com/ 4 | -------------------------------------------------------------------------------- /app/assets/stylesheets/unknown_subjects.css.scss: -------------------------------------------------------------------------------- 1 | // Place all the styles related to the UnknownSubjects controller here. 2 | // They will automatically be included in application.css. 3 | // You can use Sass (SCSS) here: http://sass-lang.com/ 4 | -------------------------------------------------------------------------------- /app/controllers/application_controller.rb: -------------------------------------------------------------------------------- 1 | # 2 | # microphisher - a spear phishing support tool 3 | # 4 | # Created by Ulisses Albuquerque & Joaquim Espinhara 5 | # Copyright (C) 2013 Trustwave Holdings, Inc. 6 | # 7 | # This program is free software: you can redistribute it and/or modify 8 | # it under the terms of the GNU General Public License as published by 9 | # the Free Software Foundation, either version 3 of the License, or 10 | # (at your option) any later version. 11 | # 12 | # This program is distributed in the hope that it will be useful, 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | # GNU General Public License for more details. 16 | # 17 | # You should have received a copy of the GNU General Public License 18 | # along with this program. If not, see . 19 | # 20 | 21 | class ApplicationController < ActionController::Base 22 | include ApplicationHelper 23 | 24 | protect_from_forgery 25 | 26 | # Restrict access to authenticated users 27 | before_filter :require_login 28 | 29 | def require_login 30 | redirect_to home_url unless logged_in? 31 | end 32 | end 33 | -------------------------------------------------------------------------------- /app/controllers/data_sources_controller.rb: -------------------------------------------------------------------------------- 1 | # 2 | # microphisher - a spear phishing support tool 3 | # 4 | # Created by Ulisses Albuquerque & Joaquim Espinhara 5 | # Copyright (C) 2013 Trustwave Holdings, Inc. 6 | # 7 | # This program is free software: you can redistribute it and/or modify 8 | # it under the terms of the GNU General Public License as published by 9 | # the Free Software Foundation, either version 3 of the License, or 10 | # (at your option) any later version. 11 | # 12 | # This program is distributed in the hope that it will be useful, 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | # GNU General Public License for more details. 16 | # 17 | # You should have received a copy of the GNU General Public License 18 | # along with this program. If not, see . 19 | # 20 | 21 | class DataSourcesController < ApplicationController 22 | before_filter :load_unknown_subject 23 | 24 | def load_unknown_subject 25 | @unknown_subject = UnknownSubject.find(params[:unknown_subject_id]) 26 | end 27 | 28 | def show 29 | @data_source = @unknown_subject.data_sources.find(params[:id]) 30 | @page = params['page'] || 1 31 | @status_updates = @data_source.status_updates.desc(:created_at).page(@page).per(15) 32 | end 33 | 34 | def new 35 | @data_source = @unknown_subject.data_sources.new 36 | end 37 | 38 | def create 39 | # TODO: Check data source type and create instance based on 40 | # corresponding class 41 | @data_source = TwitterDataSource.new(params[:data_source]) 42 | 43 | @data_source.unknown_subject = @unknown_subject 44 | @data_source.save! 45 | @data_source.delay.fetch_status_updates! 46 | redirect_to unknown_subject_data_source_path(@unknown_subject, @data_source) 47 | end 48 | 49 | def destroy 50 | @data_source = @unknown_subject.data_sources.find(params[:id]) 51 | @data_source.destroy 52 | redirect_to @unknown_subject 53 | end 54 | end 55 | -------------------------------------------------------------------------------- /app/controllers/home_controller.rb: -------------------------------------------------------------------------------- 1 | # 2 | # microphisher - a spear phishing support tool 3 | # 4 | # Created by Ulisses Albuquerque & Joaquim Espinhara 5 | # Copyright (C) 2013 Trustwave Holdings, Inc. 6 | # 7 | # This program is free software: you can redistribute it and/or modify 8 | # it under the terms of the GNU General Public License as published by 9 | # the Free Software Foundation, either version 3 of the License, or 10 | # (at your option) any later version. 11 | # 12 | # This program is distributed in the hope that it will be useful, 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | # GNU General Public License for more details. 16 | # 17 | # You should have received a copy of the GNU General Public License 18 | # along with this program. If not, see . 19 | # 20 | 21 | class HomeController < ApplicationController 22 | skip_filter :require_login, only: [ :index ] 23 | 24 | def index 25 | # We can only provide indexing metrics if the user is currently 26 | # logged in 27 | if logged_in? 28 | # Unknown subject count 29 | @unknown_subject_count = current_user.unknown_subjects.count 30 | 31 | # Data source count 32 | unknown_subject_ids = current_user.unknown_subjects.map { |us| us.id } 33 | data_sources = DataSource.in(unknown_subject_id: unknown_subject_ids) 34 | @data_source_count = data_sources.count || 0 35 | 36 | # Status update count 37 | data_source_ids = data_sources.map { |ds| ds.id } 38 | @status_update_count = StatusUpdate.in(data_source_id: data_source_ids).count || 0 39 | end 40 | end 41 | end 42 | -------------------------------------------------------------------------------- /app/controllers/oauth_controller.rb: -------------------------------------------------------------------------------- 1 | # 2 | # microphisher - a spear phishing support tool 3 | # 4 | # Created by Ulisses Albuquerque & Joaquim Espinhara 5 | # Copyright (C) 2013 Trustwave Holdings, Inc. 6 | # 7 | # This program is free software: you can redistribute it and/or modify 8 | # it under the terms of the GNU General Public License as published by 9 | # the Free Software Foundation, either version 3 of the License, or 10 | # (at your option) any later version. 11 | # 12 | # This program is distributed in the hope that it will be useful, 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | # GNU General Public License for more details. 16 | # 17 | # You should have received a copy of the GNU General Public License 18 | # along with this program. If not, see . 19 | # 20 | 21 | class OauthController < ApplicationController 22 | before_filter :setup_oauth 23 | skip_filter :require_login, :only => [ :login, :authorize ] 24 | 25 | def setup_oauth 26 | oauth_config = YAML.load_file('config/oauth.yml')[Rails.env] 27 | @consumer = OAuth::Consumer.new oauth_config['consumer_key'], 28 | oauth_config['consumer_secret'], { 29 | :site => oauth_config['site'], 30 | :request_token_path => oauth_config['request_token_path'], 31 | :access_token_path => oauth_config['access_token_path'], 32 | :authorize_path => oauth_config['authorize_path'], 33 | } 34 | end 35 | 36 | # /oauth/login should be used whenever we want to start an OAuth authorization 37 | # request using the OAuth::Consumer configured in #setup_oauth 38 | # 39 | def login 40 | oauth_callback = url_for(:action => :authorize) 41 | request_token = @consumer.get_request_token(:oauth_callback => oauth_callback) 42 | session[:request_token] = request_token 43 | redirect_to request_token.authorize_url 44 | end 45 | 46 | # /oauth/logout should be used whenever we want to remove user information 47 | # from the current session 48 | # 49 | def logout 50 | session[:user] = nil 51 | redirect_to :controller => :home 52 | end 53 | 54 | # /oauth/authorize is the callback URL where we are redirected to after the user 55 | # grants the application access to his OAuth credentials 56 | # 57 | def authorize 58 | oauth_verifier = params[:oauth_verifier] 59 | request_token = session[:request_token] 60 | access_token = request_token.get_access_token(:oauth_verifier => oauth_verifier) 61 | 62 | user = User.login(access_token) 63 | session[:user] = user.id unless user.nil? 64 | redirect_to :controller => :home 65 | end 66 | end 67 | -------------------------------------------------------------------------------- /app/controllers/profiles_controller.rb: -------------------------------------------------------------------------------- 1 | # 2 | # microphisher - a spear phishing support tool 3 | # 4 | # Created by Ulisses Albuquerque & Joaquim Espinhara 5 | # Copyright (C) 2013 Trustwave Holdings, Inc. 6 | # 7 | # This program is free software: you can redistribute it and/or modify 8 | # it under the terms of the GNU General Public License as published by 9 | # the Free Software Foundation, either version 3 of the License, or 10 | # (at your option) any later version. 11 | # 12 | # This program is distributed in the hope that it will be useful, 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | # GNU General Public License for more details. 16 | # 17 | # You should have received a copy of the GNU General Public License 18 | # along with this program. If not, see . 19 | # 20 | 21 | class ProfilesController < ApplicationController 22 | # Word delimiting regexp 23 | WORD_DELIMITER = /[\.!\?\,:; ]+/ 24 | URL_REGEXP = /\s*http(s)?\S+/ 25 | 26 | before_filter :load_unknown_subject 27 | 28 | def load_unknown_subject 29 | @unknown_subject = UnknownSubject.find(params[:unknown_subject_id]) 30 | end 31 | 32 | def search 33 | # First build a filter which restricts status updates to data sources 34 | # associated with the current unknown subject 35 | data_source_ids = @unknown_subject.data_sources.map { |ds| ds.id } 36 | source = StatusUpdate.in(data_source_id: data_source_ids) 37 | 38 | # Next we filter this list based on the user provided criteria 39 | criteria_filter = self._build_filter(params) 40 | criteria_filter.each do |criteria| 41 | if criteria[:operator] == 'match' 42 | next_source = source.all(criteria[:field].to_sym => Regexp.new(criteria[:value])) 43 | elsif criteria[:operator] == 'not_match' 44 | next_source = source.not.all(criteria[:field].to_sym => Regexp.new(criteria[:value])) 45 | end 46 | source = next_source 47 | end 48 | session[:criteria_selector] = source 49 | 50 | # Finally, provide a view-accessible cursor for all elegible 51 | # status updates 52 | @page = params[:page] || 1 53 | @status_updates = source.desc(:created_at).page(@page) 54 | end 55 | 56 | def create 57 | @profile = @unknown_subject.profiles.new(params[:profile]) 58 | @profile.selector = session[:criteria_selector].selector 59 | @profile.save! 60 | @profile.delay.build_profile! 61 | redirect_to [ @unknown_subject, @profile ] 62 | end 63 | 64 | def show 65 | @profile = @unknown_subject.profiles.find(params[:id]) 66 | end 67 | 68 | def destroy 69 | @profile = @unknown_subject.profiles.find(params[:id]) 70 | @profile.destroy 71 | redirect_to @unknown_subject 72 | end 73 | 74 | def input 75 | @profile = @unknown_subject.profiles.find(params[:id]) 76 | end 77 | 78 | def nlp_tree 79 | @profile = @unknown_subject.profiles.find(params[:id]) 80 | end 81 | 82 | def validate 83 | @profile = @unknown_subject.profiles.find(params[:id]) 84 | @text_input = params[:text] 85 | 86 | response = [ 87 | { :id => 'word_count', :label => 'Word Count', 88 | :value => _word_count_ranking(@text_input, @profile) }, 89 | { :id => 'word_usage', :label => 'Word Usage', 90 | :value => _word_usage_ranking(@text_input, @profile) }, 91 | { :id => 'user_references', :label => 'User References', 92 | :value => _user_reference_ranking(@text_input, @profile) }, 93 | { :id => 'user_references', :label => 'Hashtag References', 94 | :value => _hashtag_reference_ranking(@text_input, @profile) } 95 | ] 96 | 97 | respond_to do |format| 98 | format.json { render :json => response.to_json } 99 | end 100 | end 101 | 102 | def autocomplete 103 | @profile = @unknown_subject.profiles.find(params[:id]) 104 | @word = params[:word] 105 | 106 | dictionary = nil 107 | if @word.match(/@/) 108 | dictionary = @profile.user_mentions 109 | elsif @word.match(/#/) 110 | dictionary = @profile.hashtag_mentions 111 | else 112 | dictionary = @profile.dictionary 113 | end 114 | 115 | result = dictionary.find_all do |entry| 116 | entry[0].match(Regexp.new("^#{@word}", Regexp::IGNORECASE)) 117 | end.sort do |a,b| 118 | b[1] <=> a[1] 119 | end.map do |entry| 120 | "#{entry[0]} (#{entry[1]})" 121 | end 122 | 123 | respond_to do |format| 124 | format.json { render :json => result } 125 | end 126 | end 127 | 128 | protected 129 | def _build_filter(params) 130 | (0..(params[:profile_criteria_field].size - 1)).map do |index| 131 | { 132 | :field => params[:profile_criteria_field][index], 133 | :operator => params[:profile_criteria_operator][index], 134 | :value => params[:profile_criteria_value][index] 135 | } 136 | end 137 | end 138 | 139 | # User reference criteria is defined as the ratio of known user references 140 | # against unknown references in the input text; a text like 141 | # 'Some tweet @valid_user_1 @valid_user_2 @invalid_user' would rank 142 | # 66%, because 1/3 of its referenced users are unknown 143 | # 144 | def _user_reference_ranking(text, profile) 145 | user_references = text.split(WORD_DELIMITER).find_all do |word| 146 | word.match(/^@/) 147 | end 148 | 149 | return 100 if user_references.size == 0 150 | 151 | user_references.inject(0) do |ranking, user| 152 | ranking + (profile.user_mentions.has_key?(user) ? (100 / user_references.size) : 0) 153 | end 154 | end 155 | 156 | # Hashtag reference criteria is defined as the ratio of valid hashtag 157 | # references against invalid references in the input text; a text like 158 | # 'Some tweet #known_tag_1 #known_tag_2 #unknown_tag' would rank 159 | # 66%, because 1/3 of its referenced hashtags are unknown 160 | # 161 | def _hashtag_reference_ranking(text, profile) 162 | hashtag_references = text.split(WORD_DELIMITER).find_all do |word| 163 | word.match(/^#/) 164 | end 165 | 166 | return 100 if hashtag_references.size == 0 167 | 168 | hashtag_references.inject(0) do |ranking, hashtag| 169 | ranking + (profile.hashtag_mentions.has_key?(hashtag) ? (100 / hashtag_references.size) : 0) 170 | end 171 | end 172 | 173 | # Word count ranking is defined as the distance between the number of 174 | # words in a status update versus the average length for the profile 175 | # status update collection; anything below 1/3 or above 5/3 of the 176 | # average value is ranked 0, while other values are proportional to 177 | # how close they are to the average value 178 | # 179 | def _word_count_ranking(text, profile) 180 | # Count the number of words in current text 181 | text_word_count = text.split(WORD_DELIMITER).size 182 | 183 | return 0 if text_word_count < 1 * profile.average_word_count / 3 184 | return 0 if text_word_count > 5 * profile.average_word_count / 3 185 | 186 | 100 - (100 * (text_word_count - profile.average_word_count).abs / 187 | (2 * profile.average_word_count / 3)) 188 | end 189 | 190 | # Word usage is simple -- all words used in a status update must be 191 | # previously known, being present in a status update present in the 192 | # profile collection; every time an unknown word is used it decreases 193 | # the ranking 194 | # 195 | def _word_usage_ranking(text, profile) 196 | # The word list must be filtered of stop words, otherwise that might 197 | # affect our word usage rank incorrectly 198 | word_list = text.gsub(URL_REGEXP, '').split(WORD_DELIMITER).reject do |word| 199 | word.match(/^[@#]/) || profile.stop_words.member?(word.downcase) 200 | end 201 | 202 | return 100 if word_list.size == 0 203 | 204 | word_list.inject(0) do |ranking, word| 205 | ranking + (profile.dictionary.has_key?(word.downcase) ? (100 / word_list.size) : 0) 206 | end 207 | end 208 | end 209 | -------------------------------------------------------------------------------- /app/controllers/status_updates_controller.rb: -------------------------------------------------------------------------------- 1 | # 2 | # microphisher - a spear phishing support tool 3 | # 4 | # Created by Ulisses Albuquerque & Joaquim Espinhara 5 | # Copyright (C) 2013 Trustwave Holdings, Inc. 6 | # 7 | # This program is free software: you can redistribute it and/or modify 8 | # it under the terms of the GNU General Public License as published by 9 | # the Free Software Foundation, either version 3 of the License, or 10 | # (at your option) any later version. 11 | # 12 | # This program is distributed in the hope that it will be useful, 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | # GNU General Public License for more details. 16 | # 17 | # You should have received a copy of the GNU General Public License 18 | # along with this program. If not, see . 19 | # 20 | 21 | class StatusUpdatesController < ApplicationController 22 | before_filter :load_data_source 23 | 24 | def load_data_source 25 | @unknown_subject = UnknownSubject.find(params[:unknown_subject_id]) 26 | @data_source = @unknown_subject.data_sources.find(params[:data_source_id]) 27 | end 28 | 29 | def show 30 | @status_update = @data_source.status_updates.find(params[:id]) 31 | end 32 | end 33 | -------------------------------------------------------------------------------- /app/controllers/unknown_subjects_controller.rb: -------------------------------------------------------------------------------- 1 | # 2 | # microphisher - a spear phishing support tool 3 | # 4 | # Created by Ulisses Albuquerque & Joaquim Espinhara 5 | # Copyright (C) 2013 Trustwave Holdings, Inc. 6 | # 7 | # This program is free software: you can redistribute it and/or modify 8 | # it under the terms of the GNU General Public License as published by 9 | # the Free Software Foundation, either version 3 of the License, or 10 | # (at your option) any later version. 11 | # 12 | # This program is distributed in the hope that it will be useful, 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | # GNU General Public License for more details. 16 | # 17 | # You should have received a copy of the GNU General Public License 18 | # along with this program. If not, see . 19 | # 20 | 21 | class UnknownSubjectsController < ApplicationController 22 | def index 23 | @unknown_subjects = current_user.unknown_subjects.asc(:name) 24 | end 25 | 26 | def show 27 | @unknown_subject = current_user.unknown_subjects.find(params[:id]) 28 | 29 | # Obtain a listing of sources with geotagged status updates for the 30 | # geolocation profile analysis 31 | geo_sources = @unknown_subject.status_updates.ne(coordinates: nil) 32 | @geo_enabled_sources = sources_hash(geo_sources) 33 | 34 | # Charts 35 | @source_chart = sources_chart(@unknown_subject.status_updates) 36 | @geo_chart = geolocation_chart(@unknown_subject.status_updates) 37 | @retweet_chart = retweet_chart(@unknown_subject.status_updates) 38 | end 39 | 40 | def new 41 | @unknown_subject = current_user.unknown_subjects.new 42 | end 43 | 44 | def create 45 | @unknown_subject = current_user.unknown_subjects.new(params[:unknown_subject]) 46 | @unknown_subject.save! 47 | redirect_to @unknown_subject 48 | end 49 | 50 | def edit 51 | @unknown_subject = current_user.unknown_subjects.find(params[:id]) 52 | end 53 | 54 | def update 55 | @unknown_subject = current_user.unknown_subjects.find(params[:id]) 56 | @unknown_subject.update_attributes(params[:unknown_subject]) 57 | @unknown_subject.save! 58 | redirect_to @unknown_subject 59 | end 60 | 61 | def destroy 62 | @unknown_subject = current_user.unknown_subjects.find(params[:id]) 63 | @unknown_subject.destroy 64 | redirect_to unknown_subjects_path 65 | end 66 | 67 | def export_profile 68 | @unknown_subject = current_user.unknown_subjects.find(params[:id]) 69 | geo_updates = @unknown_subject.status_updates.ne(coordinates: nil) 70 | @status_updates = geo_updates.where(source: Regexp.new(params[:source])) 71 | 72 | respond_to do |format| 73 | format.kml 74 | end 75 | end 76 | 77 | protected 78 | # Return a listing of unique data sources along with their corresponding 79 | # counts 80 | def sources_hash(criteria) 81 | # Count sources using map/reduce 82 | # TODO: db.aggregate() would probably be easier/simpler, but apparently 83 | # Mongoid does not support that, so we use map/reduce for now 84 | map_func = %Q{ function() { emit(this.source, 1); } } 85 | reduce_func = %Q{ function(key, values) { return Array.sum(values); } } 86 | Hash[criteria.map_reduce(map_func, reduce_func).out(inline: true).map do |entry| 87 | [ entry['_id'].gsub(/^<.+>(.+)<.+>$/, '\1'), entry['value'].to_i ] 88 | end] 89 | end 90 | 91 | def sources_chart(criteria) 92 | # Fetch sources count and group low-ranking sources into 'Other' 93 | sorted_data = sources_hash(criteria).sort { |a, b| b[1] <=> a[1] } 94 | if sorted_data.size > 10 95 | other_value = [ [ 'Other', sorted_data[9..-1].inject(0) { |a,b| a + b[1] } ] ] 96 | else 97 | other_value = [] 98 | end 99 | data = sorted_data[0..8] + other_value 100 | 101 | # Produce the actual chart 102 | LazyHighCharts::HighChart.new('pie') do |f| 103 | f.chart({ :defaultSeriesType => 'pie' }) 104 | f.series({ 105 | :type => 'pie', 106 | :name => 'Status Update Source', 107 | :data => data 108 | }) 109 | f.options[:title][:text] = 'Status Update Source' 110 | f.plot_options(:pie=> { :allowPointSelect=>true }) 111 | end 112 | end 113 | 114 | def geolocation_chart(criteria) 115 | # Geolocation information is simple, so we do not resort to 116 | # MongoDB-based aggregation and/or map/reduce 117 | data = [ 118 | [ 'Present', criteria.ne(coordinates: nil).count ], 119 | [ 'Absent', criteria.where(coordinates: nil).count ] 120 | ] 121 | 122 | # Produce the actual chart 123 | LazyHighCharts::HighChart.new('pie') do |f| 124 | f.chart({ :defaultSeriesType => 'pie' }) 125 | f.series({ 126 | :type => 'pie', 127 | :name => 'Geolocation Information', 128 | :data => data 129 | }) 130 | f.options[:title][:text] = 'Geolocation Information' 131 | f.plot_options(:pie=> { :allowPointSelect=>true }) 132 | end 133 | end 134 | 135 | def retweet_chart(criteria) 136 | # Geolocation information is simple, so we do not resort to 137 | # MongoDB-based aggregation and/or map/reduce 138 | data = [ 139 | [ 'Yes', criteria.where(retweeted: true).count ], 140 | [ 'No', criteria.where(retweeted: false).count ] 141 | ] 142 | 143 | # Produce the actual chart 144 | LazyHighCharts::HighChart.new('pie') do |f| 145 | f.chart({ :defaultSeriesType => 'pie' }) 146 | f.series({ 147 | :type => 'pie', 148 | :name => 'Retweeted Content', 149 | :data => data 150 | }) 151 | f.options[:title][:text] = 'Retweeted Content' 152 | f.plot_options(:pie=> { :allowPointSelect=>true }) 153 | end 154 | end 155 | end 156 | -------------------------------------------------------------------------------- /app/helpers/application_helper.rb: -------------------------------------------------------------------------------- 1 | module ApplicationHelper 2 | def logged_in? 3 | !current_user.nil? 4 | end 5 | 6 | def current_user 7 | session[:user].nil? ? nil : User.find(session[:user]) 8 | end 9 | end 10 | -------------------------------------------------------------------------------- /app/helpers/data_sources_helper.rb: -------------------------------------------------------------------------------- 1 | module DataSourcesHelper 2 | end 3 | -------------------------------------------------------------------------------- /app/helpers/home_helper.rb: -------------------------------------------------------------------------------- 1 | module HomeHelper 2 | end 3 | -------------------------------------------------------------------------------- /app/helpers/oauth_helper.rb: -------------------------------------------------------------------------------- 1 | module OauthHelper 2 | end 3 | -------------------------------------------------------------------------------- /app/helpers/profiles_helper.rb: -------------------------------------------------------------------------------- 1 | module ProfilesHelper 2 | end 3 | -------------------------------------------------------------------------------- /app/helpers/status_updates_helper.rb: -------------------------------------------------------------------------------- 1 | module StatusUpdatesHelper 2 | end 3 | -------------------------------------------------------------------------------- /app/helpers/unknown_subjects_helper.rb: -------------------------------------------------------------------------------- 1 | module UnknownSubjectsHelper 2 | end 3 | -------------------------------------------------------------------------------- /app/mailers/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SpiderLabs/microphisher/56fefc5d30fa2ef69023266629d77af66144b7cc/app/mailers/.gitkeep -------------------------------------------------------------------------------- /app/models/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SpiderLabs/microphisher/56fefc5d30fa2ef69023266629d77af66144b7cc/app/models/.gitkeep -------------------------------------------------------------------------------- /app/models/data_source.rb: -------------------------------------------------------------------------------- 1 | # 2 | # microphisher - a spear phishing support tool 3 | # 4 | # Created by Ulisses Albuquerque & Joaquim Espinhara 5 | # Copyright (C) 2013 Trustwave Holdings, Inc. 6 | # 7 | # This program is free software: you can redistribute it and/or modify 8 | # it under the terms of the GNU General Public License as published by 9 | # the Free Software Foundation, either version 3 of the License, or 10 | # (at your option) any later version. 11 | # 12 | # This program is distributed in the hope that it will be useful, 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | # GNU General Public License for more details. 16 | # 17 | # You should have received a copy of the GNU General Public License 18 | # along with this program. If not, see . 19 | # 20 | 21 | class DataSource 22 | include Mongoid::Document 23 | 24 | # Relations 25 | belongs_to :unknown_subject 26 | has_many :status_updates 27 | 28 | # Attribute listing 29 | field :user_id, type: String 30 | field :last_crawl, type: DateTime 31 | field :status, type: String, default: 'pending' 32 | field :data_source_metadata, type: Hash, default: Hash.new 33 | field :profile_image_url, type: String 34 | 35 | def fetch_status_updates! 36 | raise NotImplementedError.new('This method must be overridden') 37 | end 38 | end 39 | 40 | -------------------------------------------------------------------------------- /app/models/facebook_data_source.rb: -------------------------------------------------------------------------------- 1 | # 2 | # microphisher - a spear phishing support tool 3 | # 4 | # Created by Ulisses Albuquerque & Joaquim Espinhara 5 | # Copyright (C) 2013 Trustwave Holdings, Inc. 6 | # 7 | # This program is free software: you can redistribute it and/or modify 8 | # it under the terms of the GNU General Public License as published by 9 | # the Free Software Foundation, either version 3 of the License, or 10 | # (at your option) any later version. 11 | # 12 | # This program is distributed in the hope that it will be useful, 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | # GNU General Public License for more details. 16 | # 17 | # You should have received a copy of the GNU General Public License 18 | # along with this program. If not, see . 19 | # 20 | 21 | class FacebookDataSource < DataSource 22 | include Mongoid::Document 23 | 24 | # Callbacks 25 | 26 | # Validations 27 | end 28 | 29 | -------------------------------------------------------------------------------- /app/models/profile.rb: -------------------------------------------------------------------------------- 1 | # 2 | # microphisher - a spear phishing support tool 3 | # 4 | # Created by Ulisses Albuquerque & Joaquim Espinhara 5 | # Copyright (C) 2013 Trustwave Holdings, Inc. 6 | # 7 | # This program is free software: you can redistribute it and/or modify 8 | # it under the terms of the GNU General Public License as published by 9 | # the Free Software Foundation, either version 3 of the License, or 10 | # (at your option) any later version. 11 | # 12 | # This program is distributed in the hope that it will be useful, 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | # GNU General Public License for more details. 16 | # 17 | # You should have received a copy of the GNU General Public License 18 | # along with this program. If not, see . 19 | # 20 | 21 | class Profile 22 | include Mongoid::Document 23 | 24 | # Attribute listing 25 | field :name, type: String 26 | field :description, type: String 27 | field :collection_yaml, type: String 28 | field :selector, type: String 29 | field :status, type: String, default: 'pending' 30 | 31 | # Validations 32 | validates :name, :selector, presence: true 33 | validates :name, uniqueness: true 34 | 35 | # Relations 36 | belongs_to :unknown_subject 37 | 38 | def treat_collection 39 | return nil if self.collection_yaml.nil? 40 | return @treat_collection unless @treat_collection.nil? 41 | @treat_collection = YAML.load(self.collection_yaml) 42 | end 43 | 44 | def treat_collection=(collection_object = nil) 45 | Dir.mktmpdir do |directory| 46 | yaml_filename = File.join(directory, 'collection_yaml') 47 | collection_object.serialize :yaml, :file => yaml_filename 48 | self.collection_yaml = File.read(yaml_filename) 49 | end 50 | @treat_collection = collection_object 51 | end 52 | 53 | def status_updates 54 | StatusUpdate.where(eval self.selector) 55 | end 56 | 57 | # All users mentioned in status updates associated with this profile, 58 | # ranked by frequency in a hashtable 59 | def user_mentions 60 | return {} if self.collection_yaml.nil? 61 | return self[:user_mentions] unless self[:user_mentions].nil? 62 | 63 | # Obtain all user mention entities from all eligible status updates 64 | mentions = status_updates.map do |status_update| 65 | status_update['entities']['user_mentions'] 66 | end 67 | 68 | # Count all user references in all entity entries 69 | self[:user_mentions] = Hash.new { 0 } 70 | mentions.each do |mention| 71 | mention.each do |user| 72 | self[:user_mentions]["@#{user['screen_name']}"] += 1 73 | end 74 | end 75 | self.save! 76 | 77 | self[:user_mentions] 78 | end 79 | 80 | # All hashtags mentioned in status updates associated with this profile, 81 | # ranked by frequency in a hashtable 82 | def hashtag_mentions 83 | return {} if self.collection_yaml.nil? 84 | return self[:hashtag_mentions] unless self[:hashtag_mentions].nil? 85 | 86 | # Obtain all hashtag mention entities from all eligible status updates 87 | mentions = status_updates.map do |status_update| 88 | status_update['entities']['hashtags'] 89 | end 90 | 91 | # Count all hashtag references in all entity entries 92 | self[:hashtag_mentions] = Hash.new { 0 } 93 | mention_counter = mentions.each do |mention| 94 | mention.each do |hashtag| 95 | self[:hashtag_mentions]["\##{hashtag['text']}"] += 1 96 | end 97 | end 98 | self.save! 99 | 100 | self[:hashtag_mentions] 101 | end 102 | 103 | # Average number of words per status update associated with this profile 104 | def average_word_count 105 | return 0 if self.collection_yaml.nil? 106 | return self[:average_word_count] unless self[:average_word_count].nil? 107 | 108 | self[:average_word_count] = (self.treat_collection.documents.inject(0) do |t, doc| 109 | t + doc.words.size 110 | end) / self.treat_collection.documents.size 111 | self.save! 112 | 113 | self[:average_word_count] 114 | end 115 | 116 | # All words present in all documents, ranked by their tf_idf scores in a hash 117 | def dictionary 118 | return {} if self.collection_yaml.nil? 119 | return self[:dictionary] unless self[:dictionary].nil? 120 | 121 | document_words = self.treat_collection.documents.map { |doc| doc.words } 122 | 123 | self[:dictionary] = Hash.new 124 | document_words.each do |word_list| 125 | word_list.each do |word| 126 | dictionary[word.to_s.downcase] = word.tf_idf 127 | end 128 | end 129 | self.save! 130 | 131 | self[:dictionary] 132 | end 133 | 134 | # Return the list of stop words associated with the language identified by Treat 135 | # as the primary language used in the status update collection 136 | def stop_words 137 | return {} if self.collection_yaml.nil? 138 | return self[:stop_words] unless self[:stop_words].nil? 139 | 140 | self[:stop_words] = Treat.languages[self.treat_collection.language].stop_words 141 | self.save! 142 | 143 | self[:stop_words] 144 | end 145 | 146 | def build_profile! 147 | # Update current status 148 | self.set(:status, 'indexing') 149 | 150 | Dir.mktmpdir do |directory| 151 | # Create a treat collection for the documents associated with each tweet 152 | collection_path = File.join(directory, 'profile') 153 | status_update_collection = Treat::Entities::Collection.new(collection_path) 154 | 155 | # Iterate over all tweets 156 | self.status_updates.each do |status_update| 157 | status_update_collection << Treat::Entities::Document.new(status_update.text) 158 | end 159 | 160 | # Fully parse and extract entities from the documents contained in the collection 161 | logger.info("Processing #{status_update_collection.documents.size} status updates") 162 | # self.treat_collection = status_update_collection.apply(:chunk, :segment, :tokenize, 163 | # :category, :keywords) 164 | self.treat_collection = status_update_collection.apply(:chunk, :segment, :tokenize, 165 | :category, :name_tag, :keywords, :tag => :stanford) 166 | 167 | self.save! 168 | end 169 | 170 | # Update current status 171 | self.set(:status, 'complete') 172 | end 173 | end 174 | -------------------------------------------------------------------------------- /app/models/status_update.rb: -------------------------------------------------------------------------------- 1 | # 2 | # microphisher - a spear phishing support tool 3 | # 4 | # Created by Ulisses Albuquerque & Joaquim Espinhara 5 | # Copyright (C) 2013 Trustwave Holdings, Inc. 6 | # 7 | # This program is free software: you can redistribute it and/or modify 8 | # it under the terms of the GNU General Public License as published by 9 | # the Free Software Foundation, either version 3 of the License, or 10 | # (at your option) any later version. 11 | # 12 | # This program is distributed in the hope that it will be useful, 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | # GNU General Public License for more details. 16 | # 17 | # You should have received a copy of the GNU General Public License 18 | # along with this program. If not, see . 19 | # 20 | 21 | # Check out https://dev.twitter.com/docs/platform-objects/tweets for the general 22 | # structure of Twitter fields; we just provide field types relevant to the 23 | # filtering/profiling process 24 | class StatusUpdate 25 | include Mongoid::Document 26 | 27 | # Filter criteria field listing 28 | CRITERIA_FIELDS = [ :text, :source, :created_at ] 29 | 30 | # Filter criteria operator listing 31 | CRITERIA_OPERATORS = [ :match, :greater_than, :less_than ] 32 | 33 | # Attribute listing 34 | field :created_at, type: DateTime 35 | 36 | # Relations 37 | belongs_to :data_source 38 | end 39 | -------------------------------------------------------------------------------- /app/models/twitter_data_source.rb: -------------------------------------------------------------------------------- 1 | # 2 | # microphisher - a spear phishing support tool 3 | # 4 | # Created by Ulisses Albuquerque & Joaquim Espinhara 5 | # Copyright (C) 2013 Trustwave Holdings, Inc. 6 | # 7 | # This program is free software: you can redistribute it and/or modify 8 | # it under the terms of the GNU General Public License as published by 9 | # the Free Software Foundation, either version 3 of the License, or 10 | # (at your option) any later version. 11 | # 12 | # This program is distributed in the hope that it will be useful, 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | # GNU General Public License for more details. 16 | # 17 | # You should have received a copy of the GNU General Public License 18 | # along with this program. If not, see . 19 | # 20 | 21 | class TwitterDataSource < DataSource 22 | include Mongoid::Document 23 | 24 | # Callbacks 25 | before_validation do |document| 26 | self.data_source_metadata = user_information(user_id) 27 | end 28 | 29 | before_save do |document| 30 | self.profile_image_url = self.data_source_metadata['profile_image_url'] || '/default_profile_image.png' 31 | end 32 | 33 | # Validations 34 | validates :user_id, presence: true 35 | validate :valid_twitter_user 36 | 37 | def valid_twitter_user 38 | if data_source_metadata.has_key?('errors') 39 | errors.add(:user_id, "#{user_id} is not a valid Twitter user") 40 | end 41 | end 42 | 43 | def twitter_profile_url 44 | "https://www.twitter.com/#{user_id}" 45 | end 46 | 47 | def fetch_status_updates! 48 | self.set(:status, 'indexing') 49 | 50 | begin 51 | self.tweets(self.user_id) do |tweet| 52 | unless status_updates.find_by(id_str: tweet['id_str']) 53 | # HACK: We skip the ID attribute because those should be 54 | # autogenerated by Mongoid/MongoDB, otherwise relations do 55 | # not work properly 56 | status_update = status_updates.new 57 | tweet.each { |k,v| status_update[k] = v unless k.match(/^id$/) } 58 | status_update.save 59 | end 60 | end 61 | rescue Exception => exception 62 | logger.error("Exception indexing #{self.user_id}: #{exception.to_s}") 63 | end 64 | 65 | self.set(:status, 'complete') 66 | self.set(:last_crawl, DateTime.now) 67 | self.save! 68 | end 69 | 70 | protected 71 | USER_TIMELINE_SIZE = 200 72 | USER_TIMELINE_PATH = '/1.1/statuses/user_timeline.json' 73 | USER_INFORMATION_PATH = '/1.1/users/show.json' 74 | 75 | # TODO: Proper error handling 76 | # TODO: Twitter only returns the latest 3,200 tweets for a given 77 | # user timeline; is there some way to overcome this? 78 | def tweets(username) 79 | access_token = unknown_subject.user.access_token 80 | tweet_request = self._tweets_request(username) 81 | tweet_list_json = access_token.get(tweet_request).body 82 | tweet_list = JSON.parse(tweet_list_json) 83 | 84 | # TODO: We should act proactively in order to avoid being 85 | # rate limited; Twitter explicitly mentions applications which 86 | # consistently exceed rate limiting will be blacklisted 87 | while tweet_list.size > 0 && tweet_list.first['errors'].nil? 88 | tweet_list.each { |tweet| yield tweet } 89 | max_id = tweet_list.min_by { |tweet| tweet['id'].to_i }['id'] 90 | tweet_request = self._tweets_request(username, max_id) 91 | tweet_list_json = access_token.get(tweet_request).body 92 | tweet_list = JSON.parse(tweet_list_json) 93 | end 94 | end 95 | 96 | def user_information(screen_name) 97 | access_token = unknown_subject.user.access_token 98 | user_request = self._user_information_request(screen_name) 99 | user_information_json = access_token.get(user_request).body 100 | JSON.parse(user_information_json) 101 | end 102 | 103 | def _tweets_request(username, max_id = nil) 104 | get_parameters = { 'screen_name' => username, 'count' => USER_TIMELINE_SIZE } 105 | get_parameters['max_id'] = max_id unless max_id.nil? 106 | get_request = Net::HTTP::Get.new(USER_TIMELINE_PATH) 107 | get_request.set_form_data(get_parameters) 108 | "#{USER_TIMELINE_PATH}?#{get_request.body}" 109 | end 110 | 111 | def _user_information_request(screen_name) 112 | get_parameters = { 'screen_name' => screen_name } 113 | get_request = Net::HTTP::Get.new(USER_INFORMATION_PATH) 114 | get_request.set_form_data(get_parameters) 115 | "#{USER_INFORMATION_PATH}?#{get_request.body}" 116 | end 117 | end 118 | 119 | -------------------------------------------------------------------------------- /app/models/unknown_subject.rb: -------------------------------------------------------------------------------- 1 | # 2 | # microphisher - a spear phishing support tool 3 | # 4 | # Created by Ulisses Albuquerque & Joaquim Espinhara 5 | # Copyright (C) 2013 Trustwave Holdings, Inc. 6 | # 7 | # This program is free software: you can redistribute it and/or modify 8 | # it under the terms of the GNU General Public License as published by 9 | # the Free Software Foundation, either version 3 of the License, or 10 | # (at your option) any later version. 11 | # 12 | # This program is distributed in the hope that it will be useful, 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | # GNU General Public License for more details. 16 | # 17 | # You should have received a copy of the GNU General Public License 18 | # along with this program. If not, see . 19 | # 20 | 21 | class UnknownSubject 22 | include Mongoid::Document 23 | 24 | # Attribute listing 25 | field :name, type: String 26 | field :description, type: String 27 | 28 | # Validations 29 | validates :name, presence: true 30 | 31 | # Relations 32 | belongs_to :user 33 | has_many :data_sources 34 | has_many :profiles 35 | 36 | def profile_image_url 37 | if data_sources.count > 0 38 | data_sources.first.profile_image_url 39 | else 40 | '/default_profile_image.png' 41 | end 42 | end 43 | 44 | def status_updates 45 | StatusUpdate.in(data_source_id: data_sources.pluck(:id)) 46 | end 47 | end 48 | -------------------------------------------------------------------------------- /app/models/user.rb: -------------------------------------------------------------------------------- 1 | # 2 | # microphisher - a spear phishing support tool 3 | # 4 | # Created by Ulisses Albuquerque & Joaquim Espinhara 5 | # Copyright (C) 2013 Trustwave Holdings, Inc. 6 | # 7 | # This program is free software: you can redistribute it and/or modify 8 | # it under the terms of the GNU General Public License as published by 9 | # the Free Software Foundation, either version 3 of the License, or 10 | # (at your option) any later version. 11 | # 12 | # This program is distributed in the hope that it will be useful, 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | # GNU General Public License for more details. 16 | # 17 | # You should have received a copy of the GNU General Public License 18 | # along with this program. If not, see . 19 | # 20 | 21 | class User 22 | include Mongoid::Document 23 | 24 | # Attribute listing 25 | field :name, type: String 26 | field :real_name, type: String 27 | field :profile_image_url, type: String 28 | field :oauth_token, type: String 29 | field :oauth_secret, type: String 30 | 31 | # Validations 32 | validates :name, :real_name, :profile_image_url, 33 | :oauth_token, :oauth_secret, presence: true 34 | validates :name, uniqueness: true 35 | 36 | # Relations 37 | has_many :unknown_subjects 38 | 39 | # Return an OAuth::AccessToken for the current user 40 | def access_token 41 | oauth_config = YAML.load_file('config/oauth.yml')[Rails.env] 42 | oauth_consumer = OAuth::Consumer.new oauth_config['consumer_key'], 43 | oauth_config['consumer_secret'], { :site => oauth_config['site'] } 44 | OAuth::AccessToken.new(oauth_consumer, oauth_token, oauth_secret) 45 | end 46 | 47 | # Return a user instance based on an OAuth::AccessToken instance 48 | def self.login(access_token) 49 | user_identity_json = access_token.get('/1.1/account/verify_credentials.json').body 50 | user_identity = JSON.parse(user_identity_json) 51 | 52 | # First, check if user already exists in database 53 | user = User.find_by(name: user_identity['screen_name']) 54 | 55 | # In case it does not, create the user based on information from 56 | # OAuth provider (Twitter, in this case) 57 | if user.nil? 58 | # Create the User instance 59 | user = User.create(:id => user_identity['id'], 60 | :name => user_identity['screen_name'], 61 | :real_name => user_identity['name'], 62 | :profile_image_url => user_identity['profile_image_url'], 63 | :oauth_token => access_token.token, 64 | :oauth_secret => access_token.secret) 65 | else 66 | # If the user unlinks microphisher from this Twitter account 67 | # and tries to use the app again, a new oauth_token/oauth_secret 68 | # tuple will be issued, and we need to update the persisted 69 | # instance accordingly 70 | if user.oauth_token != access_token.token || 71 | user.oauth_secret != access_token.secret 72 | user.oauth_token = access_token.token 73 | user.oauth_secret = access_token.secret 74 | user.save! 75 | end 76 | end 77 | 78 | user 79 | end 80 | end 81 | 82 | -------------------------------------------------------------------------------- /app/views/data_sources/_data_source_form.html.erb: -------------------------------------------------------------------------------- 1 | <%# 2 | microphisher - a spear phishing support tool 3 | 4 | Created by Ulisses Albuquerque & Joaquim Espinhara 5 | Copyright (C) 2013 Trustwave Holdings, Inc. 6 | 7 | This program is free software: you can redistribute it and/or modify 8 | it under the terms of the GNU General Public License as published by 9 | the Free Software Foundation, either version 3 of the License, or 10 | (at your option) any later version. 11 | 12 | This program is distributed in the hope that it will be useful, 13 | but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | GNU General Public License for more details. 16 | 17 | You should have received a copy of the GNU General Public License 18 | along with this program. If not, see . 19 | %> 20 | 21 | <%# unknown_subject_form %> 22 | <%= form_for [ unknown_subject, data_source ] do |f| %> 23 |
24 | New Social Network ID 25 | <%= f.label :user_id, 'Screen Name' %> 26 | <%= f.text_field :user_id, :placeholder => 'Twitter user ID' %> 27 |
28 | <%= f.submit %> 29 | <% end -%> 30 | -------------------------------------------------------------------------------- /app/views/data_sources/edit.html.erb: -------------------------------------------------------------------------------- 1 | <%# 2 | microphisher - a spear phishing support tool 3 | 4 | Created by Ulisses Albuquerque & Joaquim Espinhara 5 | Copyright (C) 2013 Trustwave Holdings, Inc. 6 | 7 | This program is free software: you can redistribute it and/or modify 8 | it under the terms of the GNU General Public License as published by 9 | the Free Software Foundation, either version 3 of the License, or 10 | (at your option) any later version. 11 | 12 | This program is distributed in the hope that it will be useful, 13 | but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | GNU General Public License for more details. 16 | 17 | You should have received a copy of the GNU General Public License 18 | along with this program. If not, see . 19 | %> 20 | 21 | <%# page title %> 22 |

<%= link_to 'Unknown Subjects', unknown_subjects_path %> > 23 | <%= link_to @unknown_subject.name, @unknown_subject %> > Edit Social Network ID

24 | 25 | <%# page contents %> 26 | <%= render :partial => 'data_source_form', 27 | :locals => { :unknown_subject => @unknown_subject, :data_source => @data_source } %> 28 | -------------------------------------------------------------------------------- /app/views/data_sources/new.html.erb: -------------------------------------------------------------------------------- 1 | <%# 2 | microphisher - a spear phishing support tool 3 | 4 | Created by Ulisses Albuquerque & Joaquim Espinhara 5 | Copyright (C) 2013 Trustwave Holdings, Inc. 6 | 7 | This program is free software: you can redistribute it and/or modify 8 | it under the terms of the GNU General Public License as published by 9 | the Free Software Foundation, either version 3 of the License, or 10 | (at your option) any later version. 11 | 12 | This program is distributed in the hope that it will be useful, 13 | but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | GNU General Public License for more details. 16 | 17 | You should have received a copy of the GNU General Public License 18 | along with this program. If not, see . 19 | %> 20 | 21 | <%# page title %> 22 |

<%= link_to 'Unknown Subjects', unknown_subjects_path %> > 23 | <%= link_to @unknown_subject.name, @unknown_subject %> > New Social Network ID

24 | 25 | <%# page contents %> 26 | <%= render :partial => 'data_source_form', 27 | :locals => { :unknown_subject => @unknown_subject, :data_source => @data_source } %> 28 | -------------------------------------------------------------------------------- /app/views/data_sources/show.html.erb: -------------------------------------------------------------------------------- 1 | <%# 2 | microphisher - a spear phishing support tool 3 | 4 | Created by Ulisses Albuquerque & Joaquim Espinhara 5 | Copyright (C) 2013 Trustwave Holdings, Inc. 6 | 7 | This program is free software: you can redistribute it and/or modify 8 | it under the terms of the GNU General Public License as published by 9 | the Free Software Foundation, either version 3 of the License, or 10 | (at your option) any later version. 11 | 12 | This program is distributed in the hope that it will be useful, 13 | but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | GNU General Public License for more details. 16 | 17 | You should have received a copy of the GNU General Public License 18 | along with this program. If not, see . 19 | %> 20 | 21 | <%# page title %> 22 |

<%= link_to 'Unknown Subjects', unknown_subjects_path %> > 23 | <%= link_to @unknown_subject.name, @unknown_subject %> > <%= @data_source.user_id %>

24 | 25 | <%= content_tag_for :div, @data_source, :class => 'well media' do -%> 26 | 27 | <%= image_tag @data_source.profile_image_url, :class => 'media-object' %> 28 | 29 |
30 | <%# page contents %> 31 |

32 | <%=h @data_source.user_id %> 33 | 34 | <%= link_to unknown_subject_data_source_path(@unknown_subject, @data_source), 35 | :confirm => "Remove #{@data_source.user_id} and his/her #{@data_source.status_updates.count} posts?", 36 | :method => :delete do %> 37 |
  •  Delete 38 | <% end -%> 39 |     40 |

    41 |
    42 | <% end -%> 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 61 | 62 | 63 | 64 | 66 | 67 |
    Status<%=h @data_source[:status] %>
    Twitter Profile<%= link_to @data_source.user_id, @data_source.twitter_profile_url %>
    Posts Indexed<%=h @data_source.status_updates.count %>
    Earliest Post<%=h @data_source.status_updates.empty? ? 'n/a' : 60 | time_ago_in_words(@data_source.status_updates.min(:created_at)) %>
    Latest Post<%=h @data_source.status_updates.empty? ? 'n/a' : 65 | time_ago_in_words(@data_source.status_updates.max(:created_at)) %>
    68 | 69 | -------------------------------------------------------------------------------- /app/views/home/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SpiderLabs/microphisher/56fefc5d30fa2ef69023266629d77af66144b7cc/app/views/home/.DS_Store -------------------------------------------------------------------------------- /app/views/home/index.html.erb: -------------------------------------------------------------------------------- 1 | <%# 2 | microphisher - a spear phishing support tool 3 | 4 | Created by Ulisses Albuquerque & Joaquim Espinhara 5 | Copyright (C) 2013 Trustwave Holdings, Inc. 6 | 7 | This program is free software: you can redistribute it and/or modify 8 | it under the terms of the GNU General Public License as published by 9 | the Free Software Foundation, either version 3 of the License, or 10 | (at your option) any later version. 11 | 12 | This program is distributed in the hope that it will be useful, 13 | but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | GNU General Public License for more details. 16 | 17 | You should have received a copy of the GNU General Public License 18 | along with this program. If not, see . 19 | %> 20 | 21 | <%# page title %> 22 |

    Home

    23 | 24 | <%# page contents %> 25 | <% if logged_in? -%> 26 | <% if current_user.unknown_subjects.count > 0 -%> 27 |

    Welcome, <%=h current_user.name %>. 28 |

    You have <%= link_to "#{@unknown_subject_count} unknown subjects", 29 | unknown_subjects_path %> registered, with <%=h @data_source_count %> 30 | associated data sources and <%=h @status_update_count %> status 31 | updates indexed.

    32 | <% else -%> 33 |

    Welcome, <%= current_user.name %>. You do not 34 | have any unknown subjects registered, 35 | <%= link_to 'add a new one', new_unknown_subject_path %>.

    36 | <% end -%> 37 | <% else -%> 38 |

    Please <%= link_to 'login with Twitter', :controller => :oauth, 39 | :action => :login %>

    40 | <% end -%> 41 | -------------------------------------------------------------------------------- /app/views/kaminari/_first_page.html.erb: -------------------------------------------------------------------------------- 1 | <%# 2 | microphisher - a spear phishing support tool 3 | 4 | Created by Ulisses Albuquerque & Joaquim Espinhara 5 | Copyright (C) 2013 Trustwave Holdings, Inc. 6 | 7 | This program is free software: you can redistribute it and/or modify 8 | it under the terms of the GNU General Public License as published by 9 | the Free Software Foundation, either version 3 of the License, or 10 | (at your option) any later version. 11 | 12 | This program is distributed in the hope that it will be useful, 13 | but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | GNU General Public License for more details. 16 | 17 | You should have received a copy of the GNU General Public License 18 | along with this program. If not, see . 19 | %> 20 | 21 | <%# Link to the "First" page 22 | - available local variables 23 | url: url to the first page 24 | current_page: a page object for the currently displayed page 25 | total_pages: total number of pages 26 | per_page: number of items to fetch per page 27 | remote: data-remote 28 | -%> 29 | 30 | <%= link_to_unless current_page.first?, raw(t 'views.pagination.first'), url, :remote => remote %> 31 | 32 | -------------------------------------------------------------------------------- /app/views/kaminari/_gap.html.erb: -------------------------------------------------------------------------------- 1 | <%# 2 | microphisher - a spear phishing support tool 3 | 4 | Created by Ulisses Albuquerque & Joaquim Espinhara 5 | Copyright (C) 2013 Trustwave Holdings, Inc. 6 | 7 | This program is free software: you can redistribute it and/or modify 8 | it under the terms of the GNU General Public License as published by 9 | the Free Software Foundation, either version 3 of the License, or 10 | (at your option) any later version. 11 | 12 | This program is distributed in the hope that it will be useful, 13 | but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | GNU General Public License for more details. 16 | 17 | You should have received a copy of the GNU General Public License 18 | along with this program. If not, see . 19 | %> 20 | 21 | <%# Non-link tag that stands for skipped pages... 22 | - available local variables 23 | current_page: a page object for the currently displayed page 24 | total_pages: total number of pages 25 | per_page: number of items to fetch per page 26 | remote: data-remote 27 | -%> 28 | <%= raw(t 'views.pagination.truncate') %> 29 | -------------------------------------------------------------------------------- /app/views/kaminari/_last_page.html.erb: -------------------------------------------------------------------------------- 1 | <%# 2 | microphisher - a spear phishing support tool 3 | 4 | Created by Ulisses Albuquerque & Joaquim Espinhara 5 | Copyright (C) 2013 Trustwave Holdings, Inc. 6 | 7 | This program is free software: you can redistribute it and/or modify 8 | it under the terms of the GNU General Public License as published by 9 | the Free Software Foundation, either version 3 of the License, or 10 | (at your option) any later version. 11 | 12 | This program is distributed in the hope that it will be useful, 13 | but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | GNU General Public License for more details. 16 | 17 | You should have received a copy of the GNU General Public License 18 | along with this program. If not, see . 19 | %> 20 | 21 | <%# Link to the "Last" page 22 | - available local variables 23 | url: url to the last page 24 | current_page: a page object for the currently displayed page 25 | total_pages: total number of pages 26 | per_page: number of items to fetch per page 27 | remote: data-remote 28 | -%> 29 | 30 | <%= link_to_unless current_page.last?, raw(t 'views.pagination.last'), url, {:remote => remote} %> 31 | 32 | -------------------------------------------------------------------------------- /app/views/kaminari/_next_page.html.erb: -------------------------------------------------------------------------------- 1 | <%# 2 | microphisher - a spear phishing support tool 3 | 4 | Created by Ulisses Albuquerque & Joaquim Espinhara 5 | Copyright (C) 2013 Trustwave Holdings, Inc. 6 | 7 | This program is free software: you can redistribute it and/or modify 8 | it under the terms of the GNU General Public License as published by 9 | the Free Software Foundation, either version 3 of the License, or 10 | (at your option) any later version. 11 | 12 | This program is distributed in the hope that it will be useful, 13 | but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | GNU General Public License for more details. 16 | 17 | You should have received a copy of the GNU General Public License 18 | along with this program. If not, see . 19 | %> 20 | 21 | <%# Link to the "Next" page 22 | - available local variables 23 | url: url to the next page 24 | current_page: a page object for the currently displayed page 25 | total_pages: total number of pages 26 | per_page: number of items to fetch per page 27 | remote: data-remote 28 | -%> 29 | 30 | <%= link_to_unless current_page.last?, raw(t 'views.pagination.next'), url, :rel => 'next', :remote => remote %> 31 | 32 | -------------------------------------------------------------------------------- /app/views/kaminari/_page.html.erb: -------------------------------------------------------------------------------- 1 | <%# 2 | microphisher - a spear phishing support tool 3 | 4 | Created by Ulisses Albuquerque & Joaquim Espinhara 5 | Copyright (C) 2013 Trustwave Holdings, Inc. 6 | 7 | This program is free software: you can redistribute it and/or modify 8 | it under the terms of the GNU General Public License as published by 9 | the Free Software Foundation, either version 3 of the License, or 10 | (at your option) any later version. 11 | 12 | This program is distributed in the hope that it will be useful, 13 | but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | GNU General Public License for more details. 16 | 17 | You should have received a copy of the GNU General Public License 18 | along with this program. If not, see . 19 | %> 20 | 21 | <%# Link showing page number 22 | - available local variables 23 | page: a page object for "this" page 24 | url: url to this page 25 | current_page: a page object for the currently displayed page 26 | total_pages: total number of pages 27 | per_page: number of items to fetch per page 28 | remote: data-remote 29 | -%> 30 | 31 | <%= link_to_unless page.current?, page, url, {:remote => remote, :rel => page.next? ? 'next' : page.prev? ? 'prev' : nil} %> 32 | 33 | -------------------------------------------------------------------------------- /app/views/kaminari/_paginator.html.erb: -------------------------------------------------------------------------------- 1 | <%# 2 | microphisher - a spear phishing support tool 3 | 4 | Created by Ulisses Albuquerque & Joaquim Espinhara 5 | Copyright (C) 2013 Trustwave Holdings, Inc. 6 | 7 | This program is free software: you can redistribute it and/or modify 8 | it under the terms of the GNU General Public License as published by 9 | the Free Software Foundation, either version 3 of the License, or 10 | (at your option) any later version. 11 | 12 | This program is distributed in the hope that it will be useful, 13 | but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | GNU General Public License for more details. 16 | 17 | You should have received a copy of the GNU General Public License 18 | along with this program. If not, see . 19 | %> 20 | 21 | <%# The container tag 22 | - available local variables 23 | current_page: a page object for the currently displayed page 24 | total_pages: total number of pages 25 | per_page: number of items to fetch per page 26 | remote: data-remote 27 | paginator: the paginator that renders the pagination tags inside 28 | -%> 29 | <%= paginator.render do -%> 30 |