├── spec ├── spec.opts ├── spec_helper.rb ├── bankjob_cli_spec.rb ├── transaction_spec.rb └── statement_spec.rb ├── PostInstall.txt ├── config ├── website.yml └── website.yml.sample ├── bin └── bankjob ├── lib ├── bankjob.rb └── bankjob │ ├── payee.rb │ ├── bankjob_runner.rb │ ├── support.rb │ ├── cli.rb │ ├── transaction.rb │ ├── statement.rb │ └── scraper.rb ├── script ├── destroy ├── generate ├── console └── txt2html ├── Manifest.txt ├── tasks └── rspec.rake ├── History.txt ├── Rakefile ├── website ├── template.html.erb ├── stylesheets │ └── screen.css ├── index.txt └── javascripts │ └── rounded_corners_lite.inc.js ├── README.rdoc └── scrapers ├── base_scraper.rb └── bpi_scraper.rb /spec/spec.opts: -------------------------------------------------------------------------------- 1 | --colour -------------------------------------------------------------------------------- /PostInstall.txt: -------------------------------------------------------------------------------- 1 | 2 | For more information on bankjob, see http://bankjob.rubyforge.org 3 | 4 | 5 | -------------------------------------------------------------------------------- /config/website.yml: -------------------------------------------------------------------------------- 1 | host: rhubarb@rubyforge.org 2 | remote_dir: /var/www/gforge-projects/bankjob 3 | -------------------------------------------------------------------------------- /config/website.yml.sample: -------------------------------------------------------------------------------- 1 | host: unknown@rubyforge.org 2 | remote_dir: /var/www/gforge-projects/bankjob -------------------------------------------------------------------------------- /spec/spec_helper.rb: -------------------------------------------------------------------------------- 1 | begin 2 | require 'spec' 3 | rescue LoadError 4 | require 'rubygems' 5 | gem 'rspec' 6 | require 'spec' 7 | end 8 | 9 | $:.unshift(File.dirname(__FILE__) + '/../lib') 10 | require 'bankjob.rb' 11 | -------------------------------------------------------------------------------- /bin/bankjob: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | # 3 | # Created on 2009-2-13. 4 | # Copyright (c) 2009. All rights reserved. 5 | 6 | require File.expand_path(File.dirname(__FILE__) + "/../lib/bankjob") 7 | 8 | require "bankjob/cli" 9 | 10 | Bankjob::CLI.execute(STDOUT, ARGV) 11 | -------------------------------------------------------------------------------- /lib/bankjob.rb: -------------------------------------------------------------------------------- 1 | $:.unshift(File.dirname(__FILE__)) unless 2 | $:.include?(File.dirname(__FILE__)) || $:.include?(File.expand_path(File.dirname(__FILE__))) 3 | 4 | require 'bankjob/support.rb' 5 | require 'bankjob/statement.rb' 6 | require 'bankjob/transaction.rb' 7 | require 'bankjob/scraper.rb' 8 | require 'bankjob/payee.rb' 9 | 10 | module Bankjob 11 | BANKJOB_VERSION = '0.5.2' unless defined?(BANKJOB_VERSION) 12 | end 13 | -------------------------------------------------------------------------------- /spec/bankjob_cli_spec.rb: -------------------------------------------------------------------------------- 1 | equire File.expand_path(File.dirname(__FILE__) + '/spec_helper') 2 | require 'bankjob/cli' 3 | 4 | describe Bankjob::CLI, "execute" do 5 | before(:each) do 6 | @stdout_io = StringIO.new 7 | Bankjob::CLI.execute(@stdout_io, []) 8 | @stdout_io.rewind 9 | @stdout = @stdout_io.read 10 | end 11 | 12 | it "should do something" do 13 | @stdout.should_not =~ /To update this executable/ 14 | end 15 | end -------------------------------------------------------------------------------- /script/destroy: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | APP_ROOT = File.expand_path(File.join(File.dirname(__FILE__), '..')) 3 | 4 | begin 5 | require 'rubigen' 6 | rescue LoadError 7 | require 'rubygems' 8 | require 'rubigen' 9 | end 10 | require 'rubigen/scripts/destroy' 11 | 12 | ARGV.shift if ['--help', '-h'].include?(ARGV[0]) 13 | RubiGen::Base.use_component_sources! [:rubygems, :newgem, :newgem_theme, :test_unit] 14 | RubiGen::Scripts::Destroy.new.run(ARGV) 15 | -------------------------------------------------------------------------------- /script/generate: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | APP_ROOT = File.expand_path(File.join(File.dirname(__FILE__), '..')) 3 | 4 | begin 5 | require 'rubigen' 6 | rescue LoadError 7 | require 'rubygems' 8 | require 'rubigen' 9 | end 10 | require 'rubigen/scripts/generate' 11 | 12 | ARGV.shift if ['--help', '-h'].include?(ARGV[0]) 13 | RubiGen::Base.use_component_sources! [:rubygems, :newgem, :newgem_theme, :test_unit] 14 | RubiGen::Scripts::Generate.new.run(ARGV) 15 | -------------------------------------------------------------------------------- /script/console: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | # File: script/console 3 | irb = RUBY_PLATFORM =~ /(:?mswin|mingw)/ ? 'irb.bat' : 'irb' 4 | 5 | libs = " -r irb/completion" 6 | # Perhaps use a console_lib to store any extra methods I may want available in the cosole 7 | # libs << " -r #{File.dirname(__FILE__) + '/../lib/console_lib/console_logger.rb'}" 8 | libs << " -r #{File.dirname(__FILE__) + '/../lib/bankjob.rb'}" 9 | puts "Loading bankjob gem" 10 | exec "#{irb} #{libs} --simple-prompt" -------------------------------------------------------------------------------- /Manifest.txt: -------------------------------------------------------------------------------- 1 | History.txt 2 | PostInstall.txt 3 | README.rdoc 4 | bin/bankjob 5 | lib/bankjob.rb 6 | lib/bankjob/bankjob_runner.rb 7 | lib/bankjob/cli.rb 8 | lib/bankjob/payee.rb 9 | lib/bankjob/scraper.rb 10 | lib/bankjob/statement.rb 11 | lib/bankjob/support.rb 12 | lib/bankjob/transaction.rb 13 | scrapers/base_scraper.rb 14 | scrapers/bpi_scraper.rb 15 | spec/bankjob_cli_spec.rb 16 | spec/spec.opts 17 | spec/spec_helper.rb 18 | spec/statement_spec.rb 19 | spec/transaction_spec.rb 20 | -------------------------------------------------------------------------------- /tasks/rspec.rake: -------------------------------------------------------------------------------- 1 | begin 2 | require 'spec' 3 | rescue LoadError 4 | require 'rubygems' 5 | require 'spec' 6 | end 7 | begin 8 | require 'spec/rake/spectask' 9 | rescue LoadError 10 | puts <<-EOS 11 | To use rspec for testing you must install rspec gem: 12 | gem install rspec 13 | EOS 14 | exit(0) 15 | end 16 | 17 | desc "Run the specs under spec/models" 18 | Spec::Rake::SpecTask.new do |t| 19 | t.spec_opts = ['--options', "spec/spec.opts"] 20 | t.spec_files = FileList['spec/**/*_spec.rb'] 21 | end 22 | -------------------------------------------------------------------------------- /History.txt: -------------------------------------------------------------------------------- 1 | == 0.5.2 2009-05-18 2 | * 3 minor enhancements: 3 | * Added the ability to set fake timestamps on transactions so that transactions that have 00:00 for their time stamp that occur on the same day will be in the correct order in Wesabe 4 | To use this in your scraper, call statement#finish(true, true) 5 | * bpi_scraper.rb now sets the account number on the statement after scraping it. It turns out this is important for uploading multiple accounts to Wesabe - the accounts get mixed if the account numbers are wrong 6 | * the Statement#finish method also updates the balance and start and end dates of the statement after the last transaction is scraped. This used to be done automatically in the attribute getters but did not support statements with most-recent transaction last (only first). 7 | 8 | === 0.5.1 2009-04-20 9 | * 1 minor enhancement: 10 | * bpi_scraper.rb now accepts an optional third argument for the account number. Entering a number will cause the scraper to scrape that account rather than the default account. 11 | 12 | == 0.5.0 2009-04-14 13 | * 1 major enhancement: 14 | * Initial release 15 | -------------------------------------------------------------------------------- /Rakefile: -------------------------------------------------------------------------------- 1 | %w[rubygems rake rake/clean fileutils newgem rubigen].each { |f| require f } 2 | require File.dirname(__FILE__) + '/lib/bankjob' 3 | 4 | # Generate all the Rake tasks 5 | # Run 'rake -T' to see list of generated tasks (from gem root directory) 6 | $hoe = Hoe.new('bankjob', Bankjob::BANKJOB_VERSION) do |p| 7 | p.developer('rhubarb', 'rhubarb.bankjob@gmail.com') 8 | p.changes = p.paragraphs_of("History.txt", 0..1).join("\n\n") 9 | p.post_install_message = 'PostInstall.txt' # TODO remove if post-install message not required 10 | p.rubyforge_name = p.name # TODO this is default value 11 | p.extra_deps = [ 12 | ['hpricot', '>= 0.6'], 13 | ['mechanize', '>= 0.7.5'], 14 | ['builder', '>= 2.1.2'], 15 | ['fastercsv', '>= 1.2.3'] 16 | ] 17 | # p.extra_dev_deps = [ 18 | # ['newgem', ">= #{::Newgem::VERSION}"] 19 | # ] 20 | 21 | p.clean_globs |= %w[**/.DS_Store tmp *.log] 22 | path = (p.rubyforge_name == p.name) ? p.rubyforge_name : "\#{p.rubyforge_name}/\#{p.name}" 23 | p.remote_rdoc_dir = File.join(path.gsub(/^#{p.rubyforge_name}\/?/,''), 'rdoc') 24 | p.rsync_args = '-av --delete --ignore-errors' 25 | end 26 | 27 | require 'newgem/tasks' # load /tasks/*.rake 28 | Dir['tasks/**/*.rake'].each { |t| load t } 29 | 30 | # TODO - want other tests/tasks run by default? Add them to the list 31 | # task :default => [:spec, :features] 32 | -------------------------------------------------------------------------------- /website/template.html.erb: -------------------------------------------------------------------------------- 1 | 3 | 4 | 5 | 6 | 7 | 8 | <%= title %> 9 | 10 | 11 | 14 | 29 | 30 | 31 |
32 | 33 |

<%= title %>

34 |
35 |
36 |

Get Version

37 | <%= version %> 38 |
39 |
40 | <%= body %> 41 |

42 | Rhubarb, <%= modified.pretty %>
43 | Theme extended from Paul Battley 44 |

45 |
46 | 47 | 48 | 49 | 50 | 51 | -------------------------------------------------------------------------------- /script/txt2html: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | 3 | load File.dirname(__FILE__) + "/../Rakefile" 4 | require 'rubyforge' 5 | require 'redcloth' 6 | require 'syntax/convertors/html' 7 | require 'erb' 8 | 9 | download = "http://rubyforge.org/projects/#{$hoe.rubyforge_name}" 10 | version = $hoe.version 11 | 12 | def rubyforge_project_id 13 | RubyForge.new.configure.autoconfig["group_ids"][$hoe.rubyforge_name] 14 | end 15 | 16 | class Fixnum 17 | def ordinal 18 | # teens 19 | return 'th' if (10..19).include?(self % 100) 20 | # others 21 | case self % 10 22 | when 1: return 'st' 23 | when 2: return 'nd' 24 | when 3: return 'rd' 25 | else return 'th' 26 | end 27 | end 28 | end 29 | 30 | class Time 31 | def pretty 32 | return "#{mday}#{mday.ordinal} #{strftime('%B')} #{year}" 33 | end 34 | end 35 | 36 | def convert_syntax(syntax, source) 37 | return Syntax::Convertors::HTML.for_syntax(syntax).convert(source).gsub(%r!^
|
$!,'') 38 | end 39 | 40 | if ARGV.length >= 1 41 | src, template = ARGV 42 | template ||= File.join(File.dirname(__FILE__), '/../website/template.html.erb') 43 | else 44 | puts("Usage: #{File.split($0).last} source.txt [template.html.erb] > output.html") 45 | exit! 46 | end 47 | 48 | template = ERB.new(File.open(template).read) 49 | 50 | title = nil 51 | body = nil 52 | File.open(src) do |fsrc| 53 | title_text = fsrc.readline 54 | body_text_template = fsrc.read 55 | body_text = ERB.new(body_text_template).result(binding) 56 | syntax_items = [] 57 | body_text.gsub!(%r!<(pre|code)[^>]*?syntax=['"]([^'"]+)[^>]*>(.*?)!m){ 58 | ident = syntax_items.length 59 | element, syntax, source = $1, $2, $3 60 | syntax_items << "<#{element} class='syntax'>#{convert_syntax(syntax, source)}" 61 | "syntax-temp-#{ident}" 62 | } 63 | title = RedCloth.new(title_text).to_html.gsub(%r!<.*?>!,'').strip 64 | body = RedCloth.new(body_text).to_html 65 | body.gsub!(%r!(?:
)?syntax-temp-(\d+)(?:
)?!){ syntax_items[$1.to_i] } 66 | end 67 | stat = File.stat(src) 68 | created = stat.ctime 69 | modified = stat.mtime 70 | 71 | $stdout << template.result(binding) 72 | -------------------------------------------------------------------------------- /spec/transaction_spec.rb: -------------------------------------------------------------------------------- 1 | require File.expand_path(File.dirname(__FILE__) + '/spec_helper') 2 | 3 | require 'bankjob.rb' 4 | include Bankjob 5 | 6 | describe Transaction do 7 | before(:each) do 8 | @tx1 = Transaction.new() 9 | @tx1.date = "30-7-2008" 10 | @tx1.value_date = "20080731145906" 11 | @tx1.raw_description = "Some tax thing 10493" 12 | @tx1.amount = "-2,40" 13 | @tx1.new_balance = "1.087,43" 14 | 15 | @tx1_copy = Transaction.new() 16 | @tx1_copy.date = "30-7-2008" 17 | @tx1_copy.value_date = "20080731145906" 18 | @tx1_copy.raw_description = "Some tax thing 10493" 19 | @tx1_copy.amount = "-2,40" 20 | @tx1_copy.new_balance = "1.087,43" 21 | 22 | @tx1_dup = @tx1.dup 23 | 24 | @tx2 = Transaction.new() 25 | @tx2.date = "0080729000000" 26 | @tx2.value_date = "20080731145906" 27 | @tx2.raw_description = "Interest payment" 28 | @tx2.amount = "-59,94" 29 | @tx2.new_balance = "1.089,83" 30 | end 31 | 32 | it "should generate the same ofx_id as its copy" do 33 | puts "tx1: #{@tx1.to_s}\n-----" 34 | puts "tx1_copy: #{@tx1.to_s}" 35 | @tx1.ofx_id.should == @tx1_copy.ofx_id 36 | puts "#{@tx1.ofx_id} == #{@tx1_copy.ofx_id}" 37 | end 38 | 39 | it "should generate the same ofx_id as its duplicate" do 40 | @tx1.ofx_id.should == @tx1_dup.ofx_id 41 | end 42 | 43 | 44 | it "should be == to its duplicate" do 45 | @tx1.should == @tx1_dup 46 | end 47 | 48 | it "should be == to its identical copy" do 49 | @tx1.should == @tx1_copy 50 | end 51 | 52 | it "should not == a different transaction" do 53 | @tx1.should_not == @tx2 54 | end 55 | 56 | it "should be eql to its duplicate (necessary for merging)" do 57 | @tx1.should eql(@tx1_dup) 58 | end 59 | 60 | it "should not be equal to its duplicate" do 61 | @tx1.should_not equal(@tx1_dup) 62 | end 63 | 64 | it "should be === to its duplicate" do 65 | @tx1.should === @tx1_dup 66 | end 67 | 68 | it "should have the same hash as its duplicate" do 69 | @tx1.hash.should == @tx1_dup.hash 70 | end 71 | 72 | it "should convert 1,000,000.32 to 1000000.32 when decimal separator is ." do 73 | Bankjob.string_to_float("1,000,000.32", ".").should == 1000000.32 74 | end 75 | 76 | it "should convert 1.000.000,32 to 1000000.32 when decimal separator is ," do 77 | Bankjob.string_to_float("1.000.000,32", ",").should == 1000000.32 78 | end 79 | 80 | end 81 | 82 | -------------------------------------------------------------------------------- /website/stylesheets/screen.css: -------------------------------------------------------------------------------- 1 | body { 2 | background-color: #8DBD82; 3 | font-family: "Georgia", sans-serif; 4 | font-size: 16px; 5 | line-height: 1.6em; 6 | padding: 1.6em 0 0 0; 7 | color: #333; 8 | } 9 | h1, h2, h3, h4, h5, h6 { 10 | color: #444; 11 | } 12 | h1 { 13 | font-family: sans-serif; 14 | font-weight: normal; 15 | font-size: 4em; 16 | line-height: 0.8em; 17 | letter-spacing: -0.1ex; 18 | margin: 5px; 19 | } 20 | li { 21 | padding: 0; 22 | margin: 0; 23 | list-style-type: square; 24 | } 25 | a { 26 | color: #5E5AFF; 27 | background-color: #A1DDB1; 28 | font-weight: normal; 29 | text-decoration: underline; 30 | } 31 | blockquote { 32 | font-size: 90%; 33 | font-style: italic; 34 | border-left: 1px solid #111; 35 | padding-left: 1em; 36 | } 37 | .caps { 38 | font-size: 80%; 39 | } 40 | 41 | #main { 42 | width: 55em; 43 | padding: 0; 44 | margin: 0 auto; 45 | } 46 | .coda { 47 | text-align: right; 48 | color: #77f; 49 | font-size: smaller; 50 | } 51 | 52 | table { 53 | font-size: 90%; 54 | line-height: 1.4em; 55 | color: #ff8; 56 | background-color: #111; 57 | padding: 2px 10px 2px 10px; 58 | border-style: dashed; 59 | } 60 | 61 | th { 62 | color: #fff; 63 | } 64 | 65 | td { 66 | padding: 2px 10px 2px 10px; 67 | } 68 | 69 | .success { 70 | color: #0CC52B; 71 | } 72 | 73 | .failed { 74 | color: #E90A1B; 75 | } 76 | 77 | .unknown { 78 | color: #995000; 79 | } 80 | pre, code { 81 | font-family: monospace; 82 | font-size: 90%; 83 | line-height: 1.4em; 84 | color: #ff8; 85 | background-color: #111; 86 | width: 40em; 87 | padding: 2px 10px 2px 10px; 88 | } 89 | .comment { color: #aaa; font-style: italic; } 90 | .keyword { color: #eff; font-weight: bold; } 91 | .punct { color: #eee; font-weight: bold; } 92 | .symbol { color: #0bb; } 93 | .string { color: #6b4; } 94 | .ident { color: #ff8; } 95 | .constant { color: #66f; } 96 | .regex { color: #ec6; } 97 | .number { color: #F99; } 98 | .expr { color: #227; } 99 | 100 | .sidebar { 101 | float: right; 102 | } 103 | 104 | #version { 105 | width: 217px; 106 | text-align: right; 107 | font-family: sans-serif; 108 | font-weight: normal; 109 | color: #141331; 110 | padding: 15px 20px 10px 20px; 111 | margin: 0 auto; 112 | margin-top: 15px; 113 | background-color: #9A5535; 114 | border: 3px solid #7E393E; 115 | } 116 | 117 | #version .numbers { 118 | display: block; 119 | font-size: 4em; 120 | line-height: 0.8em; 121 | letter-spacing: -0.1ex; 122 | margin-bottom: 15px; 123 | } 124 | 125 | #version p { 126 | text-decoration: none; 127 | color: #F1F4FF; 128 | background-color: #9A5535; 129 | margin: 0; 130 | padding: 0; 131 | } 132 | 133 | #version a { 134 | text-decoration: none; 135 | color: #F1F4FF; 136 | background-color: #9A5535; 137 | } 138 | 139 | .clickable { 140 | cursor: pointer; 141 | cursor: hand; 142 | } 143 | 144 | #twitter_search { 145 | margin: 40px 0 10px 15px; 146 | color: #F1F4FF; 147 | background-color: #9A5535; 148 | border: 3px solid #7E393E; 149 | } 150 | 151 | #twitter_search h3 { 152 | color: #F1F4FF; 153 | margin-bottom: 0px; 154 | } 155 | 156 | #twitter_search center b { 157 | display: none; 158 | } 159 | 160 | -------------------------------------------------------------------------------- /spec/statement_spec.rb: -------------------------------------------------------------------------------- 1 | require File.expand_path(File.dirname(__FILE__) + '/spec_helper') 2 | #require File.expand_path(File.dirname(__FILE__) + '/../lib/bankjob.rb') 3 | 4 | include Bankjob 5 | 6 | # Test the Statement merging in particular 7 | describe Statement do 8 | before(:each) do 9 | 10 | @tx1 = Transaction.new(",") 11 | @tx1.date = "20080730000000" 12 | @tx1.value_date = "20080731145906" 13 | @tx1.raw_description = "1 Stamp duty 001" 14 | @tx1.amount = "-2,40" 15 | @tx1.new_balance = "1.087,43" 16 | 17 | 18 | @tx2 = Transaction.new(",") 19 | @tx2.date = "0080729000000" 20 | @tx2.value_date = "20080731145906" 21 | @tx2.raw_description = "2 Interest payment 001" 22 | @tx2.amount = "-59,94" 23 | @tx2.new_balance = "1.089,83" 24 | 25 | 26 | @tx3 = Transaction.new(",") 27 | @tx3.date = "20080208000000" 28 | @tx3.value_date = "20080731145906" 29 | @tx3.raw_description = "3 Load payment 001" 30 | @tx3.amount = "-256,13" 31 | @tx3.new_balance = "1.149,77" 32 | 33 | 34 | @tx4 = Transaction.new(",") 35 | @tx4.date = "20080207000000" 36 | @tx4.value_date = "20080731145906" 37 | @tx4.raw_description = "4 Transfer to bank 2" 38 | @tx4.amount = "-1.000,00" 39 | @tx4.new_balance = "1.405,90" 40 | 41 | 42 | @tx5 = Transaction.new(",") 43 | @tx5.date = "20080209000000" 44 | @tx5.value_date = "20080731145906" 45 | @tx5.raw_description = "5 Internet payment 838" 46 | @tx5.amount = "-32,07" 47 | @tx5.new_balance = "1.405,90" 48 | 49 | # the lot 50 | @s12345 = Statement.new 51 | @s12345.transactions = [ @tx1.dup, @tx2.dup, @tx3.dup, @tx4.dup, @tx5.dup] 52 | 53 | # first 2 54 | @s12 = Statement.new 55 | @s12.transactions = [ @tx1.dup, @tx2.dup] 56 | 57 | # middle 1 58 | @s3 = Statement.new 59 | @s3.transactions = [ @tx3.dup] 60 | 61 | # last 2 62 | @s45 = Statement.new 63 | @s45.transactions = [ @tx4.dup, @tx5.dup] 64 | 65 | # first 3 66 | @s123 = Statement.new 67 | @s123.transactions = [ @tx1.dup, @tx2.dup, @tx3.dup] 68 | 69 | # last 4, overlaps with 23 of s123 70 | @s2345 = Statement.new 71 | @s2345.transactions = [ @tx2.dup, @tx3.dup, @tx4.dup, @tx5.dup] 72 | 73 | # 2nd and last - overlaps non-contiguously with s123 74 | @s25 = Statement.new 75 | @s25.transactions = [ @tx2.dup, @tx5.dup] 76 | 77 | end 78 | 79 | it "should merge consecutive satements properly" do 80 | @s123.merge(@s45).should == @s12345 81 | end 82 | 83 | it "should merge overlapping statments properly" do 84 | #@s123.merge(@s2345).transactions.each { |tx| print "#{tx.to_s}, "} 85 | @s123.merge(@s2345).should == @s12345 86 | end 87 | 88 | it "should merge a statement with a duplicate of itself without changing it" do 89 | @s123.merge(@s123.dup).should == @s123 90 | end 91 | 92 | 93 | it "should merge non-contiguous with an error" do 94 | m = @s123.merge(@s25) 95 | m.transactions.each { |tx| print "#{tx.to_s}, "} 96 | end 97 | 98 | it "should read back a satement from csv as it was written" do 99 | csv = @s123.to_csv 100 | statement = Statement.new() 101 | statement.from_csv(csv, ",") 102 | statement.should == @s123 103 | end 104 | 105 | it "should read back and merge a statement with itself without change" do 106 | csv = @s123.to_csv 107 | statement = Statement.new() 108 | statement.from_csv(csv, ",") 109 | m = @s123.merge(statement) 110 | m.should == @s123 111 | end 112 | 113 | it "should write, read, merge and write a statement without changing it" do 114 | csv = @s123.to_csv 115 | statement = Statement.new() 116 | m = @s123.merge(statement) 117 | m_csv = m.to_csv 118 | m_csv.should == csv 119 | end 120 | end 121 | 122 | -------------------------------------------------------------------------------- /README.rdoc: -------------------------------------------------------------------------------- 1 | = bankjob 2 | 3 | http://bankjob.rubyforge.org/ 4 | 5 | == DESCRIPTION: 6 | 7 | Bankjob is a command-line ruby program for scraping online banking sites and producing statements in OFX (Open Fincancial Exchange) or CSV (Comma Separated Values) formats. 8 | 9 | Bankjob was created for people like me who want to get their bank data into a 3rd party application but whose bank does not support downloads in OFX format. 10 | It's also useful for keeping a permanent store of bank statements on your computer for reading in Excel (vs filing paper statements) 11 | 12 | == FEATURES: 13 | 14 | * Scrapes an online banking website to produce a bank statement 15 | * Stores bank statements locally in CSV files, which can be loaded directly in spreadsheets like Microsoft Excel 16 | * Stores bank statements locally in OFX files, which can be imported by many programs such as Quicken, MS Money, Gnu Cash and uploaded to some web applications 17 | * Built-in support for uploading to your Wesabe account (www.wesabe.com) 18 | * Supports coding of simple rules in ruby for modifying transaction details. E.g. automatically change "pment inst 3245003" to "paid home loan interest" 19 | 20 | == SYNOPSIS: 21 | 22 | bankjob --csv c:\bank\csv --scraper c:\bank\my_bpi_scraper.rb 23 | --scraper-args " " 24 | --wesabe " 25 | --log c:\bank\bankjob.log --debug 26 | 27 | I have this command in a .bat file which is launched daily by a scheduled task on my windows Media Center PC (which, since it's always on and connected to the internet, makes a useful home server) 28 | 29 | This one command will: 30 | * scrape my online banking website after logging in as me and navigating to the page with recent transactions 31 | * apply some rules, coded in the my_bpi_scraper.rb file that make the descriptions more readable 32 | * produce a statement in comma-separated-value format, keeping the original raw data as well as the new descriptions, 33 | storing that in a file with a name like "20090327-20090406.csv" in my local directory c:\bank\csv (a permanent record) 34 | * produce an OFX document with the same statement information 35 | * upload the OFX statement to my wesabe account 36 | * log debug-level details in bankjob.log 37 | 38 | == REQUIREMENTS: 39 | 40 | * Runs in ruby so you need to have ruby installed 41 | * Requires a scraper for your online bank site 42 | Some examples come packaged with Bankjob but you will probably need to write your own scraper in ruby. 43 | For help go to http://groups.google.com/group/bankjob, but read http://bankjob.rubyforge.org first. 44 | 45 | == INSTALL: 46 | 47 | Mac OSX (linux): 48 | 49 | sudo gem install bankjob 50 | 51 | Windows: 52 | gem install bankjob 53 | 54 | == LICENSE: 55 | 56 | (The MIT License) 57 | 58 | Copyright (c) 2009 rubarb.bankjob@gmail.com 59 | 60 | Permission is hereby granted, free of charge, to any person obtaining 61 | a copy of this software and associated documentation files (the 62 | 'Software'), to deal in the Software without restriction, including 63 | without limitation the rights to use, copy, modify, merge, publish, 64 | distribute, sublicense, and/or sell copies of the Software, and to 65 | permit persons to whom the Software is furnished to do so, subject to 66 | the following conditions: 67 | 68 | The above copyright notice and this permission notice shall be 69 | included in all copies or substantial portions of the Software. 70 | 71 | THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND, 72 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 73 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. 74 | IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY 75 | CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, 76 | TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE 77 | SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 78 | -------------------------------------------------------------------------------- /lib/bankjob/payee.rb: -------------------------------------------------------------------------------- 1 | 2 | require 'rubygems' 3 | require 'builder' 4 | require 'digest/md5' 5 | 6 | module Bankjob 7 | 8 | ## 9 | # A Payee object represents an entity in a in a bank Transaction that receives a payment. 10 | # 11 | # A Scraper will create Payees while scraping web pages in an online banking site. 12 | # In many cases Payees will not be distinguished in the online bank site in which case 13 | # rules will have to be applied to separate the Payees 14 | # 15 | # A Payee object knows how to write itself as a record in a CSV 16 | # (Comma Separated Values) file using +to_csv+ or as an XML element in an 17 | # OFX (Open Financial eXchange http://www.ofx.net) file using +to_ofx+ 18 | # 19 | class Payee 20 | 21 | # name of the payee 22 | # Translates to OFX element NAME 23 | attr_accessor :name 24 | 25 | # address of the payee 26 | # Translates to OFX element ADDR1 27 | #-- TODO Consider ADDR2,3 28 | attr_accessor :address 29 | 30 | # city in which the payee is located 31 | # Translates to OFX element CITY 32 | attr_accessor :city 33 | 34 | # state in which the payee is located 35 | # Translates to OFX element STATE 36 | attr_accessor :state 37 | 38 | # post code or zip in which the payee is located 39 | # Translates to OFX element POSTALCODE 40 | attr_accessor :postalcode 41 | 42 | # country in which the payee is located 43 | # Translates to OFX element COUNTRY 44 | attr_accessor :country 45 | 46 | # phone number of the payee 47 | # Translates to OFX element PHONE 48 | attr_accessor :phone 49 | 50 | ## 51 | # Generates a string representing this Payee as a single string for use 52 | # in a comma separated values column 53 | # 54 | def to_csv 55 | name 56 | end 57 | 58 | ## 59 | # Generates an XML string adhering to the OFX standard 60 | # (see Open Financial Exchange http://www.ofx.net) 61 | # representing a single Payee XML element. 62 | # 63 | # The schema for the OFX produced is 64 | # 65 | # 66 | # 67 | # 68 | # The OFX element "PAYEE" is of type "Payee" 69 | # 70 | # 71 | # 72 | # 73 | # 74 | # 75 | # 76 | # 77 | # 78 | # 79 | # 80 | # 81 | # 82 | # 83 | # 84 | # 85 | # 86 | # 87 | # 88 | def to_ofx 89 | buf = "" 90 | # Set margin=6 to indent it nicely within the output from Transaction.to_ofx 91 | x = Builder::XmlMarkup.new(:target => buf, :indent => 2, :margin=>6) 92 | x.PAYEE { 93 | x.NAME name 94 | x.ADDR1 address 95 | x.CITY city 96 | x.STATE state 97 | x.POSTALCODE postalcode 98 | x.COUNTRY country unless country.nil? # minOccurs="0" in schema (above) 99 | x.PHONE phone 100 | } 101 | return buf 102 | end 103 | 104 | ## 105 | # Produces the Payee as a row of comma separated values 106 | # (delegates to +to_csv+) 107 | # 108 | def to_s 109 | to_csv 110 | end 111 | 112 | end # class Payee 113 | end # module 114 | 115 | -------------------------------------------------------------------------------- /scrapers/base_scraper.rb: -------------------------------------------------------------------------------- 1 | require 'rubygems' 2 | require 'mechanize' 3 | require 'hpricot' 4 | require 'bankjob' 5 | 6 | # Later versions of Mechanize no longer use Hpricot by default 7 | # but have an attribute we can set to use it 8 | begin 9 | WWW::Mechanize.html_parser = Hpricot 10 | rescue NoMethodError 11 | end 12 | 13 | include Bankjob 14 | 15 | ## 16 | # BaseScraper is a specific example of a Bankjob Scraper that can be used as a base 17 | # class for scrapers that follow a typical pattern. 18 | # 19 | # In fact, it does not add much functionality and you could just as readily subclass 20 | # the Scraper class as this class, but here is what it does add: 21 | # *+scraper_args+ attribute holds the array of args specified by the -scraper_args command line option 22 | # *+scrape_statement+ is implemented to use the --input command line option to specify a file for input 23 | # so that you can save a web-page to a file for debugging 24 | # *+scrape_statement+ instantiates a Mechanize agent and delegates to two other 25 | # simple methods that must be overridden in a subclass. 26 | # 27 | # Specifically +scrape_statement+ passes the Mechanize agent to +fetch_transactions_page+ 28 | # then passes the resulting page to +parse_transactions_page. Subclasses must implement these two methods. 29 | # See the documentation for these methods for more details on how to implement them. 30 | # Note that failure to override either method will result in an exception. 31 | # 32 | class BaseScraper < Scraper 33 | 34 | # +scraper_args+ holds the array of arguments specified on the command line with 35 | # the -scraper_args option. It is not used here, but it is set in the scrape_statement 36 | # method so that you can access it in your subclass. 37 | attr_accessor :scraper_args 38 | 39 | # This rule goes last and sets the type of any transactions 40 | # that are still set to OTHER to be the generic CREDIT or DEBIT 41 | # depending on the real amount of the transaction 42 | # +prioirity+ set to -999 to ensure it's last 43 | transaction_rule(-999) do |tx| 44 | if (tx.type == Transaction::OTHER) 45 | if tx.real_amount > 0 46 | tx.type = Transaction::CREDIT 47 | elsif tx.real_amount < 0 48 | tx.type = Transaction::DEBIT 49 | end 50 | # else leave it as OTHER if it's exactly zero 51 | end 52 | end 53 | 54 | ## 55 | # Override +fetch_transactions_page+ to use the mechanize +agent+ to 56 | # load the page holding your bank statement on your online banking website. 57 | # By using agent.get(url) to fetch the page, the returned page will be 58 | # an Hpricot document ready for parsing. 59 | # 60 | # Typically you will need to log-in using a form on a login page first. 61 | # Your implementation may look something like this: 62 | # 63 | # # My online banking app has a logon page with a standard HTML form. 64 | # # by looking at the source of the page I see that the form is named 65 | # # 'MyLoginFormName' and the two text fields for user name and password 66 | # # are called 'USERNAME' and 'PASSWORD' respectively. 67 | # login_page = agent.get("http://mybankapp.com/login.html") 68 | # form = login_page.forms.name('MyLoginFormName').first 69 | # # Mechanize automatically makes constants for the form elements based on their names. 70 | # form.USERNAME = "me" 71 | # form.PASSWORD = "foo" 72 | # agent.submit(form) 73 | # sleep 3 #wait while the login takes effect 74 | # 75 | # # Now that I've logged in and waited a bit, navigate to the page that lists 76 | # # my recent transactions and return it 77 | # return agent.get("http://mybankapp.com/latesttransactions.html") 78 | # 79 | def fetch_transactions_page(agent) 80 | raise "You must override fetch_transactions_page in your subclass of BaseScraper " + 81 | "or just subclass Scraper instead and override scrape_statement" 82 | end 83 | 84 | ## 85 | # Override +parse_transactions_page+ to take the Hpricot document passed in 86 | # as +page+, parse it using Hpricot directives, and create a Statement object 87 | # holding a set of Transaction objects for it. 88 | # 89 | def parse_transactions_page(page) 90 | raise "You must override parse_transactions_page in your subclass of BaseScraper " + 91 | "or just subclass Scraper instead and override scrape_statement" 92 | end 93 | 94 | ## 95 | # Implements the one essential method of a scraper +scrape_statement+ 96 | # by calling +fetch_transactions_page+ to get a web page holding a bank 97 | # statement followed by a call to +parse_transactions_page+ that returns 98 | # the +Statement+ object. 99 | # 100 | # Do not override this method in a subclass. (If you want to override it 101 | # you should be subclassing Scraper instead of this class) 102 | # 103 | # If the --input argument has been used to specify and input html file to 104 | # use, this will be parsed directly instead of calling +fetch_transaction_page+. 105 | # This allows for easy debugging without slow web-scraping (simply view 106 | # the page in a regular browser and use Save Page As to save a local copy 107 | # of it, then specify thiswith the --input command-line arg) 108 | # 109 | # +args+ holds the array of arguments specified on the command line with 110 | # the -scraper_args option. It is not used here, but it is set on an 111 | # attribute called scraper_args and is thus accessible in your subclass. 112 | # 113 | def scrape_statement(args) 114 | self.scraper_args = args 115 | if (not options.input.nil?) then 116 | # used for debugging - load the page from a file instead of the web 117 | logger.debug("Reading debug input html from #{options.input} instead of scraping the real website.") 118 | page = Hpricot(open(options.input)) 119 | else 120 | # not debugging use the actual scraper 121 | # First create a mechanize agent: a sort of pretend web browser 122 | agent = WWW::Mechanize.new 123 | agent.user_agent_alias = 'Windows IE 6' # pretend that we're IE 6.0 124 | 125 | page = fetch_transactions_page(agent) 126 | end 127 | raise "BaseScraper failed to load the transactions page" if page.nil? 128 | # Now that we've feteched the page, parse it to get a statement 129 | statement = parse_transactions_page(page) 130 | return statement 131 | end 132 | end # BaseScraper 133 | 134 | -------------------------------------------------------------------------------- /lib/bankjob/bankjob_runner.rb: -------------------------------------------------------------------------------- 1 | require 'rubygems' 2 | require 'logger' 3 | require 'bankjob.rb' 4 | 5 | module Bankjob 6 | class BankjobRunner 7 | 8 | # Runs the bankjob application, loading and running the 9 | # scraper specified in the command line args and generating 10 | # the output file. 11 | def run(options, stdout) 12 | logger = options.logger 13 | 14 | if options.wesabe_help 15 | Bankjob.wesabe_help(options.wesabe_args, logger) 16 | exit(0) # Wesabe help describes to the user how to use the wesabe options then quits 17 | end 18 | 19 | # Load the scraper object dynamically, then scrape the web 20 | # to get a new bank statement 21 | scraper = Scraper.load_scraper(options.scraper, options, logger) 22 | 23 | begin 24 | statement = scraper.scrape_statement(options.scraper_args) 25 | statement = Scraper.post_process_transactions(statement) 26 | rescue Exception => e 27 | logger.fatal(e) 28 | puts "Failed to scrape a statement successfully with #{options.scraper} due to: #{e.message}\n" 29 | puts "Use --debug --log bankjob.log then check the log for more details" 30 | exit (1) 31 | end 32 | 33 | # a lot of if's here but we allow for the user to generate ofx 34 | # and csv to files while simultaneously uploading to wesabe 35 | 36 | if options.csv 37 | if options.csv_out.nil? 38 | puts write_csv_doc([statement], true) # dump to console with header, no file specified 39 | else 40 | csv_file = file_name_from_option(options.csv_out, statement, "csv") 41 | 42 | # Output data as comma separated values possibly merging 43 | if File.file?(csv_file) 44 | # TODO until we fix merging csv files are appended 45 | open(csv_file, "a") do |f| 46 | f.puts(write_csv_doc([statement])) 47 | end 48 | logger.info("Statement is being appended as csv to #{csv_file}") 49 | # 50 | # TODO fix the merging then uncomment this 51 | # old_file_path = csv_file 52 | # # The file already exists, lets load it and merge with the new data 53 | # old_statement = scraper.create_statement() 54 | # old_statement.from_csv(old_file_path, scraper.decimal) 55 | # begin 56 | # old_statement.merge!(statement) 57 | # statement = old_statement 58 | # rescue Exception => e 59 | # # the merge failed, so leave the statement as the original and store it separately 60 | # output_file = output_file + "_#{date_range}_merge_failed" 61 | # logger.warn("Merge failed, storing new data in #{output_file} instead of appending it to #{old_file_path}") 62 | # logger.debug("Merge failed due to: #{e.message}") 63 | # end 64 | else 65 | open(csv_file, "w") do |f| 66 | f.puts(write_csv_doc([statement], true)) # true = write with header 67 | end 68 | logger.info("Statement is being written as csv to #{csv_file}") 69 | end 70 | end 71 | end # if csv 72 | 73 | # Create an ofx document and write it if necessary 74 | if (options.ofx or options.wesabe_upload) 75 | ofx_doc = write_ofx_doc([statement]) 76 | end 77 | 78 | # Output ofx file 79 | if options.ofx 80 | if options.ofx_out.nil? 81 | puts ofx_doc # dump to console, no file specified 82 | else 83 | ofx_file = file_name_from_option(options.ofx_out, statement, "ofx") 84 | open(ofx_file, "w") do |f| 85 | f.puts(ofx_doc) 86 | end 87 | logger.info("Statement is being output as ofx to #{ofx_file}") 88 | end 89 | end 90 | 91 | # Upload to wesabe if requested 92 | if options.wesabe_upload 93 | begin 94 | Bankjob.wesabe_upload(options.wesabe_args, ofx_doc, logger) 95 | rescue Exception => e 96 | logger.fatal("Failed to upload to Wesabe") 97 | logger.fatal(e) 98 | puts "Failed to upload to Wesabe: #{e.message}\n" 99 | puts "Try bankjob --wesabe-help for help on this feature." 100 | exit(1) 101 | end 102 | end 103 | end # run 104 | 105 | ## 106 | # Generates an OFX document to a string that starts with the stanadard 107 | # OFX header and contains the XML for the specified +statements+ 108 | # 109 | def write_ofx_doc(statements) 110 | ofx = generate_ofx2_header 111 | statements.each do |statement| 112 | ofx << statement.to_ofx 113 | end 114 | return ofx 115 | end 116 | 117 | ## 118 | # Generates a CSV document to a string containing the transactions in 119 | # all of the specified +statements+ 120 | # 121 | def write_csv_doc(statements, header = false) 122 | csv = "" 123 | csv << Statement.csv_header if header 124 | statements.each do |statement| 125 | csv << statement.to_csv 126 | end 127 | return csv 128 | end 129 | 130 | ## 131 | # Generates the (XML) OFX2 header lines that allow the OFX 2.0 document 132 | # to be recognized. 133 | # 134 | # (Note that this is crucial for www.wesabe.com to accept the OFX 135 | # document in an upload) 136 | # 137 | def generate_ofx2_header 138 | return <<-EOF 139 | 140 | 141 | EOF 142 | end 143 | 144 | ## 145 | # Generates the (non-XML) OFX header lines that allow the OFX 1.0 document 146 | # to be recognized. 147 | # 148 | # (Note that this is crucial for www.wesabe.com to accept the OFX 149 | # document in an upload) 150 | # 151 | def generate_ofx_header 152 | return <<-EOF 153 | OFXHEADER:100 154 | DATA:OFXSGML 155 | VERSION:102 156 | SECURITY:NONE 157 | ENCODING:USASCII 158 | CHARSET:1252 159 | COMPRESSION:NONE 160 | OLDFILEUID:NONE 161 | NEWFILEUID:NONE 162 | EOF 163 | end 164 | 165 | ## 166 | # Takes a name or path for an output file and a Statement and if the file 167 | # path is a directory, creates a new file name based on the date range 168 | # of the statement and returns a path to that file. 169 | # If +output_file+ is not a directory it is returned as-is. 170 | # 171 | def file_name_from_option(output_file, statement, type) 172 | # if the output_file is a directory, we create a new file name 173 | if (output_file and File.directory?(output_file)) 174 | # Create a date range string for the first and last transactions in the statement 175 | # This will looks something like: 20090130000000-20090214000000 176 | date_range = "#{Bankjob.date_time_to_ofx(statement.from_date)[0..7]}-#{Bankjob.date_time_to_ofx(statement.to_date)[0..7]}" 177 | filename = "#{date_range}.#{type}" 178 | output_file = File.join(output_file, filename) 179 | end 180 | # else we assume output_file is a file name/path already 181 | return output_file 182 | end 183 | end # class BankjobRunner 184 | end # module Bankjob 185 | -------------------------------------------------------------------------------- /website/index.txt: -------------------------------------------------------------------------------- 1 | h1. bankjob 2 | 3 | Bankjob is a command line ruby program for scraping online banking sites and producing statements as local files in OFX ("Open Fincancial Exchange":http://www.ofx.net) or CSV (Comma Separated Values) formats. 4 | 5 | Bankjob was created for people like me who want to get their bank data into a 3rd party application but whose bank does not support downloads in OFX format. 6 | It's also useful for keeping a permanent store of bank statements on your computer for reading in Excel. 7 | 8 | h2. Installing 9 | 10 | Mac OSX / Linux: 11 | 
sudo gem install bankjob
12 | 13 | Windows: 14 | 
gem install bankjob
15 | 16 | h2. The basics 17 | 18 | * Scrapes an online banking website to produce a bank statement 19 | * Stores bank statements locally in CSV files, which can be loaded directly in spreadsheets like Microsoft Excel 20 | * Stores bank statements locally in OFX files, which can be imported by many programs such as Quicken, MS Money, Gnu Cash and uploaded to some web applications 21 | * Built-in support for uploading to your Wesabe account (www.wesabe.com) 22 | * Supports coding of simple rules in ruby for modifying transaction details. E.g. automatically change "pment inst 3245003" to "paid home loan interest" 23 | 24 | h2. Usage 25 | 26 | 
 27 | bankjob --csv c:\bank\csv --scraper c:\bank\my_bpi_scraper.rb
 28 |         --scraper-args " "
 29 |         --wesabe "  "
 30 |         --log c:\bank\bankjob.log --debug
 31 |
32 | 33 | I have this command in a .bat file which is launched daily by a scheduled task on my windows Media Center PC (which, since it's always on and connected to the internet, makes a useful home server) 34 | 35 | This one command will: 36 | * scrape my online banking website after logging in as me and navigating to the page with recent transactions 37 | * apply some rules, coded in the my_bpi_scraper.rb file that make the descriptions more readable 38 | * produce a statement in comma-separated-value format, keeping the original raw data as well as the new descriptions, 39 | storing that in a file with a name like 20090327-20090406.csv in my local directory c:\bank\csv (serves as a sort of permanent 'audit' record) 40 | * produce an OFX document with the same statement information 41 | * upload the OFX statement to my wesabe account 42 | * log debug-level details in bankjob.log 43 | 44 | Use bankjob -h to see all of the options. 45 | 46 | h2. Wesabe 47 | 48 | Wesabe ("www.wesabe.com":www.wesabe.com) is a sort of free web.2.0-money-manager-financial-social-networking application that I'm using to track my bank acounts. 49 | I like it better than my bank's online banking apps because it: 50 | * shows me all my accounts at once 51 | * lets me track certain kinds of spending, with automatically applied tags and nice graphs 52 | * has an iphone-web-app (and a non-iphone one too) 53 | * even has a nifty dashboard widget for Mac OS X. 54 | * does much more than this but you should go to their site to find out more. 55 | 56 | At this point I should point out that Bankjob and its developers (okay developer - there's just me, I admit it. God it's lonely here) have no affiliation with Wesabe whatsoever. I just happen to use Wesabe and like what it has to offer. 57 | In fact I just happened to _want_ to use Wesabe, but couldn't because my bank doesn't offer OFX downloads and is not supported by the automatic updater. Hence I developed Bankjob to allow me to use Wesabe. 58 | 59 | I think there are a lot of other non-US would-be Wesabe users out there in the same boat: Wesabe and its brethren do not support our banks, and our banks don't make it any easier by supporting OFX downloads. 60 | 61 | There are similar services out there like Mint, Quicken Online, Yodlee and others (see one comparison "here":http://digg.com/business_finance/Late_2008_comparison_of_Wesabe_Mint_and_Quicken_Online), or just google them. 62 | The problem with most of these is that they offer even less support than Wesabe for banks outside of the US. Wesabe supports some non-US banks, but more importantly offers a public API which allows apps like Bankjob to upload accounts - among other things. (Actually I think Yodlee might have more support for non-US banks but still no public API - if your Bank is not officially supported you're out of luck) 63 | 64 | To use the Wesabe upload feature of Bankjob you need a Wesabe username and password. If you have more than one account with Wesabe you need to know the ID that Wesabe gives the account. This is easy to find out: if you have one a account it's "1" and you don't need to specify it. If you have more than one account, Bankjob will help you out. Use: 65 | bankjob --wesabe-help 66 | to see the general help information, but to test your Wesabe account, use 67 | bankjob --wesabe-help " " 68 | (you'll need to use the quotes, with a space between the username and password) 69 | 70 | With this command bankjob will list out all of your Wesabe accounts and tell you how to upload to them. 71 | 72 | I have only one account at the moment, and here's what happens when I use --wesabe-help (the text in angle brackets is added by me to protect my account, but otherwise this is real input/output) 73 | 74 | 
 75 | [me:~] bankjob --wesabe-help " "
 76 | Connecting to Wesabe...
 77 | You have 1 Wesabe accounts:
 78 |  Account Name: 
 79 |     wesabe id: 1
 80 |    account no: 0001
 81 |          type: Checking
 82 |       balance: 2196.12
 83 |          bank: 
 84 | To upload to this account use:
 85 |   bankjob [other bankjob args] --wesabe "  1"
 86 | 
 87 | Since you have one account you do not need to specify the id number, use:
 88 |   bankjob [other bankjob args] --wesabe " "
 89 |
90 | 91 | Again, if in any doubt bankjob --wesabe-help should set you straight. 92 | 93 | 94 | 95 | h2. Rubydoc 96 | 97 | "http://bankjob.rubyforge.org/rdoc":http://bankjob.rubyforge.org/rdoc 98 | 99 | h2. Forum 100 | 101 | "http://groups.google.com/group/bankjob":http://groups.google.com/group/bankjob 102 | 103 | h2. Creating your own Scraper 104 | 105 | Unfortunately Bankjob isn't much good to you in its natural state unless you happen to bank with the same bank as me. 106 | To make it useful you need a scraper for your own online bank site. How hard that will be depends on how handy you are with "Ruby":http://www.ruby-lang.org programming and with using "Hpricot":http://wiki.github.com/why/hpricot/an-hpricot-showcase and "Mechanize":http://mechanize.rubyforge.org/mechanize 107 | 108 | You'll get some help from: 109 | * "The Mechanize docs":http://mechanize.rubyforge.org/mechanize 110 | * "The Hpricot wiki":http://wiki.github.com/why/hpricot/an-hpricot-showcase 111 | * The BpiScraper is included in the scrapers directory and is well documented as an example. 112 | * The BaseScraper is the superclass of BpiScraper. Using it as a base class for your own scraper is optional, but it saves you a little work by starting up Mechanize for you. 113 | * The Scraper class itself, which you _must_ subclass with your own Scraper, has more to say on creating your own scraper in its "rubydoc":http://bankjob.rubyforge.org/rdoc/classes/Bankjob/Scraper.html 114 | * Look for help on the "forum":http://groups.google.com/group/bankjob 115 | * Ask me directly "rhubarb":mailto:rhubarb.bankjob@gmail.com 116 | 117 | Warning: Don't upload or email anyone (including me) your private online banking information. If you create your own scraper, please share it, but remove any account details from it first. 118 | 119 | If and when you do develop a scraper for your banksite, send it to me (without any passwords or private banking info of course!) and I'll add it to the built-in scrapers. 120 | 121 | h2. How to submit patches 122 | 123 | You can fetch the source from github at: 124 | 125 | "http://github.com/rhubarb/bankjob/tree/master":http://github.com/rhubarb/bankjob/tree/master 126 | 127 | using: 128 | 
git clone git://github.com/rhubarb/bankjob.git
129 | 130 | and submit any patches to the "forum":http://groups.google.com/group/bankjob. 131 | 132 | h2. License 133 | 134 | This code is free to use under the terms of the MIT license. 135 | 136 | As far as rhubarb is concerned, this means you can: 137 | - Use it in your private projects 138 | - Use it in your public projects 139 | - Use it in your commercial projects and make big bucks 140 | - Mulch it up and use it in your garden as fertilizer 141 | - Generally go crazy with it 142 | 143 | h2. Contact 144 | 145 | Comments are welcome as are feature requests and patches. 146 | 147 | Send an email to "rhubarb":mailto:rhubarb.bankjob@gmail.com 148 | Or check out the "forum":http://groups.google.com/group/bankjob 149 | 150 | -------------------------------------------------------------------------------- /scrapers/bpi_scraper.rb: -------------------------------------------------------------------------------- 1 | 2 | require 'rubygems' 3 | require 'bankjob' # this require will pull in all the classes we need 4 | require 'base_scraper' # this defines scraper that BpiScraper extends 5 | 6 | include Bankjob # access the namespace of Bankjob 7 | 8 | ## 9 | # BpiScraper is a scraper tailored to the BPI bank in Portugal (www.bpinet.pt). 10 | # It takes advantage of the BaseScraper to create the mechanize agent, 11 | # then followins the basic recipe there of first loading the tranasctions page 12 | # then parsing it. 13 | # 14 | # In addition to actually working for the BPI online banking, this class serves 15 | # as an example of how to build your own scraper. 16 | # 17 | # BpiScraper expects the user name and password to be passed on the command line 18 | # using --scraper-args "user password" (with a space between them). 19 | # Optionally, the account number can also be specified with the 3rd argument so: 20 | # --scraper-args "user password 803030000001" causing that account to be selected 21 | # before scraping the statement 22 | # 23 | class BpiScraper < BaseScraper 24 | 25 | currency "EUR" # Set the currency as euros 26 | decimal "," # BPI statements use commas as separators - this is used by the real_amount method 27 | account_number "1234567" # override this with a real account number 28 | account_type Statement::CHECKING # this is the default anyway 29 | 30 | # This rule detects ATM withdrawals and modifies 31 | # the description and sets the the type 32 | transaction_rule do |tx| 33 | if (tx.real_amount < 0) 34 | if tx.raw_description =~ /LEV.*ATM ELEC\s+\d+\/\d+\s+/i 35 | tx.description = "Multibanco withdrawal at #{$'}" 36 | tx.type = Transaction::ATM 37 | end 38 | end 39 | end 40 | 41 | # This rule detects checque payments and modifies the description 42 | # and sets the type 43 | transaction_rule do |tx| 44 | if tx.raw_description =~ /CHEQUE\s+(\d+)/i 45 | cheque_number = $+ # $+ holds the last group of the match which is (\d+) 46 | # change the description but append $' in case there was trailing text after the cheque no 47 | tx.description = "Cheque ##{cheque_number} withdrawn #{$'}" 48 | tx.type = Transaction::CHECK 49 | tx.check_number = cheque_number 50 | end 51 | end 52 | 53 | # This rule goes last and sets the description of transactions 54 | # that haven't had their description to the raw description after 55 | # changing the words to have capital letters only on the first word. 56 | # (Note that +description+ will default to being the same as +raw_description+ 57 | # anyway - this rule is only for making the all uppercase output less ugly) 58 | # The payee is also fixed in this way 59 | transaction_rule(-999) do |tx| 60 | if (tx.description == tx.raw_description) 61 | tx.description = Bankjob.capitalize_words(tx.raw_description) 62 | end 63 | end 64 | 65 | # Some constants for the URLs and main elements in the BPI bank app 66 | LOGIN_URL = 'https://www.bpinet.pt/' 67 | TRANSACTIONS_URL = 'https://www.bpinet.pt/areaInf/consultas/Movimentos/Movimentos.asp' 68 | 69 | ## 70 | # Uses the mechanize web +agent+ to fetch the page holding the most recent 71 | # bank transactions and returns it. 72 | # This overrides (implements) +fetch_transactions_page+ in BaseScraper 73 | # 74 | def fetch_transactions_page(agent) 75 | login(agent) 76 | logger.info("Logged in, now navigating to transactions on #{TRANSACTIONS_URL}.") 77 | transactions_page = agent.get(TRANSACTIONS_URL) 78 | if (transactions_page.nil?) 79 | raise "BPI Scraper failed to load the transactions page at #{TRANSACTIONS_URL}" 80 | end 81 | 82 | # If there is a third scraper arg, it is the account number and we use it 83 | # to select the account on the transactions page 84 | if (scraper_args and scraper_args.length > 2) 85 | account = scraper_args[2] 86 | # the account selector is the field 'contaCorrente' in the form 'form_mov' 87 | Bankjob.select_and_submit(transactions_page, 'form_mov', 'contaCorrente', account) 88 | sleep 1 89 | # refetch the transactions page after selecting the account 90 | transactions_page = agent.get(TRANSACTIONS_URL) 91 | end 92 | 93 | return transactions_page 94 | end 95 | 96 | 97 | ## 98 | # Parses the BPI page listing about a weeks worth of transactions 99 | # and creates a Transaction for each one, putting them together 100 | # in a Statement. 101 | # Overrides (implements) +parse_transactions_page+ in BaseScraper. 102 | # 103 | def parse_transactions_page(transactions_page) 104 | begin 105 | statement = create_statement 106 | 107 | account_number = get_account_number(transactions_page) 108 | statement.account_number = account_number unless account_number.nil? 109 | 110 | # Find the closing balance avaliable and accountable 111 | # Get from this: 112 | # Saldo Disponível: 113 | # 1.751,31 EUR 114 | # to 1751,31 115 | # Commenting out balances for now to let the balance be taken from the 116 | # top-most transaction - this keeps balances in synch with actual transactions 117 | # and allows for statements created for past dates (the balance at the top of the 118 | # page is always the current one, not the one for the last transaction on that page) 119 | #available_cell = (transactions_page/"td").select { |ele| ele.inner_text =~ /^Saldo Dispon/ }.first.next_sibling 120 | #statement.closing_available = available_cell.inner_text.scan(/[\d.,]+/)[0].gsub(/\./,"") 121 | #account_balance_cell = (transactions_page/"td").select { |ele| ele.inner_text =~ /^Saldo Contab/ }.first.next_sibling 122 | #statement.closing_balance = account_balance_cell.inner_text.scan(/[\d.,]+/)[0].gsub(/\./,"") 123 | 124 | #transactions = [] 125 | 126 | # find the first header with the CSS class "Laranja" as this will be the first 127 | # header in the transactions table 128 | header = (transactions_page/"td.Laranja").first 129 | 130 | # the table element is the grandparent element of this header (the row is the parent) 131 | table = header.parent.parent 132 | 133 | # each row with the valign attribute set to "top" holds a transaction 134 | rows = (table/"tr[@valign=top]") 135 | rows.each do |row| 136 | transaction = create_transaction # use the support method because it sets the separator 137 | 138 | # collect all of the table cells' inner html in an array (stripping leading/trailing spaces) 139 | data = (row/"td").collect{ |cell| cell.inner_html.strip } 140 | 141 | # the first (0th) column holds the date 142 | transaction.date = data[0] 143 | 144 | # the 2nd column holds the value date - but it's often empty 145 | # in which case we set it to nil 146 | vdate = data[1] 147 | if vdate.nil? or vdate.length == 0 or vdate.strip == " " 148 | transaction.value_date = nil 149 | else 150 | transaction.value_date = vdate 151 | end 152 | 153 | # the transaction raw_description is in the 3rd column 154 | transaction.raw_description = data[2] 155 | 156 | # the 4th column holds the transaction amount (with comma as decimal place) 157 | transaction.amount = data[3] 158 | 159 | # the new balance is in the last column 160 | transaction.new_balance=data[4] 161 | 162 | # add thew new transaction to the array 163 | statement.add_transaction(transaction) 164 | # break if $debug 165 | end 166 | rescue => exception 167 | msg = "Failed to parse the transactions page at due to exception: #{exception.message}\nCheck your user name and password." 168 | logger.fatal(msg); 169 | logger.debug(exception) 170 | logger.debug("Failed parsing transactions page:") 171 | logger.debug("--------------------------------") 172 | logger.debug(transactions_page) #.body 173 | logger.debug("--------------------------------") 174 | abort(msg) 175 | end 176 | 177 | # finish the statement to set the balances and dates 178 | # and to fake the times since the bpi web pages 179 | # don't hold the transaction times 180 | statement.finish(true, true) # most_recent_first, fake_times 181 | 182 | return statement 183 | end 184 | 185 | def get_account_number(transactions_page) 186 | # make sure the page is a mechanize page, not hpricot 187 | if transactions_page.kind_of?(Hpricot::Doc) then 188 | page = WWW::Mechanize::Page.new(nil, {'content-type'=>'text/html'}, 189 | transactions_page.html, nil, nil) 190 | else 191 | page = transactions_page 192 | end 193 | 194 | # find the form for selecting an account -it's called 'form_mov' 195 | form_mov = page.form('form_mov') 196 | # the field for selecting the current account is in this form 197 | account_selector = form_mov.field('contaCorrente') 198 | # the selected account value is the account number but it has "|NR|" on the end so strip 199 | # everything that's not a number 200 | account_number = account_selector.value.gsub(/[^0-9]/,"") 201 | return account_number 202 | end 203 | 204 | ## 205 | # Logs into the BPI banking app by finding the form 206 | # setting the name and password and submitting it then 207 | # waits a bit. 208 | # 209 | def login(agent) 210 | logger.info("Logging in to #{LOGIN_URL}.") 211 | if (scraper_args) 212 | username, password = *scraper_args 213 | end 214 | raise "Login failed for BPI Scraper - pass user name and password using -scraper_args \"user pass\"" unless (username and password) 215 | 216 | # navigate to the login page 217 | login_page = agent.get(LOGIN_URL) 218 | 219 | # find login form - it's called 'signOn' - fill it out and submit it 220 | form = login_page.form('signOn') 221 | 222 | # username and password are taken from the commandline args, set them 223 | # on USERID and PASSWORD which are the element names that the web page 224 | # form uses to identify the form fields 225 | form.USERID = username 226 | form.PASSWORD = password 227 | 228 | # submit the form - same as the user hitting the Login button 229 | agent.submit(form) 230 | sleep 3 # wait while the login takes effect 231 | end 232 | end # class BpiScraper 233 | 234 | 235 | -------------------------------------------------------------------------------- /lib/bankjob/support.rb: -------------------------------------------------------------------------------- 1 | 2 | require 'rubygems' 3 | require 'bankjob' 4 | 5 | module Bankjob 6 | 7 | ## 8 | # Takes a date-time as a string or as a Time or DateTime object and returns 9 | # it as either a Time object. 10 | # 11 | # This is useful in the setter method of a date attribute allowing the date 12 | # to be set as any type but stored internally as an object compatible with 13 | # conversion through +strftime()+ 14 | # (Bankjob::Transaction uses this internally in the setter for +date+ for example 15 | # 16 | def self.create_date_time(date_time_raw) 17 | if (date_time_raw.is_a?(Time)) then 18 | # It's already a Time 19 | return date_time_raw 20 | elsif (date_time_raw.to_s.strip.empty?) 21 | # Nil or non dates are returned as nil 22 | return nil 23 | else 24 | # Assume it can be converted to a time 25 | return Time.parse(date_time_raw.to_s) 26 | end 27 | end 28 | 29 | ## 30 | # Takes a Time or DateTime and formats it in the correct format for OFX date elements. 31 | # 32 | # The OFX format is a string of digits in the format "YYMMDDHHMMSS". 33 | # For example, the 1st of February 2009 at 2:34PM and 56 second becomes "20090201143456" 34 | # 35 | # Note must use a Time, or DateTime, not a String, nor a Date. 36 | # 37 | def self.date_time_to_ofx(time) 38 | time.nil? ? "" : "#{time.strftime( '%Y%m%d%H%M%S' )}" 39 | end 40 | 41 | ## 42 | # Takes a Time or DateTime and formats in a suitable format for comma separated values files. 43 | # The format produced is suitable for loading into an Excel-like spreadsheet program 44 | # being automatically treated as a date. 45 | # 46 | # A string is returned with the format "YY-MM-DD HH:MM:SS". 47 | # For example, the 1st of February 2009 at 2:34PM and 56 second becomes "2009-02-01 14:34:56" 48 | # 49 | # Note must use a Time, or DateTime, not a String, nor a Date. 50 | # 51 | def self.date_time_to_csv(time) 52 | time.nil? ? "" : "#{time.strftime( '%Y-%m-%d %H:%M:%S' )}" 53 | end 54 | 55 | ## 56 | # Takes a string and capitalizes the first letter of every word 57 | # and forces the rest of the word to be lowercase. 58 | # 59 | # This is a utility method for use in scrapers to make descriptions 60 | # more readable. 61 | # 62 | def self.capitalize_words(message) 63 | message.downcase.gsub(/\b\w/){$&.upcase} 64 | end 65 | 66 | ## 67 | # converts a numeric +string+ to a float given the specified +decimal+ 68 | # separator. 69 | # 70 | def self.string_to_float(string, decimal) 71 | return nil if string.nil? 72 | amt = string.gsub(/\s/, '') 73 | if (decimal == ',') # E.g. "1.000.030,99" 74 | amt.gsub!(/\./, '') # strip out . 1000s separator 75 | amt.gsub!(/,/, '.') # replace decimal , with . 76 | elsif (decimal == '.') 77 | amt.gsub!(/,/, '') # strip out comma 1000s separator 78 | end 79 | return amt.to_f 80 | end 81 | 82 | ## 83 | # Finds a selector field in a named +form+ in the given Mechanize +page+, selects 84 | # the suggested +label+ 85 | def select_and_submit(page, form_name, select_name, selection) 86 | option = nil 87 | form = page.form(form_name) 88 | unless form.nil? 89 | selector = form.field(select_name) 90 | unless selector.nil? 91 | option = select_option(selector, selection) 92 | form.submit 93 | end 94 | end 95 | return option 96 | end 97 | 98 | ## 99 | # Given a Mechanize::Form:SelectList +selector+ will attempt to select the option 100 | # specified by +selection+. 101 | # This algorithm is used: 102 | # The first option with a label equal to the +selection+ is selected. 103 | # - if none is found then - 104 | # The first option with a value equal to the +selection+ is selected. 105 | # - if none is found then - 106 | # The first option with a label or value that equal to the +selection+ is selected 107 | # after removing non-alphanumeric characters from the label or value 108 | # - if none is found then - 109 | # The first option with a lable or value that _contains_ the +selection+ 110 | # 111 | # If matching option is found, the #select is called on it. 112 | # If no option is found, nil is returned - otherwise the option is returned 113 | # 114 | def select_option(selector, selection) 115 | options = selector.options.select { |o| o.text == selection } 116 | options = selector.options.select { |o| o.value == selection } if options.empty? 117 | options = selector.options.select { |o| o.text.gsub(/[^a-zA-Z0-9]/,"") == selection } if options.empty? 118 | options = selector.options.select { |o| o.value.gsub(/[^a-zA-Z0-9]/,"") == selection } if options.empty? 119 | options = selector.options.select { |o| o.text.include?(selection) } if options.empty? 120 | options = selector.options.select { |o| o.value.include?(selection) } if options.empty? 121 | 122 | option = options.first 123 | option.select() unless option.nil? 124 | return option 125 | end 126 | 127 | ## 128 | # Uploads the given OFX document to the Wesabe account specified in the +wesabe_args+ 129 | # 130 | def self.wesabe_upload(wesabe_args, ofx_doc, logger) 131 | if (wesabe_args.nil? or (wesabe_args.length < 2 and wesabe_args.length > 3)) 132 | raise "Incorrect number of args for Wesabe (#{wesabe_args}), should be 2 or 3." 133 | else 134 | load_wesabe 135 | wuser, wpass, windex = *wesabe_args 136 | wesabe = Wesabe.new(wuser, wpass) 137 | num_accounts = wesabe.accounts.length 138 | if num_accounts == 0 139 | raise "The user \"#{wuser}\" has no Wesabe accounts. Create one at www.wesabe.com before attempting to upload a statement." 140 | elsif (not windex.nil? and (num_accounts < windex.to_i)) 141 | raise "The user \"#{wuser}\" has only #{num_accounts} Wesabe accounts, but the account index #{windex} was specified." 142 | elsif windex.nil? 143 | if num_accounts > 1 144 | raise "The user \"#{wuser}\" has #{num_accounts} Wesabe accounts, so the account index must be specified in the WESABE_ARGS." 145 | else 146 | # we have only one account, no need to specify the index 147 | windex = 1 148 | end 149 | elsif windex.to_i == 0 150 | raise "The Wesabe account index must be between 1 and #{num_accounts}. #{windex} is not acceptable" 151 | end 152 | logger.debug("Attempting to upload statement to the ##{windex} Wesabe account for user #{wuser}...") 153 | # Get the account at the index (which is not necessarily the index in the array 154 | # so we use the account(index) method to get it 155 | account = wesabe.account(windex.to_i) 156 | uploader = account.new_upload 157 | uploader.statement = ofx_doc 158 | uploader.upload! 159 | logger.info("Uploaded statement to Wesabe account #{account.name}, the ##{windex} account for user #{wuser}, with the result: #{uploader.status}") 160 | end 161 | end 162 | 163 | ## 164 | # Helps the user determine how to upload to their Wesabe account. 165 | # 166 | # When used with no args, will give generic help information. 167 | # When used with Wesabe account and password, will log into Wesabe and list 168 | # the users accounts, and suggest command line args to upload to each account. 169 | # 170 | def self.wesabe_help(wesabe_args, logger) 171 | if (wesabe_args.nil? or wesabe_args.length != 2) 172 | puts <<-EOF 173 | Wesabe (http://www.wesabe.com) is an online bank account management tool (like Mint) 174 | that allows you to upload (in some cases automatically) your bank statements and 175 | automatically convert them into a more readable format to allow you to track 176 | your spending and much more. Wesabe comes with its own community attached. 177 | 178 | Bankjob has no affiliation with Wesabe, but allows you to upload the statements it 179 | generates to your Wesabe account automatically. 180 | 181 | To use Wesabe you need the Wesabe Ruby gem installed: 182 | See the gem at http://github.com/wesabe/wesabe-rubygem 183 | Install the gem with: 184 | $ sudo gem install -r --source http://gems.github.com/ wesabe-wesabe 185 | (on Windows, omit the "sudo") 186 | 187 | You also need your Wesabe login name and password, and, if you have 188 | more than one account on Wesabe, the id number of the account. 189 | This is not a real account number - it's simply a counter that Wesabe uses. 190 | If you have a single account it will be '1', if you have two accounts the 191 | second account will be '2', etc. 192 | 193 | Bankjob will help you find this number by listing your Wesabe accounts for you. 194 | Simply use: 195 | bankjob -wesabe_help "username password" 196 | (The quotes are important - this is a single argument to Bankjob with two words) 197 | 198 | If you already know the number of the account and you want to start uploading use: 199 | 200 | bankjob [other bankjob args] --wesabe "username password id" 201 | 202 | E.g. 203 | bankjob --scraper bpi_scraper.rb --wesabe "bloggsy pw123 2" 204 | 205 | If you only have a single account, you don't need to specify the id number 206 | (but Bankjob will check and will fail with an error if you have more than one account) 207 | 208 | bankjob [other bankjob args] --wesabe "username password" 209 | 210 | If in any doubt --wesabe-help "username password" will set you straight. 211 | 212 | Troubleshooting: 213 | - If you see an error like Wesabe::Request::Unauthorized, then chances 214 | are your username or password for Wesabe is incorrect. 215 | 216 | - If you see an error "end of file reached" then it may be that you are logged 217 | into the Wesabe account to which you are trying to upload - perhaps in a browser. 218 | In this case, log out from Wesabe in the browser, _wait a minute_, then try again. 219 | EOF 220 | else 221 | load_wesabe 222 | begin 223 | puts "Connecting to Wesabe...\n" 224 | wuser, wpass = *wesabe_args 225 | wesabe = Wesabe.new(wuser, wpass) 226 | puts "You have #{wesabe.accounts.length} Wesabe accounts:" 227 | wesabe.accounts.each do |account| 228 | puts " Account Name: #{account.name}" 229 | puts " wesabe id: #{account.id}" 230 | puts " account no: #{account.number}" 231 | puts " type: #{account.type}" 232 | puts " balance: #{account.balance}" 233 | puts " bank: #{account.financial_institution.name}" 234 | puts "To upload to this account use:" 235 | puts " bankjob [other bankjob args] --wesabe \"#{wuser} password #{account.id}\"" 236 | puts "" 237 | if wesabe.accounts.length == 1 238 | puts "Since you have one account you do not need to specify the id number, use:" 239 | puts " bankjob [other bankjob args] --wesabe \"#{wuser} password\"" 240 | end 241 | end 242 | rescue Exception => e 243 | msg =<<-EOF 244 | Failed to get Wesabe account information due to: #{e.message}. 245 | Check your username and password or use: 246 | bankjob --wesabe-help 247 | with no arguments for more details. 248 | EOF 249 | logger.debug(msg) 250 | logger.debug(e) 251 | raise msg 252 | end 253 | end 254 | end # wesabe_help 255 | 256 | private 257 | 258 | def self.load_wesabe(logger = nil) 259 | begin 260 | require 'wesabe' 261 | rescue LoadError => error 262 | msg = <<-EOF 263 | Failed to load the Wesabe gem due to #{error.module} 264 | See the gem at http://github.com/wesabe/wesabe-rubygem 265 | Install the gem with: 266 | $ sudo gem install -r --source http://gems.github.com/ wesabe-wesabe 267 | EOF 268 | logger.fatal(msg) unless logger.nil? 269 | raise msg 270 | end 271 | end 272 | end # module Bankjob 273 | 274 | 275 | -------------------------------------------------------------------------------- /lib/bankjob/cli.rb: -------------------------------------------------------------------------------- 1 | require 'rubygems' 2 | require 'ostruct' 3 | require 'optparse' 4 | require 'logger' 5 | 6 | $:.unshift(File.dirname(__FILE__)) unless 7 | $:.include?(File.dirname(__FILE__)) || $:.include?(File.expand_path(File.dirname(__FILE__))) 8 | 9 | require 'bankjob_runner.rb' 10 | 11 | module Bankjob 12 | class CLI 13 | 14 | NEEDED = "Needed" # constant to indicate compulsory options 15 | NOT_NEEDED = "Not Needed" # constant to indicate no-longer compulsory options 16 | 17 | def self.execute(stdout, argv) 18 | # The BanjobOptions module above, through the magic of OptiFlags 19 | # has augmented ARGV with the command line options accessible through 20 | # ARGV.flags. 21 | runner = BankjobRunner.new() 22 | runner.run(parse(argv), stdout) 23 | end # execute 24 | 25 | ## 26 | # Parses the command line arguments using OptionParser and returns 27 | # an open struct with an attribute for each option 28 | # 29 | def self.parse(args) 30 | options = OpenStruct.new 31 | 32 | # Set the default options 33 | options.scraper = NEEDED 34 | options.scraper_args = [] 35 | options.log_level = Logger::WARN 36 | options.log_file = nil 37 | options.debug = false 38 | options.input = nil 39 | options.ofx = false # ofx is the default but only if csv is false 40 | options.ofx_out = false 41 | options.csv = false 42 | options.csv_out = nil # allow for separate csv and ofx output files 43 | options.wesabe_help = false 44 | options.wesabe_upload = false 45 | options.wesabe_args = nil 46 | options.logger = nil 47 | 48 | opt = OptionParser.new do |opt| 49 | 50 | opt.banner = "Bankjob - scrapes your online banking website and produces an OFX or CSV document.\n" + 51 | "Usage: bankjob [options]\n" 52 | 53 | opt.version = Bankjob::BANKJOB_VERSION 54 | 55 | opt.on('-s', '--scraper SCRAPER', 56 | "The name of the ruby file that scrapes the website.\n") do |file| 57 | options.scraper = file 58 | end 59 | 60 | opt.on('--scraper-args ARGS', 61 | "Any arguments you want to pass on to your scraper.", 62 | "The entire set of arguments must be quoted and separated by spaces", 63 | "but you can use single quotes to specify multi-word arguments for", 64 | "your scraper. E.g.", 65 | " -scraper-args \"-user Joe -password Joe123 -arg3 'two words'\""," ", 66 | "This assumes your scraper accepts an array of args and knows what", 67 | "to do with them, it will vary from scraper to scraper.\n") do |sargs| 68 | options.scraper_args = sub_args_to_array(sargs) 69 | end 70 | 71 | opt.on('-i', '--input INPUT_HTML_FILE', 72 | "An html file used as the input instead of scraping the website -", 73 | "useful for debugging.\n") do |file| 74 | options.input = file 75 | end 76 | 77 | opt.on('-l', '--log LOG_FILE', 78 | "Specify a file to log information and debug messages.", 79 | "If --debug is used, log info will go to the console, but if neither", 80 | "this nor --debug is specfied, there will be no log.", 81 | "Note that the log is rolled over once per week\n") do |log_file| 82 | options.log_file = log_file 83 | end 84 | 85 | opt.on('q', '--quiet', "Suppress all messages, warnings and errors.", 86 | "Only fatal errors will go in the log") do 87 | options.log_level = Logger::FATAL 88 | end 89 | 90 | opt.on( '--verbose', "Log detailed informational messages.\n") do 91 | options.log_level = Logger::INFO 92 | end 93 | 94 | opt.on('--debug', 95 | "Log debug-level information to the log", 96 | "if here is one and put debug info in log\n") do 97 | options.log_level = Logger::DEBUG 98 | options.debug = true 99 | end 100 | 101 | opt.on('--ofx [FILE]', 102 | "Write out the statement as an OFX2 compliant XML document."," ", 103 | "If FILE is not specified, the XML is dumped to the console.", 104 | "If FILE specifies a directory then a new file will be created with a", 105 | "name generated from the dates of the first and last transactions.", 106 | "If FILE specifies a file that already exists it will be overwritten."," ", 107 | "(Note that ofx is the default format unless --csv is specified,", 108 | "and that both CSV and OFX documents can be produced by specifying", 109 | "both options.)\n") do |file| 110 | options.ofx = true 111 | options.ofx_out = file 112 | end 113 | 114 | opt.on('--csv [FILE]', 115 | "Writes out the statement as a CSV (comma separated values) document.", 116 | "All of the information available including numeric values for amount,", 117 | "raw and rule-generated descriptions, etc, are produced in the CSV document.", " ", 118 | "The document produced is suitable for loading into a spreadsheet like", 119 | "Microsoft Excel with the dates formatted to allow for auto recognition.", 120 | "This option can be used in conjunction with --ofx or --wesabe to produce", 121 | "a local permanent log of all the data scraped over time.", " ", 122 | "If FILE is not specified, the CSV is dumped to the console.", 123 | "If FILE specifies a directory then a new file will be created with a", 124 | "name generated from the dates of the first and last transactions.", 125 | "If FILE specifies a file that already exists then the new statement", 126 | "will be appended to the existing one in that file with care taken to", 127 | "merge removing duplicate entries.\n", 128 | "[WARNING - this merging does not yet function properly - its best to specify a directory for now.]\n" 129 | ) do |file| 130 | # TODO update this warning when we have merging working 131 | options.csv = true 132 | options.csv_out = file 133 | end 134 | 135 | opt.on('--wesabe-help [WESABE_ARGS]', 136 | "Show help information on how to use Bankjob to upload to Wesabe.", 137 | "Optionally use with \"wesabe-user password\" to get Wesabe account info.", 138 | "Note that the quotes around the WESABE_ARGS to send both username", 139 | "and password are necessary.", " ", 140 | "Use --wesabe-help with no args for more details.\n") do |wargs| 141 | options.wesabe_args = sub_args_to_array(wargs) 142 | options.wesabe_help = true 143 | options.scraper = NOT_NEEDED # scraper is not NEEDED when this option is set 144 | end 145 | 146 | opt.on('--wesabe WESABE_ARGS', 147 | "Produce an OFX document from the statement and upload it to a Wesabe account.", 148 | "WESABE_ARGS must be quoted and space-separated, specifying the wesabe account", 149 | "username, password and - if there is more than one - the wesabe account number.", " ", 150 | "Before trying this, use bankjob --wesabe-help to get more information.\n" 151 | ) do |wargs| 152 | options.wesabe_args = sub_args_to_array(wargs) 153 | options.wesabe_upload = true 154 | end 155 | 156 | opt.on('--version', "Display program version and exit.\n" ) do 157 | puts opt.version 158 | exit 159 | end 160 | 161 | opt.on_tail('-h', '--help', "Display this usage message and exit.\n" ) do 162 | puts opt 163 | puts <<-EOF 164 | 165 | Some common options: 166 | 167 | o Debugging: 168 | --debug --scraper bpi_scraper.rb --input /tmp/DownloadedPage.html --ofx 169 | 170 | o Regular use: (output ofx documents to a directory called 'bank') 171 | --scraper /bank/mybank_scraper.rb --scraper-args "me mypass123" --ofx /bank --log /bank/bankjob.log --verbose 172 | 173 | o Abbreviated options with CSV output: (output csv appended continuously to a file) 174 | -s /bank/otherbank_scraper.rb --csv /bank/statements.csv -l /bank/bankjob.log -q 175 | 176 | o Get help on using Wesabe: 177 | --wesabe-help 178 | 179 | o Upload to Wesabe: (I have 4 Wesabe accounts and am uploading to the 3rd) 180 | -s /bank/mybank_scraper.rb --wesabe "mywesabeuser password 3" -l /bank/bankjob.log --debug 181 | EOF 182 | exit! 183 | end 184 | 185 | end #OptionParser.new 186 | 187 | begin 188 | opt.parse!(args) 189 | _validate_options(options) # will raise exceptions if options are invalid 190 | _init_logger(options) # sets the logger 191 | rescue Exception => e 192 | puts e, "", opt 193 | exit 194 | end 195 | 196 | return options 197 | end #self.parse 198 | 199 | private 200 | 201 | # Checks if the options are valid, raising exceptiosn if they are not. 202 | # If the --debug option is true, then messages are dumped but flow continues 203 | def self._validate_options(options) 204 | begin 205 | #Note that OptionParser doesn't really handle compulsory arguments so we use 206 | #our own mechanism 207 | if options.scraper == NEEDED 208 | raise "Incomplete arguments: You must specify a scaper ruby script with --scraper" 209 | end 210 | 211 | # Add in the --ofx option if it is not already specified and if --csv is not specified either 212 | options.ofx = true unless options.csv or options.wesabe_upload 213 | rescue Exception => e 214 | if options.debug 215 | # just dump the message and eat the exception - 216 | # we may be using dummy values for debugging 217 | puts "Ignoring error in options due to --debug flag: #{e}" 218 | else 219 | raise e 220 | end 221 | end #begin/rescue 222 | 223 | end #_validate_options 224 | 225 | ## 226 | # Initializes the logger taking the log-level and the log 227 | # file name from the command line +options+ and setting the logger back on 228 | # the options struct as +options.logger+ 229 | # 230 | # Note that the level is not set explicitly in options but derived from 231 | # flag options like --verbose (INFO), --quiet (FATAL) and --debug (DEBUG) 232 | # 233 | def self._init_logger(options) 234 | # the log log should roll over weekly 235 | if options.log_file.nil? 236 | if options.debug 237 | # if debug is on but no logfile is specified then log to console 238 | options.log_file = STDOUT 239 | else 240 | # Setting the log level to UNKNOWN effectively turns logging off 241 | options.log_level = Logger::UNKNOWN 242 | end 243 | end 244 | options.logger = Logger.new(options.log_file, 'weekly') # roll over weekly 245 | options.logger.level = options.log_level 246 | end 247 | 248 | # Takes a string of arguments and splits it into an array, allowing for 'single quotes' 249 | # to join words into a single argument. 250 | # (Note that parentheses are used to group to exclude the single quotes themselves, but grouping 251 | # results in scan creating an array of arrays with some nil elements hence flatten and delete) 252 | def self.sub_args_to_array(subargs) 253 | return nil if subargs.nil? 254 | return subargs.scan(/([^\s']+)|'([^']*)'/).flatten.delete_if { |x| x.nil?} 255 | end 256 | 257 | end #class CLI 258 | end 259 | -------------------------------------------------------------------------------- /lib/bankjob/transaction.rb: -------------------------------------------------------------------------------- 1 | 2 | require 'rubygems' 3 | require 'builder' 4 | require 'digest/md5' 5 | require 'bankjob.rb' 6 | 7 | module Bankjob 8 | 9 | ## 10 | # A Transaction object represents a transaction in a bank account (a withdrawal, deposit, 11 | # transfer, etc) and is generally the result of running a Bankjob scraper. 12 | # 13 | # A Scraper will create Transactions while scraping web pages in an online banking site. 14 | # These Transactions will be collected in a Statement object which will then be written 15 | # to a file. 16 | # 17 | # A Transaction object knows how to write itself as a record in a CSV 18 | # (Comma Separated Values) file using +to_csv+ or as an XML element in an 19 | # OFX (Open Financial eXchange http://www.ofx.net) file using +to_ofx+ 20 | # 21 | class Transaction 22 | 23 | # OFX transaction type for Generic credit 24 | CREDIT = "CREDIT" 25 | 26 | # OFX transaction type for Generic debit 27 | DEBIT = "DEBIT" 28 | 29 | # OFX transaction type for Interest earned or paid. (Depends on signage of amount) 30 | INT = "INT" 31 | 32 | # OFX transaction type for Dividend 33 | DIV = "DIV" 34 | 35 | # OFX transaction type for FI fee 36 | FEE = "FEE" 37 | 38 | # OFX transaction type for Service charge 39 | SRVCHG = "SRVCHG" 40 | 41 | # OFX transaction type for Deposit 42 | DEP = "DEP" 43 | 44 | # OFX transaction type for ATM debit or credit. (Depends on signage of amount) 45 | ATM = "ATM" 46 | 47 | # OFX transaction type for Point of sale debit or credit. (Depends on signage of amount) 48 | POS = "POS" 49 | 50 | # OFX transaction type for Transfer 51 | XFER = "XFER" 52 | 53 | # OFX transaction type for Check 54 | CHECK = "CHECK" 55 | 56 | # OFX transaction type for Electronic payment 57 | PAYMENT = "PAYMENT" 58 | 59 | # OFX transaction type for Cash withdrawal 60 | CASH = "CASH" 61 | 62 | # OFX transaction type for Direct deposit 63 | DIRECTDEP = "DIRECTDEP" 64 | 65 | # OFX transaction type for Merchant initiated debit 66 | DIRECTDEBIT = "DIRECTDEBIT" 67 | 68 | # OFX transaction type for Repeating payment/standing order 69 | REPEATPMT = "REPEATPMT" 70 | 71 | # OFX transaction type for Other 72 | OTHER = "OTHER" 73 | 74 | # OFX type of the transaction (credit, debit, atm withdrawal, etc) 75 | # Translates to the OFX element TRNTYPE and according to the OFX 2.0.3 schema this can be one of 76 | # * CREDIT 77 | # * DEBIT 78 | # * INT 79 | # * DIV 80 | # * FEE 81 | # * SRVCHG 82 | # * DEP 83 | # * ATM 84 | # * POS 85 | # * XFER 86 | # * CHECK 87 | # * PAYMENT 88 | # * CASH 89 | # * DIRECTDEP 90 | # * DIRECTDEBIT 91 | # * REPEATPMT 92 | # * OTHER 93 | attr_accessor :type 94 | 95 | # date of the transaction 96 | # Translates to OFX element DTPOSTED 97 | attr_accessor :date 98 | 99 | # the date the value affects the account (e.g. funds become available) 100 | # Translates to OFX element DTUSER 101 | attr_accessor :value_date 102 | 103 | # description of the transaction 104 | # This description is typically set by taking the raw description and 105 | # applying rules. If it is not set explicitly it returns the same 106 | # value as +raw_description+ 107 | # Translates to OFX element MEMO 108 | attr_accessor :description 109 | 110 | # the original format of the description as scraped from the bank site 111 | # This allows the raw information to be preserved when modifying the 112 | # +description+ with transaction rules (see Scraper#transaction_rule) 113 | # This does _not_ appear in the OFX output, only +description+ does. 114 | attr_accessor :raw_description 115 | 116 | # amount of the credit or debit (negative for debits) 117 | # Translates to OFX element TRNAMT 118 | attr_accessor :amount 119 | 120 | # account balance after the transaction 121 | # Not used in OFX but important for working out statement balances 122 | attr_accessor :new_balance 123 | 124 | # account balance after the transaction as a numeric Ruby Float 125 | # Not used in OFX but important for working out statement balances 126 | # in calculations (see #real_amount) 127 | attr_reader :real_new_balance 128 | 129 | # the generated unique id for this transaction in an OFX record 130 | # Translates to OFX element FITID this is generated if not set 131 | attr_accessor :ofx_id 132 | 133 | # the payee of an expenditure (ie a debit or transfer) 134 | # This is of type Payee and translates to complex OFX element PAYEE 135 | attr_accessor :payee 136 | 137 | # the cheque number of a cheque transaction 138 | # This is of type Payee and translates to OFX element CHECKNUM 139 | attr_accessor :check_number 140 | 141 | ## 142 | # the numeric real-number amount of the transaction. 143 | # 144 | # The transaction amount is typically a string and may hold commas for 145 | # 1000s or for decimal separators, making it unusable for mathematical 146 | # operations. 147 | # 148 | # This attribute returns the amount converted to a Ruby Float, which can 149 | # be used in operations like: 150 | # 151 | # if (transaction.real_amount < 0) 152 | # puts "It's a debit!" 153 | # end 154 | # 155 | # The +real_amount+ attribute is calculated using the +decimal+ separator 156 | # passed into the constructor (defaults to ".") 157 | # See Scraper#decimal 158 | # 159 | # This attribute is not used in OFX. 160 | # 161 | attr_reader :real_amount 162 | 163 | ## 164 | # Creates a new Transaction with the specified attributes. 165 | # 166 | def initialize(decimal = ".") 167 | @ofx_id = nil 168 | @date = nil 169 | @value_date = nil 170 | @raw_description = nil 171 | @description = nil 172 | @amount = 0 173 | @new_balance = 0 174 | @decimal = decimal 175 | 176 | # Always create a Payee even if it doesn't get used - this ensures an empty 177 | # element in the OFX output which is more correct and, for one thing, 178 | # stops Wesabe from adding UNKNOWN PAYEE to every transaction (even deposits) 179 | @payee = Payee.new() 180 | @check_number = nil 181 | @type = OTHER 182 | end 183 | 184 | def date=(raw_date_time) 185 | @date = Bankjob.create_date_time(raw_date_time) 186 | end 187 | 188 | def value_date=(raw_date_time) 189 | @value_date = Bankjob.create_date_time(raw_date_time) 190 | end 191 | 192 | ## 193 | # Creates a unique ID for the transaction for use in OFX documents, unless 194 | # one has already been set. 195 | # All OFX transactions need a unique identifier. 196 | # 197 | # Note that this is generated by creating an MD5 digest of the transaction 198 | # date, raw description, type, amount and new_balance. Which means that two 199 | # identical transactions will always produce the same +ofx_id+. 200 | # (This is important so that repeated scrapes of the same transaction value 201 | # produce identical ofx_id values) 202 | # 203 | def ofx_id() 204 | if @ofx_id.nil? 205 | text = "#{@date}:#{@raw_description}:#{@type}:#{@amount}:#{@new_balance}" 206 | @ofx_id= Digest::MD5.hexdigest(text) 207 | end 208 | return @ofx_id 209 | end 210 | 211 | ## 212 | # Returns the description, defaulting to the +raw_description+ if no 213 | # specific description has been set by the user. 214 | # 215 | def description() 216 | @description.nil? ? raw_description : @description 217 | end 218 | 219 | ## 220 | # Returns the Transaction amount attribute as a ruby Float after 221 | # replacing the decimal separator with a . and stripping any other 222 | # separators. 223 | # 224 | def real_amount() 225 | Bankjob.string_to_float(amount, @decimal) 226 | end 227 | 228 | ## 229 | # Returns the new balance after the transaction as a ruby Float after 230 | # replacing the decimal separator with a . and stripping any other 231 | # separators. 232 | # 233 | def real_new_balance() 234 | Bankjob.string_to_float(new_balance, @decimal) 235 | end 236 | 237 | ## 238 | # Generates a string representing this Transaction as comma separated values 239 | # in the form: 240 | # 241 | # date, value_date, description, real_amount, real_new_balance, amount, new_balance, raw_description, ofx_id 242 | # 243 | def to_csv 244 | # if there's a payee, prepend their name to the description - otherwise skip it 245 | if (not payee.nil? and (not payee.name.nil?)) 246 | desc = payee.name + " - " + description 247 | else 248 | desc = description 249 | end 250 | [Bankjob.date_time_to_csv(date), Bankjob.date_time_to_csv(value_date), desc, real_amount, real_new_balance, amount, new_balance, raw_description, ofx_id].to_csv 251 | end 252 | 253 | ## 254 | # Generates a string for use as a header in a CSV file for transactions. 255 | # This will produce the following string: 256 | # 257 | # date, value_date, description, real_amount, real_new_balance, amount, new_balance, raw_description, ofx_id 258 | # 259 | def self.csv_header 260 | %w{ Date Value-Date Description Amount New-Balance Raw-Amount Raw-New-Balance Raw-Description OFX-ID }.to_csv 261 | end 262 | 263 | ## 264 | # Creates a new Transaction from a string that defines a row in a CSV file. 265 | # 266 | # +csv_row+ must hold an array of values in precisely this order: 267 | # 268 | # date, value_date, description, real_amount, real_new_balance, amount, new_balance, raw_description, ofx_id 269 | # 270 | # (The format should be the same as that produced by +to_csv+) 271 | # 272 | def self.from_csv(csv_row, decimal) 273 | if (csv_row.length != 9) # must have 9 cols 274 | csv_lines = csv_row.join("\n\t") 275 | msg = "Failed to create Transaction from csv row: \n\t#{csv_lines}\n" 276 | msg << " - 9 columns are required in the form: date, value_date, " 277 | msg << "description, real_amount, real_new_balance, amount, new_balance, " 278 | msg << "raw_description, ofx_id" 279 | raise msg 280 | end 281 | tx = Transaction.new(decimal) 282 | tx.date, tx.value_date, tx.description = csv_row[0..2] 283 | # skip real_amount and real_new_balance, they're read only and calculated 284 | tx.amount, tx.new_balance, tx.raw_description, tx.ofx_id = csv_row[5..8] 285 | return tx 286 | end 287 | 288 | ## 289 | # Generates an XML string adhering to the OFX standard 290 | # (see Open Financial Exchange http://www.ofx.net) 291 | # representing a single Transaction XML element. 292 | # 293 | # The OFX 2 schema defines a STMTTRN (SatementTransaction) as follows: 294 | # 295 | # 296 | # 297 | # 298 | # The OFX element "STMTTRN" is of type "StatementTransaction" 299 | # 300 | # 301 | # 302 | # 303 | # 304 | # 305 | # 306 | # 307 | # 308 | # 309 | # 310 | # 311 | # 312 | # 313 | # 314 | # 315 | # 316 | # 317 | # 318 | # 319 | # 320 | # 321 | # 322 | # 323 | # 324 | # 325 | # 326 | # 327 | # 328 | # 329 | # 330 | # 331 | # 332 | # 333 | # 334 | def to_ofx 335 | buf = "" 336 | # Set margin=5 to indent it nicely within the output from Statement.to_ofx 337 | x = Builder::XmlMarkup.new(:target => buf, :indent => 2, :margin=>5) 338 | x.STMTTRN { # transaction statement 339 | x.TRNTYPE type 340 | x.DTPOSTED Bankjob.date_time_to_ofx(date) #Date transaction was posted to account, [datetime] yyyymmdd or yyyymmddhhmmss 341 | x.TRNAMT amount #Ammount of transaction [amount] can be , or . separated 342 | x.FITID ofx_id 343 | x.CHECKNUM check_number unless check_number.nil? 344 | buf << payee.to_ofx unless payee.nil? 345 | #x.NAME description 346 | x.MEMO description 347 | } 348 | return buf 349 | end 350 | 351 | ## 352 | # Produces a string representation of the transaction 353 | # 354 | def to_s 355 | "#{self.class} - ofx_id: #{@ofx_id}, date:#{@date}, raw description: #{@raw_description}, type: #{@type} amount: #{@amount}, new balance: #{@new_balance}" 356 | end 357 | 358 | ## 359 | # Overrides == to allow comparison of Transaction objects so that they can 360 | # be merged in Statements. See Statement#merge 361 | # 362 | def ==(other) #:nodoc: 363 | if other.kind_of?(Transaction) 364 | # sometimes the same date, when written and read back will not appear equal so convert to 365 | # a canonical string first 366 | return (Bankjob.date_time_to_ofx(@date) == Bankjob.date_time_to_ofx(other.date) and 367 | # ignore value date - it may be updated between statements 368 | # (consider using ofx_id here later) 369 | @raw_description == other.raw_description and 370 | @amount == other.amount and 371 | @type == other.type and 372 | @new_balance == other.new_balance) 373 | end 374 | end 375 | 376 | # 377 | # Overrides eql? so that array union will work when merging statements 378 | # 379 | def eql?(other) #:nodoc: 380 | return self == other 381 | end 382 | 383 | ## 384 | # Overrides hash so that array union will work when merging statements 385 | # 386 | def hash() #:nodoc: 387 | prime = 31; 388 | result = 1; 389 | result = prime * result + @amount.to_i 390 | result = prime * result + @new_balance.to_i 391 | result = prime * result + (@date.nil? ? 0 : Bankjob.date_time_to_ofx(@date).hash); 392 | result = prime * result + (@raw_description.nil? ? 0 : @raw_description.hash); 393 | result = prime * result + (@type.nil? ? 0 : @type.hash); 394 | # don't use value date 395 | return result; 396 | end 397 | 398 | end # class Transaction 399 | end # module 400 | 401 | -------------------------------------------------------------------------------- /lib/bankjob/statement.rb: -------------------------------------------------------------------------------- 1 | require 'rubygems' 2 | require 'builder' 3 | require 'fastercsv' 4 | require 'bankjob' 5 | 6 | module Bankjob 7 | 8 | ## 9 | # A Statement object represents a bank statement and is generally the result of running a Bankjob scraper. 10 | # The Statement holds an array of Transaction objects and specifies the closing balance and the currency in use. 11 | # 12 | # A Scraper will create a Statement by scraping web pages in an online banking site. 13 | # The Statement can then be stored as a file in CSV (Comma Separated Values) format 14 | # using +to_csv+ or in OFX (Open Financial eXchange http://www.ofx.net) format 15 | # using +to_ofx+. 16 | # 17 | # One special ability of Statement is the ability to merge with an existing statement, 18 | # automatically eliminating overlapping transactions. 19 | # This means that when writing subsequent statements to the same CSV file 20 | # (note well: CSV only) a continous transaction record can be built up 21 | # over a long period. 22 | # 23 | class Statement 24 | 25 | # OFX value for the ACCTTYPE of a checking account 26 | CHECKING = "CHECKING" 27 | 28 | # OFX value for the ACCTTYPE of a savings account 29 | SAVINGS = "SAVINGS" 30 | 31 | # OFX value for the ACCTTYPE of a money market account 32 | MONEYMRKT = "MONEYMRKT" 33 | 34 | # OFX value for the ACCTTYPE of a loan account 35 | CREDITLINE = "CREDITLINE" 36 | 37 | # the account balance after the last transaction in the statement 38 | # Translates to the OFX element BALAMT in LEDGERBAL 39 | attr_accessor :closing_balance 40 | 41 | # the avaliable funds in the account after the last transaction in the statement (generally the same as closing_balance) 42 | # Translates to the OFX element BALAMT in AVAILBAL 43 | attr_accessor :closing_available 44 | 45 | # the array of Transaction objects that comprise the statement 46 | attr_accessor :transactions 47 | 48 | # the three-letter currency symbol generated into the OFX output (defaults to EUR) 49 | # This is passed into the initializer (usually by the Scraper - see Scraper#currency) 50 | attr_reader :currency 51 | 52 | # the identifier of the bank - a 1-9 char string (may be empty) 53 | # Translates to the OFX element BANKID 54 | attr_accessor :bank_id 55 | 56 | # the account number of the statement - a 1-22 char string that must be passed 57 | # into the initalizer of the Statement 58 | # Translates to the OFX element ACCTID 59 | attr_accessor :account_number 60 | 61 | # the type of bank account the statement is for 62 | # Tranlsates to the OFX type ACCTTYPE and must be one of 63 | # * CHECKING 64 | # * SAVINGS 65 | # * MONEYMRKT 66 | # * CREDITLINE 67 | # Use a constant to set this - defaults to CHECKING 68 | attr_accessor :account_type 69 | 70 | # the last date of the period the statement covers 71 | # Translates to the OFX element DTEND 72 | attr_accessor :to_date 73 | 74 | # the first date of the period the statement covers 75 | # Translates to the OFX element DTSTART 76 | attr_accessor :from_date 77 | 78 | ## 79 | # Creates a new empty Statement with no transactions. 80 | # The +account_number+ must be specified as a 1-22 character string. 81 | # The specified +currency+ defaults to EUR if nothing is passed in. 82 | # 83 | def initialize(account_number, currency = "EUR") 84 | @account_number = account_number 85 | @currency = currency 86 | @transactions = [] 87 | @account_type = CHECKING 88 | @closing_balance = nil 89 | @closing_available = nil 90 | end 91 | 92 | ## 93 | # Appends a new Transaction to the end of this Statement 94 | # 95 | def add_transaction(transaction) 96 | @transactions << transaction 97 | end 98 | 99 | ## 100 | # Overrides == to allow comparison of Statement objects. 101 | # Two Statements are considered equal (that is, ==) if 102 | # and only iff they have the same values for: 103 | # * +to_date+ 104 | # * +from_date+ 105 | # * +closing_balance+ 106 | # * +closing_available+ 107 | # * each and every transaction. 108 | # Note that the transactions are compared with Transaction.== 109 | # 110 | def ==(other) # :nodoc: 111 | if other.kind_of?(Statement) 112 | return (from_date == other.from_date and 113 | to_date == other.to_date and 114 | closing_balance == other.closing_balance and 115 | closing_available == other.closing_available and 116 | transactions == other.transactions) 117 | end 118 | return false 119 | end 120 | 121 | ## 122 | # Merges the transactions of +other+ into the transactions of this statement 123 | # and returns the resulting array of transactions 124 | # Raises an exception if the two statements overlap in a discontiguous fashion. 125 | # 126 | def merge_transactions(other) 127 | if (other.kind_of?(Statement)) 128 | union = transactions | other.transactions # the set union of both 129 | # now check that the union contains all of the originals, otherwise 130 | # we have merged some sort of non-contiguous range 131 | raise "Failed to merge transactions properly." unless union.first(@transactions.length) == @transactions 132 | return union 133 | end 134 | end 135 | 136 | ## 137 | # Merges the transactions of +other+ into the transactions of this statement 138 | # and returns the result. 139 | # Neither statement is changed. See #merge! if you want to modify the statement. 140 | # Raises an exception if the two statements overlap in a discontiguous fashion. 141 | # 142 | def merge(other) 143 | union = merge_transactions(other) 144 | merged = self.dup 145 | merged.closing_balance = nil 146 | merged.closing_available = nil 147 | merged.transactions = union 148 | return merged 149 | end 150 | 151 | ## 152 | # Merges the transactions of +other+ into the transactions of this statement. 153 | # Causes this statement to be changed. See #merge for details. 154 | # 155 | def merge!(other) 156 | @closing_balance = nil 157 | @closing_available = nil 158 | @transactions = merge_transactions(other) 159 | end 160 | 161 | ## 162 | # Generates a CSV (comma separated values) string with a single 163 | # row for each transaction. 164 | # Note that no header row is generated as it would make it 165 | # difficult to concatenate and merge subsequent CSV strings 166 | # (but we should consider it as a user option in the future) 167 | # 168 | def to_csv 169 | buf = "" 170 | transactions.each do |transaction| 171 | buf << transaction.to_csv 172 | end 173 | return buf 174 | end 175 | 176 | ## 177 | # Generates a string for use as a header in a CSV file for a statement. 178 | # 179 | # Delegates to Transaction#csv_header 180 | # 181 | def self.csv_header 182 | return Transaction.csv_header 183 | end 184 | 185 | ## 186 | # Reads in transactions from a CSV file or string specified by +source+ 187 | # and adds them to this statement. 188 | # 189 | # Uses a simple (dumb) heuristic to determine if the +source+ is a file 190 | # or a string: if it contains a comma (,) then it is a string 191 | # otherwise it is treated as a file path. 192 | # 193 | def from_csv(source, decimal = ".") 194 | if (source =~ /,/) 195 | # assume source is a string 196 | FasterCSV.parse(source) do |row| 197 | add_transaction(Transaction.from_csv(row, decimal)) 198 | end 199 | else 200 | # assume source is a filepath 201 | FasterCSV.foreach(source) do |row| 202 | add_transaction(Transaction.from_csv(row, decimal)) 203 | end 204 | end 205 | end 206 | 207 | ## 208 | # Generates an XML string adhering to the OFX standard 209 | # (see Open Financial eXchange http://www.ofx.net) 210 | # representing a single bank statement holding a list 211 | # of transactions. 212 | # The XML for the individual transactions is generated 213 | # by the Transaction class itself. 214 | # 215 | # The OFX 2 schema for a statement response (STMTRS) is: 216 | # 217 | # 218 | # 219 | # 220 | # The OFX element "STMTRS" is of type "StatementResponse" 221 | # 222 | # 223 | # 224 | # 225 | # 226 | # 227 | # 228 | # 229 | # 230 | # 231 | # 232 | # 233 | # 234 | # 235 | # Where the BANKTRANLIST (Bank Transaction List) is defined as: 236 | # 237 | # 238 | # 239 | # 240 | # The OFX element "BANKTRANLIST" is of type "BankTransactionList" 241 | # 242 | # 243 | # 244 | # 245 | # 246 | # 247 | # 248 | # 249 | # 250 | # And this is the definition of the type BankAccount. 251 | # 252 | # 253 | # 254 | # 255 | # The OFX elements BANKACCTFROM and BANKACCTTO are of type "BankAccount" 256 | # 257 | # 258 | # 259 | # 260 | # 261 | # 262 | # 263 | # 264 | # 265 | # 266 | # 267 | # 268 | # 269 | # 270 | # 271 | # The to_ofx method will only generate the essential elements which are 272 | # * BANKID - the bank identifier (a 1-9 char string - may be empty) 273 | # * ACCTID - the account number (a 1-22 char string - may not be empty!) 274 | # * ACCTTYPE - the type of account - must be one of: 275 | # "CHECKING", "SAVINGS", "MONEYMRKT", "CREDITLINE" 276 | # 277 | # (See Transaction for a definition of STMTTRN) 278 | # 279 | def to_ofx 280 | buf = "" 281 | # Use Builder to generate XML. Builder works by catching missing_method 282 | # calls and generating an XML element with the name of the missing method, 283 | # nesting according to the nesting of the calls and using arguments for content 284 | x = Builder::XmlMarkup.new(:target => buf, :indent => 2) 285 | x.OFX { 286 | x.BANKMSGSRSV1 { #Bank Message Response 287 | x.STMTTRNRS { #Statement-transaction aggregate response 288 | x.STMTRS { #Statement response 289 | x.CURDEF currency #Currency 290 | x.BANKACCTFROM { 291 | x.BANKID bank_id # bank identifier 292 | x.ACCTID account_number 293 | x.ACCTTYPE account_type # acct type: checking/savings/... 294 | } 295 | x.BANKTRANLIST { #Transactions 296 | x.DTSTART Bankjob.date_time_to_ofx(from_date) 297 | x.DTEND Bankjob.date_time_to_ofx(to_date) 298 | transactions.each { |transaction| 299 | buf << transaction.to_ofx 300 | } 301 | } 302 | x.LEDGERBAL { # the final balance at the end of the statement 303 | x.BALAMT closing_balance # balance amount 304 | x.DTASOF Bankjob.date_time_to_ofx(to_date) # balance date 305 | } 306 | x.AVAILBAL { # the final Available balance 307 | x.BALAMT closing_available 308 | x.DTASOF Bankjob.date_time_to_ofx(to_date) 309 | } 310 | } 311 | } 312 | } 313 | } 314 | return buf 315 | end 316 | 317 | ONE_MINUTE = 60 318 | ELEVEN_59_PM = 23 * 60 * 60 + 59 * 60 # seconds at 23:59 319 | MIDDAY = 12 * 60 * 60 320 | 321 | ## 322 | # Finishes the statement after scraping in two ways depending on the information 323 | # that the scraper was able to obtain. Optionally have your scraper class call 324 | # this after scraping is finished. 325 | # 326 | # This method: 327 | # 328 | # 1. Sets the closing balance and available_balance and the to_ and from_dates 329 | # by using the first and last transactions in the list. Which transaction is 330 | # used depends on whether +most_recent_first+ is true or false. 331 | # The scraper may just set these directly in which case this may not be necessary. 332 | # 333 | # 2. If +fake_times+ is true time-stamps are invented and added to the transaction 334 | # date attributes. This is useful if the website beings scraped shows dates, but 335 | # not times, but has transactions listed in chronoligical arder. 336 | # Without this process, the ofx generated has no proper no indication of the order of 337 | # transactions that occurred in the same day other than the order in the statement 338 | # and this may be ignored by the client. (Specifically, Wesabe will reorder transactions 339 | # in the same day if they all appear to occur at the same time). 340 | # 341 | # Note that the algorithm to set the fake times is a little tricky. Assuming 342 | # the transactionsa are most-recent-first, the first last transaction on each 343 | # day is set at 11:59pm each transaction prior to that is one minute earlier. 344 | # 345 | # But for the first transactions in the statement, the first is set at a few 346 | # minutes after midnight, then we count backward. (The actual number of minutes 347 | # is based on the number of transactions + 1 to be sure it doesnt pass midnight) 348 | # 349 | # This is crucial because transactions for a given day will often span 2 or more 350 | # statement. By starting just after midnight and going back to just before midnight 351 | # we reduce the chance of overlap. 352 | # 353 | # If the to-date is the same as the from-date for a transaction, then we start at 354 | # midday, so that prior and subsequent statements don't overlap. 355 | # 356 | # This simple algorithm basically guarantees no overlaps so long as: 357 | # i. The number of transactions is small compared to the number of minutes in a day 358 | # ii. A single day will not span more than 3 statements 359 | # 360 | # If the statement is most-recent-last (+most_recent_first = false+) the same 361 | # algorithm is applied, only in reverse 362 | # 363 | def finish(most_recent_first, fake_times=false) 364 | if !@transactions.empty? then 365 | # if the user hasn't set the balances, set them to the first or last 366 | # transaction balance depending on the order 367 | if most_recent_first then 368 | @closing_balance ||= transactions.first.new_balance 369 | @closing_available ||= transactions.first.new_balance 370 | @to_date ||= transactions.first.date 371 | @from_date ||= transactions.last.date 372 | else 373 | @closing_balance ||= transactions.last.new_balance 374 | @closing_available ||= transactions.last.new_balance 375 | @to_date ||= transactions.last.date 376 | @from_date ||= transactions.first.date 377 | end 378 | 379 | if fake_times and to_date.hour == 0 then 380 | # the statement was unable to scrape times to go with the dates, but the 381 | # client (say wesabe) will get the transaction order wrong if there are no 382 | # times, so here we add times that order the transactions according to the 383 | # order of the array of transactions 384 | 385 | # the delta is 1 minute forward or backward fr 386 | if to_date == from_date then 387 | # all of the statement's transactions occur in the same day - to try to 388 | # avoid overlap with subsequent or previous transacitons we group order them 389 | # from 11am onward 390 | seconds = MIDDAY 391 | else 392 | seconds = (transactions.length + 1) * 60 393 | end 394 | 395 | if most_recent_first then 396 | yday = transactions.first.date.yday 397 | start = 0 398 | delta = 1 399 | finish = transactions.length 400 | else 401 | yday = transactions.last.date.yday 402 | start = transactions.length - 1 403 | finish = -1 404 | delta = -1 405 | end 406 | 407 | i = start 408 | until i == finish 409 | tx = transactions[i] 410 | if tx.date.yday != yday 411 | # starting a new day, begin the countdown from 23:59 again 412 | yday = tx.date.yday 413 | seconds = ELEVEN_59_PM 414 | end 415 | tx.date += seconds unless tx.date.hour > 0 416 | seconds -= ONE_MINUTE 417 | i += delta 418 | end 419 | end 420 | end 421 | end 422 | 423 | def to_s 424 | buf = "#{self.class}: close_bal = #{closing_balance}, avail = #{closing_available}, curr = #{currency}, transactions:" 425 | transactions.each do |tx| 426 | buf << "\n\t\t#{tx.to_s}" 427 | end 428 | buf << "\n---\n" 429 | return buf 430 | end 431 | end # class Statement 432 | end # module 433 | -------------------------------------------------------------------------------- /lib/bankjob/scraper.rb: -------------------------------------------------------------------------------- 1 | 2 | require 'rubygems' 3 | require 'mechanize' 4 | require 'logger' 5 | require 'bankjob' 6 | 7 | module Bankjob 8 | 9 | ## 10 | # The Scraper class is the basis of all Bankjob web scrapers for scraping specific 11 | # bank websites. 12 | # 13 | # To create your own scraper simply subclass Scraper and be sure to override 14 | # the method +scrape_statement+ to perform the scraping and return a 15 | # Bankjob::Statement object. 16 | # 17 | # Scraper provides some other optional methods to help you build Statements: 18 | # 19 | # +currency+:: use this class attribute to set the OFX currency at the top of 20 | # your Scraper subclass definition. E.g.: 21 | # 22 | # 23 | # class MyScraper < Scraper 24 | # currency "USD" 25 | # ... 26 | # 27 | # It defaults to "EUR" for euros. 28 | # 29 | # +decimal+:: use this class attribute to set the decimal separator at the top of 30 | # your Scraper subclass definition. E.g.: 31 | # 32 | # class MyScraper < Scraper 33 | # decimal "," 34 | # ... 35 | # 36 | # It defaults to "." (period), the common alternative being "," (comma) 37 | # 38 | # Note that this should be set to the separator used in the +amount+ 39 | # attribute of the Transaction objects your Scraper creates. If, say, 40 | # you deliberately scrape values like "12,34" and convert them to 41 | # "12.34" before storing them in your Transaction, then leave the 42 | # decimal as ".". 43 | # If you choose to store the Transaction amount with as "12,34", 44 | # however, the +decimal+ setting becomes important when calling 45 | # Transaction#real_amount to get the amount as a Float upon which 46 | # calculations can be performed. 47 | # 48 | # +options+:: holds the command line options provided when Bankjob was launched. 49 | # Use this attribute to get access to global options. For your scraper 50 | # specific options use the array passed into +scrape_statement+ instead. 51 | # (See #options below for more advice on how to use this) 52 | # 53 | # +logger+:: holds the logger initialized by Bankjob based on the command line 54 | # options. Use this to attribute to log information, warnings and debug messages 55 | # from your logger. 56 | # (See #logger below for more advice on how to use this) 57 | # 58 | # +create_statement+:: creates a new empty Statement object with the appropriate 59 | # default attributes (that is, the right currency) 60 | # Use this in your Scraper to instantiate new Statement objects. 61 | # 62 | # +create_transaction+:: creates a new empty Transaction object with the appropriate 63 | # default attributes (that is, the right decimal separator) 64 | # Use this in your Scraper to instantiate new Transaction objects. 65 | # 66 | # +transaction_rule+:: registers a rule to be applied to all transactions after the 67 | # statement has been scraped. 68 | # Define as many of these as you need in your craper to build better 69 | # organized Transaction objects with clearer descriptions of the 70 | # transaction, etc. 71 | # 72 | # +finish+ :: finishes a transaction by setting the balances and to and from dates 73 | # based on the first and last transactions. Also, optionally, generates 74 | # fake timestamps for transactions that have no time component in their 75 | # dates. This is important for clients that use the timestamps to order 76 | # the transactions correctly, and would otherwise mess up the order 77 | # if all transactions on the same day were at the same time (E.g. Wesabe) 78 | # 79 | # Here is an example of a simple (but incomplete) scraper. 80 | # Note that all of the scraping and parsing is in the +scrape_statement+ method, although 81 | # a lot of the details of Hpricot parsing are left up to the imagination of the reader. 82 | # 83 | # When creating a scraper yourself look in the +scrapers+ directory of the bankjob gem 84 | # to see some more useful examples. 85 | # 86 | # class AcmeBankScraper < Scraper 87 | # ##### 88 | # # 1. Set up the Scraper properties for currency and separator 89 | # # (this is optional) 90 | # 91 | # currency "EUR" # set the currency (EUR is the default anyway but just to demo..) 92 | # decimal "," # set the decimal separator to comma instead of . 93 | # 94 | # ##### 95 | # # 2. Create some rules to post-process my transactions 96 | # # (this is optional but is easier to maintain than manipulating 97 | # # the values in the scraper itself) 98 | # 99 | # # rule to set negative transactions as debits 100 | # transaction_rule do |tx| 101 | # tx.type = "DEBIT" if (tx.real_amount < 0 and tx.type == "OTHER") 102 | # end 103 | # 104 | # # General description parsing rule 105 | # transaction_rule do |tx| 106 | # case tx.description 107 | # when /ATM/i 108 | # tx.type = "ATM" 109 | # when /ELEC PURCHASE/ 110 | # tx.description.gsub!(/ELEC PURCHASE \d+/, "spent with ATM card: ") 111 | # end 112 | # end 113 | # 114 | # ##### 115 | # # 3. Implement main engine of the scraper 116 | # # (this is essential and where 99% of the work is) 117 | # 118 | # def scrape_statement(args) 119 | # 120 | # logger.debug("Reading debug input html from #{options.input} instead of scraping the real website.") 121 | # agent = WWW::Mechanize.new 122 | # agent.user_agent_alias = 'Windows IE 6' # pretend that we're IE 6.0 123 | # # navigate to the login page 124 | # login_page = agent.get("http://mybank.com/login") 125 | # # find login form, fill it out and submit it 126 | # form = login_page.forms.name('myBanksLoginForm').first 127 | # # Mechanize creates constants like USERNAME for the form element it finds with that name 128 | # form.USERNAME = args[0] # assuming -scraper_args "user password" 129 | # form.PASSWORD = args[1] 130 | # agent.submit(form) 131 | # sleep 3 #wait while the login takes effect 132 | # 133 | # transactions_page = agent.get("http://mybank.com/transactions") 134 | # statement = create_statement 135 | # 136 | # # ... go read the Hpricot documentation to work out how to get your transactions out of 137 | # # the transactions_page and create a new transaction object for each one 138 | # # We're going to gloss over that part here .... 139 | # 140 | # table = # use Hpricot to get the html table element assuming your transactions are in a table 141 | # rows = (table/"tr[@valign=top]") # works for a table where the rows needed have the valign attr set to top 142 | # rows.each do |row| 143 | # transaction = create_transaction 144 | # transaction.date = #... scrape a date here 145 | # ... 146 | # statement.transactions << transaction 147 | # end 148 | # end 149 | # end 150 | # 151 | #-- 152 | # (Non RDOC comment) There are two parts to the Scraper class: 153 | # - the public part which defines the 154 | # method to be overridden in subclasses and provides utility methods and attributes; 155 | # - the private internal part which handles the mechanics of registering a 156 | # subclass as the scraper to be used, setting the currency and decimal attributes 157 | # and registering transaction rules 158 | # 159 | # 160 | class Scraper 161 | 162 | ## 163 | # Provides access to a logger instance created in the BankjobRunner which 164 | # subclasses can use for logging if they need to. 165 | # 166 | # To use this in your own scraper, use code like: 167 | # 168 | # include 'logger' 169 | # ... 170 | # logger.debug("MyScraper is scraping the page at #{my_url}") 171 | # logger.info("MyScraper fetched new statement from MyBank and has been sitting in my chair") 172 | # logger.warn("MyScraper's been sitting in MY chair!") 173 | # logger.fatal("MyScraper's been sitting in MY CHAIR and IT'S ALL BROKEN!") 174 | # 175 | attr_accessor :logger 176 | 177 | ## 178 | # Provides access to the command line options which subclasses can use it if 179 | # they need access to the global options used to launch Bankjob 180 | # 181 | # To use this in your own scraper, use code like: 182 | # 183 | # if (options.input?) then 184 | # print "the input html file for debugging is #{options.input} 185 | # end 186 | # 187 | attr_accessor :options 188 | 189 | ## 190 | # Returns the decimal separator for this scraper 191 | # This is typically set in the scraper class using the "decimal" directive. 192 | # 193 | def decimal 194 | @@decimal 195 | end 196 | 197 | ## 198 | # Returns the OFX currency for this scraper. 199 | # This is typically set in the scraper class using the "currency" directive. 200 | # 201 | def currency 202 | @@currency 203 | end 204 | 205 | ## 206 | # Sets the decimal separator for the money amounts used in the data fetched 207 | # by this scraper. 208 | # The scraper class can use this as a directive to set the separator so: 209 | # decimal "," 210 | # 211 | # Defaults to period ".", but will typically need to be set as a comma in 212 | # european websites 213 | # 214 | def self.decimal(decimal) 215 | @@decimal = decimal 216 | end 217 | 218 | ## 219 | # Sets the OFX currency name for use in the OFX statements produced by 220 | # this scraper. 221 | # 222 | # The scraper class can use this as a directive to set the separator so: 223 | # currency "USD" 224 | # 225 | # Defaults to EUR 226 | # 227 | def self.currency(currency) 228 | @@currency = currency 229 | end 230 | 231 | ## 232 | # Sets the account number for statements produced by this statement. 233 | # 234 | # The scraper class can use this as a directive to set the number so: 235 | # account_number "12345678" 236 | # 237 | # Must be a string from 1 to 22 chars in length 238 | # 239 | # This will be used by the create_statement method to set the account, 240 | # but the scraper may ignore this and simply construct its own statements 241 | # or change the number using the accessor: statement.account_number = 242 | # after constructing it. 243 | # 244 | # The scraper class can use this as a directive to set the separator so: 245 | # currency "USD" 246 | # 247 | # Defaults to EUR 248 | # 249 | def self.account_number(account_number) 250 | @@account_number = account_number 251 | end 252 | 253 | ## 254 | # Sets the account type for statements produced by this statement. 255 | # 256 | # The scraper class can use this as a directive to set the type so: 257 | # account_type Statement::SAVINGS 258 | # 259 | # Must be a string based on one of the constants in Statement 260 | # 261 | # This will be used by the create_statement method to set the account type, 262 | # but the scraper may ignore this and simply construct its own statements 263 | # or change the type using the accessor: statement.account_type = 264 | # after constructing it. 265 | # 266 | # Defaults to Statement::CHECKING 267 | # 268 | def self.account_type(account_type) 269 | @@account_type = account_type 270 | end 271 | 272 | ## 273 | # Sets the bank identifier for statements produced by this statement. 274 | # 275 | # The scraper class can use this as a directive to set the number so: 276 | # bank_id "12345678" 277 | # 278 | # Must be a string from 1 to 9 chars in length 279 | # 280 | # This will be used by the create_statement method to set the bank id, 281 | # but the scraper may ignore this and simply construct its own statements 282 | # or change the number using the accessor: statement.bank_id = 283 | # after constructing it. 284 | # 285 | # Defaults to blank 286 | # 287 | def self.bank_id(bank_id) 288 | @@bank_id = bank_id 289 | end 290 | 291 | ## 292 | # ScraperRule is a struct used for holding a rule body with its priority. 293 | # Users can create transaction rules in their Scraper subclasses using 294 | # the Scraper#ransaction_rule method. 295 | ScraperRule = Struct.new(:priority, :rule_body) 296 | 297 | ## 298 | # Processes a transaction after it has been created to allow it to be manipulated 299 | # into a more useful form for the client. 300 | # 301 | # For example, the transaction description might be simplified to remove certain 302 | # common strings, or the Payee details might be extracted from the description. 303 | # 304 | # Implementing this as a class method using a block permits the user to add 305 | # implement transaction processing rules by calling this method several times 306 | # rather than implementing a single method (gives it a sort of DSL look) 307 | # 308 | # E.g. 309 | # # This rule detects ATM withdrawals and modifies 310 | # # the description and sets the the type it uses 311 | # transaction_rule do |tx| 312 | # if (tx.real_amount < 0) 313 | # if tx.raw_description =~ /WDR.*ATM\s+\d+\s+/i 314 | # # $' holds whatever is after the pattern match - usually the ATM location 315 | # tx.description = "ATM withdrawal at #{$'}" 316 | # tx.type = Transaction::ATM 317 | # end 318 | # end 319 | # end 320 | # 321 | # 322 | # A transaction rule can optionally specifiy a +priority+ - any integer value. 323 | # The default priority is zero, with lower priority rules being executed last. 324 | # 325 | # The final order in which transaction rules will be executed is thus: 326 | # * rules with a higher priority value will be executed before rules with 327 | # a lower priority no matter where they are declared 328 | # * rules of the same priority declared in the same class wil be executed in 329 | # the order in which they are declared - top rules first 330 | # * rules in parent classes are executed before rules in subclasses of the 331 | # same priority. 332 | # 333 | # If you really want a rule to be fired last, and you want to allow for 334 | # subclasses to your scraper, use a negative priority like this: 335 | # 336 | # transaction_rule(-999) do |tx| 337 | # puts "I get executed last" 338 | # end 339 | # 340 | def self.transaction_rule(priority = 0, &rule_body) 341 | @@transaction_rules ||= [] 342 | rule = ScraperRule.new(priority, rule_body) 343 | # Using Array#sort won't work on here (or later) because it doesn't preserve 344 | # the order of the rules with equal priorty - thus breaking the 345 | # rules of priority detailed above. So we have to sort as we insert 346 | # each new rule in order without messing up the equal-priority order 347 | # which is first come, first in. 348 | # Imagine we have a set of rule already inorder of priority such as: 349 | # A:999, B:999, C:0, D:0, E:-999, F:-999 350 | # we're now adding X:0, which should come after D since it's added later 351 | # First we reverse the array to get 352 | # F:-999, E:-999, D:0, C:0, B:999, A:999 353 | # then we find the first element with priority greater than or equal to 354 | # X's priority of 0. Just greater than won't work because we'll end up 355 | # putting X between B and C whereas it was added after D. 356 | # So we find D, then get it's index in the original array which is 3 357 | # which tells us we can insert X at 4 into the forward-sorted rules 358 | # 359 | rev = @@transaction_rules.reverse 360 | last_higher_or_equal = rev.find { |r| r.priority.to_i >= priority } 361 | if last_higher_or_equal.nil? 362 | # insert a the start of the list 363 | @@transaction_rules.insert(0, rule) 364 | else 365 | index_of_last = @@transaction_rules.index(last_higher_or_equal) 366 | # now insert it after the last higher or equal priority rule 367 | @@transaction_rules.insert(index_of_last + 1, rule) 368 | end 369 | end 370 | 371 | ## 372 | # Runs through all of the rules registered with calls to +transaction_rule+ 373 | # and applies them to each Transaction in the specified +statement+. 374 | # 375 | # Bankjob calls this after +scrape_statement+ and before writing out the 376 | # statement to CSV or OFX 377 | # 378 | def self.post_process_transactions(statement) #:nodoc: 379 | if defined?(@@transaction_rules) 380 | @@transaction_rules.each do |rule| 381 | statement.transactions.each do |transaction| 382 | rule.rule_body.call(transaction) 383 | end 384 | end 385 | end 386 | return statement 387 | end 388 | 389 | ## 390 | # Scrapes a website to produce a new Statement object. 391 | # 392 | # This is the one method which a Scraper *must* implement by overriding 393 | # this method. 394 | # 395 | # Override this in your own Scraper to use Mechanize and Hpricot (or 396 | # some other mechanism if you prefer) to parse your bank website 397 | # and create a Bankjob::Statement object to hold the data. 398 | # 399 | # The implementation here will raise an error if not overridden. 400 | # 401 | def scrape_statement 402 | raise "You must override the instance method scrape_statement in your scraper!" 403 | end 404 | 405 | ## 406 | # Creates a new Statement. 407 | # 408 | # Calling this method is the preferred way of creating a new Statement object 409 | # since it sets the OFX currency (and possibly other attributes) based on the 410 | # values set in the definition of the Scraper subclass. 411 | # It is otherwise no different, however, than calling Statement.new() yourself. 412 | # 413 | def create_statement 414 | statement = Statement.new(@@account_number, @@currency) 415 | statement.bank_id = @@bank_id if defined?(@@bank_id) 416 | statement.account_type = @@account_type if defined?(@@account_type) 417 | return statement 418 | end 419 | 420 | ## 421 | # Creates a new Transaction. 422 | # 423 | # Calling this method is the preferred way of creating a new Transaction object 424 | # since it sets the decimal separator (and possibly other attributes) based on the 425 | # values set in the definition of the Scraper subclass. 426 | # 427 | # It is otherwise no different, however, than calling Transaction.new() yourself. 428 | # 429 | def create_transaction 430 | Transaction.new(@@decimal) 431 | end 432 | 433 | ## 434 | # Private 435 | # 436 | # The internal workings of the Scraper come after this point - they 437 | # are not documented in RDOC 438 | ## 439 | 440 | #SCRAPER_INTERFACE is the list of methods that a scraper must define 441 | SCRAPER_INTERFACE = [:scrape_statement] 442 | 443 | # set up the directories in which user's scrapers will be sought 444 | HOME_DIR = File.dirname(__FILE__); 445 | SCRAPERS_DIR = File.join(HOME_DIR, "..", "..", "scrapers") 446 | 447 | ## 448 | # +inherited+ is always called when a class extends Scraper. 449 | # The subclass itself is passed in as +scraper_class+ alllowing 450 | # us to register it to be instantiated later 451 | # 452 | def self.inherited(scraper_class) #:nodoc: 453 | # verify that the scraper class indeed defines the necessary methods 454 | SCRAPER_INTERFACE.each do |method| 455 | if (not scraper_class.public_method_defined?(method)) 456 | raise "Invalid scraper: the scraper class #{scraper_class.name} does not define the method #{method}" 457 | end 458 | end 459 | # in the future we might keep a registry of scrapers but for now 460 | # we assume there will always be one, and just register that class 461 | @@last_scraper_class = scraper_class 462 | end 463 | 464 | ## 465 | # This is the main method of the dynamic Scraper-loader: It loads 466 | # the actual scraper ruby file and initializes the class therein. 467 | # 468 | # Note that no assumption is made about the name of the class 469 | # defined within the specified +scraper_filename+. Rather, the 470 | # +self.inherited+ method will hold a reference to the last 471 | # class loaded that extends Bankjob::Scraper and that reference 472 | # is used here to initialize the class immediately after load() 473 | # is called on the specified file. 474 | # 475 | def self.load_scraper(scraper_filename, options, logger) #:nodoc: 476 | # temporarily add the same dir as bankjob and the scrapers dir 477 | # to the ruby LOAD_PATH for finding the scraper 478 | begin 479 | $:.unshift(HOME_DIR) 480 | $:.unshift(SCRAPERS_DIR) 481 | logger.debug("About to load the scraper file named #{scraper_filename}") 482 | load(scraper_filename) 483 | rescue Exception => e 484 | logger.error("Failed to load the scraper file #{scraper_filename} due to #{e.message}.\n\t#{e.backtrace[0]}") 485 | ensure 486 | $:.delete(SCRAPERS_DIR) 487 | $:.delete(HOME_DIR) 488 | end 489 | 490 | if (not defined?(@@last_scraper_class) or @@last_scraper_class.nil?) 491 | raise "Cannot initialize the scraper as none was loaded successfully." 492 | else 493 | logger.debug("About to instantiate scraper class: #{@@last_scraper_class.name}\n") 494 | scraper = @@last_scraper_class.new() 495 | scraper.logger = logger 496 | scraper.options = options 497 | end 498 | 499 | return scraper 500 | end # init_scraper 501 | end # Scraper 502 | end # module Bankjob -------------------------------------------------------------------------------- /website/javascripts/rounded_corners_lite.inc.js: -------------------------------------------------------------------------------- 1 | 2 | /**************************************************************** 3 | * * 4 | * curvyCorners * 5 | * ------------ * 6 | * * 7 | * This script generates rounded corners for your divs. * 8 | * * 9 | * Version 1.2.9 * 10 | * Copyright (c) 2006 Cameron Cooke * 11 | * By: Cameron Cooke and Tim Hutchison. * 12 | * * 13 | * * 14 | * Website: http://www.curvycorners.net * 15 | * Email: info@totalinfinity.com * 16 | * Forum: http://www.curvycorners.net/forum/ * 17 | * * 18 | * * 19 | * This library is free software; you can redistribute * 20 | * it and/or modify it under the terms of the GNU * 21 | * Lesser General Public License as published by the * 22 | * Free Software Foundation; either version 2.1 of the * 23 | * License, or (at your option) any later version. * 24 | * * 25 | * This library is distributed in the hope that it will * 26 | * be useful, but WITHOUT ANY WARRANTY; without even the * 27 | * implied warranty of MERCHANTABILITY or FITNESS FOR A * 28 | * PARTICULAR PURPOSE. See the GNU Lesser General Public * 29 | * License for more details. * 30 | * * 31 | * You should have received a copy of the GNU Lesser * 32 | * General Public License along with this library; * 33 | * Inc., 59 Temple Place, Suite 330, Boston, * 34 | * MA 02111-1307 USA * 35 | * * 36 | ****************************************************************/ 37 | 38 | var isIE = navigator.userAgent.toLowerCase().indexOf("msie") > -1; var isMoz = document.implementation && document.implementation.createDocument; var isSafari = ((navigator.userAgent.toLowerCase().indexOf('safari')!=-1)&&(navigator.userAgent.toLowerCase().indexOf('mac')!=-1))?true:false; function curvyCorners() 39 | { if(typeof(arguments[0]) != "object") throw newCurvyError("First parameter of curvyCorners() must be an object."); if(typeof(arguments[1]) != "object" && typeof(arguments[1]) != "string") throw newCurvyError("Second parameter of curvyCorners() must be an object or a class name."); if(typeof(arguments[1]) == "string") 40 | { var startIndex = 0; var boxCol = getElementsByClass(arguments[1]);} 41 | else 42 | { var startIndex = 1; var boxCol = arguments;} 43 | var curvyCornersCol = new Array(); if(arguments[0].validTags) 44 | var validElements = arguments[0].validTags; else 45 | var validElements = ["div"]; for(var i = startIndex, j = boxCol.length; i < j; i++) 46 | { var currentTag = boxCol[i].tagName.toLowerCase(); if(inArray(validElements, currentTag) !== false) 47 | { curvyCornersCol[curvyCornersCol.length] = new curvyObject(arguments[0], boxCol[i]);} 48 | } 49 | this.objects = curvyCornersCol; this.applyCornersToAll = function() 50 | { for(var x = 0, k = this.objects.length; x < k; x++) 51 | { this.objects[x].applyCorners();} 52 | } 53 | } 54 | function curvyObject() 55 | { this.box = arguments[1]; this.settings = arguments[0]; this.topContainer = null; this.bottomContainer = null; this.masterCorners = new Array(); this.contentDIV = null; var boxHeight = get_style(this.box, "height", "height"); var boxWidth = get_style(this.box, "width", "width"); var borderWidth = get_style(this.box, "borderTopWidth", "border-top-width"); var borderColour = get_style(this.box, "borderTopColor", "border-top-color"); var boxColour = get_style(this.box, "backgroundColor", "background-color"); var backgroundImage = get_style(this.box, "backgroundImage", "background-image"); var boxPosition = get_style(this.box, "position", "position"); var boxPadding = get_style(this.box, "paddingTop", "padding-top"); this.boxHeight = parseInt(((boxHeight != "" && boxHeight != "auto" && boxHeight.indexOf("%") == -1)? boxHeight.substring(0, boxHeight.indexOf("px")) : this.box.scrollHeight)); this.boxWidth = parseInt(((boxWidth != "" && boxWidth != "auto" && boxWidth.indexOf("%") == -1)? boxWidth.substring(0, boxWidth.indexOf("px")) : this.box.scrollWidth)); this.borderWidth = parseInt(((borderWidth != "" && borderWidth.indexOf("px") !== -1)? borderWidth.slice(0, borderWidth.indexOf("px")) : 0)); this.boxColour = format_colour(boxColour); this.boxPadding = parseInt(((boxPadding != "" && boxPadding.indexOf("px") !== -1)? boxPadding.slice(0, boxPadding.indexOf("px")) : 0)); this.borderColour = format_colour(borderColour); this.borderString = this.borderWidth + "px" + " solid " + this.borderColour; this.backgroundImage = ((backgroundImage != "none")? backgroundImage : ""); this.boxContent = this.box.innerHTML; if(boxPosition != "absolute") this.box.style.position = "relative"; this.box.style.padding = "0px"; if(isIE && boxWidth == "auto" && boxHeight == "auto") this.box.style.width = "100%"; if(this.settings.autoPad == true && this.boxPadding > 0) 56 | this.box.innerHTML = ""; this.applyCorners = function() 57 | { for(var t = 0; t < 2; t++) 58 | { switch(t) 59 | { case 0: 60 | if(this.settings.tl || this.settings.tr) 61 | { var newMainContainer = document.createElement("DIV"); newMainContainer.style.width = "100%"; newMainContainer.style.fontSize = "1px"; newMainContainer.style.overflow = "hidden"; newMainContainer.style.position = "absolute"; newMainContainer.style.paddingLeft = this.borderWidth + "px"; newMainContainer.style.paddingRight = this.borderWidth + "px"; var topMaxRadius = Math.max(this.settings.tl ? this.settings.tl.radius : 0, this.settings.tr ? this.settings.tr.radius : 0); newMainContainer.style.height = topMaxRadius + "px"; newMainContainer.style.top = 0 - topMaxRadius + "px"; newMainContainer.style.left = 0 - this.borderWidth + "px"; this.topContainer = this.box.appendChild(newMainContainer);} 62 | break; case 1: 63 | if(this.settings.bl || this.settings.br) 64 | { var newMainContainer = document.createElement("DIV"); newMainContainer.style.width = "100%"; newMainContainer.style.fontSize = "1px"; newMainContainer.style.overflow = "hidden"; newMainContainer.style.position = "absolute"; newMainContainer.style.paddingLeft = this.borderWidth + "px"; newMainContainer.style.paddingRight = this.borderWidth + "px"; var botMaxRadius = Math.max(this.settings.bl ? this.settings.bl.radius : 0, this.settings.br ? this.settings.br.radius : 0); newMainContainer.style.height = botMaxRadius + "px"; newMainContainer.style.bottom = 0 - botMaxRadius + "px"; newMainContainer.style.left = 0 - this.borderWidth + "px"; this.bottomContainer = this.box.appendChild(newMainContainer);} 65 | break;} 66 | } 67 | if(this.topContainer) this.box.style.borderTopWidth = "0px"; if(this.bottomContainer) this.box.style.borderBottomWidth = "0px"; var corners = ["tr", "tl", "br", "bl"]; for(var i in corners) 68 | { if(i > -1 < 4) 69 | { var cc = corners[i]; if(!this.settings[cc]) 70 | { if(((cc == "tr" || cc == "tl") && this.topContainer != null) || ((cc == "br" || cc == "bl") && this.bottomContainer != null)) 71 | { var newCorner = document.createElement("DIV"); newCorner.style.position = "relative"; newCorner.style.fontSize = "1px"; newCorner.style.overflow = "hidden"; if(this.backgroundImage == "") 72 | newCorner.style.backgroundColor = this.boxColour; else 73 | newCorner.style.backgroundImage = this.backgroundImage; switch(cc) 74 | { case "tl": 75 | newCorner.style.height = topMaxRadius - this.borderWidth + "px"; newCorner.style.marginRight = this.settings.tr.radius - (this.borderWidth*2) + "px"; newCorner.style.borderLeft = this.borderString; newCorner.style.borderTop = this.borderString; newCorner.style.left = -this.borderWidth + "px"; break; case "tr": 76 | newCorner.style.height = topMaxRadius - this.borderWidth + "px"; newCorner.style.marginLeft = this.settings.tl.radius - (this.borderWidth*2) + "px"; newCorner.style.borderRight = this.borderString; newCorner.style.borderTop = this.borderString; newCorner.style.backgroundPosition = "-" + (topMaxRadius + this.borderWidth) + "px 0px"; newCorner.style.left = this.borderWidth + "px"; break; case "bl": 77 | newCorner.style.height = botMaxRadius - this.borderWidth + "px"; newCorner.style.marginRight = this.settings.br.radius - (this.borderWidth*2) + "px"; newCorner.style.borderLeft = this.borderString; newCorner.style.borderBottom = this.borderString; newCorner.style.left = -this.borderWidth + "px"; newCorner.style.backgroundPosition = "-" + (this.borderWidth) + "px -" + (this.boxHeight + (botMaxRadius + this.borderWidth)) + "px"; break; case "br": 78 | newCorner.style.height = botMaxRadius - this.borderWidth + "px"; newCorner.style.marginLeft = this.settings.bl.radius - (this.borderWidth*2) + "px"; newCorner.style.borderRight = this.borderString; newCorner.style.borderBottom = this.borderString; newCorner.style.left = this.borderWidth + "px" 79 | newCorner.style.backgroundPosition = "-" + (botMaxRadius + this.borderWidth) + "px -" + (this.boxHeight + (botMaxRadius + this.borderWidth)) + "px"; break;} 80 | } 81 | } 82 | else 83 | { if(this.masterCorners[this.settings[cc].radius]) 84 | { var newCorner = this.masterCorners[this.settings[cc].radius].cloneNode(true);} 85 | else 86 | { var newCorner = document.createElement("DIV"); newCorner.style.height = this.settings[cc].radius + "px"; newCorner.style.width = this.settings[cc].radius + "px"; newCorner.style.position = "absolute"; newCorner.style.fontSize = "1px"; newCorner.style.overflow = "hidden"; var borderRadius = parseInt(this.settings[cc].radius - this.borderWidth); for(var intx = 0, j = this.settings[cc].radius; intx < j; intx++) 87 | { if((intx +1) >= borderRadius) 88 | var y1 = -1; else 89 | var y1 = (Math.floor(Math.sqrt(Math.pow(borderRadius, 2) - Math.pow((intx+1), 2))) - 1); if(borderRadius != j) 90 | { if((intx) >= borderRadius) 91 | var y2 = -1; else 92 | var y2 = Math.ceil(Math.sqrt(Math.pow(borderRadius,2) - Math.pow(intx, 2))); if((intx+1) >= j) 93 | var y3 = -1; else 94 | var y3 = (Math.floor(Math.sqrt(Math.pow(j ,2) - Math.pow((intx+1), 2))) - 1);} 95 | if((intx) >= j) 96 | var y4 = -1; else 97 | var y4 = Math.ceil(Math.sqrt(Math.pow(j ,2) - Math.pow(intx, 2))); if(y1 > -1) this.drawPixel(intx, 0, this.boxColour, 100, (y1+1), newCorner, -1, this.settings[cc].radius); if(borderRadius != j) 98 | { for(var inty = (y1 + 1); inty < y2; inty++) 99 | { if(this.settings.antiAlias) 100 | { if(this.backgroundImage != "") 101 | { var borderFract = (pixelFraction(intx, inty, borderRadius) * 100); if(borderFract < 30) 102 | { this.drawPixel(intx, inty, this.borderColour, 100, 1, newCorner, 0, this.settings[cc].radius);} 103 | else 104 | { this.drawPixel(intx, inty, this.borderColour, 100, 1, newCorner, -1, this.settings[cc].radius);} 105 | } 106 | else 107 | { var pixelcolour = BlendColour(this.boxColour, this.borderColour, pixelFraction(intx, inty, borderRadius)); this.drawPixel(intx, inty, pixelcolour, 100, 1, newCorner, 0, this.settings[cc].radius, cc);} 108 | } 109 | } 110 | if(this.settings.antiAlias) 111 | { if(y3 >= y2) 112 | { if (y2 == -1) y2 = 0; this.drawPixel(intx, y2, this.borderColour, 100, (y3 - y2 + 1), newCorner, 0, 0);} 113 | } 114 | else 115 | { if(y3 >= y1) 116 | { this.drawPixel(intx, (y1 + 1), this.borderColour, 100, (y3 - y1), newCorner, 0, 0);} 117 | } 118 | var outsideColour = this.borderColour;} 119 | else 120 | { var outsideColour = this.boxColour; var y3 = y1;} 121 | if(this.settings.antiAlias) 122 | { for(var inty = (y3 + 1); inty < y4; inty++) 123 | { this.drawPixel(intx, inty, outsideColour, (pixelFraction(intx, inty , j) * 100), 1, newCorner, ((this.borderWidth > 0)? 0 : -1), this.settings[cc].radius);} 124 | } 125 | } 126 | this.masterCorners[this.settings[cc].radius] = newCorner.cloneNode(true);} 127 | if(cc != "br") 128 | { for(var t = 0, k = newCorner.childNodes.length; t < k; t++) 129 | { var pixelBar = newCorner.childNodes[t]; var pixelBarTop = parseInt(pixelBar.style.top.substring(0, pixelBar.style.top.indexOf("px"))); var pixelBarLeft = parseInt(pixelBar.style.left.substring(0, pixelBar.style.left.indexOf("px"))); var pixelBarHeight = parseInt(pixelBar.style.height.substring(0, pixelBar.style.height.indexOf("px"))); if(cc == "tl" || cc == "bl"){ pixelBar.style.left = this.settings[cc].radius -pixelBarLeft -1 + "px";} 130 | if(cc == "tr" || cc == "tl"){ pixelBar.style.top = this.settings[cc].radius -pixelBarHeight -pixelBarTop + "px";} 131 | switch(cc) 132 | { case "tr": 133 | pixelBar.style.backgroundPosition = "-" + Math.abs((this.boxWidth - this.settings[cc].radius + this.borderWidth) + pixelBarLeft) + "px -" + Math.abs(this.settings[cc].radius -pixelBarHeight -pixelBarTop - this.borderWidth) + "px"; break; case "tl": 134 | pixelBar.style.backgroundPosition = "-" + Math.abs((this.settings[cc].radius -pixelBarLeft -1) - this.borderWidth) + "px -" + Math.abs(this.settings[cc].radius -pixelBarHeight -pixelBarTop - this.borderWidth) + "px"; break; case "bl": 135 | pixelBar.style.backgroundPosition = "-" + Math.abs((this.settings[cc].radius -pixelBarLeft -1) - this.borderWidth) + "px -" + Math.abs((this.boxHeight + this.settings[cc].radius + pixelBarTop) -this.borderWidth) + "px"; break;} 136 | } 137 | } 138 | } 139 | if(newCorner) 140 | { switch(cc) 141 | { case "tl": 142 | if(newCorner.style.position == "absolute") newCorner.style.top = "0px"; if(newCorner.style.position == "absolute") newCorner.style.left = "0px"; if(this.topContainer) this.topContainer.appendChild(newCorner); break; case "tr": 143 | if(newCorner.style.position == "absolute") newCorner.style.top = "0px"; if(newCorner.style.position == "absolute") newCorner.style.right = "0px"; if(this.topContainer) this.topContainer.appendChild(newCorner); break; case "bl": 144 | if(newCorner.style.position == "absolute") newCorner.style.bottom = "0px"; if(newCorner.style.position == "absolute") newCorner.style.left = "0px"; if(this.bottomContainer) this.bottomContainer.appendChild(newCorner); break; case "br": 145 | if(newCorner.style.position == "absolute") newCorner.style.bottom = "0px"; if(newCorner.style.position == "absolute") newCorner.style.right = "0px"; if(this.bottomContainer) this.bottomContainer.appendChild(newCorner); break;} 146 | } 147 | } 148 | } 149 | var radiusDiff = new Array(); radiusDiff["t"] = Math.abs(this.settings.tl.radius - this.settings.tr.radius) 150 | radiusDiff["b"] = Math.abs(this.settings.bl.radius - this.settings.br.radius); for(z in radiusDiff) 151 | { if(z == "t" || z == "b") 152 | { if(radiusDiff[z]) 153 | { var smallerCornerType = ((this.settings[z + "l"].radius < this.settings[z + "r"].radius)? z +"l" : z +"r"); var newFiller = document.createElement("DIV"); newFiller.style.height = radiusDiff[z] + "px"; newFiller.style.width = this.settings[smallerCornerType].radius+ "px" 154 | newFiller.style.position = "absolute"; newFiller.style.fontSize = "1px"; newFiller.style.overflow = "hidden"; newFiller.style.backgroundColor = this.boxColour; switch(smallerCornerType) 155 | { case "tl": 156 | newFiller.style.bottom = "0px"; newFiller.style.left = "0px"; newFiller.style.borderLeft = this.borderString; this.topContainer.appendChild(newFiller); break; case "tr": 157 | newFiller.style.bottom = "0px"; newFiller.style.right = "0px"; newFiller.style.borderRight = this.borderString; this.topContainer.appendChild(newFiller); break; case "bl": 158 | newFiller.style.top = "0px"; newFiller.style.left = "0px"; newFiller.style.borderLeft = this.borderString; this.bottomContainer.appendChild(newFiller); break; case "br": 159 | newFiller.style.top = "0px"; newFiller.style.right = "0px"; newFiller.style.borderRight = this.borderString; this.bottomContainer.appendChild(newFiller); break;} 160 | } 161 | var newFillerBar = document.createElement("DIV"); newFillerBar.style.position = "relative"; newFillerBar.style.fontSize = "1px"; newFillerBar.style.overflow = "hidden"; newFillerBar.style.backgroundColor = this.boxColour; newFillerBar.style.backgroundImage = this.backgroundImage; switch(z) 162 | { case "t": 163 | if(this.topContainer) 164 | { if(this.settings.tl.radius && this.settings.tr.radius) 165 | { newFillerBar.style.height = topMaxRadius - this.borderWidth + "px"; newFillerBar.style.marginLeft = this.settings.tl.radius - this.borderWidth + "px"; newFillerBar.style.marginRight = this.settings.tr.radius - this.borderWidth + "px"; newFillerBar.style.borderTop = this.borderString; if(this.backgroundImage != "") 166 | newFillerBar.style.backgroundPosition = "-" + (topMaxRadius + this.borderWidth) + "px 0px"; this.topContainer.appendChild(newFillerBar);} 167 | this.box.style.backgroundPosition = "0px -" + (topMaxRadius - this.borderWidth) + "px";} 168 | break; case "b": 169 | if(this.bottomContainer) 170 | { if(this.settings.bl.radius && this.settings.br.radius) 171 | { newFillerBar.style.height = botMaxRadius - this.borderWidth + "px"; newFillerBar.style.marginLeft = this.settings.bl.radius - this.borderWidth + "px"; newFillerBar.style.marginRight = this.settings.br.radius - this.borderWidth + "px"; newFillerBar.style.borderBottom = this.borderString; if(this.backgroundImage != "") 172 | newFillerBar.style.backgroundPosition = "-" + (botMaxRadius + this.borderWidth) + "px -" + (this.boxHeight + (topMaxRadius + this.borderWidth)) + "px"; this.bottomContainer.appendChild(newFillerBar);} 173 | } 174 | break;} 175 | } 176 | } 177 | if(this.settings.autoPad == true && this.boxPadding > 0) 178 | { var contentContainer = document.createElement("DIV"); contentContainer.style.position = "relative"; contentContainer.innerHTML = this.boxContent; contentContainer.className = "autoPadDiv"; var topPadding = Math.abs(topMaxRadius - this.boxPadding); var botPadding = Math.abs(botMaxRadius - this.boxPadding); if(topMaxRadius < this.boxPadding) 179 | contentContainer.style.paddingTop = topPadding + "px"; if(botMaxRadius < this.boxPadding) 180 | contentContainer.style.paddingBottom = botMaxRadius + "px"; contentContainer.style.paddingLeft = this.boxPadding + "px"; contentContainer.style.paddingRight = this.boxPadding + "px"; this.contentDIV = this.box.appendChild(contentContainer);} 181 | } 182 | this.drawPixel = function(intx, inty, colour, transAmount, height, newCorner, image, cornerRadius) 183 | { var pixel = document.createElement("DIV"); pixel.style.height = height + "px"; pixel.style.width = "1px"; pixel.style.position = "absolute"; pixel.style.fontSize = "1px"; pixel.style.overflow = "hidden"; var topMaxRadius = Math.max(this.settings["tr"].radius, this.settings["tl"].radius); if(image == -1 && this.backgroundImage != "") 184 | { pixel.style.backgroundImage = this.backgroundImage; pixel.style.backgroundPosition = "-" + (this.boxWidth - (cornerRadius - intx) + this.borderWidth) + "px -" + ((this.boxHeight + topMaxRadius + inty) -this.borderWidth) + "px";} 185 | else 186 | { pixel.style.backgroundColor = colour;} 187 | if (transAmount != 100) 188 | setOpacity(pixel, transAmount); pixel.style.top = inty + "px"; pixel.style.left = intx + "px"; newCorner.appendChild(pixel);} 189 | } 190 | function insertAfter(parent, node, referenceNode) 191 | { parent.insertBefore(node, referenceNode.nextSibling);} 192 | function BlendColour(Col1, Col2, Col1Fraction) 193 | { var red1 = parseInt(Col1.substr(1,2),16); var green1 = parseInt(Col1.substr(3,2),16); var blue1 = parseInt(Col1.substr(5,2),16); var red2 = parseInt(Col2.substr(1,2),16); var green2 = parseInt(Col2.substr(3,2),16); var blue2 = parseInt(Col2.substr(5,2),16); if(Col1Fraction > 1 || Col1Fraction < 0) Col1Fraction = 1; var endRed = Math.round((red1 * Col1Fraction) + (red2 * (1 - Col1Fraction))); if(endRed > 255) endRed = 255; if(endRed < 0) endRed = 0; var endGreen = Math.round((green1 * Col1Fraction) + (green2 * (1 - Col1Fraction))); if(endGreen > 255) endGreen = 255; if(endGreen < 0) endGreen = 0; var endBlue = Math.round((blue1 * Col1Fraction) + (blue2 * (1 - Col1Fraction))); if(endBlue > 255) endBlue = 255; if(endBlue < 0) endBlue = 0; return "#" + IntToHex(endRed)+ IntToHex(endGreen)+ IntToHex(endBlue);} 194 | function IntToHex(strNum) 195 | { base = strNum / 16; rem = strNum % 16; base = base - (rem / 16); baseS = MakeHex(base); remS = MakeHex(rem); return baseS + '' + remS;} 196 | function MakeHex(x) 197 | { if((x >= 0) && (x <= 9)) 198 | { return x;} 199 | else 200 | { switch(x) 201 | { case 10: return "A"; case 11: return "B"; case 12: return "C"; case 13: return "D"; case 14: return "E"; case 15: return "F";} 202 | } 203 | } 204 | function pixelFraction(x, y, r) 205 | { var pixelfraction = 0; var xvalues = new Array(1); var yvalues = new Array(1); var point = 0; var whatsides = ""; var intersect = Math.sqrt((Math.pow(r,2) - Math.pow(x,2))); if ((intersect >= y) && (intersect < (y+1))) 206 | { whatsides = "Left"; xvalues[point] = 0; yvalues[point] = intersect - y; point = point + 1;} 207 | var intersect = Math.sqrt((Math.pow(r,2) - Math.pow(y+1,2))); if ((intersect >= x) && (intersect < (x+1))) 208 | { whatsides = whatsides + "Top"; xvalues[point] = intersect - x; yvalues[point] = 1; point = point + 1;} 209 | var intersect = Math.sqrt((Math.pow(r,2) - Math.pow(x+1,2))); if ((intersect >= y) && (intersect < (y+1))) 210 | { whatsides = whatsides + "Right"; xvalues[point] = 1; yvalues[point] = intersect - y; point = point + 1;} 211 | var intersect = Math.sqrt((Math.pow(r,2) - Math.pow(y,2))); if ((intersect >= x) && (intersect < (x+1))) 212 | { whatsides = whatsides + "Bottom"; xvalues[point] = intersect - x; yvalues[point] = 0;} 213 | switch (whatsides) 214 | { case "LeftRight": 215 | pixelfraction = Math.min(yvalues[0],yvalues[1]) + ((Math.max(yvalues[0],yvalues[1]) - Math.min(yvalues[0],yvalues[1]))/2); break; case "TopRight": 216 | pixelfraction = 1-(((1-xvalues[0])*(1-yvalues[1]))/2); break; case "TopBottom": 217 | pixelfraction = Math.min(xvalues[0],xvalues[1]) + ((Math.max(xvalues[0],xvalues[1]) - Math.min(xvalues[0],xvalues[1]))/2); break; case "LeftBottom": 218 | pixelfraction = (yvalues[0]*xvalues[1])/2; break; default: 219 | pixelfraction = 1;} 220 | return pixelfraction;} 221 | function rgb2Hex(rgbColour) 222 | { try{ var rgbArray = rgb2Array(rgbColour); var red = parseInt(rgbArray[0]); var green = parseInt(rgbArray[1]); var blue = parseInt(rgbArray[2]); var hexColour = "#" + IntToHex(red) + IntToHex(green) + IntToHex(blue);} 223 | catch(e){ alert("There was an error converting the RGB value to Hexadecimal in function rgb2Hex");} 224 | return hexColour;} 225 | function rgb2Array(rgbColour) 226 | { var rgbValues = rgbColour.substring(4, rgbColour.indexOf(")")); var rgbArray = rgbValues.split(", "); return rgbArray;} 227 | function setOpacity(obj, opacity) 228 | { opacity = (opacity == 100)?99.999:opacity; if(isSafari && obj.tagName != "IFRAME") 229 | { var rgbArray = rgb2Array(obj.style.backgroundColor); var red = parseInt(rgbArray[0]); var green = parseInt(rgbArray[1]); var blue = parseInt(rgbArray[2]); obj.style.backgroundColor = "rgba(" + red + ", " + green + ", " + blue + ", " + opacity/100 + ")";} 230 | else if(typeof(obj.style.opacity) != "undefined") 231 | { obj.style.opacity = opacity/100;} 232 | else if(typeof(obj.style.MozOpacity) != "undefined") 233 | { obj.style.MozOpacity = opacity/100;} 234 | else if(typeof(obj.style.filter) != "undefined") 235 | { obj.style.filter = "alpha(opacity:" + opacity + ")";} 236 | else if(typeof(obj.style.KHTMLOpacity) != "undefined") 237 | { obj.style.KHTMLOpacity = opacity/100;} 238 | } 239 | function inArray(array, value) 240 | { for(var i = 0; i < array.length; i++){ if (array[i] === value) return i;} 241 | return false;} 242 | function inArrayKey(array, value) 243 | { for(key in array){ if(key === value) return true;} 244 | return false;} 245 | function addEvent(elm, evType, fn, useCapture) { if (elm.addEventListener) { elm.addEventListener(evType, fn, useCapture); return true;} 246 | else if (elm.attachEvent) { var r = elm.attachEvent('on' + evType, fn); return r;} 247 | else { elm['on' + evType] = fn;} 248 | } 249 | function removeEvent(obj, evType, fn, useCapture){ if (obj.removeEventListener){ obj.removeEventListener(evType, fn, useCapture); return true;} else if (obj.detachEvent){ var r = obj.detachEvent("on"+evType, fn); return r;} else { alert("Handler could not be removed");} 250 | } 251 | function format_colour(colour) 252 | { var returnColour = "#ffffff"; if(colour != "" && colour != "transparent") 253 | { if(colour.substr(0, 3) == "rgb") 254 | { returnColour = rgb2Hex(colour);} 255 | else if(colour.length == 4) 256 | { returnColour = "#" + colour.substring(1, 2) + colour.substring(1, 2) + colour.substring(2, 3) + colour.substring(2, 3) + colour.substring(3, 4) + colour.substring(3, 4);} 257 | else 258 | { returnColour = colour;} 259 | } 260 | return returnColour;} 261 | function get_style(obj, property, propertyNS) 262 | { try 263 | { if(obj.currentStyle) 264 | { var returnVal = eval("obj.currentStyle." + property);} 265 | else 266 | { if(isSafari && obj.style.display == "none") 267 | { obj.style.display = ""; var wasHidden = true;} 268 | var returnVal = document.defaultView.getComputedStyle(obj, '').getPropertyValue(propertyNS); if(isSafari && wasHidden) 269 | { obj.style.display = "none";} 270 | } 271 | } 272 | catch(e) 273 | { } 274 | return returnVal;} 275 | function getElementsByClass(searchClass, node, tag) 276 | { var classElements = new Array(); if(node == null) 277 | node = document; if(tag == null) 278 | tag = '*'; var els = node.getElementsByTagName(tag); var elsLen = els.length; var pattern = new RegExp("(^|\s)"+searchClass+"(\s|$)"); for (i = 0, j = 0; i < elsLen; i++) 279 | { if(pattern.test(els[i].className)) 280 | { classElements[j] = els[i]; j++;} 281 | } 282 | return classElements;} 283 | function newCurvyError(errorMessage) 284 | { return new Error("curvyCorners Error:\n" + errorMessage) 285 | } 286 | --------------------------------------------------------------------------------