├── test └── .bacon ├── Gemfile ├── .gitignore ├── Rakefile ├── lib ├── larch │ ├── db │ │ ├── mailbox.rb │ │ ├── migrate │ │ │ ├── 002_add_timestamps.rb │ │ │ └── 001_create_schema.rb │ │ ├── message.rb │ │ └── account.rb │ ├── version.rb │ ├── errors.rb │ ├── logger.rb │ ├── monkeypatch │ │ └── net │ │ │ └── imap.rb │ ├── config.rb │ ├── imap.rb │ └── imap │ │ └── mailbox.rb └── larch.rb ├── Gemfile.lock ├── larch.gemspec ├── HISTORY ├── bin └── larch ├── README.rdoc └── LICENSE /test/.bacon: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /Gemfile: -------------------------------------------------------------------------------- 1 | source "http://rubygems.org" 2 | gemspec 3 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | doc/ 2 | pkg/ 3 | .DS_Store 4 | ._* 5 | exclude 6 | imap.log 7 | -------------------------------------------------------------------------------- /Rakefile: -------------------------------------------------------------------------------- 1 | require 'bundler' 2 | require 'rake/clean' 3 | require 'rake/testtask' 4 | 5 | Bundler::GemHelper.install_tasks 6 | -------------------------------------------------------------------------------- /lib/larch/db/mailbox.rb: -------------------------------------------------------------------------------- 1 | module Larch; module Database 2 | 3 | class Mailbox < Sequel::Model(:mailboxes) 4 | plugin :hook_class_methods 5 | one_to_many :messages, :class => Larch::Database::Message 6 | 7 | before_destroy do 8 | Larch::Database::Message.filter(:mailbox_id => id).destroy 9 | end 10 | end 11 | 12 | end; end 13 | -------------------------------------------------------------------------------- /lib/larch/version.rb: -------------------------------------------------------------------------------- 1 | module Larch 2 | APP_NAME = 'Larch' 3 | APP_VERSION = '1.1.2' 4 | APP_AUTHOR = 'Ryan Grove' 5 | APP_EMAIL = 'ryan@wonko.com' 6 | APP_URL = 'https://github.com/rgrove/larch/' 7 | APP_COPYRIGHT = 'Copyright (c) 2013 Ryan Grove . All ' << 8 | 'rights reserved.' 9 | end 10 | -------------------------------------------------------------------------------- /lib/larch/errors.rb: -------------------------------------------------------------------------------- 1 | module Larch 2 | class Error < StandardError; end 3 | 4 | class Config 5 | class Error < Larch::Error; end 6 | end 7 | 8 | class IMAP 9 | class Error < Larch::Error; end 10 | class FatalError < Error; end 11 | class MailboxNotFoundError < Error; end 12 | class MessageNotFoundError < Error; end 13 | end 14 | end 15 | -------------------------------------------------------------------------------- /lib/larch/db/migrate/002_add_timestamps.rb: -------------------------------------------------------------------------------- 1 | class AddTimestamps < Sequel::Migration 2 | def down 3 | alter_table :accounts do 4 | drop_column :created_at 5 | drop_column :updated_at 6 | end 7 | end 8 | 9 | def up 10 | alter_table :accounts do 11 | add_column :created_at, :integer 12 | add_column :updated_at, :integer 13 | end 14 | end 15 | end 16 | -------------------------------------------------------------------------------- /Gemfile.lock: -------------------------------------------------------------------------------- 1 | PATH 2 | remote: . 3 | specs: 4 | larch (1.1.2) 5 | highline (~> 1.5) 6 | sequel (~> 3.14) 7 | sqlite3 (~> 1.3) 8 | trollop (~> 1.13) 9 | 10 | GEM 11 | remote: http://rubygems.org/ 12 | specs: 13 | highline (1.6.15) 14 | sequel (3.43.0) 15 | sqlite3 (1.3.7) 16 | trollop (1.16.2) 17 | 18 | PLATFORMS 19 | ruby 20 | 21 | DEPENDENCIES 22 | larch! 23 | -------------------------------------------------------------------------------- /lib/larch/db/message.rb: -------------------------------------------------------------------------------- 1 | module Larch; module Database 2 | 3 | class Message < Sequel::Model(:messages) 4 | def flags 5 | self[:flags].split(',').sort.map do |f| 6 | # Flags beginning with $ should be strings; all others should be symbols. 7 | f[0,1] == '$' ? f : f.to_sym 8 | end 9 | end 10 | 11 | def flags_str 12 | self[:flags] 13 | end 14 | 15 | def flags=(flags) 16 | self[:flags] = flags.map{|f| f.to_s }.join(',') 17 | end 18 | end 19 | 20 | end; end 21 | -------------------------------------------------------------------------------- /lib/larch/db/account.rb: -------------------------------------------------------------------------------- 1 | module Larch; module Database 2 | 3 | class Account < Sequel::Model(:accounts) 4 | plugin :hook_class_methods 5 | 6 | one_to_many :mailboxes, :class => Larch::Database::Mailbox 7 | 8 | before_create do 9 | now = Time.now.to_i 10 | 11 | self.created_at = now 12 | self.updated_at = now 13 | end 14 | 15 | before_destroy do 16 | Mailbox.filter(:account_id => id).destroy 17 | end 18 | 19 | before_save do 20 | now = Time.now.to_i 21 | 22 | self.created_at = now if self.created_at.nil? 23 | self.updated_at = now 24 | end 25 | 26 | def touch 27 | update(:updated_at => Time.now.to_i) 28 | end 29 | end 30 | 31 | end; end 32 | -------------------------------------------------------------------------------- /larch.gemspec: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | require './lib/larch/version' 3 | 4 | Gem::Specification.new do |s| 5 | s.name = 'larch' 6 | s.summary = 'Larch copies messages from one IMAP server to another. Awesomely.' 7 | s.version = Larch::APP_VERSION 8 | s.authors = ['Ryan Grove'] 9 | s.email = 'ryan@wonko.com' 10 | s.homepage = 'https://github.com/rgrove/larch' 11 | s.platform = Gem::Platform::RUBY 12 | 13 | s.executables = ['larch'] 14 | s.require_path = 'lib' 15 | s.required_ruby_version = '>= 1.8.6' 16 | 17 | # s.add_dependency('amalgalite', '~> 1.0') 18 | s.add_dependency('highline', '~> 1.5') 19 | s.add_dependency('sequel', '~> 3.14') 20 | s.add_dependency('sqlite3', '~> 1.3') 21 | s.add_dependency('trollop', '~> 1.13') 22 | 23 | s.files = [ 24 | 'HISTORY', 25 | 'LICENSE', 26 | 'README.rdoc', 27 | 'bin/larch' 28 | ] + Dir.glob('lib/**/*.rb') 29 | end 30 | -------------------------------------------------------------------------------- /lib/larch/db/migrate/001_create_schema.rb: -------------------------------------------------------------------------------- 1 | class CreateSchema < Sequel::Migration 2 | def down 3 | drop_table :accounts, :mailboxes, :messages 4 | end 5 | 6 | def up 7 | create_table :accounts do 8 | primary_key :id 9 | text :hostname, :null => false 10 | text :username, :null => false 11 | 12 | unique [:hostname, :username] 13 | end 14 | 15 | create_table :mailboxes do 16 | primary_key :id 17 | foreign_key :account_id, :table => :accounts 18 | text :name, :null => false 19 | text :delim, :null => false 20 | text :attr, :null => false, :default => '' 21 | integer :subscribed, :null => false, :default => 0 22 | integer :uidvalidity 23 | integer :uidnext 24 | 25 | unique [:account_id, :name, :uidvalidity] 26 | end 27 | 28 | create_table :messages do 29 | primary_key :id 30 | foreign_key :mailbox_id, :table => :mailboxes 31 | integer :uid, :null => false 32 | text :guid, :null => false 33 | text :message_id 34 | integer :rfc822_size, :null => false 35 | integer :internaldate, :null => false 36 | text :flags, :null => false, :default => '' 37 | 38 | index :guid 39 | unique [:mailbox_id, :uid] 40 | end 41 | end 42 | end 43 | -------------------------------------------------------------------------------- /lib/larch/logger.rb: -------------------------------------------------------------------------------- 1 | module Larch 2 | 3 | class Logger 4 | attr_reader :level, :output 5 | 6 | LEVELS = { 7 | :fatal => 0, 8 | :error => 1, 9 | :warn => 2, 10 | :warning => 2, 11 | :info => 3, 12 | :debug => 4, 13 | :insane => 5 14 | } 15 | 16 | def initialize(level = :info, output = $stdout) 17 | self.level = level.to_sym 18 | self.output = output 19 | end 20 | 21 | def const_missing(name) 22 | return LEVELS[name] if LEVELS.key?(name) 23 | raise NameError, "uninitialized constant: #{name}" 24 | end 25 | 26 | def method_missing(name, *args) 27 | return log(name, *args) if LEVELS.key?(name) 28 | raise NoMethodError, "undefined method: #{name}" 29 | end 30 | 31 | def level=(level) 32 | raise ArgumentError, "invalid log level: #{level}" unless LEVELS.key?(level) 33 | @level = level 34 | end 35 | 36 | def log(level, msg) 37 | return true if LEVELS[level] > LEVELS[@level] || msg.nil? || msg.empty? 38 | @output.puts "[#{Time.new.strftime('%H:%M:%S')}] [#{level}] #{msg}" 39 | true 40 | 41 | rescue => e 42 | false 43 | end 44 | 45 | def output=(output) 46 | raise ArgumentError, "output must be an instance of class IO" unless output.is_a?(IO) 47 | @output = output 48 | end 49 | end 50 | 51 | end 52 | -------------------------------------------------------------------------------- /lib/larch/monkeypatch/net/imap.rb: -------------------------------------------------------------------------------- 1 | # Monkeypatches for Net::IMAP. 2 | 3 | module Net # :nodoc: 4 | class IMAP # :nodoc: 5 | class ResponseParser # :nodoc: 6 | private 7 | 8 | # Fixes an issue with bogus STATUS responses from Exchange that contain 9 | # trailing whitespace. This monkeypatch works cleanly against Ruby 1.8.x 10 | # and 1.9.x. 11 | def status_response 12 | token = match(T_ATOM) 13 | name = token.value.upcase 14 | match(T_SPACE) 15 | mailbox = astring 16 | match(T_SPACE) 17 | match(T_LPAR) 18 | attr = {} 19 | while true 20 | token = lookahead 21 | case token.symbol 22 | when T_RPAR 23 | shift_token 24 | break 25 | when T_SPACE 26 | shift_token 27 | end 28 | token = match(T_ATOM) 29 | key = token.value.upcase 30 | match(T_SPACE) 31 | val = number 32 | attr[key] = val 33 | end 34 | 35 | # Monkeypatch starts here... 36 | token = lookahead 37 | shift_token if token.symbol == T_SPACE 38 | # ...and ends here. 39 | 40 | data = StatusData.new(mailbox, attr) 41 | return UntaggedResponse.new(name, data, @str) 42 | end 43 | 44 | if RUBY_VERSION <= '1.9.1' 45 | 46 | # Monkeypatches Net::IMAP in Ruby <= 1.9.1 to fix broken response 47 | # handling, particularly when changing mailboxes on a Dovecot 1.2+ 48 | # server. 49 | # 50 | # This monkeypatch shouldn't be necessary in Ruby 1.9.2 and higher. 51 | # It's included in Ruby 1.9 SVN trunk as of 2010-02-08. 52 | def resp_text_code 53 | @lex_state = EXPR_BEG 54 | match(T_LBRA) 55 | token = match(T_ATOM) 56 | name = token.value.upcase 57 | case name 58 | when /\A(?:ALERT|PARSE|READ-ONLY|READ-WRITE|TRYCREATE|NOMODSEQ)\z/n 59 | result = ResponseCode.new(name, nil) 60 | when /\A(?:PERMANENTFLAGS)\z/n 61 | match(T_SPACE) 62 | result = ResponseCode.new(name, flag_list) 63 | when /\A(?:UIDVALIDITY|UIDNEXT|UNSEEN)\z/n 64 | match(T_SPACE) 65 | result = ResponseCode.new(name, number) 66 | else 67 | token = lookahead 68 | if token.symbol == T_SPACE 69 | shift_token 70 | @lex_state = EXPR_CTEXT 71 | token = match(T_TEXT) 72 | @lex_state = EXPR_BEG 73 | result = ResponseCode.new(name, token.value) 74 | else 75 | result = ResponseCode.new(name, nil) 76 | end 77 | end 78 | match(T_RBRA) 79 | @lex_state = EXPR_RTEXT 80 | return result 81 | end 82 | end 83 | 84 | end 85 | 86 | end 87 | end 88 | -------------------------------------------------------------------------------- /HISTORY: -------------------------------------------------------------------------------- 1 | Larch History 2 | ================================================================================ 3 | 4 | Version 1.1.2 (2013-01-24) 5 | * Fixed an issue in which a folder containing only one message would not be 6 | copied. 7 | * Switched to the sqlite3 gem instead of amalgalite, since the latter fails 8 | to compile against Ruby 1.9.3. This means you'll need to have SQLite 9 | available on your system, since it's no longer bundled. 10 | 11 | Version 1.1.1 (2011-06-26) 12 | * Relaxed dependency versions. 13 | 14 | Version 1.1.0 (2011-01-22) 15 | * Mailbox and message state information is now stored in a local SQLite 16 | database, which allows Larch to resync and resume interrupted syncs much 17 | more quickly without having to rescan all messages. As a result, SQLite 3 is 18 | now a dependency. 19 | * Larch now loads config options from ~/.larch/config.yaml if it exists, or 20 | from the file specified by the --config command-line option. This file may 21 | contain multiple sections. If a section name is specified via the 22 | command-line, Larch will use the options in that section for the session; 23 | otherwise it will use the options in the "default" section. See the README 24 | for more details. 25 | * Added experimental support for Yahoo! Mail IMAP when connecting to 26 | imap.mail.yahoo.com or imap-ssl.mail.yahoo.com. See the README for caveats 27 | and known issues. 28 | * Folders are now copied recursively by default. Use the --no-recurse option 29 | for the old behavior. 30 | * Progress information is now displayed regularly while scanning large 31 | mailboxes. 32 | * Added --delete option to delete messages from the source after copying them 33 | to the destination, or if they already exist at the destination. 34 | * Added --expunge option to expunge deleted messages from the source. 35 | * Added --sync-flags option to synchronize message flags (like Seen, Flagged, 36 | etc.) from the source server to the destination server for messages that 37 | already exist on the destination. 38 | * Added short versions of common command-line options. 39 | * The --fast-scan option has been removed. 40 | * The --to-folder option can now be used in conjunction with --all or 41 | --all-subscribed to copy messages from multiple source folders to a single 42 | destination folder. 43 | * More concise log messages to reduce visual clutter in the log. 44 | * Monkeypatched Net::IMAP to fix broken response handling with certain server 45 | responses, particularly when changing mailboxes on a Dovecot 1.2+ server. 46 | * Fixed encoding issues when creating mailboxes and getting mailbox lists. 47 | * Fixed incorrect case-sensitive treatment of the 'INBOX' folder name. 48 | * Fixed a bug in which Larch would try to copy flags that weren't supported on 49 | the destination server. 50 | * Fixed an issue when trying to copy a folder with leading or trailing 51 | whitespace in the name to Gmail, since Gmail doesn't allow whitespace around 52 | folder names. 53 | * Fixed an issue with bogus Exchange STATUS responses containing trailing 54 | spaces. 55 | 56 | Version 1.0.2 (2009-08-05) 57 | * Fixed a bug that caused Larch to try to set the read-only \Recent flag on 58 | the destination server. 59 | 60 | Version 1.0.1 (2009-05-10) 61 | * Ruby 1.9.1 support. 62 | * Much more robust handling of unexpected server disconnects and dropped 63 | connections. 64 | * Added --all option to copy all folders recursively. 65 | * Added --all-subscribed option to copy all subscribed folders recursively. 66 | * Added --dry-run option to simulate changes without actually making them. 67 | * Added --exclude and --exclude-file options to specify folders that should 68 | not be copied. 69 | * Added --ssl-certs option to specify a bundle of trusted SSL certificates. 70 | * Added --ssl-verify option to verify server SSL certificates. 71 | * Added a new "insane" logging level, which will output all IMAP commands and 72 | responses to STDERR. 73 | * Fixed excessive post-scan processing times for very large mailboxes. 74 | * Fixed potential scan problems with very large mailboxes on certain servers. 75 | * POSIX signals are no longer trapped on platforms that aren't likely to 76 | support them. 77 | 78 | Version 1.0.0 (2009-03-17) 79 | * First release. 80 | -------------------------------------------------------------------------------- /lib/larch/config.rb: -------------------------------------------------------------------------------- 1 | module Larch 2 | 3 | class Config 4 | attr_reader :filename, :section 5 | 6 | DEFAULT = { 7 | 'all' => false, 8 | 'all-subscribed' => false, 9 | 'config' => File.join('~', '.larch', 'config.yaml'), 10 | 'database' => File.join('~', '.larch', 'larch.db'), 11 | 'delete' => false, 12 | 'dry-run' => false, 13 | 'exclude' => [], 14 | 'exclude-file' => nil, 15 | 'expunge' => false, 16 | 'from' => nil, 17 | 'from-folder' => nil, # actually INBOX; see validate() 18 | 'from-pass' => nil, 19 | 'from-user' => nil, 20 | 'max-retries' => 3, 21 | 'no-create-folder' => false, 22 | 'no-recurse' => false, 23 | 'ssl-certs' => nil, 24 | 'ssl-verify' => false, 25 | 'sync-flags' => false, 26 | 'to' => nil, 27 | 'to-folder' => nil, # actually INBOX; see validate() 28 | 'to-pass' => nil, 29 | 'to-user' => nil, 30 | 'verbosity' => 'info' 31 | }.freeze 32 | 33 | def initialize(section = 'default', filename = DEFAULT['config'], override = {}) 34 | @section = section.to_s 35 | @override = {} 36 | 37 | override.each do |k, v| 38 | opt = k.to_s.gsub('_', '-') 39 | @override[opt] = v if DEFAULT.has_key?(opt) && override["#{k}_given".to_sym] && v != DEFAULT[opt] 40 | end 41 | 42 | load_file(filename) 43 | validate 44 | end 45 | 46 | def fetch(name) 47 | (@cached || {})[name.to_s.gsub('_', '-')] || nil 48 | end 49 | alias [] fetch 50 | 51 | def load_file(filename) 52 | @filename = File.expand_path(filename) 53 | 54 | config = {} 55 | 56 | if File.exist?(@filename) 57 | begin 58 | config = YAML.load_file(@filename) 59 | rescue => e 60 | raise Larch::Config::Error, "config error in #{filename}: #{e}" 61 | end 62 | end 63 | 64 | @lookup = [@override, config[@section] || {}, config['default'] || {}, DEFAULT] 65 | cache_config 66 | end 67 | 68 | def method_missing(name) 69 | fetch(name) 70 | end 71 | 72 | # Validates the config and resolves conflicting settings. 73 | def validate 74 | ['from', 'to'].each do |s| 75 | raise Error, "'#{s}' must be a valid IMAP URI (e.g. imap://example.com)" unless fetch(s) =~ IMAP::REGEX_URI 76 | end 77 | 78 | unless Logger::LEVELS.has_key?(verbosity.to_sym) 79 | raise Error, "'verbosity' must be one of: #{Logger::LEVELS.keys.join(', ')}" 80 | end 81 | 82 | if exclude_file 83 | raise Error, "exclude file not found: #{exclude_file}" unless File.file?(exclude_file) 84 | raise Error, "exclude file cannot be read: #{exclude_file}" unless File.readable?(exclude_file) 85 | end 86 | 87 | if @cached['all'] || @cached['all-subscribed'] 88 | # A specific source folder wins over 'all' and 'all-subscribed' 89 | if @cached['from-folder'] 90 | @cached['all'] = false 91 | @cached['all-subscribed'] = false 92 | @cached['to-folder'] ||= @cached['from-folder'] 93 | 94 | elsif @cached['all'] && @cached['all-subscribed'] 95 | # 'all' wins over 'all-subscribed' 96 | @cached['all-subscribed'] = false 97 | end 98 | 99 | # 'no-recurse' is not compatible with 'all' and 'all-subscribed' 100 | raise Error, "'no-recurse' option cannot be used with 'all' or 'all-subscribed'" if @cached['no-recurse'] 101 | 102 | else 103 | @cached['from-folder'] ||= 'INBOX' 104 | @cached['to-folder'] ||= 'INBOX' 105 | end 106 | 107 | @cached['exclude'].flatten! 108 | end 109 | 110 | private 111 | 112 | # Merges configs such that those earlier in the lookup chain override those 113 | # later in the chain. 114 | def cache_config 115 | @cached = {} 116 | 117 | @lookup.reverse.each do |c| 118 | c.each {|k, v| @cached[k] = config_merge(@cached[k] || {}, v) } 119 | end 120 | end 121 | 122 | def config_merge(master, value) 123 | if value.is_a?(Hash) 124 | value.each {|k, v| master[k] = config_merge(master[k] || {}, v) } 125 | return master 126 | end 127 | 128 | value 129 | end 130 | 131 | end 132 | 133 | end 134 | -------------------------------------------------------------------------------- /bin/larch: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | 3 | $0 = 'larch' # hide arguments from ps and top 4 | 5 | require 'rubygems' 6 | require 'highline/import' # optional dep: termios 7 | require 'trollop' 8 | 9 | require 'larch' 10 | 11 | module Larch 12 | 13 | # Parse command-line options. 14 | options = Trollop.options do 15 | version "Larch #{APP_VERSION}\n" << APP_COPYRIGHT 16 | banner <<-EOS 17 | Larch copies messages from one IMAP server to another. Awesomely. 18 | 19 | Usage: 20 | larch [config section] [options] 21 | larch --from --to [options] 22 | 23 | Server Options: 24 | EOS 25 | opt :from, "URI of the source IMAP server", :short => '-f', :type => :string 26 | opt :from_folder, "Source folder to copy from (default: INBOX)", :short => '-F', :default => Config::DEFAULT['from-folder'], :type => :string 27 | opt :from_pass, "Source server password (default: prompt)", :short => '-p', :type => :string 28 | opt :from_user, "Source server username (default: prompt)", :short => '-u', :type => :string 29 | opt :to, "URI of the destination IMAP server", :short => '-t', :type => :string 30 | opt :to_folder, "Destination folder to copy to (default: INBOX)", :short => '-T', :default => Config::DEFAULT['to-folder'], :type => :string 31 | opt :to_pass, "Destination server password (default: prompt)", :short => '-P', :type => :string 32 | opt :to_user, "Destination server username (default: prompt)", :short => '-U', :type => :string 33 | 34 | text "\nCopy Options:" 35 | opt :all, "Copy all folders recursively", :short => '-a' 36 | opt :all_subscribed, "Copy all subscribed folders recursively", :short => '-s' 37 | opt :delete, "Delete messages from the source after copying them, or if they already exist at the destination", :short => '-d' 38 | opt :exclude, "List of mailbox names/patterns that shouldn't be copied", :short => :none, :type => :strings, :multi => true 39 | opt :exclude_file, "Filename containing mailbox names/patterns that shouldn't be copied", :short => :none, :type => :string 40 | opt :expunge, "Expunge deleted messages from the source", :short => '-x' 41 | opt :no_recurse, "Don't copy subfolders recursively (cannot be used with --all or --all_subscribed)", :short => :none 42 | opt :sync_flags, "Sync message flags from the source to the destination for messages that already exist at the destination", :short => '-S' 43 | 44 | text "\nGeneral Options:" 45 | opt :config, "Specify a non-default config file to use", :short => '-c', :default => Config::DEFAULT['config'] 46 | opt :database, "Specify a non-default message database to use", :short => :none, :default => Config::DEFAULT['database'] 47 | opt :dry_run, "Don't actually make any changes", :short => '-n' 48 | opt :max_retries, "Maximum number of times to retry after a recoverable error", :short => :none, :default => Config::DEFAULT['max-retries'] 49 | opt :no_create_folder, "Don't create destination folders that don't already exist", :short => :none 50 | opt :ssl_certs, "Path to a trusted certificate bundle to use to verify server SSL certificates", :short => :none, :type => :string 51 | opt :ssl_verify, "Verify server SSL certificates", :short => :none 52 | opt :verbosity, "Output verbosity: debug, info, warn, error, or fatal", :short => '-V', :default => Config::DEFAULT['verbosity'] 53 | end 54 | 55 | if options[:config_given] 56 | Trollop.die :config, ": file not found: #{options[:config]}" unless File.exist?(options[:config]) 57 | end 58 | 59 | # Load config. 60 | begin 61 | config = Config.new(ARGV.shift || 'default', options[:config], options) 62 | rescue Config::Error => e 63 | abort "Config error: #{e}" 64 | end 65 | 66 | # Create URIs. 67 | uri_from = URI(config.from) 68 | uri_to = URI(config.to) 69 | 70 | # Use --from-folder and --to-folder unless folders were specified in the URIs. 71 | uri_from.path = uri_from.path.empty? ? '/' + CGI.escape(config.from_folder.gsub(/^\//, '')) : uri_from.path if config.from_folder 72 | uri_to.path = uri_to.path.empty? ? '/' + CGI.escape(config.to_folder.gsub(/^\//, '')) : uri_to.path if config.to_folder 73 | 74 | # --all and --all-subscribed options override folders 75 | if config.all || config.all_subscribed 76 | uri_from.path = '' 77 | end 78 | 79 | # Usernames and passwords specified as arguments override those in the URIs 80 | uri_from.user = CGI.escape(config.from_user) if config.from_user 81 | uri_from.password = CGI.escape(config.from_pass) if config.from_pass 82 | uri_to.user = CGI.escape(config.to_user) if config.to_user 83 | uri_to.password = CGI.escape(config.to_pass) if config.to_pass 84 | 85 | # If usernames/passwords aren't specified in either URIs or config, then prompt. 86 | uri_from.user ||= CGI.escape(ask("Source username (#{uri_from.host}): ")) 87 | uri_from.password ||= CGI.escape(ask("Source password (#{uri_from.host}): ") {|q| q.echo = false }) 88 | uri_to.user ||= CGI.escape(ask("Destination username (#{uri_to.host}): ")) 89 | uri_to.password ||= CGI.escape(ask("Destination password (#{uri_to.host}): ") {|q| q.echo = false }) 90 | 91 | # Go go go! 92 | init(config) 93 | 94 | imap_from = Larch::IMAP.new(uri_from, 95 | :dry_run => config[:dry_run], 96 | :log_label => '[<]', 97 | :max_retries => config[:max_retries], 98 | :ssl_certs => config[:ssl_certs] || nil, 99 | :ssl_verify => config[:ssl_verify] 100 | ) 101 | 102 | imap_to = Larch::IMAP.new(uri_to, 103 | :create_mailbox => !config[:no_create_folder] && !config[:dry_run], 104 | :dry_run => config[:dry_run], 105 | :log_label => '[>]', 106 | :max_retries => config[:max_retries], 107 | :ssl_certs => config[:ssl_certs] || nil, 108 | :ssl_verify => config[:ssl_verify] 109 | ) 110 | 111 | unless RUBY_PLATFORM =~ /mswin|mingw|bccwin|wince|java/ 112 | begin 113 | for sig in [:SIGINT, :SIGQUIT, :SIGTERM] 114 | trap(sig) { @log.fatal "Interrupted (#{sig})"; Kernel.exit } 115 | end 116 | rescue => e 117 | end 118 | end 119 | 120 | if config.all 121 | copy_all(imap_from, imap_to) 122 | elsif config.all_subscribed 123 | copy_all(imap_from, imap_to, true) 124 | else 125 | copy_folder(imap_from, imap_to) 126 | end 127 | end 128 | -------------------------------------------------------------------------------- /lib/larch.rb: -------------------------------------------------------------------------------- 1 | require 'cgi' 2 | require 'digest/md5' 3 | require 'fileutils' 4 | require 'net/imap' 5 | require 'time' 6 | require 'uri' 7 | require 'yaml' 8 | 9 | require 'sequel' 10 | require 'sequel/extensions/migration' 11 | 12 | require 'larch/monkeypatch/net/imap' 13 | 14 | require 'larch/config' 15 | require 'larch/errors' 16 | require 'larch/imap' 17 | require 'larch/imap/mailbox' 18 | require 'larch/logger' 19 | require 'larch/version' 20 | 21 | module Larch 22 | 23 | class << self 24 | attr_reader :config, :db, :exclude, :log 25 | 26 | EXCLUDE_COMMENT = /#.*$/ 27 | EXCLUDE_REGEX = /^\s*\/(.*)\/\s*/ 28 | GLOB_PATTERNS = {'*' => '.*', '?' => '.'} 29 | LIB_DIR = File.join(File.dirname(File.expand_path(__FILE__)), 'larch') 30 | 31 | def init(config) 32 | raise ArgumentError, "config must be a Larch::Config instance" unless config.is_a?(Config) 33 | 34 | @config = config 35 | @log = Logger.new(@config[:verbosity]) 36 | @db = open_db(@config[:database]) 37 | 38 | parse_exclusions 39 | 40 | Net::IMAP.debug = true if @log.level == :insane 41 | 42 | # Stats 43 | @copied = 0 44 | @deleted = 0 45 | @failed = 0 46 | @total = 0 47 | end 48 | 49 | # Recursively copies all messages in all folders from the source to the 50 | # destination. 51 | def copy_all(imap_from, imap_to, subscribed_only = false) 52 | raise ArgumentError, "imap_from must be a Larch::IMAP instance" unless imap_from.is_a?(IMAP) 53 | raise ArgumentError, "imap_to must be a Larch::IMAP instance" unless imap_to.is_a?(IMAP) 54 | 55 | @copied = 0 56 | @deleted = 0 57 | @failed = 0 58 | @total = 0 59 | 60 | imap_from.each_mailbox do |mailbox_from| 61 | next if excluded?(mailbox_from.name) 62 | next if subscribed_only && !mailbox_from.subscribed? 63 | 64 | if imap_to.uri_mailbox 65 | mailbox_to = imap_to.mailbox(imap_to.uri_mailbox) 66 | else 67 | mailbox_to = imap_to.mailbox(mailbox_from.name, mailbox_from.delim) 68 | end 69 | 70 | mailbox_to.subscribe if mailbox_from.subscribed? 71 | 72 | copy_messages(mailbox_from, mailbox_to) 73 | end 74 | 75 | rescue => e 76 | @log.fatal e.message 77 | 78 | ensure 79 | summary 80 | db_maintenance 81 | end 82 | 83 | # Copies the messages in a single IMAP folder and all its subfolders 84 | # (recursively) from the source to the destination. 85 | def copy_folder(imap_from, imap_to) 86 | raise ArgumentError, "imap_from must be a Larch::IMAP instance" unless imap_from.is_a?(IMAP) 87 | raise ArgumentError, "imap_to must be a Larch::IMAP instance" unless imap_to.is_a?(IMAP) 88 | 89 | @copied = 0 90 | @deleted = 0 91 | @failed = 0 92 | @total = 0 93 | 94 | mailbox_from = imap_from.mailbox(imap_from.uri_mailbox || 'INBOX') 95 | mailbox_to = imap_to.mailbox(imap_to.uri_mailbox || 'INBOX') 96 | 97 | copy_mailbox(mailbox_from, mailbox_to) 98 | 99 | imap_from.disconnect 100 | imap_to.disconnect 101 | 102 | rescue => e 103 | @log.fatal e.message 104 | 105 | ensure 106 | summary 107 | db_maintenance 108 | end 109 | 110 | # Opens a connection to the Larch message database, creating it if 111 | # necessary. 112 | def open_db(database) 113 | unless database == ':memory:' 114 | filename = File.expand_path(database) 115 | directory = File.dirname(filename) 116 | 117 | unless File.exist?(directory) 118 | FileUtils.mkdir_p(directory) 119 | File.chmod(0700, directory) 120 | end 121 | end 122 | 123 | begin 124 | db = Sequel.sqlite(:database => filename) 125 | db.test_connection 126 | rescue => e 127 | @log.fatal "unable to open message database: #{e}" 128 | abort 129 | end 130 | 131 | # Ensure that the database schema is up to date. 132 | migration_dir = File.join(LIB_DIR, 'db', 'migrate') 133 | 134 | begin 135 | Sequel::Migrator.apply(db, migration_dir) 136 | rescue => e 137 | @log.fatal "unable to migrate message database: #{e}" 138 | abort 139 | end 140 | 141 | require 'larch/db/message' 142 | require 'larch/db/mailbox' 143 | require 'larch/db/account' 144 | 145 | db 146 | end 147 | 148 | def summary 149 | @log.info "#{@copied} message(s) copied, #{@failed} failed, #{@deleted} deleted out of #{@total} total" 150 | end 151 | 152 | 153 | private 154 | 155 | 156 | def copy_mailbox(mailbox_from, mailbox_to) 157 | raise ArgumentError, "mailbox_from must be a Larch::IMAP::Mailbox instance" unless mailbox_from.is_a?(Larch::IMAP::Mailbox) 158 | raise ArgumentError, "mailbox_to must be a Larch::IMAP::Mailbox instance" unless mailbox_to.is_a?(Larch::IMAP::Mailbox) 159 | 160 | return if excluded?(mailbox_from.name) || excluded?(mailbox_to.name) 161 | 162 | mailbox_to.subscribe if mailbox_from.subscribed? 163 | copy_messages(mailbox_from, mailbox_to) 164 | 165 | unless @config['no-recurse'] 166 | mailbox_from.each_mailbox do |child_from| 167 | next if excluded?(child_from.name) 168 | child_to = mailbox_to.imap.mailbox(child_from.name, child_from.delim) 169 | copy_mailbox(child_from, child_to) 170 | end 171 | end 172 | end 173 | 174 | def copy_messages(mailbox_from, mailbox_to) 175 | raise ArgumentError, "mailbox_from must be a Larch::IMAP::Mailbox instance" unless mailbox_from.is_a?(Larch::IMAP::Mailbox) 176 | raise ArgumentError, "mailbox_to must be a Larch::IMAP::Mailbox instance" unless mailbox_to.is_a?(Larch::IMAP::Mailbox) 177 | 178 | return if excluded?(mailbox_from.name) || excluded?(mailbox_to.name) 179 | 180 | imap_from = mailbox_from.imap 181 | imap_to = mailbox_to.imap 182 | 183 | @log.info "#{imap_from.host}/#{mailbox_from.name} -> #{imap_to.host}/#{mailbox_to.name}" 184 | 185 | @total += mailbox_from.length 186 | 187 | mailbox_from.each_db_message do |from_db_message| 188 | guid = from_db_message.guid 189 | uid = from_db_message.uid 190 | 191 | if mailbox_to.has_guid?(guid) 192 | begin 193 | if @config['sync_flags'] 194 | to_db_message = mailbox_to.fetch_db_message(guid) 195 | 196 | if to_db_message.flags != from_db_message.flags 197 | new_flags = from_db_message.flags_str 198 | new_flags = '(none)' if new_flags.empty? 199 | 200 | @log.info "[>] syncing flags: uid #{uid}: #{new_flags}" 201 | mailbox_to.set_flags(guid, from_db_message.flags) 202 | end 203 | end 204 | 205 | if @config['delete'] && !from_db_message.flags.include?(:Deleted) 206 | @log.info "[<] deleting uid #{uid} (already exists at destination)" 207 | @deleted += 1 if mailbox_from.delete_message(guid) 208 | end 209 | 210 | rescue Larch::IMAP::Error => e 211 | @log.error e.message 212 | end 213 | 214 | next 215 | end 216 | 217 | begin 218 | unless msg = mailbox_from.peek(guid) 219 | @failed += 1 220 | next 221 | end 222 | 223 | if msg.envelope.from 224 | env_from = msg.envelope.from.first 225 | from = "#{env_from.mailbox}@#{env_from.host}" 226 | else 227 | from = '?' 228 | end 229 | 230 | @log.info "[>] copying uid #{uid}: #{from} - #{msg.envelope.subject}" 231 | 232 | mailbox_to << msg 233 | @copied += 1 234 | 235 | if @config['delete'] 236 | @log.info "[<] deleting uid #{uid}" 237 | @deleted += 1 if mailbox_from.delete_message(guid) 238 | end 239 | 240 | rescue Larch::IMAP::Error => e 241 | @failed += 1 242 | @log.error e.message 243 | next 244 | end 245 | end 246 | 247 | if @config['expunge'] 248 | begin 249 | @log.debug "[<] expunging deleted messages" 250 | mailbox_from.expunge 251 | rescue Larch::IMAP::Error => e 252 | @log.error e.message 253 | end 254 | end 255 | 256 | rescue Larch::IMAP::Error => e 257 | @log.error e.message 258 | 259 | end 260 | 261 | def db_maintenance 262 | @log.debug 'performing database maintenance' 263 | 264 | # Remove accounts that haven't been used in over 30 days. 265 | Database::Account.filter(:updated_at => nil).destroy 266 | Database::Account.filter('? - updated_at >= 2592000', Time.now.to_i).destroy 267 | 268 | # Release unused disk space and defragment the database. 269 | @db.run('VACUUM') 270 | end 271 | 272 | def excluded?(name) 273 | name = name.downcase 274 | 275 | @exclude.each do |e| 276 | return true if (e.is_a?(Regexp) ? !!(name =~ e) : File.fnmatch?(e, name)) 277 | end 278 | 279 | return false 280 | end 281 | 282 | def glob_to_regex(str) 283 | str.gsub!(/(.)/) {|c| GLOB_PATTERNS[$1] || Regexp.escape(c) } 284 | Regexp.new("^#{str}$", Regexp::IGNORECASE) 285 | end 286 | 287 | def load_exclude_file(filename) 288 | @exclude ||= [] 289 | lineno = 0 290 | 291 | File.open(filename, 'rb') do |f| 292 | f.each do |line| 293 | lineno += 1 294 | 295 | # Strip comments. 296 | line.sub!(EXCLUDE_COMMENT, '') 297 | line.strip! 298 | 299 | # Skip empty lines. 300 | next if line.empty? 301 | 302 | if line =~ EXCLUDE_REGEX 303 | @exclude << Regexp.new($1, Regexp::IGNORECASE) 304 | else 305 | @exclude << glob_to_regex(line) 306 | end 307 | end 308 | end 309 | 310 | rescue => e 311 | raise Larch::IMAP::FatalError, "error in exclude file at line #{lineno}: #{e}" 312 | end 313 | 314 | def parse_exclusions 315 | @exclude = @config[:exclude].map do |e| 316 | if e =~ EXCLUDE_REGEX 317 | Regexp.new($1, Regexp::IGNORECASE) 318 | else 319 | glob_to_regex(e.strip) 320 | end 321 | end 322 | 323 | load_exclude_file(@config[:exclude_file]) if @config[:exclude_file] 324 | end 325 | end 326 | 327 | end 328 | -------------------------------------------------------------------------------- /lib/larch/imap.rb: -------------------------------------------------------------------------------- 1 | module Larch 2 | 3 | # Manages a connection to an IMAP server and all the glorious fun that entails. 4 | # 5 | # This class borrows heavily from Sup, the source code of which should be 6 | # required reading if you're doing anything with IMAP in Ruby: 7 | # http://sup.rubyforge.org 8 | class IMAP 9 | attr_reader :conn, :db_account, :mailboxes, :options, :quirks 10 | 11 | # URI format validation regex. 12 | REGEX_URI = URI.regexp(['imap', 'imaps']) 13 | 14 | # Larch::IMAP::Message represents a transferable IMAP message which can be 15 | # passed between Larch::IMAP instances. 16 | Message = Struct.new(:guid, :envelope, :rfc822, :flags, :internaldate) 17 | 18 | # Initializes a new Larch::IMAP instance that will connect to the specified 19 | # IMAP URI. 20 | # 21 | # In addition to the URI, the following options may be specified: 22 | # 23 | # [:create_mailbox] 24 | # If +true+, mailboxes that don't already exist will be created if 25 | # necessary. 26 | # 27 | # [:dry_run] 28 | # If +true+, read-only operations will be performed as usual and all change 29 | # operations will be simulated, but no changes will actually be made. Note 30 | # that it's not actually possible to simulate mailbox creation, so 31 | # +:dry_run+ mode always behaves as if +:create_mailbox+ is +false+. 32 | # 33 | # [:log_label] 34 | # Label to use for this connection in log output. If not specified, the 35 | # default label is "[username@host]". 36 | # 37 | # [:max_retries] 38 | # After a recoverable error occurs, retry the operation up to this many 39 | # times. Default is 3. 40 | # 41 | # [:ssl_certs] 42 | # Path to a trusted certificate bundle to use to verify server SSL 43 | # certificates. You can download a bundle of certificate authority root 44 | # certs at http://curl.haxx.se/ca/cacert.pem (it's up to you to verify that 45 | # this bundle hasn't been tampered with, however; don't trust it blindly). 46 | # 47 | # [:ssl_verify] 48 | # If +true+, server SSL certificates will be verified against the trusted 49 | # certificate bundle specified in +ssl_certs+. By default, server SSL 50 | # certificates are not verified. 51 | # 52 | def initialize(uri, options = {}) 53 | raise ArgumentError, "not an IMAP URI: #{uri}" unless uri.is_a?(URI) || uri =~ REGEX_URI 54 | raise ArgumentError, "options must be a Hash" unless options.is_a?(Hash) 55 | 56 | @uri = uri.is_a?(URI) ? uri : URI(uri) 57 | @options = { 58 | :log_label => "[#{username}@#{host}]", 59 | :max_retries => 3, 60 | :ssl_verify => false 61 | }.merge(options) 62 | 63 | raise ArgumentError, "must provide a username and password" unless @uri.user && @uri.password 64 | 65 | @conn = nil 66 | @mailboxes = {} 67 | 68 | @quirks = { 69 | :gmail => false, 70 | :yahoo => false 71 | } 72 | 73 | @db_account = Database::Account.find_or_create( 74 | :hostname => host, 75 | :username => username 76 | ) 77 | 78 | @db_account.touch 79 | 80 | # Create private convenience methods (debug, info, warn, etc.) to make 81 | # logging easier. 82 | Logger::LEVELS.each_key do |level| 83 | next if IMAP.private_method_defined?(level) 84 | 85 | IMAP.class_eval do 86 | define_method(level) do |msg| 87 | Larch.log.log(level, "#{@options[:log_label]} #{msg}") 88 | end 89 | 90 | private level 91 | end 92 | end 93 | end 94 | 95 | # Connects to the IMAP server and logs in if a connection hasn't already been 96 | # established. 97 | def connect 98 | return if @conn 99 | safely {} # connect, but do nothing else 100 | end 101 | 102 | # Gets the server's mailbox hierarchy delimiter. 103 | def delim 104 | @delim ||= safely { @conn.list('', '')[0].delim || '.'} 105 | end 106 | 107 | # Closes the IMAP connection if one is currently open. 108 | def disconnect 109 | return unless @conn 110 | 111 | begin 112 | @conn.disconnect 113 | rescue Errno::ENOTCONN => e 114 | debug "#{e.class.name}: #{e.message}" 115 | end 116 | 117 | reset 118 | 119 | info "disconnected" 120 | end 121 | 122 | # Iterates through all mailboxes in the account, yielding each one as a 123 | # Larch::IMAP::Mailbox instance to the given block. 124 | def each_mailbox 125 | update_mailboxes 126 | @mailboxes.each_value {|mailbox| yield mailbox } 127 | end 128 | 129 | # Gets the IMAP hostname. 130 | def host 131 | @uri.host 132 | end 133 | 134 | # Gets a Larch::IMAP::Mailbox instance representing the specified mailbox. If 135 | # the mailbox doesn't exist and the :create_mailbox option is 136 | # +false+, or if :create_mailbox is +true+ and mailbox creation 137 | # fails, a Larch::IMAP::MailboxNotFoundError will be raised. 138 | def mailbox(name, delim = '/') 139 | retries = 0 140 | 141 | name.gsub!(/^(inbox\/?)/i){ $1.upcase } 142 | name.gsub!(delim, self.delim) 143 | 144 | # Gmail doesn't allow folders with leading or trailing whitespace. 145 | name.strip! if @quirks[:gmail] 146 | 147 | #Rackspace namespaces everything under INDEX. 148 | name.sub!(/^|inbox\./i, "INBOX.") if @quirks[:rackspace] && name != 'INBOX' 149 | 150 | begin 151 | @mailboxes.fetch(name) do 152 | update_mailboxes 153 | return @mailboxes[name] if @mailboxes.has_key?(name) 154 | raise MailboxNotFoundError, "mailbox not found: #{name}" 155 | end 156 | 157 | rescue MailboxNotFoundError => e 158 | raise unless @options[:create_mailbox] && retries == 0 159 | 160 | info "creating mailbox: #{name}" 161 | safely { @conn.create(Net::IMAP.encode_utf7(name)) } unless @options[:dry_run] 162 | 163 | retries += 1 164 | retry 165 | end 166 | end 167 | 168 | # Sends an IMAP NOOP command. 169 | def noop 170 | safely { @conn.noop } 171 | end 172 | 173 | # Gets the IMAP password. 174 | def password 175 | CGI.unescape(@uri.password) 176 | end 177 | 178 | # Gets the IMAP port number. 179 | def port 180 | @uri.port || (ssl? ? 993 : 143) 181 | end 182 | 183 | # Connect if necessary, execute the given block, retry if a recoverable error 184 | # occurs, die if an unrecoverable error occurs. 185 | def safely 186 | safe_connect 187 | 188 | retries = 0 189 | 190 | begin 191 | yield 192 | 193 | rescue Errno::ECONNABORTED, 194 | Errno::ECONNRESET, 195 | Errno::ENOTCONN, 196 | Errno::EPIPE, 197 | Errno::ETIMEDOUT, 198 | IOError, 199 | Net::IMAP::ByeResponseError, 200 | OpenSSL::SSL::SSLError => e 201 | 202 | raise unless (retries += 1) <= @options[:max_retries] 203 | 204 | warning "#{e.class.name}: #{e.message} (reconnecting)" 205 | 206 | reset 207 | sleep 1 * retries 208 | safe_connect 209 | retry 210 | 211 | rescue Net::IMAP::BadResponseError, 212 | Net::IMAP::NoResponseError, 213 | Net::IMAP::ResponseParseError => e 214 | 215 | raise unless (retries += 1) <= @options[:max_retries] 216 | 217 | warning "#{e.class.name}: #{e.message} (will retry)" 218 | 219 | sleep 1 * retries 220 | retry 221 | end 222 | 223 | rescue Larch::Error => e 224 | raise 225 | 226 | rescue Net::IMAP::Error => e 227 | raise Error, "#{e.class.name}: #{e.message} (giving up)" 228 | 229 | rescue => e 230 | raise FatalError, "#{e.class.name}: #{e.message} (cannot recover)" 231 | end 232 | 233 | # Gets the SSL status. 234 | def ssl? 235 | @uri.scheme == 'imaps' 236 | end 237 | 238 | # Gets the IMAP URI. 239 | def uri 240 | @uri.to_s 241 | end 242 | 243 | # Gets the IMAP mailbox specified in the URI, or +nil+ if none. 244 | def uri_mailbox 245 | mb = @uri.path[1..-1] 246 | mb.nil? || mb.empty? ? nil : CGI.unescape(mb) 247 | end 248 | 249 | # Gets the IMAP username. 250 | def username 251 | CGI.unescape(@uri.user) 252 | end 253 | 254 | private 255 | 256 | # Tries to identify server implementations with certain quirks that we'll need 257 | # to work around. 258 | def check_quirks 259 | return unless @conn && 260 | @conn.greeting.kind_of?(Net::IMAP::UntaggedResponse) && 261 | @conn.greeting.data.kind_of?(Net::IMAP::ResponseText) 262 | 263 | if @conn.greeting.data.text =~ /^Gimap ready/ 264 | @quirks[:gmail] = true 265 | debug "looks like Gmail" 266 | 267 | elsif host =~ /^imap(?:-ssl)?\.mail\.yahoo\.com$/ 268 | @quirks[:yahoo] = true 269 | debug "looks like Yahoo! Mail" 270 | 271 | elsif host =~ /emailsrvr\.com/ 272 | @quirks[:rackspace] = true 273 | debug "looks like Rackspace Mail" 274 | end 275 | end 276 | 277 | # Resets the connection and mailbox state. 278 | def reset 279 | @conn = nil 280 | @mailboxes.each_value {|mb| mb.reset } 281 | end 282 | 283 | def safe_connect 284 | return if @conn 285 | 286 | retries = 0 287 | 288 | begin 289 | unsafe_connect 290 | 291 | rescue Errno::ECONNRESET, 292 | Errno::EPIPE, 293 | Errno::ETIMEDOUT, 294 | OpenSSL::SSL::SSLError => e 295 | 296 | raise unless (retries += 1) <= @options[:max_retries] 297 | 298 | # Special check to ensure that we don't retry on OpenSSL certificate 299 | # verification errors. 300 | raise if e.is_a?(OpenSSL::SSL::SSLError) && e.message =~ /certificate verify failed/ 301 | 302 | warning "#{e.class.name}: #{e.message} (will retry)" 303 | 304 | reset 305 | sleep 1 * retries 306 | retry 307 | end 308 | 309 | rescue => e 310 | raise FatalError, "#{e.class.name}: #{e.message} (cannot recover)" 311 | end 312 | 313 | def unsafe_connect 314 | debug "connecting..." 315 | 316 | exception = nil 317 | 318 | Thread.new do 319 | begin 320 | @conn = Net::IMAP.new(host, port, ssl?, 321 | ssl? && @options[:ssl_verify] ? @options[:ssl_certs] : nil, 322 | @options[:ssl_verify]) 323 | 324 | info "connected to #{host} on port #{port}" << (ssl? ? ' using SSL' : '') 325 | 326 | check_quirks 327 | 328 | # If this is Yahoo! Mail, we have to send a special command before 329 | # it'll let us authenticate. 330 | if @quirks[:yahoo] 331 | @conn.instance_eval { send_command('ID ("guid" "1")') } 332 | end 333 | 334 | auth_methods = ['PLAIN'] 335 | tried = [] 336 | capability = @conn.capability 337 | 338 | ['LOGIN', 'CRAM-MD5'].each do |method| 339 | auth_methods << method if capability.include?("AUTH=#{method}") 340 | end 341 | 342 | begin 343 | tried << method = auth_methods.pop 344 | 345 | debug "authenticating using #{method}" 346 | 347 | if method == 'PLAIN' 348 | @conn.login(username, password) 349 | else 350 | @conn.authenticate(method, username, password) 351 | end 352 | 353 | debug "authenticated using #{method}" 354 | 355 | rescue Net::IMAP::BadResponseError, Net::IMAP::NoResponseError => e 356 | debug "#{method} auth failed: #{e.message}" 357 | retry unless auth_methods.empty? 358 | 359 | raise e, "#{e.message} (tried #{tried.join(', ')})" 360 | end 361 | 362 | rescue => e 363 | exception = e 364 | end 365 | end.join 366 | 367 | raise exception if exception 368 | end 369 | 370 | def update_mailboxes 371 | debug "updating mailboxes" 372 | 373 | all = safely { @conn.list('', '*') } || [] 374 | subscribed = safely { @conn.lsub('', '*') } || [] 375 | 376 | # Remove cached mailboxes that no longer exist. 377 | @mailboxes.delete_if {|k, v| !all.any?{|mb| Net::IMAP.decode_utf7(mb.name) == k}} 378 | 379 | # Update cached mailboxes. 380 | all.each do |mb| 381 | name = Net::IMAP.decode_utf7(mb.name) 382 | name = 'INBOX' if name.downcase == 'inbox' 383 | 384 | @mailboxes[name] ||= Mailbox.new(self, name, mb.delim || '.', 385 | subscribed.any?{|s| s.name == mb.name}, mb.attr) 386 | end 387 | 388 | # Remove mailboxes that no longer exist from the database. 389 | @db_account.mailboxes_dataset.all do |db_mailbox| 390 | db_mailbox.destroy unless @mailboxes.has_key?(db_mailbox.name) 391 | end 392 | end 393 | 394 | end 395 | 396 | end 397 | -------------------------------------------------------------------------------- /README.rdoc: -------------------------------------------------------------------------------- 1 | = Larch 2 | 3 | *Note:* I no longer actively develop or use Larch, so it is effectively 4 | unmaintained. Many people claim it still works well for them, so feel free 5 | to use it, but please don't expect support, bug fixes, or new features. 6 | 7 | Larch is a tool to copy messages from one IMAP server to another quickly and 8 | safely. It's smart enough not to copy messages that already exist on the 9 | destination and robust enough to deal with interruptions caused by flaky 10 | connections or misbehaving servers. 11 | 12 | Larch is particularly well-suited for copying email to, from, or between Gmail 13 | accounts. 14 | 15 | *Author*:: Ryan Grove (mailto:ryan@wonko.com) 16 | *Version*:: 1.1.2 (2013-01-24) 17 | *Copyright*:: Copyright (c) 2013 Ryan Grove. All rights reserved. 18 | *License*:: GPL 2.0 (http://opensource.org/licenses/gpl-2.0.php) 19 | *Website*:: http://github.com/rgrove/larch 20 | 21 | == Installation 22 | 23 | Latest stable release: 24 | 25 | gem install larch 26 | 27 | Latest development version: 28 | 29 | gem install larch --pre 30 | 31 | == Usage 32 | 33 | larch [config section] [options] 34 | larch --from --to [options] 35 | 36 | Server Options: 37 | --from, -f : URI of the source IMAP server 38 | --from-folder, -F : Source folder to copy from (default: INBOX) 39 | --from-pass, -p : Source server password (default: prompt) 40 | --from-user, -u : Source server username (default: prompt) 41 | --to, -t : URI of the destination IMAP server 42 | --to-folder, -T : Destination folder to copy to (default: INBOX) 43 | --to-pass, -P : Destination server password (default: prompt) 44 | --to-user, -U : Destination server username (default: prompt) 45 | 46 | Copy Options: 47 | --all, -a: Copy all folders recursively 48 | --all-subscribed, -s: Copy all subscribed folders recursively 49 | --delete, -d: Delete messages from the source after copying 50 | them, or if they already exist at the destination 51 | --exclude : List of mailbox names/patterns that shouldn't be 52 | copied 53 | --exclude-file : Filename containing mailbox names/patterns that 54 | shouldn't be copied 55 | --expunge, -x: Expunge deleted messages from the source 56 | --sync-flags, -S: Sync message flags from the source to the 57 | destination for messages that already exist at the 58 | destination 59 | 60 | General Options: 61 | --config, -c : Specify a non-default config file to use (default: 62 | ~/.larch/config.yaml) 63 | --database : Specify a non-default message database to use 64 | (default: ~/.larch/larch.db) 65 | --dry-run, -n: Don't actually make any changes 66 | --max-retries : Maximum number of times to retry after a 67 | recoverable error (default: 3) 68 | --no-create-folder: Don't create destination folders that don't 69 | already exist 70 | --ssl-certs : Path to a trusted certificate bundle to use to 71 | verify server SSL certificates 72 | --ssl-verify: Verify server SSL certificates 73 | --verbosity, -V : Output verbosity: debug, info, warn, error, or 74 | fatal (default: info) 75 | --version, -v: Print version and exit 76 | --help, -h: Show this message 77 | 78 | == Usage Examples 79 | 80 | Larch is run from the command line. The following examples demonstrate how to 81 | run Larch using only command line arguments, but you may also place these 82 | options in a config file and run Larch without any arguments if you prefer. See 83 | the "Configuration" section below for more details. 84 | 85 | For an overview of all available options, run: 86 | 87 | larch -h 88 | 89 | At a minimum, you must specify a source server and a destination server 90 | in the form of IMAP URIs: 91 | 92 | larch --from imap://mail.example.com --to imap://imap.gmail.com 93 | 94 | Larch will prompt you for the necessary usernames and passwords, then sync the 95 | contents of the source's +INBOX+ folder to the destination's INBOX folder. 96 | 97 | To connect using SSL, specify a URI beginning with imaps://: 98 | 99 | larch --from imaps://mail.example.com --to imaps://imap.gmail.com 100 | 101 | If you'd like to sync a specific folder other than +INBOX+, specify the 102 | source and destination folders using --from-folder and 103 | --to-folder. Folder names containing spaces must be enclosed in quotes: 104 | 105 | larch --from imaps://mail.example.com --to imaps://imap.gmail.com \ 106 | --from-folder 'Sent Mail' --to-folder 'Sent Mail' 107 | 108 | To sync all folders, use the --all option (or 109 | --all-subscribed if you only want to sync subscribed folders): 110 | 111 | larch --from imaps://mail.example.com --to imaps://imap.gmail.com --all 112 | 113 | By default Larch will create folders on the destination server if they don't 114 | already exist. To prevent this, add the --no-create-folder option: 115 | 116 | larch --from imaps://mail.example.com --to imaps://imap.gmail.com --all \ 117 | --no-create-folder 118 | 119 | You can prevent Larch from syncing one or more folders by using the 120 | --exclude option, which accepts multiple arguments: 121 | 122 | larch --from imaps://mail.example.com --to imaps://imap.gmail.com --all \ 123 | --exclude Spam Trash Drafts "[Gmail]/*" 124 | 125 | If your exclusion list is long or complex, create a text file with one exclusion 126 | pattern per line and tell Larch to load it with the --exclude-file 127 | option: 128 | 129 | larch --from imaps://mail.example.com --to imaps://imap.gmail.com --all \ 130 | --exclude-file exclude.txt 131 | 132 | The wildcard characters * and ? are supported in exclusion 133 | lists. You may also use a regular expression by enclosing a pattern in 134 | forward slashes, so the previous example could be achieved with the 135 | pattern /(Spam|Trash|Drafts|\[Gmail\]\/.*)/ 136 | 137 | == Configuration 138 | 139 | While it's possible to control Larch entirely from the command line, this can be 140 | inconvenient if you need to specify a lot of options or if you run Larch 141 | frequently and can't always remember which options to use. Using a configuration 142 | file can simplify things. 143 | 144 | By default, Larch looks for a config file at ~/.larch/config.yaml and 145 | uses it if found. You may specify a custom config file using the 146 | --config command line option. 147 | 148 | The Larch configuration file is a simple YAML[http://yaml.org/] file that may 149 | contain multiple sections, each with a different set of options, as well as a 150 | special +default+ section. The options in the +default+ section will be used 151 | unless they're overridden either in another config section or on the command 152 | line. 153 | 154 | === Example 155 | 156 | Here's a sample Larch config file: 157 | 158 | default: 159 | all-subscribed: true # Copy all subscribed folders by default 160 | 161 | # Copy mail from Gmail to my server, excluding stuff I don't want. 162 | gmail to my server: 163 | from: imaps://imap.gmail.com 164 | from-user: example 165 | from-pass: secret 166 | 167 | to: imaps://mail.example.com 168 | to-user: example 169 | to-pass: secret 170 | 171 | exclude: 172 | - "[Gmail]/Sent Mail" 173 | - "[Gmail]/Spam" 174 | - "[Gmail]/Trash" 175 | 176 | # Copy mail from my INBOX to Gmail's INBOX 177 | my inbox to gmail inbox: 178 | all-subscribed: false 179 | 180 | from: imaps://mail.example.com 181 | from-folder: INBOX 182 | from-user: example 183 | from-pass: secret 184 | 185 | to: imaps://imap.gmail.com 186 | to-folder: INBOX 187 | to-user: example 188 | to-pass: secret 189 | 190 | This file contains three sections. The options from +default+ will be used in 191 | all other sections as well unless they're overridden. 192 | 193 | To specify which config section you want Larch to use, just pass its name on the 194 | command line (use quotes if the name contains spaces): 195 | 196 | larch 'gmail to my server' 197 | 198 | If you specify additional command line options, they'll override options in the 199 | config file: 200 | 201 | larch 'gmail to my server' --from-user anotheruser 202 | 203 | Running Larch with no command line arguments will cause the +default+ section 204 | to be used. With the example above, this will result in an error since the 205 | +default+ section doesn't contain the required +from+ and +to+ options, but if 206 | you only need to use Larch with a single configuration, you could use the 207 | +default+ section for everything and save yourself some typing on the command 208 | line. 209 | 210 | == Server Compatibility 211 | 212 | Larch should work well with any server that properly supports 213 | IMAP4rev1[http://tools.ietf.org/html/rfc3501], and does its best to get along 214 | with servers that have buggy, unreliable, or incomplete IMAP implementations. 215 | 216 | Larch has been tested on and is known to work well with the following IMAP 217 | servers: 218 | 219 | * Dovecot 220 | * Gmail 221 | * Microsoft Exchange 2003 222 | 223 | The following servers are known to work, but with caveats: 224 | 225 | * Yahoo! Mail 226 | 227 | The following servers do not work well with Larch: 228 | 229 | * BlitzMail - Buggy server implementation; fails to properly quote or escape 230 | some IMAP responses, which can cause Net::IMAP to hang waiting for a 231 | terminating character that will never arrive. 232 | 233 | === Gmail Quirks 234 | 235 | Gmail's IMAP implementation is quirky. Larch does its best to work around these 236 | quirks whenever possible, but here are a few things to watch out for: 237 | 238 | ==== "Some messages could not be FETCHed" error 239 | 240 | This error indicates that a message on Gmail is corrupt, and Gmail itself is 241 | unable to read it. The message will continue to show up in the mailbox, but all 242 | attempts to access it via IMAP, POP, or the Gmail web interface will result in 243 | errors. Larch will try to skip these messages and continue processing others 244 | if possible. 245 | 246 | It's not clear how this corruption occurs or exactly what kind of corruption 247 | causes these errors, although in every case I'm aware of, the corrupt message 248 | has originated outside of Gmail (Gmail itself does not corrupt the message). 249 | There is currently no known solution for this problem apart from deleting the 250 | corrupted messages. 251 | 252 | ==== Folder names cannot contain leading or trailing whitespace 253 | 254 | Most IMAP servers allow folder names to contain leading and trailing whitespace, 255 | such as " folder ". Gmail does not. When copying folders to Gmail, Larch will 256 | automatically remove leading and trailing whitespace in folder names to prevent 257 | errors. 258 | 259 | === Yahoo! Mail Quirks 260 | 261 | Yahoo! doesn't officially support IMAP access for general usage, but Larch is 262 | able to connect to imap.mail.yahoo.com and imap-ssl.mail.yahoo.com by using a 263 | fairly well-known trick. That said, as with anything tricky, there are caveats. 264 | 265 | ==== No hierarchical folders 266 | 267 | Similar to Gmail, Yahoo! Mail doesn't allow hierarchical (nested) folders. If 268 | you try to copy a folder hierarchy to Yahoo!, it will work, but you'll end up 269 | with a set of folders named "folder" and "folder.subfolder" rather than seeing 270 | "subfolder" as an actual subfolder of "folder". 271 | 272 | ==== No custom flags 273 | 274 | Yahoo! Mail IMAP doesn't support custom message flags, such as the tags and 275 | junk/not junk flags used by Thunderbird. When transferring messages with custom 276 | flags to a Yahoo! Mail IMAP account, the custom flags will be lost. 277 | 278 | ==== Here there be dragons 279 | 280 | Larch's support for Yahoo! Mail is very new and very lightly tested. Given its 281 | newness and the fact that Yahoo!'s IMAP gateway isn't official, there are likely 282 | to be other quirks we're not yet aware of. There's also no guarantee that Yahoo! 283 | won't shut down its IMAP gateway, deprecate the trick Larch uses to connect, or 284 | just outright block Larch. Use at your own risk. 285 | 286 | == Known Issues 287 | 288 | * Larch uses Ruby's Net::IMAP standard library for all IMAP operations. While 289 | Net::IMAP is generally a very solid library, it contains a bug that can 290 | cause a deadlock to occur if a connection drops unexpectedly (either due to 291 | network issues or because the server closed the connection without warning) 292 | when the server has already begun sending a response and Net::IMAP is 293 | waiting to receive more data. If this happens, Net::IMAP will continue waiting 294 | forever without passing control back to Larch, and you will need to manually 295 | kill and restart Larch. 296 | 297 | Net::IMAP in Ruby 1.8 has also been known to hang when it can't parse a server 298 | response, either because the response itself is malformed or because of a 299 | bug in Net::IMAP's parser. This is rare, but it happens. Unfortunately there's 300 | nothing Larch can do about this. 301 | 302 | * The Ruby package on Debian, Ubuntu, and some other Debian-based Linux 303 | distributions doesn't include the OpenSSL standard library. If you see an 304 | error like uninitialized constant Larch::IMAP::OpenSSL (NameError) 305 | when running Larch, you may need to install the libopenssl-ruby 306 | package. Please feel free to complain to the maintainer of your distribution's 307 | Ruby packages. 308 | 309 | == Support 310 | 311 | The Larch mailing list is the best place for questions, comments, and discussion 312 | about Larch. You can join the list or view the archives at 313 | http://groups.google.com/group/larch 314 | 315 | First-time senders to the list are moderated to prevent spam, so there may be a 316 | delay before your first message shows up. 317 | 318 | == Contributors 319 | 320 | Larch was created and is maintained by Ryan Grove . 321 | 322 | The following lovely people have also contributed to Larch: 323 | 324 | * Torey Heinz 325 | * Edgardo Hames 326 | * Andrew Hobson 327 | * Justin Mazzi 328 | 329 | == Credit 330 | 331 | The Larch::IMAP class borrows heavily from Sup[http://sup.rubyforge.org] by 332 | William Morgan, the source code of which should be required reading if you're 333 | doing anything with IMAP in Ruby. 334 | 335 | Larch uses the excellent Trollop[http://trollop.rubyforge.org] command-line 336 | option parser (also by William Morgan) and the 337 | HighLine[http://highline.rubyforge.org] command-line IO library (by James Edward 338 | Gray II). 339 | 340 | == License 341 | 342 | Copyright (c) 2013 Ryan Grove 343 | 344 | Licensed under the GNU General Public License version 2.0. 345 | 346 | This program is free software; you can redistribute it and/or modify it under 347 | the terms of version 2.0 of the GNU General Public License as published by the 348 | Free Software Foundation. 349 | 350 | This program is distributed in the hope that it will be useful, but WITHOUT ANY 351 | WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A 352 | PARTICULAR PURPOSE. See the GNU General Public License for more details. 353 | 354 | You should have received a copy of the GNU General Public License along with 355 | this program; if not, visit http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt 356 | or write to the Free Software Foundation, Inc., 59 Temple Place, Suite 330, 357 | Boston, MA 02111-1307 USA. 358 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | GNU GENERAL PUBLIC LICENSE 2 | Version 2, June 1991 3 | 4 | Copyright (C) 1989, 1991 Free Software Foundation, Inc., 5 | 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA 6 | Everyone is permitted to copy and distribute verbatim copies 7 | of this license document, but changing it is not allowed. 8 | 9 | Preamble 10 | 11 | The licenses for most software are designed to take away your 12 | freedom to share and change it. By contrast, the GNU General Public 13 | License is intended to guarantee your freedom to share and change free 14 | software--to make sure the software is free for all its users. This 15 | General Public License applies to most of the Free Software 16 | Foundation's software and to any other program whose authors commit to 17 | using it. (Some other Free Software Foundation software is covered by 18 | the GNU Lesser General Public License instead.) You can apply it to 19 | your programs, too. 20 | 21 | When we speak of free software, we are referring to freedom, not 22 | price. Our General Public Licenses are designed to make sure that you 23 | have the freedom to distribute copies of free software (and charge for 24 | this service if you wish), that you receive source code or can get it 25 | if you want it, that you can change the software or use pieces of it 26 | in new free programs; and that you know you can do these things. 27 | 28 | To protect your rights, we need to make restrictions that forbid 29 | anyone to deny you these rights or to ask you to surrender the rights. 30 | These restrictions translate to certain responsibilities for you if you 31 | distribute copies of the software, or if you modify it. 32 | 33 | For example, if you distribute copies of such a program, whether 34 | gratis or for a fee, you must give the recipients all the rights that 35 | you have. You must make sure that they, too, receive or can get the 36 | source code. And you must show them these terms so they know their 37 | rights. 38 | 39 | We protect your rights with two steps: (1) copyright the software, and 40 | (2) offer you this license which gives you legal permission to copy, 41 | distribute and/or modify the software. 42 | 43 | Also, for each author's protection and ours, we want to make certain 44 | that everyone understands that there is no warranty for this free 45 | software. If the software is modified by someone else and passed on, we 46 | want its recipients to know that what they have is not the original, so 47 | that any problems introduced by others will not reflect on the original 48 | authors' reputations. 49 | 50 | Finally, any free program is threatened constantly by software 51 | patents. We wish to avoid the danger that redistributors of a free 52 | program will individually obtain patent licenses, in effect making the 53 | program proprietary. To prevent this, we have made it clear that any 54 | patent must be licensed for everyone's free use or not licensed at all. 55 | 56 | The precise terms and conditions for copying, distribution and 57 | modification follow. 58 | 59 | GNU GENERAL PUBLIC LICENSE 60 | TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION 61 | 62 | 0. This License applies to any program or other work which contains 63 | a notice placed by the copyright holder saying it may be distributed 64 | under the terms of this General Public License. The "Program", below, 65 | refers to any such program or work, and a "work based on the Program" 66 | means either the Program or any derivative work under copyright law: 67 | that is to say, a work containing the Program or a portion of it, 68 | either verbatim or with modifications and/or translated into another 69 | language. (Hereinafter, translation is included without limitation in 70 | the term "modification".) Each licensee is addressed as "you". 71 | 72 | Activities other than copying, distribution and modification are not 73 | covered by this License; they are outside its scope. The act of 74 | running the Program is not restricted, and the output from the Program 75 | is covered only if its contents constitute a work based on the 76 | Program (independent of having been made by running the Program). 77 | Whether that is true depends on what the Program does. 78 | 79 | 1. You may copy and distribute verbatim copies of the Program's 80 | source code as you receive it, in any medium, provided that you 81 | conspicuously and appropriately publish on each copy an appropriate 82 | copyright notice and disclaimer of warranty; keep intact all the 83 | notices that refer to this License and to the absence of any warranty; 84 | and give any other recipients of the Program a copy of this License 85 | along with the Program. 86 | 87 | You may charge a fee for the physical act of transferring a copy, and 88 | you may at your option offer warranty protection in exchange for a fee. 89 | 90 | 2. You may modify your copy or copies of the Program or any portion 91 | of it, thus forming a work based on the Program, and copy and 92 | distribute such modifications or work under the terms of Section 1 93 | above, provided that you also meet all of these conditions: 94 | 95 | a) You must cause the modified files to carry prominent notices 96 | stating that you changed the files and the date of any change. 97 | 98 | b) You must cause any work that you distribute or publish, that in 99 | whole or in part contains or is derived from the Program or any 100 | part thereof, to be licensed as a whole at no charge to all third 101 | parties under the terms of this License. 102 | 103 | c) If the modified program normally reads commands interactively 104 | when run, you must cause it, when started running for such 105 | interactive use in the most ordinary way, to print or display an 106 | announcement including an appropriate copyright notice and a 107 | notice that there is no warranty (or else, saying that you provide 108 | a warranty) and that users may redistribute the program under 109 | these conditions, and telling the user how to view a copy of this 110 | License. (Exception: if the Program itself is interactive but 111 | does not normally print such an announcement, your work based on 112 | the Program is not required to print an announcement.) 113 | 114 | These requirements apply to the modified work as a whole. If 115 | identifiable sections of that work are not derived from the Program, 116 | and can be reasonably considered independent and separate works in 117 | themselves, then this License, and its terms, do not apply to those 118 | sections when you distribute them as separate works. But when you 119 | distribute the same sections as part of a whole which is a work based 120 | on the Program, the distribution of the whole must be on the terms of 121 | this License, whose permissions for other licensees extend to the 122 | entire whole, and thus to each and every part regardless of who wrote it. 123 | 124 | Thus, it is not the intent of this section to claim rights or contest 125 | your rights to work written entirely by you; rather, the intent is to 126 | exercise the right to control the distribution of derivative or 127 | collective works based on the Program. 128 | 129 | In addition, mere aggregation of another work not based on the Program 130 | with the Program (or with a work based on the Program) on a volume of 131 | a storage or distribution medium does not bring the other work under 132 | the scope of this License. 133 | 134 | 3. You may copy and distribute the Program (or a work based on it, 135 | under Section 2) in object code or executable form under the terms of 136 | Sections 1 and 2 above provided that you also do one of the following: 137 | 138 | a) Accompany it with the complete corresponding machine-readable 139 | source code, which must be distributed under the terms of Sections 140 | 1 and 2 above on a medium customarily used for software interchange; or, 141 | 142 | b) Accompany it with a written offer, valid for at least three 143 | years, to give any third party, for a charge no more than your 144 | cost of physically performing source distribution, a complete 145 | machine-readable copy of the corresponding source code, to be 146 | distributed under the terms of Sections 1 and 2 above on a medium 147 | customarily used for software interchange; or, 148 | 149 | c) Accompany it with the information you received as to the offer 150 | to distribute corresponding source code. (This alternative is 151 | allowed only for noncommercial distribution and only if you 152 | received the program in object code or executable form with such 153 | an offer, in accord with Subsection b above.) 154 | 155 | The source code for a work means the preferred form of the work for 156 | making modifications to it. For an executable work, complete source 157 | code means all the source code for all modules it contains, plus any 158 | associated interface definition files, plus the scripts used to 159 | control compilation and installation of the executable. However, as a 160 | special exception, the source code distributed need not include 161 | anything that is normally distributed (in either source or binary 162 | form) with the major components (compiler, kernel, and so on) of the 163 | operating system on which the executable runs, unless that component 164 | itself accompanies the executable. 165 | 166 | If distribution of executable or object code is made by offering 167 | access to copy from a designated place, then offering equivalent 168 | access to copy the source code from the same place counts as 169 | distribution of the source code, even though third parties are not 170 | compelled to copy the source along with the object code. 171 | 172 | 4. You may not copy, modify, sublicense, or distribute the Program 173 | except as expressly provided under this License. Any attempt 174 | otherwise to copy, modify, sublicense or distribute the Program is 175 | void, and will automatically terminate your rights under this License. 176 | However, parties who have received copies, or rights, from you under 177 | this License will not have their licenses terminated so long as such 178 | parties remain in full compliance. 179 | 180 | 5. You are not required to accept this License, since you have not 181 | signed it. However, nothing else grants you permission to modify or 182 | distribute the Program or its derivative works. These actions are 183 | prohibited by law if you do not accept this License. Therefore, by 184 | modifying or distributing the Program (or any work based on the 185 | Program), you indicate your acceptance of this License to do so, and 186 | all its terms and conditions for copying, distributing or modifying 187 | the Program or works based on it. 188 | 189 | 6. Each time you redistribute the Program (or any work based on the 190 | Program), the recipient automatically receives a license from the 191 | original licensor to copy, distribute or modify the Program subject to 192 | these terms and conditions. You may not impose any further 193 | restrictions on the recipients' exercise of the rights granted herein. 194 | You are not responsible for enforcing compliance by third parties to 195 | this License. 196 | 197 | 7. If, as a consequence of a court judgment or allegation of patent 198 | infringement or for any other reason (not limited to patent issues), 199 | conditions are imposed on you (whether by court order, agreement or 200 | otherwise) that contradict the conditions of this License, they do not 201 | excuse you from the conditions of this License. If you cannot 202 | distribute so as to satisfy simultaneously your obligations under this 203 | License and any other pertinent obligations, then as a consequence you 204 | may not distribute the Program at all. For example, if a patent 205 | license would not permit royalty-free redistribution of the Program by 206 | all those who receive copies directly or indirectly through you, then 207 | the only way you could satisfy both it and this License would be to 208 | refrain entirely from distribution of the Program. 209 | 210 | If any portion of this section is held invalid or unenforceable under 211 | any particular circumstance, the balance of the section is intended to 212 | apply and the section as a whole is intended to apply in other 213 | circumstances. 214 | 215 | It is not the purpose of this section to induce you to infringe any 216 | patents or other property right claims or to contest validity of any 217 | such claims; this section has the sole purpose of protecting the 218 | integrity of the free software distribution system, which is 219 | implemented by public license practices. Many people have made 220 | generous contributions to the wide range of software distributed 221 | through that system in reliance on consistent application of that 222 | system; it is up to the author/donor to decide if he or she is willing 223 | to distribute software through any other system and a licensee cannot 224 | impose that choice. 225 | 226 | This section is intended to make thoroughly clear what is believed to 227 | be a consequence of the rest of this License. 228 | 229 | 8. If the distribution and/or use of the Program is restricted in 230 | certain countries either by patents or by copyrighted interfaces, the 231 | original copyright holder who places the Program under this License 232 | may add an explicit geographical distribution limitation excluding 233 | those countries, so that distribution is permitted only in or among 234 | countries not thus excluded. In such case, this License incorporates 235 | the limitation as if written in the body of this License. 236 | 237 | 9. The Free Software Foundation may publish revised and/or new versions 238 | of the General Public License from time to time. Such new versions will 239 | be similar in spirit to the present version, but may differ in detail to 240 | address new problems or concerns. 241 | 242 | Each version is given a distinguishing version number. If the Program 243 | specifies a version number of this License which applies to it and "any 244 | later version", you have the option of following the terms and conditions 245 | either of that version or of any later version published by the Free 246 | Software Foundation. If the Program does not specify a version number of 247 | this License, you may choose any version ever published by the Free Software 248 | Foundation. 249 | 250 | 10. If you wish to incorporate parts of the Program into other free 251 | programs whose distribution conditions are different, write to the author 252 | to ask for permission. For software which is copyrighted by the Free 253 | Software Foundation, write to the Free Software Foundation; we sometimes 254 | make exceptions for this. Our decision will be guided by the two goals 255 | of preserving the free status of all derivatives of our free software and 256 | of promoting the sharing and reuse of software generally. 257 | 258 | NO WARRANTY 259 | 260 | 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY 261 | FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN 262 | OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES 263 | PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED 264 | OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 265 | MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS 266 | TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE 267 | PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, 268 | REPAIR OR CORRECTION. 269 | 270 | 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING 271 | WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR 272 | REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, 273 | INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING 274 | OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED 275 | TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY 276 | YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER 277 | PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE 278 | POSSIBILITY OF SUCH DAMAGES. 279 | 280 | END OF TERMS AND CONDITIONS 281 | -------------------------------------------------------------------------------- /lib/larch/imap/mailbox.rb: -------------------------------------------------------------------------------- 1 | module Larch; class IMAP 2 | 3 | # Represents an IMAP mailbox. 4 | class Mailbox 5 | attr_reader :attr, :db_mailbox, :delim, :flags, :imap, :name, :perm_flags, :state, :subscribed 6 | 7 | # Maximum number of message headers to fetch with a single IMAP command. 8 | FETCH_BLOCK_SIZE = 1024 9 | 10 | # Regex to capture a Message-Id header. 11 | REGEX_MESSAGE_ID = /message-id\s*:\s*(\S+)/i 12 | 13 | # Minimum time (in seconds) allowed between mailbox scans. 14 | SCAN_INTERVAL = 60 15 | 16 | def initialize(imap, name, delim, subscribed, *attr) 17 | raise ArgumentError, "must provide a Larch::IMAP instance" unless imap.is_a?(Larch::IMAP) 18 | 19 | @attr = attr.flatten 20 | @delim = delim 21 | @flags = [] 22 | @imap = imap 23 | @last_scan = nil 24 | @name = name 25 | @name_utf7 = Net::IMAP.encode_utf7(@name) 26 | @perm_flags = [] 27 | @subscribed = subscribed 28 | 29 | # Valid mailbox states are :closed (no mailbox open), :examined (mailbox 30 | # open and read-only), or :selected (mailbox open and read-write). 31 | @state = :closed 32 | 33 | # Create/update this mailbox in the database. 34 | mb_data = { 35 | :name => @name, 36 | :delim => @delim, 37 | :attr => @attr.map{|a| a.to_s }.join(','), 38 | :subscribed => @subscribed ? 1 : 0 39 | } 40 | 41 | @db_mailbox = imap.db_account.mailboxes_dataset.filter(:name => @name).first 42 | 43 | if @db_mailbox 44 | @db_mailbox.update(mb_data) 45 | else 46 | @db_mailbox = Database::Mailbox.create(mb_data) 47 | imap.db_account.add_mailbox(@db_mailbox) 48 | end 49 | 50 | # Create private convenience methods (debug, info, warn, etc.) to make 51 | # logging easier. 52 | Logger::LEVELS.each_key do |level| 53 | next if Mailbox.private_method_defined?(level) 54 | 55 | Mailbox.class_eval do 56 | define_method(level) do |msg| 57 | Larch.log.log(level, "#{@imap.options[:log_label]} #{@name}: #{msg}") 58 | end 59 | 60 | private level 61 | end 62 | end 63 | end 64 | 65 | # Appends the specified Larch::IMAP::Message to this mailbox if it doesn't 66 | # already exist. Returns +true+ if the message was appended successfully, 67 | # +false+ if the message already exists in the mailbox. 68 | def append(message) 69 | raise ArgumentError, "must provide a Larch::IMAP::Message object" unless message.is_a?(Larch::IMAP::Message) 70 | return false if has_guid?(message.guid) 71 | 72 | @imap.safely do 73 | unless imap_select(!!@imap.options[:create_mailbox]) 74 | raise Larch::IMAP::Error, "mailbox cannot contain messages: #{@name}" 75 | end 76 | 77 | debug "appending message: #{message.guid}" 78 | @imap.conn.append(@name_utf7, message.rfc822, get_supported_flags(message.flags), message.internaldate) unless @imap.options[:dry_run] 79 | end 80 | 81 | true 82 | end 83 | alias << append 84 | 85 | # Deletes the message in this mailbox with the specified guid. Returns +true+ 86 | # on success, +false+ on failure. 87 | def delete_message(guid) 88 | if @imap.quirks[:gmail] 89 | return false unless db_message = fetch_db_message(guid) 90 | 91 | debug "moving message to Gmail trash: #{guid}" 92 | 93 | @imap.safely { @imap.conn.uid_copy(db_message.uid, '[Gmail]/Trash') } && 94 | set_flags(guid, [:Deleted], true) 95 | else 96 | set_flags(guid, [:Deleted], true) 97 | end 98 | end 99 | 100 | # Iterates through messages in this mailbox, yielding a 101 | # Larch::Database::Message object for each to the provided block. 102 | def each_db_message # :yields: db_message 103 | scan 104 | @db_mailbox.messages_dataset.all {|db_message| yield db_message } 105 | end 106 | 107 | # Iterates through messages in this mailbox, yielding the Larch message guid 108 | # of each to the provided block. 109 | def each_guid # :yields: guid 110 | each_db_message {|db_message| yield db_message.guid } 111 | end 112 | 113 | # Iterates through mailboxes that are first-level children of this mailbox, 114 | # yielding a Larch::IMAP::Mailbox object for each to the provided block. 115 | def each_mailbox # :yields: mailbox 116 | mailboxes.each {|mb| yield mb } 117 | end 118 | 119 | # Expunges this mailbox, permanently removing all messages with the \Deleted 120 | # flag. 121 | def expunge 122 | return false unless imap_select 123 | 124 | @imap.safely do 125 | debug "expunging deleted messages" 126 | 127 | @last_scan = nil 128 | @imap.conn.expunge unless @imap.options[:dry_run] 129 | end 130 | end 131 | 132 | # Returns a Larch::IMAP::Message struct representing the message with the 133 | # specified Larch _guid_, or +nil+ if the specified guid was not found in this 134 | # mailbox. 135 | def fetch(guid, peek = false) 136 | scan 137 | 138 | unless db_message = fetch_db_message(guid) 139 | warning "message not found in local db: #{guid}" 140 | return nil 141 | end 142 | 143 | debug "#{peek ? 'peeking at' : 'fetching'} message: #{guid}" 144 | 145 | imap_uid_fetch([db_message.uid], [(peek ? 'BODY.PEEK[]' : 'BODY[]'), 'FLAGS', 'INTERNALDATE', 'ENVELOPE']) do |fetch_data| 146 | data = fetch_data.first 147 | check_response_fields(data, 'BODY[]', 'FLAGS', 'INTERNALDATE', 'ENVELOPE') 148 | 149 | return Message.new(guid, data.attr['ENVELOPE'], data.attr['BODY[]'], 150 | data.attr['FLAGS'], Time.parse(data.attr['INTERNALDATE'])) 151 | end 152 | 153 | warning "message not found on server: #{guid}" 154 | return nil 155 | end 156 | alias [] fetch 157 | 158 | # Returns a Larch::Database::Message object representing the message with the 159 | # specified Larch _guid_, or +nil+ if the specified guide was not found in 160 | # this mailbox. 161 | def fetch_db_message(guid) 162 | scan 163 | @db_mailbox.messages_dataset.filter(:guid => guid).first 164 | end 165 | 166 | # Returns +true+ if a message with the specified Larch guid exists in this 167 | # mailbox, +false+ otherwise. 168 | def has_guid?(guid) 169 | scan 170 | @db_mailbox.messages_dataset.filter(:guid => guid).count > 0 171 | end 172 | 173 | # Gets the number of messages in this mailbox. 174 | def length 175 | scan 176 | @db_mailbox.messages_dataset.count 177 | end 178 | alias size length 179 | 180 | # Returns an Array of Larch::IMAP::Mailbox objects representing mailboxes that 181 | # are first-level children of this mailbox. 182 | def mailboxes 183 | return [] if @attr.include?(:Noinferiors) 184 | 185 | all = @imap.safely{ @imap.conn.list('', "#{@name_utf7}#{@delim}%") } || [] 186 | subscribed = @imap.safely{ @imap.conn.lsub('', "#{@name_utf7}#{@delim}%") } || [] 187 | 188 | all.map{|mb| Mailbox.new(@imap, mb.name, mb.delim, 189 | subscribed.any?{|s| s.name == mb.name}, mb.attr) } 190 | end 191 | 192 | # Same as fetch, but doesn't mark the message as seen. 193 | def peek(guid) 194 | fetch(guid, true) 195 | end 196 | 197 | # Resets the mailbox state. 198 | def reset 199 | @state = :closed 200 | end 201 | 202 | # Fetches message headers from this mailbox. 203 | def scan 204 | now = Time.now.to_i 205 | return if @last_scan && (now - @last_scan) < SCAN_INTERVAL 206 | 207 | first_scan = @last_scan.nil? 208 | @last_scan = now 209 | 210 | # Compare the mailbox's current status with its last known status. 211 | begin 212 | return unless status = imap_status('MESSAGES', 'UIDNEXT', 'UIDVALIDITY') 213 | rescue Error => e 214 | return if @imap.options[:create_mailbox] 215 | raise 216 | end 217 | 218 | flag_range = nil 219 | full_range = nil 220 | 221 | if @db_mailbox.uidvalidity && @db_mailbox.uidnext && 222 | status['UIDVALIDITY'] == @db_mailbox.uidvalidity 223 | 224 | # The UIDVALIDITY is the same as what we saw last time we scanned this 225 | # mailbox, which means that all the existing messages in the database are 226 | # still valid. We only need to request headers for new messages. 227 | # 228 | # If this is the first scan of this mailbox during this Larch session, 229 | # then we'll also update the flags of all messages in the mailbox. 230 | 231 | flag_range = 1...@db_mailbox.uidnext if first_scan 232 | full_range = @db_mailbox.uidnext...status['UIDNEXT'] 233 | 234 | else 235 | 236 | # The UIDVALIDITY has changed or this is the first time we've scanned this 237 | # mailbox (ever). Either way, all existing messages in the database are no 238 | # longer valid, so we have to throw them out and re-request everything. 239 | 240 | @db_mailbox.remove_all_messages 241 | full_range = 1...status['UIDNEXT'] 242 | end 243 | 244 | @db_mailbox.update(:uidvalidity => status['UIDVALIDITY']) 245 | 246 | need_flag_scan = flag_range && flag_range.max && flag_range.min && flag_range.max - flag_range.min >= 0 247 | need_full_scan = full_range && full_range.max && full_range.min && full_range.max - full_range.min >= 0 248 | 249 | return unless need_flag_scan || need_full_scan 250 | 251 | fetch_flags(flag_range) if need_flag_scan 252 | 253 | if need_full_scan 254 | fetch_headers(full_range, { 255 | :progress_start => @db_mailbox.messages_dataset.count + 1, 256 | :progress_total => status['MESSAGES'] 257 | }) 258 | end 259 | 260 | @db_mailbox.update(:uidnext => status['UIDNEXT']) 261 | return 262 | end 263 | 264 | # Sets the IMAP flags for the message specified by _guid_. _flags_ should be 265 | # an array of symbols for standard flags, strings for custom flags. 266 | # 267 | # If _merge_ is +true+, the specified flags will be merged with the message's 268 | # existing flags. Otherwise, all existing flags will be cleared and replaced 269 | # with the specified flags. 270 | # 271 | # Note that the :Recent flag cannot be manually set or removed. 272 | # 273 | # Returns +true+ on success, +false+ on failure. 274 | def set_flags(guid, flags, merge = false) 275 | raise ArgumentError, "flags must be an Array" unless flags.is_a?(Array) 276 | 277 | return false unless db_message = fetch_db_message(guid) 278 | 279 | merged_flags = merge ? (db_message.flags + flags).uniq : flags 280 | supported_flags = get_supported_flags(merged_flags) 281 | 282 | return true if db_message.flags == supported_flags 283 | 284 | return false if !imap_select 285 | @imap.safely { @imap.conn.uid_store(db_message.uid, 'FLAGS.SILENT', supported_flags) } unless @imap.options[:dry_run] 286 | 287 | true 288 | end 289 | 290 | # Subscribes to this mailbox. 291 | def subscribe(force = false) 292 | return false if subscribed? && !force 293 | 294 | @imap.safely { @imap.conn.subscribe(@name_utf7) } unless @imap.options[:dry_run] 295 | @subscribed = true 296 | @db_mailbox.update(:subscribed => 1) 297 | 298 | true 299 | end 300 | 301 | # Returns +true+ if this mailbox is subscribed, +false+ otherwise. 302 | def subscribed? 303 | @subscribed 304 | end 305 | 306 | # Unsubscribes from this mailbox. 307 | def unsubscribe(force = false) 308 | return false unless subscribed? || force 309 | 310 | @imap.safely { @imap.conn.unsubscribe(@name_utf7) } unless @imap.options[:dry_run] 311 | @subscribed = false 312 | @db_mailbox.update(:subscribed => 0) 313 | 314 | true 315 | end 316 | 317 | private 318 | 319 | # Checks the specified Net::IMAP::FetchData object and raises a 320 | # Larch::IMAP::Error unless it contains all the specified _fields_. 321 | # 322 | # _data_ can be a single object or an Array of objects; if it's an Array, then 323 | # only the first object in the Array will be checked. 324 | def check_response_fields(data, *fields) 325 | check_data = data.is_a?(Array) ? data.first : data 326 | 327 | fields.each do |f| 328 | raise Error, "required data not in IMAP response: #{f}" unless check_data.attr.has_key?(f) 329 | end 330 | 331 | true 332 | end 333 | 334 | # Creates a globally unique id suitable for identifying a specific message 335 | # on any mail server (we hope) based on the given IMAP FETCH _data_. 336 | # 337 | # If the given message data includes a valid Message-Id header, then that will 338 | # be used to generate an MD5 hash. Otherwise, the hash will be generated based 339 | # on the message's RFC822.SIZE and INTERNALDATE. 340 | def create_guid(data) 341 | if message_id = parse_message_id(data.attr['BODY[HEADER.FIELDS (MESSAGE-ID)]']) 342 | Digest::MD5.hexdigest(message_id) 343 | else 344 | check_response_fields(data, 'RFC822.SIZE', 'INTERNALDATE') 345 | 346 | Digest::MD5.hexdigest(sprintf('%d%d', data.attr['RFC822.SIZE'], 347 | Time.parse(data.attr['INTERNALDATE']).to_i)) 348 | end 349 | end 350 | 351 | # Returns only the flags from the specified _flags_ array that can be set in 352 | # this mailbox. Emits a warning message for any unsupported flags. 353 | def get_supported_flags(flags) 354 | supported_flags = flags.dup 355 | 356 | supported_flags.delete_if do |flag| 357 | # The \Recent flag is read-only, so we shouldn't try to set it. 358 | next true if flag == :Recent 359 | 360 | unless @flags.include?(flag) || @perm_flags.include?(:*) || @perm_flags.include?(flag) 361 | warning "flag not supported on destination: #{flag}" 362 | true 363 | end 364 | end 365 | 366 | supported_flags 367 | end 368 | 369 | # Fetches the latest flags from the server for the specified range of message 370 | # UIDs. 371 | def fetch_flags(flag_range) 372 | return unless imap_examine 373 | 374 | info "fetching latest message flags..." 375 | 376 | # Load the expected UIDs and their flags into a Hash for quicker lookups. 377 | expected_uids = {} 378 | @db_mailbox.messages_dataset.all do |db_message| 379 | expected_uids[db_message.uid] = db_message.flags 380 | end 381 | 382 | imap_uid_fetch(flag_range, "(UID FLAGS)", 16384) do |fetch_data| 383 | # Check the fields in the first response to ensure that everything we 384 | # asked for is there. 385 | check_response_fields(fetch_data.first, 'UID', 'FLAGS') unless fetch_data.empty? 386 | 387 | Larch.db.transaction do 388 | fetch_data.each do |data| 389 | uid = data.attr['UID'] 390 | flags = data.attr['FLAGS'] 391 | local_flags = expected_uids[uid] 392 | 393 | # If we haven't seen this message before, or if its flags have 394 | # changed, update the database. 395 | unless local_flags && local_flags == flags 396 | @db_mailbox.messages_dataset.filter(:uid => uid).update(:flags => flags.map{|f| f.to_s }.join(',')) 397 | end 398 | 399 | expected_uids.delete(uid) 400 | end 401 | end 402 | end 403 | 404 | # Any UIDs that are in the database but weren't in the response have been 405 | # deleted from the server, so we need to delete them from the database as 406 | # well. 407 | unless expected_uids.empty? 408 | debug "removing #{expected_uids.length} deleted messages from the database..." 409 | 410 | Larch.db.transaction do 411 | expected_uids.each_key do |uid| 412 | @db_mailbox.messages_dataset.filter(:uid => uid).destroy 413 | end 414 | end 415 | end 416 | 417 | expected_uids = nil 418 | fetch_data = nil 419 | end 420 | 421 | # Fetches the latest headers from the server for the specified range of 422 | # message UIDs. 423 | def fetch_headers(header_range, options = {}) 424 | return unless imap_examine 425 | 426 | options = { 427 | :progress_start => 0, 428 | :progress_total => 0 429 | }.merge(options) 430 | 431 | fetched = 0 432 | progress = 0 433 | show_progress = options[:progress_total] - options[:progress_start] > FETCH_BLOCK_SIZE * 4 434 | 435 | info "fetching message headers #{options[:progress_start]} through #{options[:progress_total]}..." 436 | 437 | last_good_uid = nil 438 | 439 | imap_uid_fetch(header_range, "(UID BODY.PEEK[HEADER.FIELDS (MESSAGE-ID)] RFC822.SIZE INTERNALDATE FLAGS)") do |fetch_data| 440 | # Check the fields in the first response to ensure that everything we 441 | # asked for is there. 442 | check_response_fields(fetch_data, 'UID', 'RFC822.SIZE', 'INTERNALDATE', 'FLAGS') 443 | 444 | Larch.db.transaction do 445 | fetch_data.each do |data| 446 | uid = data.attr['UID'] 447 | 448 | Database::Message.create( 449 | :mailbox_id => @db_mailbox.id, 450 | :guid => create_guid(data), 451 | :uid => uid, 452 | :message_id => parse_message_id(data.attr['BODY[HEADER.FIELDS (MESSAGE-ID)]']), 453 | :rfc822_size => data.attr['RFC822.SIZE'].to_i, 454 | :internaldate => Time.parse(data.attr['INTERNALDATE']).to_i, 455 | :flags => data.attr['FLAGS'] 456 | ) 457 | 458 | last_good_uid = uid 459 | end 460 | 461 | # Set this mailbox's uidnext value to the last known good UID that 462 | # was stored in the database, plus 1. This will allow Larch to 463 | # resume where the error occurred on the next attempt rather than 464 | # having to start over. 465 | @db_mailbox.update(:uidnext => last_good_uid + 1) 466 | end 467 | 468 | if show_progress 469 | fetched += fetch_data.length 470 | last_progress = progress 471 | progress = ((100 / (options[:progress_total] - options[:progress_start]).to_f) * fetched).round 472 | 473 | info "#{progress}% complete" if progress > last_progress 474 | end 475 | end 476 | end 477 | 478 | # Examines this mailbox. If _force_ is true, the mailbox will be examined even 479 | # if it is already selected (which isn't necessary unless you want to ensure 480 | # that it's in a read-only state). 481 | # 482 | # Returns +false+ if this mailbox cannot be examined, which may be the case if 483 | # the \Noselect attribute is set. 484 | def imap_examine(force = false) 485 | return false if @attr.include?(:Noselect) 486 | return true if @state == :examined || (!force && @state == :selected) 487 | 488 | @imap.safely do 489 | begin 490 | @imap.conn.close unless @state == :closed 491 | @state = :closed 492 | 493 | debug "examining mailbox" 494 | @imap.conn.examine(@name_utf7) 495 | refresh_flags 496 | 497 | @state = :examined 498 | 499 | rescue Net::IMAP::NoResponseError => e 500 | raise Error, "unable to examine mailbox: #{e.message}" 501 | end 502 | end 503 | 504 | return true 505 | end 506 | 507 | # Selects the mailbox if it is not already selected. If the mailbox does not 508 | # exist and _create_ is +true+, it will be created. Otherwise, a 509 | # Larch::IMAP::Error will be raised. 510 | # 511 | # Returns +false+ if this mailbox cannot be selected, which may be the case if 512 | # the \Noselect attribute is set. 513 | def imap_select(create = false) 514 | return false if @attr.include?(:Noselect) 515 | return true if @state == :selected 516 | 517 | @imap.safely do 518 | begin 519 | @imap.conn.close unless @state == :closed 520 | @state = :closed 521 | 522 | debug "selecting mailbox" 523 | @imap.conn.select(@name_utf7) 524 | refresh_flags 525 | 526 | @state = :selected 527 | 528 | rescue Net::IMAP::NoResponseError => e 529 | raise Error, "unable to select mailbox: #{e.message}" unless create 530 | 531 | info "creating mailbox: #{@name}" 532 | 533 | begin 534 | @imap.conn.create(@name_utf7) unless @imap.options[:dry_run] 535 | retry 536 | rescue => e 537 | raise Error, "unable to create mailbox: #{e.message}" 538 | end 539 | end 540 | end 541 | 542 | return true 543 | end 544 | 545 | # Sends an IMAP STATUS command and returns the status of the requested 546 | # attributes. Supported attributes include: 547 | # 548 | # - MESSAGES 549 | # - RECENT 550 | # - UIDNEXT 551 | # - UIDVALIDITY 552 | # - UNSEEN 553 | def imap_status(*attr) 554 | @imap.safely do 555 | begin 556 | debug "getting mailbox status" 557 | @imap.conn.status(@name_utf7, attr) 558 | rescue Net::IMAP::NoResponseError => e 559 | raise Error, "unable to get status of mailbox: #{e.message}" 560 | end 561 | end 562 | end 563 | 564 | # Fetches the specified _fields_ for the specified _set_ of UIDs, which can be 565 | # a number, Range, or Array of UIDs. 566 | # 567 | # If _set_ is a number, an Array containing a single Net::IMAP::FetchData 568 | # object will be yielded to the given block. 569 | # 570 | # If _set_ is a Range or Array of UIDs, Arrays of up to block_size 571 | # Net::IMAP::FetchData objects will be yielded until all requested messages 572 | # have been fetched. 573 | # 574 | # However, if _set_ is a Range with an end value of -1, a single Array 575 | # containing all requested messages will be yielded, since it's impossible to 576 | # divide an infinite range into finite blocks. 577 | def imap_uid_fetch(set, fields, block_size = FETCH_BLOCK_SIZE, &block) # :yields: fetch_data 578 | if set.is_a?(Numeric) || (set.is_a?(Range) && set.last < 0) 579 | data = @imap.safely do 580 | imap_examine 581 | @imap.conn.uid_fetch(set, fields) 582 | end 583 | 584 | yield data unless data.nil? 585 | return 586 | end 587 | 588 | blocks = [] 589 | pos = 0 590 | 591 | if set.is_a?(Array) 592 | while pos < set.length 593 | blocks += set[pos, block_size] 594 | pos += block_size 595 | end 596 | 597 | elsif set.is_a?(Range) 598 | pos = set.min - 1 599 | 600 | while pos < set.max 601 | blocks << ((pos + 1)..[set.max, pos += block_size].min) 602 | end 603 | end 604 | 605 | blocks.each do |block| 606 | data = @imap.safely do 607 | imap_examine 608 | 609 | begin 610 | data = @imap.conn.uid_fetch(block, fields) 611 | 612 | rescue Net::IMAP::NoResponseError => e 613 | raise unless e.message == 'Some messages could not be FETCHed (Failure)' 614 | 615 | # Workaround for stupid Gmail shenanigans. 616 | warning "Gmail error: '#{e.message}'; continuing anyway" 617 | end 618 | 619 | next data 620 | end 621 | 622 | yield data unless data.nil? 623 | end 624 | end 625 | 626 | # Parses a Message-Id header out of _str_ and returns it, or +nil+ if _str_ 627 | # doesn't contain a valid Message-Id header. 628 | def parse_message_id(str) 629 | return str =~ REGEX_MESSAGE_ID ? $1 : nil 630 | end 631 | 632 | # Refreshes the list of valid flags for this mailbox. 633 | def refresh_flags 634 | return unless @imap.conn.responses.has_key?('FLAGS') && 635 | @imap.conn.responses.has_key?('PERMANENTFLAGS') 636 | 637 | @flags = Array(@imap.conn.responses['FLAGS'].first) 638 | @perm_flags = Array(@imap.conn.responses['PERMANENTFLAGS'].first) 639 | end 640 | 641 | end 642 | 643 | end; end 644 | --------------------------------------------------------------------------------