├── tmp └── .gitkeep ├── .github ├── FUNDING.yml └── workflows │ ├── main.yml │ └── publish-image.yaml ├── docs ├── files │ ├── mboxrd.md │ ├── imap.md │ └── config.md ├── images │ └── entering-connection-options-as-json.png ├── commands │ ├── local-accounts.md │ ├── remote-folders.md │ ├── local-folders.md │ ├── local-list.md │ ├── local-show.md │ ├── utils-export-to-thunderbird.md │ ├── utils-ignore-history.md │ ├── local-check.md │ ├── single-backup.md │ ├── restore.md │ ├── backup.md │ ├── copy.md │ ├── migrate.md │ ├── mirror.md │ └── setup.md ├── installation │ ├── rubygem.md │ └── source.md ├── documentation.md ├── delimiters-and-prefixes.md ├── performance.md └── howto │ └── migrate-server-keep-address.md ├── .rspec ├── container ├── .containerignore ├── README.md └── Containerfile ├── .gitignore ├── spec ├── features │ ├── helper.rb │ ├── support │ │ ├── email_server.rb │ │ ├── 10_server_message_helpers.rb │ │ ├── performance_testing.rb │ │ ├── shared │ │ │ └── message_fixtures.rb │ │ └── 30_email_server_helpers.rb │ ├── help_spec.rb │ ├── version_spec.rb │ ├── setup │ │ ├── global_options │ │ │ └── download_strategy_spec.rb │ │ └── add_account_spec.rb │ ├── remote │ │ ├── list_namespaces_spec.rb │ │ └── list_account_folders_spec.rb │ ├── local │ │ ├── list_accounts_spec.rb │ │ ├── list_folders_spec.rb │ │ ├── list_emails_spec.rb │ │ └── show_an_email_spec.rb │ ├── setup_spec.rb │ ├── regressions │ │ └── migrate_legacy_backups_spec.rb │ ├── utils │ │ ├── ignore_history_spec.rb │ │ └── export_to_thunderbird_spec.rb │ ├── single │ │ └── backup_spec.rb │ └── stats_spec.rb ├── support │ ├── configure_rspec.rb │ ├── silence_logging.rb │ ├── higline_test_helpers.rb │ ├── cli_coverage.rb │ └── shared_examples │ │ ├── an_action_that_handles_logger_options.rb │ │ └── requiring_an_existing_configuration.rb ├── spec_helper.rb ├── unit │ ├── email │ │ ├── provider │ │ │ ├── fastmail_spec.rb │ │ │ ├── gmail_spec.rb │ │ │ ├── purelymail_spec.rb │ │ │ ├── apple_mail_spec.rb │ │ │ └── base_spec.rb │ │ └── provider_spec.rb │ ├── setup │ │ ├── helpers_spec.rb │ │ ├── connection_tester_spec.rb │ │ ├── global_options │ │ │ └── download_strategy_chooser_spec.rb │ │ ├── global_options_spec.rb │ │ └── backup_path_spec.rb │ ├── cli │ │ ├── setup_spec.rb │ │ ├── single_spec.rb │ │ ├── remote_spec.rb │ │ ├── restore_spec.rb │ │ └── backup_spec.rb │ ├── file_mode_spec.rb │ ├── account │ │ ├── client_factory_spec.rb │ │ ├── folder_ensurer_spec.rb │ │ ├── local_only_folder_deleter_spec.rb │ │ ├── restore_spec.rb │ │ └── backup_folders_spec.rb │ ├── serializer │ │ ├── message_enumerator_spec.rb │ │ ├── permission_checker_spec.rb │ │ ├── unused_name_finder_spec.rb │ │ ├── directory_spec.rb │ │ ├── folder_maker_spec.rb │ │ ├── message_spec.rb │ │ └── integrity_checker_spec.rb │ ├── text │ │ └── sanitizer_spec.rb │ ├── flag_refresher_spec.rb │ ├── local_only_message_deleter_spec.rb │ ├── client │ │ └── automatic_login_wrapper_spec.rb │ ├── retry_on_error_spec.rb │ ├── naming_spec.rb │ ├── migrator_spec.rb │ └── logger_spec.rb └── performance │ └── backup_spec.rb ├── contrib ├── example_users.csv ├── list-failures-in-local-check.jq ├── README.md └── import-thunderbird-folder ├── lib └── imap │ └── backup │ ├── configuration_not_found.rb │ ├── version.rb │ ├── email │ ├── provider │ │ ├── fastmail.rb │ │ ├── unknown.rb │ │ ├── purelymail.rb │ │ ├── gmail.rb │ │ ├── apple_mail.rb │ │ └── base.rb │ └── provider.rb │ ├── setup │ ├── helpers.rb │ ├── connection_tester.rb │ ├── backup_path.rb │ ├── email_changer.rb │ ├── global_options.rb │ ├── asker.rb │ └── global_options │ │ └── download_strategy_chooser.rb │ ├── file_mode.rb │ ├── cli │ ├── setup.rb │ ├── restore.rb │ ├── backup.rb │ ├── local │ │ └── check.rb │ ├── options.rb │ ├── stats.rb │ ├── migrate.rb │ └── mirror.rb │ ├── account │ ├── client_factory.rb │ ├── folder_ensurer.rb │ ├── local_only_folder_deleter.rb │ ├── restore.rb │ ├── backup_folders.rb │ ├── backup.rb │ ├── serialized_folders.rb │ └── folder_backup.rb │ ├── serializer │ ├── message_enumerator.rb │ ├── unused_name_finder.rb │ ├── permission_checker.rb │ ├── folder_maker.rb │ ├── directory.rb │ ├── transaction.rb │ └── message.rb │ ├── retry_on_error.rb │ ├── local_only_message_deleter.rb │ ├── flag_refresher.rb │ ├── migrator.rb │ ├── naming.rb │ ├── client │ └── automatic_login_wrapper.rb │ ├── text │ └── sanitizer.rb │ ├── logger.rb │ └── uploader.rb ├── Gemfile ├── Rakefile ├── .git-blame-ignore-revs ├── bin └── imap-backup ├── .mailmap ├── .simplecov ├── dev ├── Containerfile ├── config.json ├── compose.yml ├── ruby-compose.yml └── README.md ├── .yardopts ├── LICENSE ├── imap-backup.gemspec ├── .rubocop_todo.yml └── ARCHITECTURE.md /tmp/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /.github/FUNDING.yml: -------------------------------------------------------------------------------- 1 | github: [joeyates] 2 | -------------------------------------------------------------------------------- /docs/files/mboxrd.md: -------------------------------------------------------------------------------- 1 | 4 | 5 | -------------------------------------------------------------------------------- /.rspec: -------------------------------------------------------------------------------- 1 | --color 2 | --format documentation 3 | --order random 4 | --require spec_helper 5 | -------------------------------------------------------------------------------- /container/.containerignore: -------------------------------------------------------------------------------- 1 | * 2 | !bin/imap-backup 3 | !Gemfile 4 | !imap-backup.gemspec 5 | !lib 6 | !LICENSE 7 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /.bundle 2 | /coverage 3 | /doc 4 | /Gemfile.lock 5 | /pkg 6 | /.rspec_status 7 | /tmp 8 | /vendor 9 | /.yardoc/ -------------------------------------------------------------------------------- /spec/features/helper.rb: -------------------------------------------------------------------------------- 1 | support_glob = File.expand_path("support/**/*.rb", __dir__) 2 | Dir[support_glob].each { |f| require f } 3 | -------------------------------------------------------------------------------- /docs/images/entering-connection-options-as-json.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/joeyates/imap-backup/HEAD/docs/images/entering-connection-options-as-json.png -------------------------------------------------------------------------------- /contrib/example_users.csv: -------------------------------------------------------------------------------- 1 | email,password,connection_options,folders 2 | address@example.com,pass,"{""port"":8993,""ssl"":{""verify_mode"":0}}","[""INBOX""]" 3 | -------------------------------------------------------------------------------- /lib/imap/backup/configuration_not_found.rb: -------------------------------------------------------------------------------- 1 | module Imap; end 2 | 3 | module Imap::Backup 4 | # Thrown when no configuration file is found 5 | class ConfigurationNotFound < StandardError; end 6 | end 7 | -------------------------------------------------------------------------------- /docs/commands/local-accounts.md: -------------------------------------------------------------------------------- 1 | 4 | # Local Accounts 5 | 6 | ```sh 7 | imap-backup local accounts 8 | ``` 9 | 10 | This command lists all configured accounts. 11 | -------------------------------------------------------------------------------- /docs/commands/remote-folders.md: -------------------------------------------------------------------------------- 1 | 4 | # Remote Folders 5 | 6 | ```sh 7 | imap-backup remote folders EMAIL 8 | ``` 9 | 10 | This commmand lists all the folders in an (online) account. 11 | -------------------------------------------------------------------------------- /spec/support/configure_rspec.rb: -------------------------------------------------------------------------------- 1 | RSpec.configure do |config| 2 | config.example_status_persistence_file_path = ".rspec_status" 3 | config.disable_monkey_patching! 4 | 5 | config.expect_with :rspec do |c| 6 | c.syntax = :expect 7 | end 8 | end 9 | -------------------------------------------------------------------------------- /docs/commands/local-folders.md: -------------------------------------------------------------------------------- 1 | 4 | # Local Folders 5 | 6 | ```sh 7 | imap-backup local folders EMAIL 8 | ``` 9 | 10 | This command lists the folders that have been backed up for a given account. 11 | -------------------------------------------------------------------------------- /docs/commands/local-list.md: -------------------------------------------------------------------------------- 1 | 4 | # Local List 5 | 6 | ```sh 7 | imap-backup local list EMAIL FOLDER 8 | ``` 9 | 10 | For a given account (EMAIL) and folder, this command lists 11 | all emails that have been backup up. 12 | -------------------------------------------------------------------------------- /spec/features/support/email_server.rb: -------------------------------------------------------------------------------- 1 | require_relative "10_server_message_helpers" 2 | require_relative "30_email_server_helpers" 3 | 4 | RSpec.configure do |config| 5 | config.include ServerMessageHelpers, type: :aruba 6 | config.include EmailServerHelpers, type: :aruba 7 | end 8 | -------------------------------------------------------------------------------- /Gemfile: -------------------------------------------------------------------------------- 1 | source "https://rubygems.org" 2 | 3 | gemspec 4 | 5 | group :development do 6 | gem "aruba", ">= 0.0.0" 7 | gem "pry-byebug" 8 | gem "rspec", ">= 3.0.0" 9 | gem "rubocop-rspec", "2.27.1" 10 | gem "simplecov" 11 | gem "webrick" 12 | gem "yard" 13 | end 14 | -------------------------------------------------------------------------------- /docs/commands/local-show.md: -------------------------------------------------------------------------------- 1 | 4 | # Local Show 5 | 6 | ```sh 7 | imap-backup local show EMAIL FOLDER UID[,UID] 8 | ``` 9 | 10 | Given an account (EMAIL), a folder and one or more UIDs (email ids), 11 | this command dumps their contents. 12 | -------------------------------------------------------------------------------- /docs/installation/rubygem.md: -------------------------------------------------------------------------------- 1 | 4 | # Preparation 5 | 6 | Check you have Ruby installed 7 | 8 | ```sh 9 | ruby -v 10 | ``` 11 | 12 | You need at least Ruby 3.0 installed. 13 | 14 | # Install 15 | 16 | ```sh 17 | gem install imap-backup --no-document 18 | ``` 19 | -------------------------------------------------------------------------------- /spec/spec_helper.rb: -------------------------------------------------------------------------------- 1 | require "rspec" 2 | 3 | ENV["SIMPLECOV_COMMAND_NAME"] = "RSpec tests" 4 | require "simplecov" 5 | 6 | $LOAD_PATH << File.expand_path("../lib", __dir__) 7 | 8 | support_glob = File.join(__dir__, "support", "**", "*.rb") 9 | Dir[support_glob].each { |f| require f } 10 | 11 | silence_logging 12 | -------------------------------------------------------------------------------- /spec/unit/email/provider/fastmail_spec.rb: -------------------------------------------------------------------------------- 1 | require "imap/backup/email/provider/fastmail" 2 | 3 | module Imap::Backup 4 | RSpec.describe Email::Provider::Fastmail do 5 | describe "#host" do 6 | it "returns host" do 7 | expect(subject.host).to eq("imap.fastmail.com") 8 | end 9 | end 10 | end 11 | end 12 | -------------------------------------------------------------------------------- /spec/support/silence_logging.rb: -------------------------------------------------------------------------------- 1 | require "logger" 2 | require "net/imap" 3 | 4 | require "imap/backup/logger" 5 | 6 | def silence_logging 7 | RSpec.configure do |config| 8 | config.before do 9 | Imap::Backup::Logger.logger.level = Logger::Severity::UNKNOWN 10 | Net::IMAP.debug = false 11 | end 12 | end 13 | end 14 | -------------------------------------------------------------------------------- /docs/commands/utils-export-to-thunderbird.md: -------------------------------------------------------------------------------- 1 | 4 | # Utils Export to Thunderbird 5 | 6 | ```sh 7 | imap-backup utils export-to-thunderbird EMAIL 8 | ``` 9 | 10 | This command exports backup up emails as files and directories 11 | that can be browsed using Mozilla Thunderbird. 12 | -------------------------------------------------------------------------------- /lib/imap/backup/version.rb: -------------------------------------------------------------------------------- 1 | module Imap; end 2 | 3 | module Imap::Backup 4 | # @private 5 | MAJOR = 16 6 | # @private 7 | MINOR = 2 8 | # @private 9 | REVISION = 0 10 | # @private 11 | PRE = nil 12 | # The application version 13 | VERSION = [MAJOR, MINOR, REVISION, PRE].compact.map(&:to_s).join(".") 14 | end 15 | -------------------------------------------------------------------------------- /docs/documentation.md: -------------------------------------------------------------------------------- 1 | # Documentation 2 | 3 | This project uses [yard](https://yardoc.org/) to generate its documentation. 4 | 5 | # Development 6 | 7 | Run an autoreloading document viewer 8 | 9 | ``` 10 | yard server --reload 11 | ``` 12 | 13 | # Coverage 14 | 15 | See what's not documented 16 | 17 | ``` 18 | yard stats --list-undoc 19 | ``` 20 | -------------------------------------------------------------------------------- /Rakefile: -------------------------------------------------------------------------------- 1 | require "bundler/gem_tasks" 2 | require "rspec/core/rake_task" 3 | require "rubocop/rake_task" 4 | 5 | RSpec::Core::RakeTask.new do |t| 6 | t.pattern = "spec/**/*_spec.rb" 7 | end 8 | 9 | RuboCop::RakeTask.new(:rubocop) do |task| 10 | task.options = ["--config", ".rubocop.yml"] 11 | end 12 | 13 | task default: :spec 14 | task default: :rubocop 15 | -------------------------------------------------------------------------------- /spec/support/higline_test_helpers.rb: -------------------------------------------------------------------------------- 1 | require "highline" 2 | 3 | require "imap/backup/setup" 4 | 5 | module HighLineTestHelpers 6 | def prepare_highline 7 | @input = instance_double(IO, eof?: false, gets: "q\n") 8 | @output = StringIO.new 9 | Imap::Backup::Setup.highline = HighLine.new(@input, @output) 10 | [@input, @output] 11 | end 12 | end 13 | -------------------------------------------------------------------------------- /.git-blame-ignore-revs: -------------------------------------------------------------------------------- 1 | # Nest classes under module's namespace 2 | 8c04221838f14fd1222f0e2f20176497a04e0a38 3 | # Put all code under the Imap::Backup namespace 4 | 2c818d8b7c377927e0feb0e55d44567859869c9e 5 | c83f7c3d938f0880b011c2cbe68585be7aab6a64 6 | 1e10c1d21cb9d6c9230dedecf393add3393de5c8 7 | fe01b42edb4b01a7d02baa67b4d31ece5352d831 8 | dd871eaeddfabc245ff443d4132f4bc1095bfe5c 9 | -------------------------------------------------------------------------------- /spec/features/help_spec.rb: -------------------------------------------------------------------------------- 1 | require "features/helper" 2 | 3 | RSpec.describe "imap-backup help", type: :aruba do 4 | context "when subcommands are invoked with a method" do 5 | it "outputs the method's help" do 6 | run_command_and_stop "imap-backup help remote namespaces" 7 | 8 | expect(last_command_started).to have_output(/Options:/) 9 | end 10 | end 11 | end 12 | -------------------------------------------------------------------------------- /lib/imap/backup/email/provider/fastmail.rb: -------------------------------------------------------------------------------- 1 | require "imap/backup/email/provider/base" 2 | 3 | module Imap; end 4 | 5 | module Imap::Backup 6 | # Provides overrides for Fastmail accounts 7 | class Email::Provider::Fastmail < Email::Provider::Base 8 | # @return [String] the Fastmail IMAP server host name 9 | def host 10 | "imap.fastmail.com" 11 | end 12 | end 13 | end 14 | -------------------------------------------------------------------------------- /lib/imap/backup/email/provider/unknown.rb: -------------------------------------------------------------------------------- 1 | require "imap/backup/email/provider/base" 2 | 3 | module Imap; end 4 | 5 | module Imap::Backup 6 | # Provides overrides when the IMAP provider is not known 7 | class Email::Provider::Unknown < Email::Provider::Base 8 | # We don't know how to guess the IMAP server 9 | # @return [nil] 10 | def host 11 | end 12 | end 13 | end 14 | -------------------------------------------------------------------------------- /spec/support/cli_coverage.rb: -------------------------------------------------------------------------------- 1 | class CliCoverage 2 | def self.conditionally_activate 3 | return if !ENV.key?("FEATURE_SPEC_ID") 4 | 5 | # Collect coverage separately 6 | ENV["SIMPLECOV_COMMAND_NAME"] = ENV.fetch("FEATURE_SPEC_ID") 7 | require "simplecov" 8 | 9 | # Silence output 10 | SimpleCov.formatter = SimpleCov::Formatter::SimpleFormatter 11 | SimpleCov.print_error_status = false 12 | end 13 | end 14 | -------------------------------------------------------------------------------- /spec/features/support/10_server_message_helpers.rb: -------------------------------------------------------------------------------- 1 | module ServerMessageHelpers 2 | BODY_ATTRIBUTE = "BODY[]".freeze 3 | 4 | def message_as_server_message(from:, subject:, body:, **_options) 5 | <<~MESSAGE.gsub("\n", "\r\n") 6 | From: #{from} 7 | Subject: #{subject} 8 | 9 | #{body} 10 | 11 | MESSAGE 12 | end 13 | 14 | def server_message_to_body(message) 15 | message[BODY_ATTRIBUTE] 16 | end 17 | end 18 | -------------------------------------------------------------------------------- /lib/imap/backup/setup/helpers.rb: -------------------------------------------------------------------------------- 1 | require "imap/backup/version" 2 | 3 | module Imap; end 4 | 5 | module Imap::Backup 6 | class Setup; end 7 | 8 | # Helpers for the setup system 9 | class Setup::Helpers 10 | # @return [String] the prefix for setup menus 11 | def title_prefix 12 | "imap-backup -" 13 | end 14 | 15 | # @return [String] the current application version 16 | def version 17 | VERSION 18 | end 19 | end 20 | end 21 | -------------------------------------------------------------------------------- /bin/imap-backup: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | 3 | $LOAD_PATH.unshift(File.expand_path("../lib/", __dir__)) 4 | 5 | spec_path = File.expand_path("../spec", __dir__) 6 | if File.directory?(spec_path) 7 | require_relative "../spec/support/cli_coverage" 8 | 9 | CliCoverage.conditionally_activate 10 | end 11 | 12 | require "imap/backup/cli" 13 | require "imap/backup/logger" 14 | 15 | Imap::Backup::Logger.sanitize_stderr do 16 | Imap::Backup::CLI.start(ARGV) 17 | end 18 | -------------------------------------------------------------------------------- /lib/imap/backup/email/provider/purelymail.rb: -------------------------------------------------------------------------------- 1 | require "imap/backup/email/provider/base" 2 | 3 | module Imap; end 4 | 5 | module Imap::Backup 6 | # Provides overrides for Purelymail accounts 7 | class Email::Provider::Purelymail < Email::Provider::Base 8 | # @return [String] The Purelymail IMAP server host name 9 | def host 10 | "mailserver.purelymail.com" 11 | end 12 | 13 | def sets_seen_flags_on_fetch? 14 | true 15 | end 16 | end 17 | end 18 | -------------------------------------------------------------------------------- /spec/unit/setup/helpers_spec.rb: -------------------------------------------------------------------------------- 1 | require "imap/backup/setup/helpers" 2 | 3 | module Imap::Backup 4 | RSpec.describe Setup::Helpers do 5 | describe "#title_prefix" do 6 | it "is a string" do 7 | expect(subject.title_prefix).to eq("imap-backup -") 8 | end 9 | end 10 | 11 | describe "#version" do 12 | it "is a version string" do 13 | expect(subject.version).to match(/\A\d+\.\d+\.\d+(\.\w+)?\z/) 14 | end 15 | end 16 | end 17 | end 18 | -------------------------------------------------------------------------------- /spec/unit/email/provider/gmail_spec.rb: -------------------------------------------------------------------------------- 1 | require "imap/backup/email/provider/gmail" 2 | 3 | module Imap::Backup 4 | RSpec.describe Email::Provider::GMail do 5 | describe "#folder_ignore_tags" do 6 | it "returns Noselect" do 7 | expect(subject.folder_ignore_tags).to eq([:Noselect]) 8 | end 9 | end 10 | 11 | describe "#host" do 12 | it "returns host" do 13 | expect(subject.host).to eq("imap.gmail.com") 14 | end 15 | end 16 | end 17 | end 18 | -------------------------------------------------------------------------------- /spec/features/support/performance_testing.rb: -------------------------------------------------------------------------------- 1 | # If the environment variable 'PERFORMANCE' is set, 2 | # run *only* performance specs. 3 | # Otherwise, run other specs, skipping performace ones. 4 | RSpec.configure do |config| 5 | performance_run = !!ENV["PERFORMANCE"] 6 | config.filter_run_excluding performance: !performance_run 7 | config.filter_run_when_matching performance: performance_run 8 | end 9 | 10 | Aruba.configure do |config| 11 | config.exit_timeout = 6 * 60 * 60 if !!ENV["PERFORMANCE"] 12 | end 13 | -------------------------------------------------------------------------------- /spec/unit/email/provider/purelymail_spec.rb: -------------------------------------------------------------------------------- 1 | require "imap/backup/email/provider/purelymail" 2 | 3 | module Imap::Backup 4 | RSpec.describe Email::Provider::Purelymail do 5 | describe "#host" do 6 | it "returns host" do 7 | expect(subject.host).to eq("mailserver.purelymail.com") 8 | end 9 | end 10 | 11 | describe "#sets_seen_flags_on_fetch?" do 12 | it "is true" do 13 | expect(subject.sets_seen_flags_on_fetch?).to be true 14 | end 15 | end 16 | end 17 | end 18 | -------------------------------------------------------------------------------- /.mailmap: -------------------------------------------------------------------------------- 1 | AB 2 | AD 3 | AK 4 | AN 5 | AS 6 | FR <317514+MagicFab@users.noreply.github.com> 7 | GS 8 | JM 9 | JY 10 | JY 11 | JY 12 | MW 13 | MW 14 | NJ 15 | OJ 16 | SP 17 | -------------------------------------------------------------------------------- /lib/imap/backup/file_mode.rb: -------------------------------------------------------------------------------- 1 | module Imap; end 2 | 3 | module Imap::Backup 4 | # Accesses a file's access permissions 5 | class FileMode 6 | def initialize(filename:) 7 | @filename = filename 8 | end 9 | 10 | # @return [Integer, nil] The user, group and "other" part of the file's "mode" 11 | def mode 12 | return nil if !File.exist?(filename) 13 | 14 | stat = File.stat(filename) 15 | stat.mode & 0o777 16 | end 17 | 18 | private 19 | 20 | attr_reader :filename 21 | end 22 | end 23 | -------------------------------------------------------------------------------- /lib/imap/backup/email/provider/gmail.rb: -------------------------------------------------------------------------------- 1 | require "imap/backup/email/provider/base" 2 | 3 | module Imap; end 4 | 5 | module Imap::Backup 6 | # Provides overrides for GMail accounts 7 | class Email::Provider::GMail < Email::Provider::Base 8 | # https://imap-use.u.washington.narkive.com/RYMsOHTN/imap-protocol-status-on-a-noselect-mailbox 9 | def folder_ignore_tags 10 | [:Noselect] 11 | end 12 | 13 | # @return [String] the GMail IMAP server host name 14 | def host 15 | "imap.gmail.com" 16 | end 17 | end 18 | end 19 | -------------------------------------------------------------------------------- /.simplecov: -------------------------------------------------------------------------------- 1 | SimpleCov.start do 2 | command_name ENV.fetch("SIMPLECOV_COMMAND_NAME") 3 | 4 | # Ensure SimpleCov doesn't filter out all out code 5 | root __dir__ 6 | 7 | add_filter "/spec/" 8 | 9 | coverage_dir(File.join(__dir__, "coverage")) 10 | 11 | enable_coverage :branch 12 | end 13 | 14 | SimpleCov.at_exit do 15 | File.open(File.join(SimpleCov.coverage_path, "coverage_percent.txt"), "w") do |f| 16 | rounded = (SimpleCov.result.covered_percent + 0.5).floor 17 | f.write rounded 18 | end 19 | SimpleCov.result.format! 20 | end 21 | -------------------------------------------------------------------------------- /container/README.md: -------------------------------------------------------------------------------- 1 | # Runnable imap-backup container 2 | 3 | This Containerfile is used by the `publish-image` 4 | GitHub action. It creates an image which is pushed 5 | to GitHub packages. The image can be run via 6 | Podman or Docker in order to use imap-backup without 7 | installing anything else. 8 | 9 | # Build in development 10 | 11 | The image can be build in development as follows 12 | 13 | ```sh 14 | podman build \ 15 | --file container/Containerfile \ 16 | --tag imap-backup:latest \ 17 | --ignorefile ./container/.containerignore \ 18 | . 19 | ``` 20 | -------------------------------------------------------------------------------- /spec/support/shared_examples/an_action_that_handles_logger_options.rb: -------------------------------------------------------------------------------- 1 | require "imap/backup/logger" 2 | 3 | module Imap::Backup 4 | RSpec.shared_examples "an action that handles Logger options" do |action:, &block| 5 | before do 6 | allow(Logger).to receive(:setup_logging).and_call_original 7 | action.call(subject, {quiet: true, verbose: [true]}) 8 | end 9 | 10 | it "configures the logger" do 11 | expect(Logger).to have_received(:setup_logging) 12 | end 13 | 14 | # block holds other examples 15 | block&.call 16 | end 17 | end 18 | -------------------------------------------------------------------------------- /docs/commands/utils-ignore-history.md: -------------------------------------------------------------------------------- 1 | 4 | # Utils Ignore History 5 | 6 | ```sh 7 | imap-backup utils ignore-history EMAIL 8 | ``` 9 | 10 | If you only want to download future emails for an account and skip 11 | all emails that have been received so far, this command 12 | fills the backup with small dummy emails for each existing email. 13 | 14 | The resulting backup is much smaller as emails up to a certain date 15 | do not contain the real content, which, especially if there are attachments, 16 | may amount to a lot of data. 17 | -------------------------------------------------------------------------------- /lib/imap/backup/email/provider/apple_mail.rb: -------------------------------------------------------------------------------- 1 | require "imap/backup/email/provider/base" 2 | 3 | module Imap; end 4 | 5 | module Imap::Backup 6 | # Provides overrides for Apple mail accounts 7 | class Email::Provider::AppleMail < Email::Provider::Base 8 | # @return [String] the Apple Mail IMAP server host name 9 | def host 10 | "imap.mail.me.com" 11 | end 12 | 13 | # With Apple Mails's IMAP, passing "/" to list results in an empty list 14 | def root 15 | "" 16 | end 17 | 18 | def sets_seen_flags_on_fetch? 19 | true 20 | end 21 | end 22 | end 23 | -------------------------------------------------------------------------------- /dev/Containerfile: -------------------------------------------------------------------------------- 1 | ARG RUBY_VERSION 2 | FROM docker.io/library/ruby:$RUBY_VERSION-buster 3 | 4 | ARG BUNDLER_VERSION=2.4.22 5 | 6 | # Install dependencies 7 | RUN \ 8 | apt-get update && \ 9 | apt-get install --yes less libffi-dev vim-nox 10 | 11 | # Show full path in prompt 12 | RUN echo 'PS1='\''imap-backup:$(pwd)>'\''' > /etc/bash.bashrc 13 | 14 | # Create binstubs (including one for imap-backup) so we can run it 15 | # without using `bundle exec` 16 | ENV PATH /app/bin/stubs:$PATH 17 | 18 | WORKDIR /app 19 | 20 | RUN gem install bundler --version=$BUNDLER_VERSION 21 | 22 | ENTRYPOINT ["bash"] 23 | -------------------------------------------------------------------------------- /container/Containerfile: -------------------------------------------------------------------------------- 1 | FROM docker.io/library/ruby:3.2.2-alpine3.18 AS builder 2 | 3 | WORKDIR /app 4 | 5 | COPY Gemfile . 6 | COPY imap-backup.gemspec . 7 | COPY lib/imap/backup/version.rb lib/imap/backup/ 8 | 9 | RUN \ 10 | apk add alpine-sdk && \ 11 | gem install bundler --version "2.4.21" && \ 12 | BUNDLE_WITHOUT=development bundle install 13 | 14 | FROM docker.io/library/ruby:3.2.2-alpine3.18 15 | 16 | COPY --from=builder /usr/local/bundle /usr/local/bundle 17 | 18 | WORKDIR /app 19 | 20 | COPY . . 21 | 22 | ENV PATH=${PATH}:/app/bin 23 | 24 | CMD ["imap-backup", "backup", "-c", "/config/imap-backup.json"] 25 | -------------------------------------------------------------------------------- /spec/features/version_spec.rb: -------------------------------------------------------------------------------- 1 | require "features/helper" 2 | 3 | RSpec.describe "imap-backup version", type: :aruba do 4 | context "when invoked with '--version'" do 5 | it "outputs the version" do 6 | run_command_and_stop "imap-backup --version" 7 | 8 | expect(last_command_started).to have_output(/imap-backup \d+\.\d+\.\d+/) 9 | end 10 | end 11 | 12 | context "when invoked with 'version'" do 13 | it "outputs the version" do 14 | run_command_and_stop "imap-backup version" 15 | 16 | expect(last_command_started).to have_output(/imap-backup \d+\.\d+\.\d+/) 17 | end 18 | end 19 | end 20 | -------------------------------------------------------------------------------- /docs/commands/local-check.md: -------------------------------------------------------------------------------- 1 | 4 | # Local Integrity Check 5 | 6 | ```sh 7 | imap-backup local check 8 | ``` 9 | 10 | This command checks the integrity of the local backup. 11 | 12 | See the [backup command](./backup.md), for details. 13 | 14 | For each account, each folder is listed, indicating whether 15 | the `.imap` and `.mbox` files are corrupt or not. 16 | 17 | # Options 18 | 19 | * `config` - allows supplying a non-default path for the configuration file, 20 | * `delete-corrupt` - deletes folders that are corrupt, 21 | * `format` - passing `--format json` produces JSON output. 22 | -------------------------------------------------------------------------------- /spec/unit/email/provider/apple_mail_spec.rb: -------------------------------------------------------------------------------- 1 | require "imap/backup/email/provider/apple_mail" 2 | 3 | module Imap::Backup 4 | RSpec.describe Email::Provider::AppleMail do 5 | describe "#host" do 6 | it "returns host" do 7 | expect(subject.host).to eq("imap.mail.me.com") 8 | end 9 | end 10 | 11 | describe "#root" do 12 | it "is an empty string" do 13 | expect(subject.root).to eq("") 14 | end 15 | end 16 | 17 | describe "#sets_seen_flags_on_fetch?" do 18 | it "is true" do 19 | expect(subject.sets_seen_flags_on_fetch?).to be true 20 | end 21 | end 22 | end 23 | end 24 | -------------------------------------------------------------------------------- /docs/delimiters-and-prefixes.md: -------------------------------------------------------------------------------- 1 | # Delimiters and Prefixes 2 | 3 | A simple folder name is `Friends`. 4 | 5 | Most email servers allow you to put folders inside other folders. 6 | 7 | On most email servers, the parts of a folder's name are separated with a `/` character. 8 | So you might have `People/Friends`. 9 | 10 | On the other hand, some email servers use a `.`, giving `People.Friends`. 11 | 12 | Some email servers keep most email in a parent folder, often `INBOX`, so the above folder 13 | would be `INBOX/People/Friends`. 14 | 15 | The `migrate` and `mirror` commands provide options to help "translate" between 16 | the behaviour of the source and destination servers. 17 | -------------------------------------------------------------------------------- /dev/config.json: -------------------------------------------------------------------------------- 1 | { 2 | "version": "2.0", 3 | "accounts": [ 4 | { 5 | "username": "address@example.com", 6 | "password": "pass", 7 | "local_path": "tmp/imap-backup/address_example.com", 8 | "server": "imap", 9 | "connection_options": { 10 | "port": 993, 11 | "ssl": { 12 | "verify_mode": 0 13 | } 14 | } 15 | }, 16 | { 17 | "username": "email@other.org", 18 | "password": "pass", 19 | "local_path": "tmp/imap-backup/email_other.com", 20 | "server": "other-imap", 21 | "connection_options": { 22 | "port": 993, 23 | "ssl": { 24 | "verify_mode": 0 25 | } 26 | } 27 | } 28 | ] 29 | } 30 | -------------------------------------------------------------------------------- /docs/commands/single-backup.md: -------------------------------------------------------------------------------- 1 | 4 | # Direct Config-less Backup 5 | 6 | This command is an alternative to the `imap-backup backup` command. 7 | It lets you back up a single email account without relying on a configuration file. 8 | 9 | To do so, you pass all the relevant settings as command-line parameters. 10 | 11 | For example 12 | 13 | ```sh 14 | imap-backup single backup --email me@example.com --password mysecret --server imap.example.com 15 | ``` 16 | 17 | As putting your password in a command line is obviously problematic for security 18 | reasons, there are alternatives to the `--password` parameter, 19 | see `imap-backup help single backup` for a full list of parameters. 20 | -------------------------------------------------------------------------------- /contrib/list-failures-in-local-check.jq: -------------------------------------------------------------------------------- 1 | # This is a [jq](https://jqlang.github.io/jq/) script 2 | # It can be used to list the accounts and folders 3 | # that have errors when running `imap-backup local check` 4 | # Usage: 5 | # imap-backup local check -c my_config.json --format json | jq -f contrib/list-failures-in-local-check.jq 6 | 7 | map( 8 | # Save references for later 9 | . as {account: $account, folders: $folders} 10 | | $folders 11 | # Get a list of folders which have errors 12 | | map( 13 | select(.result != "OK") 14 | ) 15 | # Save a reference to any errors 16 | | . as $errors 17 | # Skip accounts without errors 18 | | select(length > 0) 19 | # List the errors 20 | | [$account, $errors] 21 | ) 22 | 23 | -------------------------------------------------------------------------------- /lib/imap/backup/cli/setup.rb: -------------------------------------------------------------------------------- 1 | require "thor" 2 | 3 | require "imap/backup/cli/helpers" 4 | require "imap/backup/setup" 5 | 6 | module Imap; end 7 | 8 | module Imap::Backup 9 | class CLI < Thor; end 10 | 11 | # Runs the menu-driven setup program 12 | class CLI::Setup < Thor 13 | include Thor::Actions 14 | include CLI::Helpers 15 | 16 | def initialize(options) 17 | super([]) 18 | @options = options 19 | end 20 | 21 | # @!method run 22 | # @return [void] 23 | no_commands do 24 | def run 25 | config = load_config(**options, require_exists: false) 26 | Setup.new(config: config).run 27 | end 28 | end 29 | 30 | private 31 | 32 | attr_reader :options 33 | end 34 | end 35 | -------------------------------------------------------------------------------- /lib/imap/backup/account/client_factory.rb: -------------------------------------------------------------------------------- 1 | require "socket" 2 | 3 | require "imap/backup/client/automatic_login_wrapper" 4 | require "imap/backup/client/default" 5 | 6 | module Imap; end 7 | 8 | module Imap::Backup 9 | class Account; end 10 | 11 | # Returns an IMAP client set up for the supplied account 12 | class Account::ClientFactory 13 | def initialize(account:) 14 | @account = account 15 | end 16 | 17 | # @return [Client::AutomaticLoginWrapper] a client for the account 18 | def run 19 | Logger.logger.debug("Creating IMAP instance") 20 | client = Client::Default.new(account) 21 | Client::AutomaticLoginWrapper.new(client: client) 22 | end 23 | 24 | private 25 | 26 | attr_reader :account 27 | end 28 | end 29 | -------------------------------------------------------------------------------- /spec/features/setup/global_options/download_strategy_spec.rb: -------------------------------------------------------------------------------- 1 | require "features/helper" 2 | 3 | RSpec.describe "imap-backup setup - global options - download strategy", type: :aruba do 4 | before { create_config(accounts: []) } 5 | 6 | it "allows setting the strategy" do 7 | run_command "imap-backup setup" 8 | last_command_started.write "modify global options\n" 9 | last_command_started.write "change download strategy\n" 10 | last_command_started.write "write straight to disk\n" 11 | last_command_started.write "q\n" 12 | last_command_started.write "q\n" 13 | last_command_started.write "save and exit\n" 14 | last_command_started.stop 15 | 16 | config = parsed_config 17 | expect(config[:download_strategy]).to eq("direct") 18 | end 19 | end 20 | -------------------------------------------------------------------------------- /lib/imap/backup/setup/connection_tester.rb: -------------------------------------------------------------------------------- 1 | require "net/imap" 2 | 3 | module Imap; end 4 | 5 | module Imap::Backup 6 | class Setup; end 7 | 8 | # Attempts to login to an account and reports the result 9 | class Setup::ConnectionTester 10 | # @param account [Account] an Account 11 | def initialize(account) 12 | @account = account 13 | end 14 | 15 | # Carries out the attempted login and indicates 16 | # whether it was successful 17 | # @return [void] 18 | def test 19 | account.client.login 20 | "Connection successful" 21 | rescue Net::IMAP::NoResponseError 22 | "No response" 23 | rescue StandardError => e 24 | "Unexpected error: #{e}" 25 | end 26 | 27 | private 28 | 29 | attr_reader :account 30 | end 31 | end 32 | -------------------------------------------------------------------------------- /dev/compose.yml: -------------------------------------------------------------------------------- 1 | # This file adapted from github.com/antespi/docker-imap-devel 2 | version: "3" 3 | 4 | services: 5 | imap: 6 | image: ghcr.io/joeyates/docker-imap-devel@sha256:6d6a64c32e2c583222d75286aa46a04ada5aea76efa36117815bf0e19d5063b6 7 | container_name: imap 8 | ports: 9 | - "8993:993" 10 | environment: 11 | - MAILNAME=example.com 12 | - MAIL_ADDRESS=address@example.com 13 | - MAIL_PASS=pass 14 | other-imap: 15 | image: ghcr.io/joeyates/docker-imap-devel@sha256:6d6a64c32e2c583222d75286aa46a04ada5aea76efa36117815bf0e19d5063b6 16 | container_name: other-imap 17 | ports: 18 | - "9993:993" 19 | environment: 20 | - MAILNAME=other.org 21 | - MAIL_ADDRESS=email@other.org 22 | - MAIL_PASS=pass 23 | - DOVECOT_PUBLIC_NAMESPACE_PREFIX=other_public 24 | -------------------------------------------------------------------------------- /spec/unit/cli/setup_spec.rb: -------------------------------------------------------------------------------- 1 | require "imap/backup/cli/setup" 2 | 3 | require "imap/backup/configuration" 4 | 5 | module Imap::Backup 6 | RSpec.describe CLI::Setup do 7 | subject { described_class.new({}) } 8 | 9 | let(:setup) { instance_double(Setup, run: nil) } 10 | let(:config) { instance_double(Configuration) } 11 | 12 | before do 13 | allow(Configuration).to receive(:exist?) { true } 14 | allow(Configuration).to receive(:new) { config } 15 | allow(Setup).to receive(:new) { setup } 16 | end 17 | 18 | it_behaves_like( 19 | "an action that doesn't require an existing configuration", 20 | action: lambda(&:run) 21 | ) 22 | 23 | it "reruns the setup process" do 24 | subject.run 25 | 26 | expect(setup).to have_received(:run) 27 | end 28 | end 29 | end 30 | -------------------------------------------------------------------------------- /.yardopts: -------------------------------------------------------------------------------- 1 | --protected 2 | --no-private 3 | --hide-void-return 4 | - 5 | ARCHITECTURE.md 6 | CHANGELOG.md 7 | LICENSE 8 | docs/delimiters-and-prefixes.md 9 | docs/documentation.md 10 | docs/files/config.md 11 | docs/files/imap.md 12 | docs/files/mboxrd.md 13 | docs/installation/rubygem.md 14 | docs/installation/source.md 15 | docs/commands/backup.md 16 | docs/commands/local-accounts.md 17 | docs/commands/local-check.md 18 | docs/commands/local-folders.md 19 | docs/commands/local-list.md 20 | docs/commands/local-show.md 21 | docs/commands/migrate.md 22 | docs/commands/mirror.md 23 | docs/commands/remote-folders.md 24 | docs/commands/restore.md 25 | docs/commands/setup.md 26 | docs/commands/single-backup.md 27 | docs/commands/utils-export-to-thunderbird.md 28 | docs/commands/utils-ignore-history.md 29 | docs/howto/migrate-server-keep-address.md 30 | -------------------------------------------------------------------------------- /spec/features/support/shared/message_fixtures.rb: -------------------------------------------------------------------------------- 1 | RSpec.shared_context "message-fixtures" do 2 | let(:uid_one) { 123 } 3 | let(:uid_two) { 345 } 4 | let(:uid_three) { 567 } 5 | let(:uid_iso8859) { 890 } 6 | let(:message_one) do 7 | {uid: uid_one, from: "address@example.org", subject: "Test 1", body: "body 1\nHi"} 8 | end 9 | let(:message_two) do 10 | {uid: uid_two, from: "address@example.org", subject: "Test 2", body: "body 2"} 11 | end 12 | let(:message_three) do 13 | {uid: uid_three, from: "address@example.org", subject: "Test 3", body: "body 3"} 14 | end 15 | let(:msg_iso8859) do 16 | { 17 | uid: uid_iso8859, 18 | from: "address@example.org", 19 | subject: "iso8859 Body", 20 | body: "Ma, perchè?".encode(Encoding::ISO_8859_1).force_encoding("binary") 21 | } 22 | end 23 | end 24 | -------------------------------------------------------------------------------- /spec/unit/file_mode_spec.rb: -------------------------------------------------------------------------------- 1 | require "imap/backup/file_mode" 2 | 3 | module Imap::Backup 4 | RSpec.describe FileMode do 5 | subject { described_class.new(filename: filename) } 6 | 7 | let(:filename) { "filename" } 8 | let(:exists) { true } 9 | let(:stat) { instance_double(File::Stat, mode: 0o2345) } 10 | 11 | before do 12 | allow(File).to receive(:exist?).and_call_original 13 | allow(File).to receive(:exist?).with(filename) { exists } 14 | allow(File).to receive(:stat).with(filename) { stat } 15 | end 16 | 17 | it "is the last 9 bits of the file mode" do 18 | expect(subject.mode).to eq(0o345) 19 | end 20 | 21 | context "with non-existent files" do 22 | let(:exists) { false } 23 | 24 | it "is nil" do 25 | expect(subject.mode).to be_nil 26 | end 27 | end 28 | end 29 | end 30 | -------------------------------------------------------------------------------- /lib/imap/backup/email/provider/base.rb: -------------------------------------------------------------------------------- 1 | module Imap; end 2 | 3 | module Imap::Backup 4 | module Email; end 5 | class Email::Provider; end 6 | 7 | # Supplies defaults for email provider behaviour 8 | class Email::Provider::Base 9 | # @return [Array] tags to ignore when listing folders 10 | def folder_ignore_tags 11 | [] 12 | end 13 | 14 | # @return [Hash] defaults for the Net::IMAP connection 15 | def options 16 | {port: 993, ssl: {min_version: OpenSSL::SSL::TLS1_2_VERSION}} 17 | end 18 | 19 | # By default, we query the server for this value. 20 | # It is only fixed for Apple Mail accounts. 21 | # @return [String, nil] any fixed value to use when requesting the list of account folders 22 | def root 23 | end 24 | 25 | def sets_seen_flags_on_fetch? 26 | false 27 | end 28 | end 29 | end 30 | -------------------------------------------------------------------------------- /lib/imap/backup/serializer/message_enumerator.rb: -------------------------------------------------------------------------------- 1 | module Imap; end 2 | 3 | module Imap::Backup 4 | class Serializer; end 5 | 6 | # Enumerates over a list of stores messages 7 | class Serializer::MessageEnumerator 8 | # @param imap [Serializer::Imap] the metadata serializer for the folder 9 | def initialize(imap:) 10 | @imap = imap 11 | end 12 | 13 | # Enumerates over the messages 14 | # @param uids [Array] the message UIDs of the messages to iterate over 15 | # @yieldparam message [Serializer::Message] 16 | # @return [void] 17 | def run(uids:, &block) 18 | uids.each do |uid_maybe_string| 19 | uid = uid_maybe_string.to_i 20 | message = imap.get(uid) 21 | 22 | next if !message 23 | 24 | block.call(message) 25 | end 26 | end 27 | 28 | private 29 | 30 | attr_reader :imap 31 | end 32 | end 33 | -------------------------------------------------------------------------------- /spec/unit/account/client_factory_spec.rb: -------------------------------------------------------------------------------- 1 | require "imap/backup/account/client_factory" 2 | 3 | require "imap/backup/account" 4 | 5 | module Imap::Backup 6 | RSpec.describe Account::ClientFactory do 7 | subject { described_class.new(account: account) } 8 | 9 | let(:account) do 10 | instance_double( 11 | Account, 12 | connection_options: nil, 13 | username: "username@example.com", 14 | password: "password", 15 | server: "server" 16 | ) 17 | end 18 | let(:client) { instance_double(Client::Default) } 19 | let(:result) { subject.run } 20 | 21 | before do 22 | allow(Client::Default).to receive(:new) { client } 23 | allow(client).to receive(:login).with(no_args) 24 | end 25 | 26 | it "returns the AutomaticLoginWrapper" do 27 | expect(result).to be_a(Client::AutomaticLoginWrapper) 28 | end 29 | end 30 | end 31 | -------------------------------------------------------------------------------- /spec/support/shared_examples/requiring_an_existing_configuration.rb: -------------------------------------------------------------------------------- 1 | require "imap/backup/configuration" 2 | require "imap/backup/configuration_not_found" 3 | 4 | module Imap::Backup 5 | RSpec.shared_examples "an action that doesn't require an existing configuration" do |action:| 6 | before do 7 | allow(Configuration).to receive(:exist?) { false } 8 | end 9 | 10 | it "works if there is no configuration file" do 11 | expect do 12 | action.call(subject) 13 | end.to_not raise_error 14 | end 15 | end 16 | 17 | RSpec.shared_examples "an action that requires an existing configuration" do |action:| 18 | before do 19 | allow(Configuration).to receive(:exist?) { false } 20 | end 21 | 22 | it "fails if there is no configuration file" do 23 | expect do 24 | action.call(subject) 25 | end.to raise_error(ConfigurationNotFound) 26 | end 27 | end 28 | end 29 | -------------------------------------------------------------------------------- /spec/features/remote/list_namespaces_spec.rb: -------------------------------------------------------------------------------- 1 | require "features/helper" 2 | 3 | RSpec.describe "imap-backup remote namespaces", :container, type: :aruba do 4 | let(:account) { test_server_connection_parameters } 5 | let(:config_options) { {accounts: [account]} } 6 | let(:command) { "imap-backup remote namespaces #{account[:username]}" } 7 | 8 | before do 9 | create_config(**config_options) 10 | end 11 | 12 | it "lists namespaces" do 13 | run_command_and_stop command 14 | 15 | expect(last_command_started).to have_output(/personal\s+""\s+"\."/) 16 | end 17 | 18 | context "when JSON is requested" do 19 | let(:command) { "imap-backup remote namespaces #{account[:username]} --format json" } 20 | 21 | it "lists namespaces as JSON" do 22 | run_command_and_stop command 23 | 24 | expect(last_command_started).to have_output(/{"personal":{"prefix":"","delim":"."}/) 25 | end 26 | end 27 | end 28 | -------------------------------------------------------------------------------- /docs/files/imap.md: -------------------------------------------------------------------------------- 1 | 4 | Metadata about each folder is stored in `.imap` files. 5 | 6 | The current version (version 3), looks like this: 7 | 8 | ```json 9 | { 10 | "version": 3, 11 | "uid_validity": 1606316666, 12 | "messages": [ 13 | { 14 | "uid": 1, 15 | "offset": 0, 16 | "length": 3736, 17 | "flags": ["Draft"] 18 | } 19 | ] 20 | } 21 | ``` 22 | 23 | * version - the file format version, 24 | * uid_validity - the [UIDVALIDITY attribute](https://www.rfc-editor.org/rfc/rfc3501#section-2.3.1.1) for the folder on the IMAP server, 25 | * messages - metadata about the downloaded messages, 26 | * uid - the message's unique identifier, 27 | * offset - the offset of the start of the message in the accompanying `.mbox` file, 28 | * length - the length of the serialized message, 29 | * flags - any of the standard flags which were set in the message when last downloaded. 30 | -------------------------------------------------------------------------------- /spec/features/setup/add_account_spec.rb: -------------------------------------------------------------------------------- 1 | require "features/helper" 2 | 3 | RSpec.describe "imap-backup setup - adding an account", type: :aruba do 4 | let(:account) { test_server_connection_parameters } 5 | let(:config_options) { {accounts: [account]} } 6 | 7 | before do 8 | create_config(**config_options) 9 | 10 | run_command "imap-backup setup" 11 | last_command_started.write "add account\n" 12 | last_command_started.write "new@example.com\n" 13 | last_command_started.write "(q) return to main menu\n" 14 | last_command_started.write "save and exit\n" 15 | last_command_started.stop 16 | end 17 | 18 | it "creates the configuration directory" do 19 | expect(directory?(config_path)).to be true 20 | end 21 | 22 | it "saves account info" do 23 | config = parsed_config 24 | account = config[:accounts].last 25 | 26 | expect(account[:username]).to eq("new@example.com") 27 | end 28 | end 29 | -------------------------------------------------------------------------------- /spec/unit/email/provider/base_spec.rb: -------------------------------------------------------------------------------- 1 | require "imap/backup/email/provider/base" 2 | 3 | module Imap::Backup 4 | RSpec.describe Email::Provider::Base do 5 | describe "#folder_ignore_tags" do 6 | it "returns an empty array" do 7 | expect(subject.folder_ignore_tags).to eq([]) 8 | end 9 | end 10 | 11 | describe "#options" do 12 | it "returns options" do 13 | expect(subject.options).to be_a(Hash) 14 | end 15 | 16 | it "forces TLSv1_2" do 17 | expect(subject.options[:ssl][:min_version]).to eq(OpenSSL::SSL::TLS1_2_VERSION) 18 | end 19 | end 20 | 21 | describe "#root" do 22 | it "is an nil" do 23 | expect(subject.root).to be_nil 24 | end 25 | end 26 | 27 | describe "#sets_seen_flags_on_fetch?" do 28 | it "is false" do 29 | expect(subject.sets_seen_flags_on_fetch?).to be false 30 | end 31 | end 32 | end 33 | end 34 | -------------------------------------------------------------------------------- /lib/imap/backup/serializer/unused_name_finder.rb: -------------------------------------------------------------------------------- 1 | require "imap/backup/serializer" 2 | 3 | module Imap; end 4 | 5 | module Imap::Backup 6 | class Serializer; end 7 | 8 | # Finds a name that can be used to rename a serialized folder 9 | class Serializer::UnusedNameFinder 10 | # @param serializer [Serializer] a folder serializer 11 | def initialize(serializer:) 12 | @serializer = serializer 13 | end 14 | 15 | # Finds the name 16 | # @return [String] the name 17 | def run 18 | digit = 0 19 | folder = nil 20 | 21 | loop do 22 | extra = digit.zero? ? "" : "-#{digit}" 23 | folder = "#{serializer.folder}-#{serializer.uid_validity}#{extra}" 24 | test = Serializer.new(serializer.path, folder) 25 | break if !test.validate! 26 | 27 | digit += 1 28 | end 29 | 30 | folder 31 | end 32 | 33 | private 34 | 35 | attr_reader :serializer 36 | end 37 | end 38 | -------------------------------------------------------------------------------- /lib/imap/backup/account/folder_ensurer.rb: -------------------------------------------------------------------------------- 1 | require "imap/backup/serializer/directory" 2 | require "imap/backup/serializer/folder_maker" 3 | 4 | module Imap; end 5 | 6 | module Imap::Backup 7 | class Account; end 8 | 9 | # Handles creation of directories for backup storage 10 | class Account::FolderEnsurer 11 | def initialize(account:) 12 | @account = account 13 | end 14 | 15 | # Creates the account's base directory and sets its permissions 16 | # @raise [RuntimeError] is the account's backup path is not set 17 | # @return [void] 18 | def run 19 | raise "The backup path for #{account.username} is not set" if !account.local_path 20 | 21 | Serializer::FolderMaker.new( 22 | base: File.dirname(account.local_path), 23 | path: File.basename(account.local_path), 24 | permissions: Serializer::Directory::DIRECTORY_PERMISSIONS 25 | ).run 26 | end 27 | 28 | private 29 | 30 | attr_reader :account 31 | end 32 | end 33 | -------------------------------------------------------------------------------- /spec/unit/serializer/message_enumerator_spec.rb: -------------------------------------------------------------------------------- 1 | require "imap/backup/serializer/message_enumerator" 2 | require "imap/backup/serializer/imap" 3 | require "imap/backup/serializer/message" 4 | 5 | module Imap::Backup 6 | RSpec.describe Serializer::MessageEnumerator do 7 | subject { described_class.new(imap: imap) } 8 | 9 | let(:imap) { instance_double(Serializer::Imap, get: message) } 10 | let(:message) { instance_double(Serializer::Message) } 11 | let(:good_uid) { 999 } 12 | 13 | before do 14 | allow(imap).to receive(:get) { nil } 15 | allow(imap).to receive(:get).with(good_uid) { message } 16 | end 17 | 18 | it "yields messages" do 19 | expect { |b| subject.run(uids: [good_uid], &b) }. 20 | to yield_successive_args(message) 21 | end 22 | 23 | context "with UIDs that are not present" do 24 | it "skips them" do 25 | expect { |b| subject.run(uids: [good_uid, 1234], &b) }. 26 | to yield_successive_args(message) 27 | end 28 | end 29 | end 30 | end 31 | -------------------------------------------------------------------------------- /docs/installation/source.md: -------------------------------------------------------------------------------- 1 | 4 | # Installation From Source 5 | 6 | In order to run imap-backup from source, you'll need [Ruby](https://www.ruby-lang.org/en/documentation/installation/) and [Git](https://git-scm.com/book/en/v2/Getting-Started-Installing-Git). 7 | 8 | Next, clone the repository: 9 | 10 | ```sh 11 | git clone https://github.com/joeyates/imap-backup.git 12 | ``` 13 | 14 | If you want to use a branch other than `main`: 15 | 16 | ```sh 17 | git checkout --track -b BRANCH origin/BRANCH 18 | ``` 19 | 20 | Install dependencies: 21 | 22 | ```sh 23 | cd imap-backup 24 | gem install bundler --version=2.3.22 25 | bundle install 26 | ``` 27 | 28 | Check that it runs: 29 | 30 | ```sh 31 | bin/imap-backup version 32 | ``` 33 | 34 | If you get something like 35 | 36 | ``` 37 | imap-backup 14.4.4 38 | ``` 39 | 40 | congratulations, you have succesfully built imap-backup. 41 | 42 | You can now run the following to see the built-in help: 43 | 44 | ```sh 45 | bin/imap-backup help 46 | ``` 47 | -------------------------------------------------------------------------------- /spec/unit/account/folder_ensurer_spec.rb: -------------------------------------------------------------------------------- 1 | require "imap/backup/account/folder_ensurer" 2 | 3 | require "imap/backup/account" 4 | 5 | module Imap::Backup 6 | RSpec.describe Account::FolderEnsurer do 7 | subject { described_class.new(account: account) } 8 | 9 | let(:account) { instance_double(Account, local_path: local_path, username: "username") } 10 | let(:local_path) { "local_path" } 11 | 12 | context "when local_path is not set" do 13 | let(:local_path) { nil } 14 | 15 | it "fails" do 16 | expect { subject.run }.to raise_error(RuntimeError, /backup path.*?not set/) 17 | end 18 | end 19 | 20 | context "when the directory does not exist" do 21 | let(:folder_maker) { instance_double(Serializer::FolderMaker, run: nil) } 22 | 23 | before do 24 | allow(Serializer::FolderMaker).to receive(:new) { folder_maker } 25 | end 26 | 27 | it "creates it" do 28 | subject.run 29 | 30 | expect(folder_maker).to have_received(:run) 31 | end 32 | end 33 | end 34 | end 35 | -------------------------------------------------------------------------------- /lib/imap/backup/retry_on_error.rb: -------------------------------------------------------------------------------- 1 | require "imap/backup/logger" 2 | 3 | module Imap; end 4 | 5 | module Imap::Backup 6 | # Provides a mechanism for retrying blocks of code which often throw errors 7 | module RetryOnError 8 | # Calls the supplied block, 9 | # traps the given types of errors 10 | # retrying up to a given number of times 11 | # @param errors [Array] the exceptions to trap 12 | # @param limit [Integer] the maximum number of retries 13 | # @param on_error [Proc] a block to call when an error occurs 14 | # @raise any error ocurring more than `limit` times 15 | # @return the result of any successful completion of the block 16 | def retry_on_error(errors:, limit: 10, on_error: nil, &block) 17 | tries ||= 1 18 | block.call 19 | rescue *errors => e 20 | if tries < limit 21 | message = "#{e}, attempt #{tries} of #{limit}" 22 | Logger.logger.debug message 23 | on_error&.call 24 | tries += 1 25 | retry 26 | end 27 | raise e 28 | end 29 | end 30 | end 31 | -------------------------------------------------------------------------------- /lib/imap/backup/account/local_only_folder_deleter.rb: -------------------------------------------------------------------------------- 1 | require "imap/backup/account/backup_folders" 2 | require "imap/backup/account/serialized_folders" 3 | 4 | module Imap; end 5 | 6 | module Imap::Backup 7 | class Account; end 8 | 9 | # Deletes serialized folders that are not configured to be backed up. 10 | # This is used in mirror mode, where local copies are only kept as long as they 11 | # exist on the server. 12 | class Account::LocalOnlyFolderDeleter 13 | def initialize(account:) 14 | @account = account 15 | end 16 | 17 | # Runs the deletion operation 18 | # @return [void] 19 | def run 20 | backup_folders = Account::BackupFolders.new( 21 | client: account.client, account: account 22 | ) 23 | wanted = backup_folders.map(&:name) 24 | serialized_folders = Account::SerializedFolders.new(account: account) 25 | serialized_folders.each_key do |serializer| 26 | serializer.delete if !wanted.include?(serializer.folder) 27 | end 28 | end 29 | 30 | private 31 | 32 | attr_reader :account 33 | end 34 | end 35 | -------------------------------------------------------------------------------- /contrib/README.md: -------------------------------------------------------------------------------- 1 | # contrib 2 | 3 | This directory contains contributed scripts that relate to 4 | imap-backup 5 | 6 | # import-accounts-from-csv 7 | 8 | This script reads a CSV file and merges the supplied 9 | information into an imap-backup configuration file. 10 | 11 | Depending on how your CSV file is structured, 12 | you will probably need to modify the `COLUMNS` structure in the script. 13 | 14 | While importing, it checks that the provided credentials 15 | ans connection parameters work. 16 | 17 | An example CSV file `contrib/example_users.csv` is provided. 18 | 19 | You can try out the script as follows: 20 | 21 | ```sh 22 | contrib/import-accounts-from-csv --csv contrib/example_users.csv --config example-config.json --verbose 23 | ``` 24 | 25 | # import-messages-from-thunderbird 26 | 27 | This script imports all messages from a Thunderbird folder. 28 | 29 | Obviously, Thunderbird must be installed and the folder in question must 30 | have the Thunderbird setting "Select this folder for offline use". 31 | 32 | ```sh 33 | contrib/import-thunderbird-folder --config example-config.json --verbose 34 | ``` 35 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2012 Joe Yates 2 | 3 | MIT License 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining 6 | a copy of this software and associated documentation files (the 7 | "Software"), to deal in the Software without restriction, including 8 | without limitation the rights to use, copy, modify, merge, publish, 9 | distribute, sublicense, and/or sell copies of the Software, and to 10 | permit persons to whom the Software is furnished to do so, subject to 11 | the following conditions: 12 | 13 | The above copyright notice and this permission notice shall be 14 | included in all copies or substantial portions of the Software. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 17 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 18 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 19 | NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE 20 | LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 21 | OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 22 | WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -------------------------------------------------------------------------------- /lib/imap/backup/serializer/permission_checker.rb: -------------------------------------------------------------------------------- 1 | require "imap/backup/file_mode" 2 | 3 | module Imap; end 4 | 5 | module Imap::Backup 6 | class Serializer; end 7 | 8 | # Ensures a file has the desired permissions 9 | class Serializer::PermissionChecker 10 | # @param filename [String] the file name 11 | # @param limit [Integer] the maximum permission that should be set 12 | def initialize(filename:, limit:) 13 | @filename = filename 14 | @limit = limit 15 | end 16 | 17 | # Runs the check 18 | # @raise [RuntimeError] if the permissions are incorrect 19 | # @return [void] 20 | def run 21 | actual = FileMode.new(filename: filename).mode 22 | return nil if actual.nil? 23 | 24 | mask = ~limit & 0o777 25 | return if (actual & mask).zero? 26 | 27 | message = format( 28 | "Permissions on '%s' " \ 29 | "should be 0%o, not 0%o", 30 | filename: filename, limit: limit, actual: actual 31 | ) 32 | raise message 33 | end 34 | 35 | private 36 | 37 | attr_reader :filename 38 | attr_reader :limit 39 | end 40 | end 41 | -------------------------------------------------------------------------------- /lib/imap/backup/account/restore.rb: -------------------------------------------------------------------------------- 1 | require "imap/backup/account/folder_mapper" 2 | require "imap/backup/uploader" 3 | 4 | module Imap; end 5 | 6 | module Imap::Backup 7 | class Account; end 8 | 9 | # Restores all backed up folders to the server 10 | class Account::Restore 11 | def initialize(account:, delimiter: "/", prefix: "") 12 | @account = account 13 | @destination_delimiter = delimiter 14 | @destination_prefix = prefix 15 | end 16 | 17 | # Runs the restore operation 18 | # @return [void] 19 | def run 20 | folders.each do |serializer, folder| 21 | Uploader.new(folder, serializer).run 22 | end 23 | end 24 | 25 | private 26 | 27 | attr_reader :account 28 | attr_reader :destination_delimiter 29 | attr_reader :destination_prefix 30 | 31 | def enumerator_options 32 | { 33 | account: account, 34 | destination: account, 35 | destination_delimiter: destination_delimiter, 36 | destination_prefix: destination_prefix 37 | } 38 | end 39 | 40 | def folders 41 | Account::FolderMapper.new(**enumerator_options) 42 | end 43 | end 44 | end 45 | -------------------------------------------------------------------------------- /lib/imap/backup/local_only_message_deleter.rb: -------------------------------------------------------------------------------- 1 | require "imap/backup/logger" 2 | 3 | module Imap; end 4 | 5 | module Imap::Backup 6 | # Deletes locally backed-up emails that are no longer on the server 7 | class LocalOnlyMessageDeleter 8 | def initialize(folder, serializer) 9 | @folder = folder 10 | @serializer = serializer 11 | end 12 | 13 | # TODO: this method is very slow as it copies all messages. 14 | # A quicker method would only remove UIDs from the .imap file, 15 | # but that would require a garbage collection later. 16 | # @return [void] 17 | def run 18 | local_only_uids = serializer.uids - folder.uids 19 | if local_only_uids.empty? 20 | Logger.logger.debug "There are no 'local-only' messages to delete" 21 | return 22 | end 23 | 24 | Logger.logger.info "Deleting messages only present locally" 25 | Logger.logger.debug "Messages to be deleted: #{local_only_uids.inspect}" 26 | 27 | serializer.filter do |message| 28 | !local_only_uids.include?(message.uid) 29 | end 30 | end 31 | 32 | private 33 | 34 | attr_reader :folder 35 | attr_reader :serializer 36 | end 37 | end 38 | -------------------------------------------------------------------------------- /lib/imap/backup/flag_refresher.rb: -------------------------------------------------------------------------------- 1 | module Imap; end 2 | 3 | module Imap::Backup 4 | # Updates the flags on backed-up emails 5 | class FlagRefresher 6 | # The number of messages to process at a time 7 | CHUNK_SIZE = 100 8 | 9 | def initialize(folder, serializer) 10 | @folder = folder 11 | @serializer = serializer 12 | end 13 | 14 | # Runs the update 15 | # @return [void] 16 | def run 17 | uids = serializer.uids.clone 18 | 19 | uids.each_slice(CHUNK_SIZE) do |block| 20 | refresh_block block 21 | end 22 | end 23 | 24 | private 25 | 26 | attr_reader :folder 27 | attr_reader :serializer 28 | 29 | def refresh_block(uids) 30 | uids_and_flags = folder.fetch_multi(uids, ["FLAGS"]) 31 | if !uids_and_flags 32 | Logger.logger.debug( 33 | "[#{folder.name}] failed to fetch flags for #{uids} - " \ 34 | "cannot refresh flags" 35 | ) 36 | return 37 | end 38 | uids_and_flags.each do |uid_and_flags| 39 | uid = uid_and_flags[:uid] 40 | flags = uid_and_flags[:flags] 41 | serializer.update(uid, flags: flags) 42 | end 43 | end 44 | end 45 | end 46 | -------------------------------------------------------------------------------- /spec/unit/text/sanitizer_spec.rb: -------------------------------------------------------------------------------- 1 | require "imap/backup/text/sanitizer" 2 | 3 | require "stringio" 4 | 5 | module Imap::Backup 6 | RSpec.describe Text::Sanitizer do 7 | subject { described_class.new(output) } 8 | 9 | let(:output) { StringIO.new } 10 | 11 | describe "#puts" do 12 | it "delegates to output" do 13 | subject.puts("x") 14 | 15 | expect(output.string).to eq("x\n") 16 | end 17 | end 18 | 19 | describe "#write" do 20 | it "delegates to output" do 21 | subject.write("x") 22 | 23 | expect(output.string).to eq("x") 24 | end 25 | end 26 | 27 | describe "#print" do 28 | it "removes passwords from complete lines of text" do 29 | subject.print("C: RUBY99 LOGIN xx) secret!!!!\netc") 30 | 31 | expect(output.string).to eq("C: RUBY99 LOGIN xx) [PASSWORD REDACTED]\n") 32 | end 33 | end 34 | 35 | describe "#flush" do 36 | it "sanitizes remaining text" do 37 | subject.print("before\nC: RUBY99 LOGIN xx) secret!!!!") 38 | subject.flush 39 | 40 | expect(output.string).to eq("before\nC: RUBY99 LOGIN xx) [PASSWORD REDACTED]\n") 41 | end 42 | end 43 | end 44 | end 45 | -------------------------------------------------------------------------------- /lib/imap/backup/serializer/folder_maker.rb: -------------------------------------------------------------------------------- 1 | require "fileutils" 2 | 3 | module Imap; end 4 | 5 | module Imap::Backup 6 | class Serializer; end 7 | 8 | # Creates directories 9 | class Serializer::FolderMaker 10 | # @param base [String] The base directory of the account 11 | # @param path [String] The path to the folder, relative to the base 12 | # @param permissions [Integer] The permissions to set on the folder 13 | def initialize(base:, path:, permissions:) 14 | @base = base 15 | @path = path 16 | @permissions = permissions 17 | end 18 | 19 | # Creates the directory and any missing parent directories, 20 | # ensuring the desired permissions. 21 | # @return [void] 22 | def run 23 | parts = path.split("/") 24 | return if parts.empty? 25 | 26 | FileUtils.mkdir_p(full_path) 27 | full = base 28 | parts.each do |part| 29 | full = File.join(full, part) 30 | FileUtils.chmod permissions, full 31 | end 32 | end 33 | 34 | private 35 | 36 | attr_reader :base 37 | attr_reader :path 38 | attr_reader :permissions 39 | 40 | def full_path 41 | File.join(base, path) 42 | end 43 | end 44 | end 45 | -------------------------------------------------------------------------------- /spec/unit/cli/single_spec.rb: -------------------------------------------------------------------------------- 1 | require "imap/backup/cli/single" 2 | 3 | module Imap::Backup 4 | RSpec.describe CLI::Single do 5 | describe "#backup" do 6 | let(:backup) { instance_double(CLI::Single::Backup, run: nil) } 7 | 8 | before do 9 | allow(CLI::Single::Backup).to receive(:new) { backup } 10 | end 11 | 12 | it "runs a single backup" do 13 | subject.backup 14 | 15 | expect(backup).to have_received(:run) 16 | end 17 | 18 | it_behaves_like( 19 | "an action that handles Logger options", 20 | action: ->(subject, options) do 21 | with_required = options.merge({"email" => "me", "server" => "host"}) 22 | subject.invoke(:backup, [], with_required) 23 | end 24 | ) do 25 | it "passes other options to the class" do 26 | expect(CLI::Single::Backup).to have_received(:new). 27 | with(hash_including({email: "me", server: "host"})) 28 | end 29 | 30 | it "does not pass loggint options to the class" do 31 | expect(CLI::Single::Backup).to have_received(:new). 32 | with(hash_not_including([:quiet, :verbose])) 33 | end 34 | end 35 | end 36 | end 37 | end 38 | -------------------------------------------------------------------------------- /spec/unit/flag_refresher_spec.rb: -------------------------------------------------------------------------------- 1 | require "imap/backup/account/folder" 2 | require "imap/backup/flag_refresher" 3 | require "imap/backup/serializer" 4 | 5 | module Imap::Backup 6 | RSpec.describe FlagRefresher do 7 | subject { described_class.new(folder, serializer) } 8 | 9 | let(:serializer) { instance_double(Serializer, uids: [1, 2]) } 10 | let(:folder) { instance_double(Account::Folder, uids: [2], name: "my_folder") } 11 | 12 | it "refreshes the flags" do 13 | response = [{uid: 1, flags: [:Draft]}, {uid: 2, flags: [:Seen]}] 14 | allow(folder).to receive(:fetch_multi).with([1, 2], ["FLAGS"]) { response } 15 | 16 | expect(serializer).to receive(:update).with(1, flags: [:Draft]) 17 | expect(serializer).to receive(:update).with(2, flags: [:Seen]) 18 | 19 | subject.run 20 | end 21 | 22 | context "when the fetch fails" do 23 | it "logs a warning" do 24 | allow(folder).to receive(:fetch_multi).with([1, 2], ["FLAGS"]).and_return(nil) 25 | expect(Logger.logger). 26 | to receive(:debug). 27 | with("[#{folder.name}] failed to fetch flags for [1, 2] - cannot refresh flags") 28 | 29 | subject.run 30 | end 31 | end 32 | end 33 | end 34 | -------------------------------------------------------------------------------- /lib/imap/backup/migrator.rb: -------------------------------------------------------------------------------- 1 | require "imap/backup/logger" 2 | 3 | module Imap; end 4 | 5 | module Imap::Backup 6 | # Copies a folder of backed-up emails to an online folder 7 | class Migrator 8 | def initialize(serializer, folder, reset: false) 9 | @folder = folder 10 | @reset = reset 11 | @serializer = serializer 12 | end 13 | 14 | # Runs the migration 15 | # @return [void] 16 | def run 17 | count = serializer.uids.count 18 | folder.create 19 | folder.clear if reset 20 | 21 | Logger.logger.debug "[#{folder.name}] #{count} to migrate" 22 | serializer.each_message(serializer.uids).with_index do |message, i| 23 | next if message.nil? 24 | 25 | log_prefix = "[#{folder.name}] uid: #{message.uid} (#{i + 1}/#{count}) -" 26 | Logger.logger.debug( 27 | "#{log_prefix} #{message.body.size} bytes" 28 | ) 29 | 30 | begin 31 | folder.append(message) 32 | rescue StandardError => e 33 | Logger.logger.warn "#{log_prefix} append error: #{e}" 34 | end 35 | end 36 | end 37 | 38 | private 39 | 40 | attr_reader :folder 41 | attr_reader :reset 42 | attr_reader :serializer 43 | end 44 | end 45 | -------------------------------------------------------------------------------- /spec/features/support/30_email_server_helpers.rb: -------------------------------------------------------------------------------- 1 | require_relative "20_test_email_server" 2 | 3 | module EmailServerHelpers 4 | def test_server_connection_parameters 5 | { 6 | server: ENV.fetch("DOCKER_HOST_IMAP", "localhost"), 7 | username: "address@example.com", 8 | password: "pass", 9 | local_path: File.join(File.expand_path("~/.imap-backup"), "address_example.com"), 10 | connection_options: { 11 | port: ENV.fetch("DOCKER_PORT_IMAP", "8993").to_i, 12 | ssl: {verify_mode: 0} 13 | } 14 | } 15 | end 16 | 17 | def other_server_connection_parameters 18 | { 19 | server: ENV.fetch("DOCKER_HOST_OTHER_IMAP", "localhost"), 20 | username: "email@other.org", 21 | password: "pass", 22 | local_path: File.join(File.expand_path("~/.imap-backup"), "email_other.org"), 23 | connection_options: { 24 | port: ENV.fetch("DOCKER_PORT_OTHER_IMAP", "9993").to_i, 25 | ssl: {verify_mode: 0} 26 | } 27 | } 28 | end 29 | 30 | def test_server 31 | @test_server ||= TestEmailServer.new(**test_server_connection_parameters) 32 | end 33 | 34 | def other_server 35 | @other_server ||= TestEmailServer.new(**other_server_connection_parameters) 36 | end 37 | end 38 | -------------------------------------------------------------------------------- /lib/imap/backup/naming.rb: -------------------------------------------------------------------------------- 1 | module Imap; end 2 | 3 | module Imap::Backup 4 | # Maps between server and file system folder names 5 | # `/` is treated as an acceptable character 6 | class Naming 7 | # The characters that cannot be used in file names 8 | INVALID_FILENAME_CHARACTERS = ":%;".freeze 9 | # A regular expression that captures each disallowed character 10 | INVALID_FILENAME_CHARACTER_MATCH = /([#{INVALID_FILENAME_CHARACTERS}])/ 11 | 12 | # @param name [String] a folder name 13 | # @return [String] the supplied string iwth disallowed characters replaced 14 | # by their hexadecimal representation 15 | def self.to_local_path(name) 16 | name.gsub(INVALID_FILENAME_CHARACTER_MATCH) do |character| 17 | hex = 18 | character. 19 | codepoints[0]. 20 | to_s(16) 21 | "%#{hex};" 22 | end 23 | end 24 | 25 | # @param name [String] a serialized folder name 26 | # @return the supplied string with hexadecimal codes ('%xx') replaced with 27 | # the characters they represent 28 | def self.from_local_path(name) 29 | name.gsub(/%(.*?);/) do 30 | ::Regexp.last_match(1). 31 | to_i(16). 32 | chr("UTF-8") 33 | end 34 | end 35 | end 36 | end 37 | -------------------------------------------------------------------------------- /docs/commands/restore.md: -------------------------------------------------------------------------------- 1 | 4 | # Restore 5 | 6 | ```sh 7 | imap-backup restore EMAIL_ADDRESS 8 | ``` 9 | 10 | All missing messages are pushed to the IMAP server. 11 | Existing messages are left unchanged. 12 | 13 | This functionality requires that the IMAP server supports the UIDPLUS 14 | extension to IMAP4. 15 | 16 | # FAQ 17 | 18 | ## How does restore work? 19 | 20 | Backed-up emails are pushed to the IMAP server. 21 | If there are clashes, folders are renamed. 22 | 23 | ## What are all these 'INBOX.12345' files? 24 | 25 | If, when the backup is launched, the IMAP server contains a folder with 26 | the same name, but different history to the local backup, the local 27 | emails cannot simply be added to the existing folder. 28 | 29 | In this case, a numeric suffix is added to the **local** folder, 30 | before it is restored. 31 | 32 | In this way, old and new emails are kept separate. 33 | 34 | ## Will my email get overwritten? 35 | 36 | No. 37 | 38 | Emails are identified by folder and a specific email id. Any email that 39 | is already on the server is skipped. 40 | 41 | ## How do I restore e-mails to a new service while keeping the same e-mail address? 42 | 43 | See [this guide on the topic](/docs/howto/migrate-server-keep-address.md). 44 | -------------------------------------------------------------------------------- /docs/commands/backup.md: -------------------------------------------------------------------------------- 1 | 4 | # Backup 5 | 6 | ```sh 7 | imap-backup backup 8 | ``` 9 | 10 | This command runs the backup operation using information provided 11 | by a configuration file created using `imap-backup setup`. 12 | 13 | By default, emails for all *configured* accounts are copied to disk. 14 | 15 | The backup is incremental and interruptible, so backups won't get messed up 16 | if your connection goes down during an operation. 17 | 18 | # Single Account Backups 19 | 20 | As an alternative, if you only want to backup a single account, 21 | you can pass all the necessary parameters directly to the `single backup` command 22 | (see the [`single backup`](./single-backup.md) docs). 23 | 24 | # Serialized Format 25 | 26 | Emails are stored on disk in [Mbox files](../files/mboxrd.md), one for each folder, 27 | with metadata stored in [Imap files](../files/imap.md). 28 | 29 | The Imap file contains information about the email messages stored in the Mbox file. 30 | For each, it has the offset to the start of the message and its length. 31 | 32 | # Output 33 | 34 | Verbose output can be configured by adding the `--verbose` (or `-v`) parameter. 35 | Add that parameter twice will also show all network traffic between 36 | imap-backup and the IMAP server. 37 | -------------------------------------------------------------------------------- /spec/unit/email/provider_spec.rb: -------------------------------------------------------------------------------- 1 | require "imap/backup/email/provider" 2 | 3 | module Imap::Backup 4 | RSpec.describe Email::Provider do 5 | describe ".for_address" do 6 | context "with known providers" do 7 | [ 8 | ["fastmail.com", "Fastmail .com", Email::Provider::Fastmail], 9 | ["fastmail.fm", "Fastmail .fm", Email::Provider::Fastmail], 10 | ["gmail.com", "GMail", Email::Provider::GMail], 11 | ["icloud.com", "Apple Mail icloud.com", Email::Provider::AppleMail], 12 | ["mac.com", "Apple Mail mac.com", Email::Provider::AppleMail], 13 | ["me.com", "Apple Mail me.com", Email::Provider::AppleMail], 14 | ["purelymail.com", "Purelymail", Email::Provider::Purelymail] 15 | ].each do |domain, name, klass| 16 | it "recognises #{name} addresses" do 17 | address = "foo@#{domain}" 18 | expect(described_class.for_address(address)).to be_a(klass) 19 | end 20 | end 21 | end 22 | 23 | context "with unknown providers" do 24 | it "returns the Unknown provider" do 25 | result = described_class.for_address("foo@unknown.com") 26 | 27 | expect(result).to be_a(Email::Provider::Unknown) 28 | end 29 | end 30 | end 31 | end 32 | end 33 | -------------------------------------------------------------------------------- /dev/ruby-compose.yml: -------------------------------------------------------------------------------- 1 | # This file adapted from github.com/antespi/docker-imap-devel 2 | version: "3" 3 | 4 | services: 5 | imap: 6 | image: ghcr.io/joeyates/docker-imap-devel@sha256:6d6a64c32e2c583222d75286aa46a04ada5aea76efa36117815bf0e19d5063b6 7 | container_name: imap 8 | environment: 9 | - MAILNAME=example.com 10 | - MAIL_ADDRESS=address@example.com 11 | - MAIL_PASS=pass 12 | other-imap: 13 | image: ghcr.io/joeyates/docker-imap-devel@sha256:6d6a64c32e2c583222d75286aa46a04ada5aea76efa36117815bf0e19d5063b6 14 | container_name: other-imap 15 | environment: 16 | - MAILNAME=other.org 17 | - MAIL_ADDRESS=email@other.org 18 | - MAIL_PASS=pass 19 | - DOVECOT_PUBLIC_NAMESPACE_PREFIX=other_public 20 | imap-backup: 21 | build: 22 | context: . 23 | args: 24 | - BUNDLER_VERSION 25 | - RUBY_VERSION 26 | image: imap-backup:${RUBY_VERSION} 27 | container_name: imap-backup 28 | tty: true 29 | stdin_open: true 30 | environment: 31 | - RUBY_VERSION=$RUBY_VERSION 32 | - BUNDLE_PATH=/app/vendor 33 | - BUNDLE_BINSTUBS=/app/bin/stubs 34 | - DOCKER_HOST_IMAP=imap 35 | - DOCKER_PORT_IMAP=993 36 | - DOCKER_HOST_OTHER_IMAP=other-imap 37 | - DOCKER_PORT_OTHER_IMAP=993 38 | - HOME=/app 39 | volumes: 40 | - ..:/app 41 | -------------------------------------------------------------------------------- /spec/features/local/list_accounts_spec.rb: -------------------------------------------------------------------------------- 1 | require "features/helper" 2 | 3 | RSpec.describe "imap-backup local accounts", type: :aruba do 4 | let(:config_options) { {accounts: [{username: "me@example.com", password: "password1"}]} } 5 | let(:command) { "imap-backup local accounts" } 6 | 7 | before do 8 | create_config(**config_options) 9 | 10 | run_command_and_stop command 11 | end 12 | 13 | it "lists accounts" do 14 | expect(last_command_started).to have_output("me@example.com") 15 | end 16 | 17 | context "when JSON is requested" do 18 | let(:command) { "imap-backup local accounts --format json" } 19 | 20 | it "lists accounts" do 21 | expect(last_command_started).to have_output('[{"username":"me@example.com"}]') 22 | end 23 | end 24 | 25 | context "when a config path is supplied" do 26 | let(:custom_config_path) { File.join(File.expand_path("~/.imap-backup"), "foo.json") } 27 | let(:config_options) do 28 | {path: custom_config_path, accounts: [{username: "other@example.com", password: "password1"}]} 29 | end 30 | let(:command) { "imap-backup local accounts --config #{custom_config_path}" } 31 | 32 | it "lists accounts from the supplied config file" do 33 | expect(last_command_started).to have_output("other@example.com") 34 | end 35 | end 36 | end 37 | -------------------------------------------------------------------------------- /spec/unit/serializer/permission_checker_spec.rb: -------------------------------------------------------------------------------- 1 | require "imap/backup/serializer/permission_checker" 2 | 3 | module Imap::Backup 4 | RSpec.describe Serializer::PermissionChecker do 5 | subject { described_class.new(filename: "filename", limit: 0o345) } 6 | 7 | let(:file_mode) { instance_double(FileMode, mode: mode) } 8 | let(:mode) {} 9 | 10 | before do 11 | allow(FileMode).to receive(:new) { file_mode } 12 | end 13 | 14 | [ 15 | [0o100, "less than the limit", true], 16 | [0o345, "equal to the limit", true], 17 | [0o777, "over the limit", false] 18 | ].each do |mode, description, success| 19 | context "when permissions are #{description}" do 20 | let(:mode) { mode } 21 | 22 | if success 23 | it "succeeds" do 24 | expect { subject.run }.to_not raise_error 25 | end 26 | else 27 | it "fails" do 28 | message = /Permissions on '.*?' should be .*?, not .*?/ 29 | expect do 30 | subject.run 31 | end.to raise_error(RuntimeError, message) 32 | end 33 | end 34 | end 35 | end 36 | 37 | context "with non-existent files" do 38 | it "succeeds" do 39 | expect { subject.run }.to_not raise_error 40 | end 41 | end 42 | end 43 | end 44 | -------------------------------------------------------------------------------- /spec/unit/local_only_message_deleter_spec.rb: -------------------------------------------------------------------------------- 1 | require "imap/backup/local_only_message_deleter" 2 | 3 | require "imap/backup/account/folder" 4 | require "imap/backup/serializer" 5 | require "imap/backup/serializer/message" 6 | 7 | module Imap::Backup 8 | RSpec.describe LocalOnlyMessageDeleter do 9 | subject { described_class.new(folder, serializer) } 10 | 11 | let(:serializer) { instance_double(Serializer, uids: [1, 2]) } 12 | let(:folder) { instance_double(Account::Folder, uids: [2]) } 13 | let(:message_one) { instance_double(Serializer::Message, uid: 1) } 14 | let(:message_two) { instance_double(Serializer::Message, uid: 2) } 15 | let(:responses) { [] } 16 | 17 | before do 18 | allow(serializer).to receive(:filter) do |&block| 19 | responses << block.call(message_one) 20 | responses << block.call(message_two) 21 | end 22 | end 23 | 24 | context "with UIDs only present on the local backup" do 25 | it "indicates not to keep the message" do 26 | subject.run 27 | 28 | expect(responses.first).to be false 29 | end 30 | end 31 | 32 | context "with UIDs present locally and on the server" do 33 | it "indicates to keep the message" do 34 | subject.run 35 | 36 | expect(responses.last).to be true 37 | end 38 | end 39 | end 40 | end 41 | -------------------------------------------------------------------------------- /lib/imap/backup/setup/backup_path.rb: -------------------------------------------------------------------------------- 1 | module Imap; end 2 | 3 | module Imap::Backup 4 | class Setup; end 5 | 6 | # Requests an updated backup path from the user 7 | class Setup::BackupPath 8 | # @param account [Account] an Account 9 | # @param config [Configuration] the application configuration 10 | def initialize(account:, config:) 11 | @account = account 12 | @config = config 13 | end 14 | 15 | # Asks the user for a backup path 16 | # 17 | # @return [void] 18 | def run 19 | account.local_path = highline.ask("backup directory: ") do |q| 20 | q.default = account.local_path 21 | q.validate = ->(path) { path_modification_validator(path) } 22 | q.responses[:not_valid] = "Choose a different directory " 23 | end 24 | end 25 | 26 | private 27 | 28 | attr_reader :account 29 | attr_reader :config 30 | 31 | def highline 32 | Setup.highline 33 | end 34 | 35 | def path_modification_validator(path) 36 | same = config.accounts.find do |a| 37 | a.username != account.username && a.local_path == path 38 | end 39 | if same 40 | Kernel.puts "The path '#{path}' is used to backup " \ 41 | "the account '#{same.username}'" 42 | false 43 | else 44 | true 45 | end 46 | end 47 | end 48 | end 49 | -------------------------------------------------------------------------------- /docs/files/config.md: -------------------------------------------------------------------------------- 1 | 4 | By default, imap-backup's configuration is stored as `~/.imap-backup/config.json`. 5 | It is a JSON file. 6 | You can use a configuration file in another location by passing the 7 | `--config PATH` parameter to any command. 8 | 9 | A typical configuration file looks like this: 10 | 11 | ```json 12 | { 13 | "accounts": [ 14 | { 15 | "username": "my.user@gmail.com", 16 | "password": "secret", 17 | "local_path": "/path/to/backup/root", 18 | "folders": 19 | [ 20 | {"name": "[Gmail]/All Mail"}, 21 | {"name": "my_folder"} 22 | ] 23 | } 24 | ] 25 | } 26 | ``` 27 | 28 | # Security 29 | 30 | Note that email usernames and passwords are held in plain text 31 | in the configuration file. 32 | 33 | The directory ~/.imap-backup, the configuration file and all backup 34 | directories have their access permissions set to only allow access 35 | by your user. This is not done on Windows - see below. 36 | 37 | If you choose a custom path for your configuration file, 38 | make sure that is not accessible by other users. 39 | 40 | ## Windows 41 | 42 | Due to the complexity of managing permissions on Windows, 43 | directory and file access permissions are not set explicity. 44 | 45 | A pull request that implements permissions management on Windows 46 | would be welcome! 47 | -------------------------------------------------------------------------------- /docs/commands/copy.md: -------------------------------------------------------------------------------- 1 | 4 | # Copy 5 | 6 | ```sh 7 | imap-backup copy SOURCE_EMAIL DESTINATION_EMAIL 8 | ``` 9 | 10 | This command makes a local copy of the emails in the source account 11 | and then copies them to the destination account. 12 | 13 | Exactly which folders are backed up (and copied) depends on how the account is set up. 14 | 15 | Specifically, the `folder inclusion mode (whitelist/blacklist)` and 16 | `folders to include/exclude` list. 17 | 18 | Note that, any messages on the destination account that is not on the source account 19 | are left unchanged. 20 | 21 | # Options 22 | 23 | * `--source-delimiter` - the separator between the elements of folders names 24 | on the source server, defaults to `/`, 25 | * `--source-prefix` - optionally, a prefix element to remove from the name 26 | of source folders, 27 | * `--destination-delimiter` - the separator between the elements of folder 28 | names on the destination server, defaults to `/`, 29 | * `--destination-prefix` - optionally, a prefix element to add before names 30 | on the destination server, 31 | * `--automatic-namespaces` - works out the 4 parameters above by querying 32 | the source and destination IMAP servers. 33 | 34 | # Delimiters and Prefixes 35 | 36 | For details of the delimiter and prefix options, 37 | see [the note about delimiters and prefixes](../delimiters-and-prefixes.md). 38 | -------------------------------------------------------------------------------- /spec/unit/serializer/unused_name_finder_spec.rb: -------------------------------------------------------------------------------- 1 | require "imap/backup/serializer/unused_name_finder" 2 | 3 | module Imap::Backup 4 | RSpec.describe Serializer::UnusedNameFinder do 5 | subject { described_class.new(serializer: serializer) } 6 | 7 | let(:serializer) do 8 | instance_double(Serializer, folder: "folder", uid_validity: 999, path: "serializer_path") 9 | end 10 | let(:test_serializer) { instance_double(Serializer, validate!: default_serializer_validates) } 11 | let(:default_serializer_validates) { false } 12 | let(:new_name) { "folder-#{serializer.uid_validity}" } 13 | let(:result) { subject.run } 14 | 15 | before do 16 | allow(Serializer).to receive(:new).with(anything, new_name) { test_serializer } 17 | end 18 | 19 | it "returns the folder name with the uid_validity appended" do 20 | expect(result).to eq(new_name) 21 | end 22 | 23 | context "when the default rename is not possible" do 24 | let(:default_serializer_validates) { true } 25 | let(:test_serializer1) { instance_double(Serializer, validate!: false) } 26 | let(:new_name1) { "folder-#{serializer.uid_validity}-1" } 27 | 28 | before do 29 | allow(Serializer).to receive(:new).with(anything, new_name1) { test_serializer1 } 30 | end 31 | 32 | it "appends a numeral" do 33 | expect(result).to eq(new_name1) 34 | end 35 | end 36 | end 37 | end 38 | -------------------------------------------------------------------------------- /spec/unit/serializer/directory_spec.rb: -------------------------------------------------------------------------------- 1 | require "imap/backup/serializer/directory" 2 | 3 | module Imap::Backup 4 | RSpec.describe Serializer::Directory do 5 | subject { described_class.new("directory_path", "relative") } 6 | 7 | let(:windows) { false } 8 | let(:file_mode) { instance_double(FileMode, mode: 0o600) } 9 | let(:folder_maker) { instance_double(Serializer::FolderMaker, run: nil) } 10 | let(:exists) { true } 11 | 12 | before do 13 | allow(File).to receive(:directory?).with(/relative/) { exists } 14 | allow(Serializer::FolderMaker).to receive(:new) { folder_maker } 15 | allow(OS).to receive(:windows?) { windows } 16 | allow(FileMode).to receive(:new) { file_mode } 17 | allow(FileUtils).to receive(:chmod) 18 | end 19 | 20 | context "when the directory doesn't exist" do 21 | let(:exists) { false } 22 | 23 | it "makes the directory" do 24 | subject.ensure_exists 25 | 26 | expect(folder_maker).to have_received(:run) 27 | end 28 | end 29 | 30 | it "sets permissions" do 31 | subject.ensure_exists 32 | 33 | expect(FileUtils).to have_received(:chmod) 34 | end 35 | 36 | context "when on Windows" do 37 | let(:windows) { true } 38 | 39 | it "doesn't set permissions" do 40 | subject.ensure_exists 41 | 42 | expect(FileUtils).to_not have_received(:chmod) 43 | end 44 | end 45 | end 46 | end 47 | -------------------------------------------------------------------------------- /spec/unit/serializer/folder_maker_spec.rb: -------------------------------------------------------------------------------- 1 | require "imap/backup/serializer/folder_maker" 2 | 3 | module Imap::Backup 4 | RSpec.describe Serializer::FolderMaker do 5 | subject { described_class.new(base: base, path: path, permissions: permissions) } 6 | 7 | let(:base) { "base" } 8 | let(:path) { "sub/path" } 9 | let(:permissions) { 0o222 } 10 | let(:full_path) { File.join(base, path) } 11 | let(:exists) { false } 12 | 13 | before do 14 | allow(File).to receive(:stat) { stat } 15 | allow(File).to receive(:exist?).and_call_original 16 | allow(File).to receive(:exist?).with(full_path) { exists } 17 | allow(FileUtils).to receive(:mkdir_p).with(full_path) 18 | allow(FileUtils).to receive(:chmod).with(permissions, /^base/) 19 | end 20 | 21 | it "creates the path" do 22 | subject.run 23 | 24 | expect(FileUtils).to have_received(:mkdir_p).with("base/sub/path") 25 | end 26 | 27 | it "sets permissions on the path" do 28 | subject.run 29 | 30 | expect(FileUtils).to have_received(:chmod).with(0o222, "base/sub") 31 | expect(FileUtils).to have_received(:chmod).with(0o222, "base/sub/path") 32 | end 33 | 34 | context "when an empty path is supplied" do 35 | let(:path) { "" } 36 | 37 | it "does nothing" do 38 | subject.run 39 | 40 | expect(FileUtils).to_not have_received(:mkdir_p) 41 | end 42 | end 43 | end 44 | end 45 | -------------------------------------------------------------------------------- /docs/commands/migrate.md: -------------------------------------------------------------------------------- 1 | 4 | # Migrate 5 | 6 | ```sh 7 | imap-backup migrate SOURCE_EMAIL DESTINATION_EMAIL [OPTIONS] 8 | ``` 9 | 10 | This command is deprecated and will be removed in a future version. Use [copy](./copy.md). 11 | 12 | This command copies backed up emails for one account (the "source") 13 | to another account (the "destination"). 14 | 15 | # Options 16 | 17 | * `--reset` - delete all messages from destination folders before uploading, 18 | * `--source-delimiter` - the separator between the elements of folders names 19 | on the source server, defaults to `/`, 20 | * `--source-prefix` - optionally, a prefix element to remove from the name 21 | of source folders, 22 | * `--destination-delimiter` - the separator between the elements of folder 23 | names on the destination server, defaults to `/`, 24 | * `--destination-prefix` - optionally, a prefix element to add before names 25 | on the destination server, 26 | * `--automatic-namespaces` - works out the 4 parameters above by querying 27 | the source and destination IMAP servers. 28 | 29 | # FAQ 30 | 31 | ## How do I use delimiters and prefixes? 32 | 33 | For details of the delimiter and prefix options, 34 | see [the note about delimiters and prefixes](/docs/delimiters-and-prefixes.md). 35 | 36 | ## How do I migrate to a new server while keeping the same e-mail address? 37 | 38 | See [this note on the topic](/docs/howto/migrate-server-keep-address.md). 39 | -------------------------------------------------------------------------------- /lib/imap/backup/client/automatic_login_wrapper.rb: -------------------------------------------------------------------------------- 1 | require "imap/backup/retry_on_error" 2 | 3 | module Imap; end 4 | 5 | module Imap::Backup 6 | module Client; end 7 | 8 | # Transparently wraps a client instance, while delaying login until it becomes necessary 9 | class Client::AutomaticLoginWrapper 10 | include RetryOnError 11 | 12 | # @private 13 | LOGIN_RETRY_CLASSES = [::EOFError, ::Errno::ECONNRESET, ::SocketError].freeze 14 | 15 | # @return [Client] 16 | attr_reader :client 17 | 18 | def initialize(client:) 19 | @client = client 20 | @login_called = false 21 | end 22 | 23 | # Proxies calls to the client. 24 | # Before the first call does login 25 | # @return the return value of the client method called 26 | def method_missing(method_name, ...) 27 | if login_called 28 | client.send(method_name, ...) 29 | else 30 | do_first_login 31 | client.send(method_name, ...) if method_name != :login 32 | end 33 | end 34 | 35 | # @return [Boolean] whether the client responds to the method 36 | def respond_to_missing?(method_name, _include_private = false) 37 | client.respond_to?(method_name) 38 | end 39 | 40 | private 41 | 42 | attr_reader :login_called 43 | 44 | def do_first_login 45 | retry_on_error(errors: LOGIN_RETRY_CLASSES) do 46 | client.login 47 | @login_called = true 48 | end 49 | end 50 | end 51 | end 52 | -------------------------------------------------------------------------------- /imap-backup.gemspec: -------------------------------------------------------------------------------- 1 | require_relative "lib/imap/backup/version" 2 | 3 | Gem::Specification.new do |gem| 4 | gem.name = "imap-backup" 5 | gem.description = "Backup GMail, or any other IMAP email service, to disk." 6 | gem.summary = "Backup GMail (or other IMAP) accounts to disk" 7 | gem.authors = ["Joe Yates"] 8 | gem.email = ["joe.g.yates@gmail.com"] 9 | gem.homepage = "https://github.com/joeyates/imap-backup" 10 | gem.licenses = ["MIT"] 11 | gem.version = Imap::Backup::VERSION 12 | 13 | gem.files = %w[bin/imap-backup] 14 | gem.files += Dir.glob("docs/*.md") 15 | gem.files += Dir.glob("lib/**/*.rb") 16 | gem.files += %w[imap-backup.gemspec] 17 | gem.files += %w[LICENSE README.md] 18 | 19 | gem.executables = gem.files.grep(%r{^bin/}).map { |f| File.basename(f) } 20 | gem.require_paths = ["lib"] 21 | gem.required_ruby_version = ">= 3.0" 22 | 23 | gem.add_runtime_dependency "highline" 24 | gem.add_runtime_dependency "logger" 25 | gem.add_runtime_dependency "mail", "2.7.1" 26 | gem.add_runtime_dependency "net-imap", ">= 0.3.2" 27 | gem.add_runtime_dependency "net-smtp" 28 | gem.add_runtime_dependency "os" 29 | gem.add_runtime_dependency "ostruct" 30 | gem.add_runtime_dependency "rake" 31 | gem.add_runtime_dependency "thor", "~> 1.1" 32 | gem.add_runtime_dependency "thunderbird", "0.3.0" 33 | 34 | gem.metadata = { 35 | "rubygems_mfa_required" => "true" 36 | } 37 | end 38 | -------------------------------------------------------------------------------- /spec/features/setup_spec.rb: -------------------------------------------------------------------------------- 1 | require "features/helper" 2 | 3 | RSpec.describe "imap-backup setup", :container, type: :aruba do 4 | let(:config_options) { {accounts: []} } 5 | let(:command) { "imap-backup setup" } 6 | let!(:setup) do 7 | create_config(**config_options) 8 | end 9 | 10 | it "shows the main menu" do 11 | run_command command 12 | last_command_started.write "q\n" 13 | last_command_started.stop 14 | 15 | expect(last_command_started).to have_output(/imap-backup - Main Menu/) 16 | end 17 | 18 | context "when the configuration file does not exist" do 19 | let(:setup) {} 20 | 21 | it "does not raise any errors" do 22 | run_command command 23 | last_command_started.write "q\n" 24 | last_command_started.stop 25 | 26 | expect(last_command_started).to have_exit_status(0) 27 | end 28 | end 29 | 30 | context "when a config path is supplied" do 31 | let(:account) { other_server_connection_parameters } 32 | let(:custom_config_path) { File.join(File.expand_path("~/.imap-backup"), "foo.json") } 33 | let(:config_options) { {path: custom_config_path, accounts: [account]} } 34 | let(:command) { "imap-backup setup --config #{custom_config_path}" } 35 | 36 | it "shows that configuration's accounts" do 37 | run_command command 38 | last_command_started.write "q\n" 39 | last_command_started.stop 40 | 41 | expect(last_command_started).to have_output(/1. #{account[:username]}/) 42 | end 43 | end 44 | end 45 | -------------------------------------------------------------------------------- /spec/unit/setup/connection_tester_spec.rb: -------------------------------------------------------------------------------- 1 | require "imap/backup/setup/connection_tester" 2 | 3 | module Imap::Backup 4 | RSpec.describe Setup::ConnectionTester do 5 | describe "#test" do 6 | subject { described_class.new(account) } 7 | 8 | let(:account) { instance_double(Account, client: client) } 9 | let(:client) { instance_double(Client::Default, login: nil) } 10 | 11 | describe "success" do 12 | it "attempts to login" do 13 | subject.test 14 | 15 | expect(client).to have_received(:login) 16 | end 17 | 18 | it "returns success" do 19 | expect(subject.test).to match(/successful/) 20 | end 21 | end 22 | 23 | describe "failure" do 24 | before do 25 | allow(client).to receive(:login).and_raise(error) 26 | end 27 | 28 | context "with no connection" do 29 | let(:error) do 30 | data = OpenStruct.new(text: "bar") 31 | response = OpenStruct.new(data: data) 32 | Net::IMAP::NoResponseError.new(response) 33 | end 34 | 35 | it "returns error" do 36 | expect(subject.test).to match(/no response/i) 37 | end 38 | end 39 | 40 | context "when caused by other errors" do 41 | let(:error) { "Error" } 42 | 43 | it "returns error" do 44 | expect(subject.test).to match(/unexpected error/i) 45 | end 46 | end 47 | end 48 | end 49 | end 50 | end 51 | -------------------------------------------------------------------------------- /lib/imap/backup/serializer/directory.rb: -------------------------------------------------------------------------------- 1 | require "os" 2 | 3 | require "imap/backup/file_mode" 4 | require "imap/backup/serializer/folder_maker" 5 | 6 | module Imap; end 7 | 8 | module Imap::Backup 9 | class Serializer; end 10 | 11 | # Ensures that serialization directories exist and have the correct permissions. 12 | class Serializer::Directory 13 | # The desired permissions for all directories that store backups 14 | DIRECTORY_PERMISSIONS = 0o700 15 | 16 | # @param path [String] The base path of the account backup 17 | # @param relative [String] The path relative from the base 18 | # 19 | # @return [void] 20 | def initialize(path, relative) 21 | @path = path 22 | @relative = relative 23 | end 24 | 25 | # Creates the directory, if present and sets it's access permissions 26 | # 27 | # @return [void] 28 | def ensure_exists 29 | if !File.directory?(full_path) 30 | Serializer::FolderMaker.new( 31 | base: path, path: relative, permissions: DIRECTORY_PERMISSIONS 32 | ).run 33 | end 34 | 35 | return if OS.windows? 36 | return if FileMode.new(filename: full_path).mode == DIRECTORY_PERMISSIONS 37 | 38 | FileUtils.chmod DIRECTORY_PERMISSIONS, full_path 39 | end 40 | 41 | private 42 | 43 | attr_reader :relative 44 | attr_reader :path 45 | 46 | def full_path 47 | containing_directory = File.join(path, relative) 48 | File.expand_path(containing_directory) 49 | end 50 | end 51 | end 52 | -------------------------------------------------------------------------------- /spec/features/remote/list_account_folders_spec.rb: -------------------------------------------------------------------------------- 1 | require "features/helper" 2 | 3 | RSpec.describe "imap-backup remote folders", :container, type: :aruba do 4 | let(:account) { test_server_connection_parameters } 5 | let(:config_options) { {accounts: [account]} } 6 | let(:command) { "imap-backup remote folders #{account[:username]}" } 7 | 8 | before do 9 | create_config(**config_options) 10 | end 11 | 12 | it "lists folders" do 13 | run_command_and_stop command 14 | 15 | expect(last_command_started).to have_output(/"INBOX"/) 16 | end 17 | 18 | context "when JSON is requested" do 19 | let(:command) { "imap-backup remote folders #{account[:username]} --format json" } 20 | 21 | it "lists folders as JSON" do 22 | run_command_and_stop command 23 | 24 | expect(last_command_started).to have_output(/\{"name":"INBOX"\}/) 25 | end 26 | end 27 | 28 | context "when a config path is supplied" do 29 | let(:custom_config_path) { File.join(File.expand_path("~/.imap-backup"), "foo.json") } 30 | let(:config_options) do 31 | {path: custom_config_path, accounts: [other_server_connection_parameters]} 32 | end 33 | let(:account) { other_server_connection_parameters } 34 | let(:command) do 35 | "imap-backup remote folders #{account[:username]} --config #{custom_config_path}" 36 | end 37 | 38 | it "lists folders" do 39 | run_command_and_stop command 40 | 41 | expect(last_command_started).to have_output(/"INBOX"/) 42 | end 43 | end 44 | end 45 | -------------------------------------------------------------------------------- /lib/imap/backup/account/backup_folders.rb: -------------------------------------------------------------------------------- 1 | require "imap/backup/account/folder" 2 | 3 | module Imap; end 4 | 5 | module Imap::Backup 6 | class Account; end 7 | 8 | # Enumerates over the account folders that are to be backed up 9 | class Account::BackupFolders 10 | include Enumerable 11 | 12 | def initialize(client:, account:) 13 | @client = client 14 | @account = account 15 | end 16 | 17 | # Runs the enumeration 18 | # @yieldparam folder [Account::Folder] the online folder 19 | # @return [void] 20 | def each(&block) 21 | return enum_for(:each) if !block 22 | 23 | all_names = client.list 24 | 25 | configured = 26 | case 27 | when account.folders&.any? 28 | account.folders 29 | when account.folder_blacklist 30 | [] 31 | else 32 | all_names 33 | end 34 | 35 | names = 36 | if account.folder_blacklist 37 | all_names - configured 38 | else 39 | all_names & configured 40 | end 41 | 42 | names.each { |name| block.call(Account::Folder.new(client, name)) } 43 | end 44 | 45 | # Runs a map operation over the folders 46 | # @yieldparam folder [Account::Folder] the online folder 47 | # @return The results of the map operation 48 | def map(&block) 49 | each.map do |folder| 50 | block.call(folder) 51 | end 52 | end 53 | 54 | private 55 | 56 | attr_reader :account 57 | attr_reader :client 58 | end 59 | end 60 | -------------------------------------------------------------------------------- /spec/unit/setup/global_options/download_strategy_chooser_spec.rb: -------------------------------------------------------------------------------- 1 | require "imap/backup/setup/global_options/download_strategy_chooser" 2 | 3 | module Imap::Backup 4 | RSpec.describe Setup::GlobalOptions::DownloadStrategyChooser do 5 | include HighLineTestHelpers 6 | 7 | subject { described_class.new(config: config) } 8 | 9 | let(:config) do 10 | instance_double( 11 | Configuration, 12 | download_strategy: "a", 13 | "download_strategy=": nil 14 | ) 15 | end 16 | let!(:highline_streams) { prepare_highline } 17 | let(:input) { highline_streams[0] } 18 | let(:output) { highline_streams[1] } 19 | 20 | before do 21 | allow(Kernel).to receive(:system) 22 | allow(Kernel).to receive(:puts) 23 | end 24 | 25 | it "clears the screen" do 26 | subject.run 27 | 28 | expect(Kernel).to have_received(:system).with("clear") 29 | end 30 | 31 | it "shows the menu" do 32 | subject.run 33 | 34 | expect(output.string).to match(/Choose a Download Strategy/) 35 | end 36 | 37 | it "accepts choices" do 38 | allow(input).to receive(:gets).and_return("2\n", "q\n") 39 | 40 | subject.run 41 | 42 | expect(config).to have_received(:download_strategy=).with("delay_metadata") 43 | end 44 | 45 | it "shows help" do 46 | allow(input).to receive(:gets).and_return("help\n", "q\n") 47 | 48 | subject.run 49 | 50 | expect(Kernel).to have_received(:puts).with(/This setting/) 51 | end 52 | end 53 | end 54 | -------------------------------------------------------------------------------- /lib/imap/backup/text/sanitizer.rb: -------------------------------------------------------------------------------- 1 | require "forwardable" 2 | 3 | module Imap; end 4 | 5 | module Imap::Backup 6 | module Text; end 7 | 8 | # Wraps standard output and hides passwords from debug output 9 | # Any text matching Net::IMAP debug output of passwords is sanitized 10 | class Text::Sanitizer 11 | extend Forwardable 12 | 13 | delegate puts: :output 14 | delegate write: :output 15 | 16 | # @param output [IO] the stream to write output to 17 | def initialize(output) 18 | @output = output 19 | @current = "" 20 | end 21 | 22 | # Outputs everything up to the last newline character, 23 | # storing whatever follows the newline. 24 | # @param args [Array] lines of text 25 | # @return [void] 26 | def print(*args) 27 | @current << args.join 28 | loop do 29 | line, newline, rest = @current.partition("\n") 30 | break if newline != "\n" 31 | 32 | clean = sanitize(line) 33 | output.puts clean 34 | @current = rest 35 | end 36 | end 37 | 38 | # Outputs any text still not printed 39 | # @return [void] 40 | def flush 41 | return if @current == "" 42 | 43 | clean = sanitize(@current) 44 | output.puts clean 45 | end 46 | 47 | private 48 | 49 | attr_reader :output 50 | 51 | def sanitize(text) 52 | # Hide password in Net::IMAP debug output 53 | text.gsub( 54 | /\A(C: RUBY\d+ LOGIN \S+) \S+/, 55 | "\\1 [PASSWORD REDACTED]" 56 | ) 57 | end 58 | end 59 | end 60 | -------------------------------------------------------------------------------- /lib/imap/backup/email/provider.rb: -------------------------------------------------------------------------------- 1 | require "imap/backup/email/provider/apple_mail" 2 | require "imap/backup/email/provider/fastmail" 3 | require "imap/backup/email/provider/gmail" 4 | require "imap/backup/email/provider/purelymail" 5 | require "imap/backup/email/provider/unknown" 6 | 7 | module Imap; end 8 | 9 | module Imap::Backup 10 | module Email; end 11 | 12 | # Provides a class factory for email account providers 13 | class Email::Provider 14 | # @param address [String] an email address 15 | # @return [Email::Provider::Fastmail, Email::Provider::GMail, Email::Provider::AppleMail, 16 | # Email::Provider::Purelymail, Email::Provider::Unknown] 17 | # an instance supplying default values for the email's account set up 18 | def self.for_address(address) 19 | # rubocop:disable Lint/DuplicateBranch 20 | case 21 | when address.end_with?("@fastmail.com") 22 | Email::Provider::Fastmail.new 23 | when address.end_with?("@fastmail.fm") 24 | Email::Provider::Fastmail.new 25 | when address.end_with?("@gmail.com") 26 | Email::Provider::GMail.new 27 | when address.end_with?("@icloud.com") 28 | Email::Provider::AppleMail.new 29 | when address.end_with?("@mac.com") 30 | Email::Provider::AppleMail.new 31 | when address.end_with?("@me.com") 32 | Email::Provider::AppleMail.new 33 | when address.end_with?("@purelymail.com") 34 | Email::Provider::Purelymail.new 35 | else 36 | Email::Provider::Unknown.new 37 | end 38 | # rubocop:enable Lint/DuplicateBranch 39 | end 40 | end 41 | end 42 | -------------------------------------------------------------------------------- /spec/unit/account/local_only_folder_deleter_spec.rb: -------------------------------------------------------------------------------- 1 | require "imap/backup/account/local_only_folder_deleter" 2 | 3 | require "imap/backup/account" 4 | require "imap/backup/serializer" 5 | 6 | module Imap::Backup 7 | RSpec.describe Account::LocalOnlyFolderDeleter do 8 | subject { described_class.new(account: account) } 9 | 10 | let(:account) { instance_double(Account, client: nil) } 11 | let(:backup_folders) { instance_double(Account::BackupFolders, map: online_folders) } 12 | let(:online_folders) { %w(server_only both) } 13 | let(:serialized_folders) { instance_double(Account::SerializedFolders) } 14 | let(:disk_only) { instance_double(Serializer, "disk_only", folder: "disk_only", delete: nil) } 15 | let(:both) { instance_double(Serializer, "both", folder: "both", delete: nil) } 16 | 17 | before do 18 | allow(Account::BackupFolders).to receive(:new) { backup_folders } 19 | allow(Account::SerializedFolders).to receive(:new) { serialized_folders } 20 | allow(serialized_folders).to receive(:each_key). 21 | and_yield(disk_only). 22 | and_yield(both) 23 | end 24 | 25 | context "with serialized folders" do 26 | context "when they are not on the server" do 27 | it "deletes them" do 28 | subject.run 29 | 30 | expect(disk_only).to have_received(:delete) 31 | end 32 | end 33 | 34 | context "when they are on the server" do 35 | it "doesn't delete them" do 36 | subject.run 37 | 38 | expect(both).to_not have_received(:delete) 39 | end 40 | end 41 | end 42 | end 43 | end 44 | -------------------------------------------------------------------------------- /spec/features/local/list_folders_spec.rb: -------------------------------------------------------------------------------- 1 | require "features/helper" 2 | 3 | RSpec.describe "imap-backup local folders", type: :aruba do 4 | let(:account) { test_server_connection_parameters } 5 | let(:configuration_path) { File.join(config_path, "config.json") } 6 | let(:config_options) { {accounts: [account]} } 7 | let(:command) { "imap-backup local folders #{account[:username]}" } 8 | 9 | before do 10 | create_config(**config_options) 11 | append_local( 12 | configuration_path: configuration_path, 13 | email: account[:username], 14 | folder: "my_folder", 15 | body: "Hi" 16 | ) 17 | 18 | run_command_and_stop command 19 | end 20 | 21 | it "lists folders that have been backed up" do 22 | expect(last_command_started).to have_output('"my_folder"') 23 | end 24 | 25 | context "when JSON is requested" do 26 | let(:command) { "imap-backup local folders #{account[:username]} --format json" } 27 | 28 | it "lists folders as JSON" do 29 | expect(last_command_started).to have_output(/\{"name":"my_folder"\}/) 30 | end 31 | end 32 | 33 | context "when a config path is supplied" do 34 | let(:custom_config_path) { File.join(File.expand_path("~/.imap-backup"), "foo.json") } 35 | let(:configuration_path) { custom_config_path } 36 | let(:config_options) { {path: custom_config_path, accounts: [account]} } 37 | let(:command) do 38 | "imap-backup local folders #{account[:username]} --config #{custom_config_path}" 39 | end 40 | 41 | it "lists folders" do 42 | expect(last_command_started).to have_output('"my_folder"') 43 | end 44 | end 45 | end 46 | -------------------------------------------------------------------------------- /spec/features/local/list_emails_spec.rb: -------------------------------------------------------------------------------- 1 | require "features/helper" 2 | 3 | RSpec.describe "imap-backup local list", type: :aruba do 4 | let(:email) { account[:username] } 5 | let(:account) { test_server_connection_parameters } 6 | let(:config_options) { {accounts: [account]} } 7 | let!(:setup) do 8 | create_config(**config_options) 9 | append_local email: email, folder: "my_folder", subject: "Ciao" 10 | end 11 | 12 | it "lists emails" do 13 | run_command_and_stop "imap-backup local list #{email} my_folder" 14 | 15 | expect(last_command_started).to have_output(/1: Ciao/) 16 | end 17 | 18 | context "when JSON is requested" do 19 | it "lists emails" do 20 | run_command_and_stop "imap-backup local list #{email} my_folder --format json" 21 | 22 | expect(last_command_started).to have_output(/"subject":"Ciao"/) 23 | end 24 | end 25 | 26 | context "when a config path is supplied" do 27 | let(:custom_config_path) { File.join(File.expand_path("~/.imap-backup"), "foo.json") } 28 | let(:account) { other_server_connection_parameters } 29 | let(:config_options) { {path: custom_config_path, accounts: [account]} } 30 | let(:setup) do 31 | create_config(**config_options) 32 | append_local( 33 | configuration_path: custom_config_path, email: email, folder: "my_folder", subject: "Ciao" 34 | ) 35 | end 36 | 37 | it "lists emails" do 38 | run_command_and_stop( 39 | "imap-backup local list #{email} my_folder --config #{custom_config_path}" 40 | ) 41 | 42 | expect(last_command_started).to have_output(/1: Ciao/) 43 | end 44 | end 45 | end 46 | -------------------------------------------------------------------------------- /spec/unit/client/automatic_login_wrapper_spec.rb: -------------------------------------------------------------------------------- 1 | require "imap/backup/client/automatic_login_wrapper" 2 | 3 | require "imap/backup/client/default" 4 | 5 | module Imap::Backup 6 | RSpec.describe Client::AutomaticLoginWrapper do 7 | subject { described_class.new(client: client) } 8 | 9 | let(:client) { instance_double(Client::Default, examine: nil, login: nil) } 10 | 11 | context "when #login is called" do 12 | it "logs in" do 13 | subject.login 14 | 15 | expect(client).to have_received(:login).once 16 | end 17 | end 18 | 19 | context "when any other method is called" do 20 | it "logs in first" do 21 | subject.examine("foo") 22 | 23 | expect(client).to have_received(:login) 24 | end 25 | end 26 | 27 | context "when further methods are called" do 28 | it "logs in first" do 29 | subject.examine("foo") 30 | subject.examine("bar") 31 | 32 | expect(client).to have_received(:login).once 33 | end 34 | end 35 | 36 | context "when #login is called explicitly more than once" do 37 | it "calls #login" do 38 | subject.login 39 | subject.login 40 | 41 | expect(client).to have_received(:login).twice 42 | end 43 | end 44 | 45 | context "when the first login attempt fails" do 46 | before do 47 | outcomes = [-> { raise EOFError }, -> { true }] 48 | allow(client).to receive(:login) { outcomes.shift.call } 49 | end 50 | 51 | it "retries" do 52 | subject.examine("bar") 53 | 54 | expect(client).to have_received(:login).twice 55 | end 56 | end 57 | end 58 | end 59 | -------------------------------------------------------------------------------- /spec/unit/retry_on_error_spec.rb: -------------------------------------------------------------------------------- 1 | require "imap/backup/retry_on_error" 2 | 3 | class Retrier 4 | class FooError < StandardError; end 5 | 6 | include Imap::Backup::RetryOnError 7 | 8 | def do_stuff(errors:, limit: 10, on_error: nil) 9 | calls = 0 10 | 11 | retry_on_error(errors: errors, limit: limit, on_error: on_error) do 12 | calls += 1 13 | raise FooError, "Failed!" if calls < 3 14 | 15 | 42 16 | end 17 | end 18 | end 19 | 20 | module Imap::Backup 21 | RSpec.describe RetryOnError do 22 | describe "#retry_on_error" do 23 | subject { Retrier.new } 24 | 25 | it "retries" do 26 | expect(subject.do_stuff(errors: [Retrier::FooError], limit: 3)).to eq(42) 27 | end 28 | 29 | context "when the block fails more than the limit" do 30 | it "fails" do 31 | expect do 32 | subject.do_stuff(errors: [Retrier::FooError], limit: 1) 33 | end.to raise_error(Retrier::FooError, /Failed/) 34 | end 35 | end 36 | 37 | context "when unexpected errors are raised" do 38 | it "fails" do 39 | expect do 40 | subject.do_stuff(errors: [RuntimeError]) 41 | end.to raise_error(Retrier::FooError, /Failed/) 42 | end 43 | end 44 | 45 | context "when an :on_error block is passed" do 46 | it "calls the block before retrying" do 47 | on_error_calls = 0 48 | error_proc = -> { on_error_calls += 1 } 49 | subject.do_stuff(errors: [Retrier::FooError], on_error: error_proc) 50 | 51 | expect(on_error_calls).to eq(2) 52 | end 53 | end 54 | end 55 | end 56 | end 57 | -------------------------------------------------------------------------------- /lib/imap/backup/setup/email_changer.rb: -------------------------------------------------------------------------------- 1 | require "imap/backup/email/provider" 2 | require "imap/backup/setup/asker" 3 | 4 | module Imap; end 5 | 6 | module Imap::Backup 7 | class Setup; end 8 | 9 | # Asks the user for a new email address 10 | class Setup::EmailChanger 11 | # @param account [Account] an Account 12 | # @param config [Configuration] the application configuration 13 | def initialize(account:, config:) 14 | @account = account 15 | @config = config 16 | end 17 | 18 | # Asks the user for an email address, 19 | # ensuring that the supplied address is not an existing account 20 | # @return [void] 21 | def run 22 | username = Setup::Asker.email(account.username) 23 | other_accounts = config.accounts.reject { |a| a == account } 24 | others = other_accounts.map(&:username) 25 | if others.include?(username) 26 | Kernel.puts( 27 | "There is already an account set up with that email address" 28 | ) 29 | else 30 | account.username = username 31 | if account.server.nil? || (account.server == "") 32 | default = default_server(username) 33 | account.server = default if default 34 | end 35 | end 36 | end 37 | 38 | private 39 | 40 | attr_reader :account 41 | attr_reader :config 42 | 43 | def default_server(username) 44 | provider = Email::Provider.for_address(username) 45 | 46 | if provider.is_a?(Email::Provider::Unknown) 47 | Kernel.puts "Can't decide provider for email address '#{username}'" 48 | return nil 49 | end 50 | 51 | provider.host 52 | end 53 | end 54 | end 55 | -------------------------------------------------------------------------------- /lib/imap/backup/setup/global_options.rb: -------------------------------------------------------------------------------- 1 | require "imap/backup/configuration" 2 | require "imap/backup/setup/global_options/download_strategy_chooser" 3 | 4 | module Imap; end 5 | 6 | module Imap::Backup 7 | class Setup; end 8 | 9 | # Shows the menu of global options 10 | class Setup::GlobalOptions 11 | # @param config [Configuration] the application configuration 12 | def initialize(config:) 13 | @config = config 14 | end 15 | 16 | # Shows the menu 17 | # @return [void] 18 | def run 19 | catch :done do 20 | loop do 21 | Kernel.system("clear") 22 | show_menu 23 | end 24 | end 25 | end 26 | 27 | private 28 | 29 | attr_reader :config 30 | 31 | def show_menu 32 | highline.choose do |menu| 33 | menu.header = <<~MENU.chomp 34 | Global Options 35 | 36 | These settings affect all accounts. 37 | 38 | Choose an action 39 | MENU 40 | change_download_strategy menu 41 | menu.choice("(q) return to main menu") { throw :done } 42 | menu.hidden("quit") { throw :done } 43 | end 44 | end 45 | 46 | def change_download_strategy(menu) 47 | strategies = Imap::Backup::Configuration::DOWNLOAD_STRATEGIES 48 | current = strategies.find { |s| s[:key] == config.download_strategy } 49 | changed = config.download_strategy_modified? ? " *" : "" 50 | menu.choice("change download strategy (currently: '#{current[:description]}')#{changed}") do 51 | DownloadStrategyChooser.new(config: config).run 52 | end 53 | end 54 | 55 | def highline 56 | Imap::Backup::Setup.highline 57 | end 58 | end 59 | end 60 | -------------------------------------------------------------------------------- /lib/imap/backup/account/backup.rb: -------------------------------------------------------------------------------- 1 | require "imap/backup/account/backup_folders" 2 | require "imap/backup/account/folder_backup" 3 | require "imap/backup/account/folder_ensurer" 4 | require "imap/backup/account/local_only_folder_deleter" 5 | 6 | module Imap; end 7 | 8 | module Imap::Backup 9 | class Account; end 10 | 11 | # Carries out the backup of the configured folders of the account 12 | class Account::Backup 13 | def initialize(account:, refresh: false) 14 | @account = account 15 | @refresh = refresh 16 | end 17 | 18 | # Runs the backup 19 | # @return [void] 20 | def run 21 | Logger.logger.info "Running backup of account '#{account.username}'" 22 | # start the connection so we get logging messages in the right order 23 | account.client.login 24 | 25 | run_pre_backup_tasks 26 | backup_folders = Account::BackupFolders.new( 27 | client: account.client, account: account 28 | ).to_a 29 | if backup_folders.none? 30 | Logger.logger.warn "No folders found to backup for account '#{account.username}'" 31 | return 32 | end 33 | Logger.logger.debug "Starting backup of #{backup_folders.count} folders" 34 | backup_folders.each do |folder| 35 | Account::FolderBackup.new(account: account, folder: folder, refresh: refresh).run 36 | end 37 | Logger.logger.debug "Backup of account '#{account.username}' complete" 38 | end 39 | 40 | private 41 | 42 | attr_reader :account 43 | attr_reader :refresh 44 | 45 | def run_pre_backup_tasks 46 | Account::FolderEnsurer.new(account: account).run 47 | Account::LocalOnlyFolderDeleter.new(account: account).run if account.mirror_mode 48 | end 49 | end 50 | end 51 | -------------------------------------------------------------------------------- /spec/unit/account/restore_spec.rb: -------------------------------------------------------------------------------- 1 | require "imap/backup/account/restore" 2 | 3 | module Imap::Backup 4 | RSpec.describe Account::Restore do 5 | subject { described_class.new(account: account, **options) } 6 | 7 | let(:account) { "account" } 8 | let(:options) { {} } 9 | let(:folder_mapper) { instance_double(Account::FolderMapper) } 10 | let(:uploader) { instance_double(Uploader, run: nil) } 11 | 12 | before do 13 | allow(Account::FolderMapper).to receive(:new) { folder_mapper } 14 | allow(folder_mapper).to receive(:each).and_yield("serializer", "folder") 15 | allow(Uploader).to receive(:new) { uploader } 16 | end 17 | 18 | it "runs the uploader" do 19 | subject.run 20 | 21 | expect(uploader).to have_received(:run) 22 | end 23 | 24 | context "when a delimiter is provided" do 25 | let(:options) { {delimiter: "."} } 26 | let(:delimited_folder) { instance_double(Account::Folder) } 27 | let(:serializer) { instance_double(Serializer) } 28 | 29 | it "maps destination folders with the delimiter" do 30 | subject.run 31 | 32 | expect(Account::FolderMapper).to have_received(:new). 33 | with(hash_including(destination_delimiter: ".")) 34 | end 35 | end 36 | 37 | context "when a prefix is provided" do 38 | let(:options) { {prefix: "."} } 39 | let(:delimited_folder) { instance_double(Account::Folder) } 40 | let(:serializer) { instance_double(Serializer) } 41 | 42 | it "maps destination folders with the prefix" do 43 | subject.run 44 | 45 | expect(Account::FolderMapper).to have_received(:new). 46 | with(hash_including(destination_prefix: ".")) 47 | end 48 | end 49 | end 50 | end 51 | -------------------------------------------------------------------------------- /docs/commands/mirror.md: -------------------------------------------------------------------------------- 1 | 4 | # Mirror 5 | 6 | ```sh 7 | imap-backup mirror SOURCE_EMAIL DESTINATION_EMAIL 8 | ``` 9 | 10 | This command is deprecated and will be removed in a future version. Use [copy](./copy.md). 11 | 12 | This command makes a local copy of the emails in the source account 13 | and then copies them to the destination account. 14 | 15 | Exactly which folders are backed up (and mirrored) depends on how the account is set up. 16 | 17 | Specifically, the `folder inclusion mode (whitelist/blacklist)` and 18 | `folders to include/exclude` list. 19 | 20 | Note that, anything on the destination account that is not on the source account gets deleted. 21 | 22 | # Options 23 | 24 | * `--source-delimiter` - the separator between the elements of folders names 25 | on the source server, defaults to `/`, 26 | * `--source-prefix` - optionally, a prefix element to remove from the name 27 | of source folders, 28 | * `--destination-delimiter` - the separator between the elements of folder 29 | names on the destination server, defaults to `/`, 30 | * `--destination-prefix` - optionally, a prefix element to add before names 31 | on the destination server, 32 | * `--automatic-namespaces` - works out the 4 parameters above by querying 33 | the source and destination IMAP servers. 34 | 35 | ## Modes 36 | 37 | If the local copy is in 'keep all' mode, the destination account will gradually have more and more emails. 38 | 39 | On the other hand, if the local copy is in 'mirror' mode, the destination account will have the same emails 40 | as the source account. 41 | 42 | # Delimiters and Prefixes 43 | 44 | For details of the delimiter and prefix options, 45 | see [the note about delimiters and prefixes](../delimiters-and-prefixes.md). 46 | -------------------------------------------------------------------------------- /lib/imap/backup/cli/restore.rb: -------------------------------------------------------------------------------- 1 | require "thor" 2 | 3 | require "imap/backup/account/restore" 4 | require "imap/backup/cli/helpers" 5 | require "imap/backup/logger" 6 | 7 | module Imap; end 8 | 9 | module Imap::Backup 10 | class CLI < Thor; end 11 | 12 | # Restores backups for one or more accounts 13 | class CLI::Restore < Thor 14 | include Thor::Actions 15 | include CLI::Helpers 16 | 17 | def initialize(email = nil, options) 18 | super([]) 19 | @email = email 20 | @options = options 21 | end 22 | 23 | # @!method run 24 | # @raise [RuntimeError] if no email is specified 25 | # @return [void] 26 | no_commands do 27 | def run 28 | config = load_config(**options) 29 | case 30 | when email && !options.key?(:accounts) 31 | account = account(config, email) 32 | restore(account, **restore_options) 33 | when !email && !options.key?(:accounts) 34 | Logger.logger.info "Calling restore without an EMAIL parameter is deprecated" 35 | config.accounts.each { |a| restore(a) } 36 | when email && options.key?(:accounts) 37 | raise "Missing EMAIL parameter" 38 | when !email && options.key?(:accounts) 39 | Logger.logger.info( 40 | "Calling restore with the --account option is deprecated, " \ 41 | "please pass a single EMAIL parameter" 42 | ) 43 | requested_accounts(config).each { |a| restore(a) } 44 | end 45 | end 46 | end 47 | 48 | private 49 | 50 | attr_reader :email 51 | attr_reader :options 52 | 53 | def restore(account, **options) 54 | restore = Account::Restore.new(account: account, **options) 55 | restore.run 56 | end 57 | 58 | def restore_options 59 | options.slice(:delimiter, :prefix) 60 | end 61 | end 62 | end 63 | -------------------------------------------------------------------------------- /lib/imap/backup/serializer/transaction.rb: -------------------------------------------------------------------------------- 1 | module Imap; end 2 | 3 | module Imap::Backup 4 | class Serializer; end 5 | 6 | # Stores data during a transaction 7 | class Serializer::Transaction 8 | # @return the transaction's stored data 9 | attr_reader :data 10 | 11 | # @param owner [any] the class using the transaction - 12 | # this is used when raising errors 13 | def initialize(owner:) 14 | @data = nil 15 | @owner = owner 16 | @in_transaction = false 17 | end 18 | 19 | # Runs the transaction 20 | # @param data [any] the data to maintain during the transaction 21 | # @param block [block] the block to wrap with the transaction 22 | # @return [void] 23 | def begin(data, &block) 24 | @data = data 25 | @in_transaction = true 26 | block.call 27 | @in_transaction = false 28 | end 29 | 30 | # Clears rollback data 31 | # @return [void] 32 | def clear 33 | @data = nil 34 | end 35 | 36 | def in_transaction? 37 | @in_transaction 38 | end 39 | 40 | # Throws an exception if there is a current transaction 41 | # @param method [Symbol] the method where the check is run 42 | # @raise [RuntimeError] if called from inside a transaction 43 | # @return [void] 44 | def fail_in_transaction!(method, message: "not supported inside trasactions") 45 | raise "#{owner.class}##{method} #{message}" if in_transaction? 46 | end 47 | 48 | # Throws an exception if there is not a current transaction 49 | # @param method [Symbol] the method where the check is run 50 | # @raise [RuntimeError] if called from outside a transaction 51 | # @return [void] 52 | def fail_outside_transaction!(method) 53 | raise "#{owner.class}##{method} can only be called inside a transaction" if !in_transaction? 54 | end 55 | 56 | private 57 | 58 | attr_reader :owner 59 | end 60 | end 61 | -------------------------------------------------------------------------------- /lib/imap/backup/setup/asker.rb: -------------------------------------------------------------------------------- 1 | module Imap; end 2 | 3 | module Imap::Backup 4 | class Setup; end 5 | 6 | # Implements interactively requesting information from the user 7 | class Setup::Asker 8 | # @param highline [Higline] the configured Highline instance 9 | def initialize(highline) 10 | @highline = highline 11 | end 12 | 13 | # Asks for a email address 14 | # 15 | # @param default [String] the existing email address 16 | # @return [String] the email address 17 | def email(default = "") 18 | highline.ask("email address: ") do |q| 19 | q.default = default 20 | q.validate = EMAIL_MATCHER 21 | q.responses[:not_valid] = "Enter a valid email address " 22 | end 23 | end 24 | 25 | # Asks for a password 26 | # 27 | # @return [String] the password 28 | def password 29 | password = highline.ask("password: ") { |q| q.echo = false } 30 | confirmation = highline.ask("repeat password: ") { |q| q.echo = false } 31 | if password != confirmation 32 | return nil if !highline.agree( 33 | "the password and confirmation did not match.\nContinue? (y/n) " 34 | ) 35 | 36 | return self.password 37 | end 38 | password 39 | end 40 | 41 | # Asks for a email address using the configured menu handler 42 | # 43 | # @param default [String] the existing email address 44 | # @return [String] the email address 45 | def self.email(default = "") 46 | new(Setup.highline).email(default) 47 | end 48 | 49 | # Asks for a password using the configured menu handler 50 | # 51 | # @return [String] the password 52 | def self.password 53 | new(Setup.highline).password 54 | end 55 | 56 | private 57 | 58 | EMAIL_MATCHER = /^[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]+$/i 59 | 60 | attr_reader :highline 61 | end 62 | end 63 | -------------------------------------------------------------------------------- /lib/imap/backup/serializer/message.rb: -------------------------------------------------------------------------------- 1 | require "forwardable" 2 | 3 | require "imap/backup/email/mboxrd/message" 4 | 5 | module Imap; end 6 | 7 | module Imap::Backup 8 | class Serializer; end 9 | 10 | # Represents a stored message 11 | class Serializer::Message 12 | # @return [Array[Symbol]] the message's flags 13 | attr_accessor :flags 14 | # @return [Integer] the length of the message (as stored on disk) 15 | attr_reader :length 16 | # @return [Integer] the start of the message inside the mailbox file 17 | attr_reader :offset 18 | # @return [Integer] the message's UID 19 | attr_accessor :uid 20 | 21 | extend Forwardable 22 | 23 | def_delegator :message, :supplied_body, :body 24 | def_delegators :message, :imap_body, :date, :subject 25 | 26 | # @param uid [Integer] the message's UID 27 | # @param offset [Integer] the start of the message inside the mailbox file 28 | # @param length [Integer] the length of the message (as stored on disk) 29 | # @param mbox [Serializer::Mbox] the mailbox containing the message 30 | # @param flags [Array[Symbol]] the message's flags 31 | def initialize(uid:, offset:, length:, mbox:, flags: []) 32 | @uid = uid 33 | @offset = offset 34 | @length = length 35 | @mbox = mbox 36 | @flags = flags.map(&:to_sym) 37 | end 38 | 39 | # @return [Hash] the message metadata 40 | def to_h 41 | { 42 | uid: uid, 43 | offset: offset, 44 | length: length, 45 | flags: flags.map(&:to_s) 46 | } 47 | end 48 | 49 | # Reads the message text and returns the original form 50 | # @return [String] the message 51 | def message 52 | @message = 53 | begin 54 | raw = mbox.read(offset, length) 55 | Email::Mboxrd::Message.from_serialized(raw) 56 | end 57 | end 58 | 59 | private 60 | 61 | attr_reader :mbox 62 | end 63 | end 64 | -------------------------------------------------------------------------------- /spec/unit/naming_spec.rb: -------------------------------------------------------------------------------- 1 | require "imap/backup/naming" 2 | 3 | module Imap::Backup 4 | RSpec.describe Naming do 5 | describe ".to_local_path" do 6 | it "returns a String" do 7 | expect(described_class.to_local_path("ciao")).to be_a(String) 8 | end 9 | 10 | context "when there are unacceptable characters" do 11 | it "replaces them" do 12 | expect(described_class.to_local_path("c:a%")).to eq("c%3a;a%25;") 13 | end 14 | end 15 | 16 | context "when there are no unacceptable characters" do 17 | it "returns the text unchanged" do 18 | expect(described_class.to_local_path("ciao")).to eq("ciao") 19 | end 20 | end 21 | 22 | [ 23 | [":", "%3a;"], 24 | ["%", "%25;"], 25 | [";", "%3b;"], 26 | ["/", "/"], 27 | ["\\", "\\"], 28 | ["≈", "≈"] 29 | ].each do |char, expected| 30 | if char == expected 31 | it "does not convert '#{char}'" do 32 | expect(described_class.to_local_path(char)).to eq(char) 33 | end 34 | else 35 | it "converts '#{char}' to '#{expected}'" do 36 | expect(described_class.to_local_path(char)).to eq(expected) 37 | end 38 | end 39 | end 40 | end 41 | 42 | describe ".from_local_path" do 43 | it "returns a string" do 44 | expect(described_class.from_local_path("ciao")).to be_a(String) 45 | end 46 | 47 | context "when there are encoded characters" do 48 | it "reconverts them to the original" do 49 | expect(described_class.from_local_path("c%3a;a%25;")).to eq("c:a%") 50 | end 51 | end 52 | 53 | context "when there are no encoded characters" do 54 | it "returns the text unchanged" do 55 | expect(described_class.from_local_path("ciao")).to eq("ciao") 56 | end 57 | end 58 | end 59 | end 60 | end 61 | -------------------------------------------------------------------------------- /spec/features/regressions/migrate_legacy_backups_spec.rb: -------------------------------------------------------------------------------- 1 | require "features/helper" 2 | 3 | RSpec.describe "imap-backup migrate: avoid regression in migrating legacy backups", 4 | :container, type: :aruba do 5 | def overwrite_metadata_with_old_version(email, folder) 6 | content = imap_parsed(email, folder) 7 | uids = content[:messages].map { |m| m[:uid] } 8 | uid_validity = content[:uid_validity] 9 | old_metadata = {version: 2, uids: uids, uid_validity: uid_validity} 10 | path = imap_path(email, folder) 11 | File.open(path, "w") { |f| f.write(JSON.pretty_generate(old_metadata)) } 12 | end 13 | 14 | let(:email) { "me@example.com" } 15 | let(:folder) { "migrate-folder" } 16 | let(:source_account) do 17 | { 18 | username: email, 19 | password: "password1", 20 | local_path: File.join(config_path, email.gsub("@", "_")) 21 | } 22 | end 23 | let(:destination_account) { test_server_connection_parameters } 24 | let(:destination_server) { test_server } 25 | let(:config_options) { {accounts: [source_account, destination_account]} } 26 | 27 | let!(:setup) do 28 | test_server.warn_about_non_default_folders 29 | create_config(**config_options) 30 | append_local( 31 | email: email, folder: folder, subject: "Ciao", flags: [:Draft, :$CUSTOM] 32 | ) 33 | overwrite_metadata_with_old_version(email, folder) 34 | end 35 | 36 | after do 37 | destination_server.delete_folder folder 38 | destination_server.disconnect 39 | end 40 | 41 | it "copies emails to the destination account" do 42 | run_command_and_stop "imap-backup migrate #{email} #{destination_account[:username]}" 43 | 44 | messages = test_server.folder_messages(folder) 45 | expected = <<~MESSAGE.gsub("\n", "\r\n") 46 | From: sender@example.com 47 | Subject: Ciao 48 | 49 | body 50 | 51 | MESSAGE 52 | expect(messages[0]["BODY[]"]).to eq(expected) 53 | end 54 | end 55 | -------------------------------------------------------------------------------- /docs/performance.md: -------------------------------------------------------------------------------- 1 | # Performance 2 | 3 | The two performance-related settings are "Download strategy", which is a global setting, and "Multi-fetch size", which is an Account-level setting. 4 | 5 | As with all performance tweaks, there are trade-offs. 6 | 7 | # Overview 8 | 9 | The defaults, which suit most machines and play nice with servers are: 10 | 11 | * Download strategy: "delay writing metadata", 12 | * Multi-fetch size: 1. 13 | 14 | If you are using a resource-limited machine like 15 | a small virtual server or Raspberry Pi 16 | to run your backups, you can change "Download strategy". 17 | 18 | If your email provider supports it, 19 | and you don't have tight memory limits, 20 | increase "Multi-fetch size" for faster backups. 21 | 22 | # Delay download writes 23 | 24 | This is a global setting, affecting all account backups. 25 | 26 | By default, `imap-backup` uses the "delay writing metadata" strategy. 27 | As messages are being backed-up, the message *text* 28 | is written to disk, while the related metadata is stored in memory. 29 | 30 | While this uses a *little* more memory, it avoids rewiting a growing JSON 31 | file for every message, speeding things up and reducing disk wear. 32 | 33 | The alternative strategy, called "write straight to disk", 34 | writes everything to disk as it is received. 35 | This method is slower, but has the advantage 36 | of using slightly less memory, which may be important on very 37 | resource-limited systems, like Raspberry Pis. 38 | 39 | # Multi-fetch Size 40 | 41 | By default, during backup, each message is downloaded one-by-one. 42 | 43 | Using this setting, you can download chunks of emails at a time, 44 | potentially speeding up the process. 45 | 46 | Using multi-fetch means that the backup process *will* use 47 | more memory - equivalent to the size of the groups of messages 48 | that are downloaded. 49 | 50 | This behaviour may also exceed the rate limits on your email provider, 51 | so it's best to check before cranking it up! 52 | -------------------------------------------------------------------------------- /.rubocop_todo.yml: -------------------------------------------------------------------------------- 1 | # This configuration was generated by 2 | # `rubocop --auto-gen-config` 3 | # on 2025-08-11 12:48:04 UTC using RuboCop version 1.78.0. 4 | # The point is for the user to remove these configuration records 5 | # one by one as the offenses are removed from the code base. 6 | # Note that changes in the inspected code, or installation of new 7 | # versions of RuboCop, may require this file to be generated again. 8 | 9 | # Offense count: 44 10 | # Configuration parameters: AllowedMethods, AllowedPatterns, CountRepeatedAttributes. 11 | Metrics/AbcSize: 12 | Max: 35 13 | 14 | # Offense count: 2 15 | # Configuration parameters: CountComments, CountAsOne, AllowedMethods, AllowedPatterns, inherit_mode. 16 | # AllowedMethods: refine 17 | Metrics/BlockLength: 18 | Max: 47 19 | 20 | # Offense count: 13 21 | # Configuration parameters: CountComments, CountAsOne. 22 | Metrics/ClassLength: 23 | Max: 180 24 | 25 | # Offense count: 5 26 | # Configuration parameters: AllowedMethods, AllowedPatterns. 27 | Metrics/CyclomaticComplexity: 28 | Max: 11 29 | 30 | # Offense count: 80 31 | # Configuration parameters: CountComments, CountAsOne, AllowedMethods, AllowedPatterns. 32 | Metrics/MethodLength: 33 | Max: 49 34 | 35 | # Offense count: 2 36 | # Configuration parameters: CountKeywordArgs, MaxOptionalParameters. 37 | Metrics/ParameterLists: 38 | Max: 8 39 | 40 | # Offense count: 3 41 | # Configuration parameters: AllowedMethods, AllowedPatterns. 42 | Metrics/PerceivedComplexity: 43 | Max: 11 44 | 45 | # Offense count: 33 46 | # Configuration parameters: CountAsOne. 47 | RSpec/ExampleLength: 48 | Max: 16 49 | 50 | # Offense count: 2 51 | # This cop supports safe autocorrection (--autocorrect). 52 | # Configuration parameters: MinBodyLength, AllowConsecutiveConditionals. 53 | Style/GuardClause: 54 | Exclude: 55 | - 'lib/imap/backup/serializer/integrity_checker.rb' 56 | 57 | # Offense count: 1 58 | Style/OptionalArguments: 59 | Exclude: 60 | - 'lib/imap/backup/cli/restore.rb' 61 | -------------------------------------------------------------------------------- /.github/workflows/main.yml: -------------------------------------------------------------------------------- 1 | name: Tests 2 | 3 | on: 4 | push: 5 | branches: 6 | - main 7 | - feature/* 8 | 9 | pull_request: 10 | 11 | jobs: 12 | build: 13 | runs-on: ubuntu-latest 14 | name: Ruby ${{ matrix.ruby }} 15 | strategy: 16 | matrix: 17 | ruby: 18 | - '3.0' 19 | - '3.1' 20 | - '3.2' 21 | - '3.3' 22 | - '3.4' 23 | 24 | steps: 25 | - uses: actions/checkout@v4 26 | - name: Set up Ruby 27 | uses: ruby/setup-ruby@v1 28 | with: 29 | ruby-version: ${{ matrix.ruby }} 30 | rubygems: 3.0.3 31 | bundler-cache: true 32 | - name: Run the default task 33 | run: bundle exec rake 34 | - name: Read Coverage 35 | run: | 36 | echo "COVERAGE=$(cat coverage/coverage_percent.txt)%" >> $GITHUB_ENV 37 | - name: Create Coverage Badge 38 | if: ${{ github.ref == 'refs/heads/main' && matrix.ruby == '3.2' }} 39 | uses: schneegans/dynamic-badges-action@v1.6.0 40 | with: 41 | auth: ${{ secrets.BADGES_GIST_ACCESS }} 42 | gistID: b54fe758bfb405c04bef72dad293d707 43 | filename: coverage.json 44 | label: Coverage 45 | message: ${{ env.COVERAGE }} 46 | color: brightgreen 47 | 48 | services: 49 | imap: 50 | image: ghcr.io/joeyates/docker-imap-devel@sha256:6d6a64c32e2c583222d75286aa46a04ada5aea76efa36117815bf0e19d5063b6 51 | ports: 52 | - "8993:993" 53 | env: 54 | MAILNAME: example.com 55 | MAIL_ADDRESS: address@example.com 56 | MAIL_PASS: pass 57 | other-imap: 58 | image: ghcr.io/joeyates/docker-imap-devel@sha256:6d6a64c32e2c583222d75286aa46a04ada5aea76efa36117815bf0e19d5063b6 59 | ports: 60 | - "9993:993" 61 | env: 62 | MAILNAME: other.org 63 | MAIL_ADDRESS: email@other.org 64 | MAIL_PASS: pass 65 | DOVECOT_PUBLIC_NAMESPACE_PREFIX: other_public 66 | -------------------------------------------------------------------------------- /spec/features/local/show_an_email_spec.rb: -------------------------------------------------------------------------------- 1 | require "features/helper" 2 | 3 | RSpec.describe "imap-backup local show", type: :aruba do 4 | include_context "message-fixtures" 5 | 6 | let(:email) { account[:username] } 7 | let(:account) { test_server_connection_parameters } 8 | let(:config_options) { {accounts: [account]} } 9 | let!(:setup) do 10 | create_config(**config_options) 11 | append_local email: account[:username], folder: "my_folder", **message_one, uid: 99 12 | end 13 | 14 | it "shows the email" do 15 | run_command_and_stop "imap-backup local show #{email} my_folder 99" 16 | 17 | expect(last_command_started).to have_output(to_serialized(**message_one)) 18 | end 19 | 20 | context "when JSON is requested" do 21 | it "shows the email" do 22 | run_command_and_stop "imap-backup local show #{email} my_folder 99 --format json" 23 | 24 | expected = /"body":"#{to_serialized(**message_one)}\n"/ 25 | expect(last_command_started).to have_output(expected) 26 | end 27 | end 28 | 29 | context "when a config path is supplied" do 30 | let(:account) { other_server_connection_parameters } 31 | let(:custom_config_path) { File.join(File.expand_path("~/.imap-backup"), "foo.json") } 32 | let(:config_options) { {path: custom_config_path, accounts: [account]} } 33 | let!(:setup) do 34 | create_config(**config_options) 35 | append_local( 36 | configuration_path: custom_config_path, 37 | email: account[:username], 38 | folder: "my_folder", 39 | **message_one, 40 | uid: 99 41 | ) 42 | end 43 | let(:command) { "imap-backup local show #{email} --config #{custom_config_path}" } 44 | 45 | it "shows emails correctly" do 46 | run_command_and_stop( 47 | "imap-backup local show #{email} my_folder 99 --config #{custom_config_path}" 48 | ) 49 | 50 | expect(last_command_started).to have_output(to_serialized(**message_one)) 51 | end 52 | end 53 | end 54 | -------------------------------------------------------------------------------- /spec/unit/setup/global_options_spec.rb: -------------------------------------------------------------------------------- 1 | require "imap/backup/setup/global_options" 2 | 3 | module Imap::Backup 4 | RSpec.describe Setup::GlobalOptions do 5 | include HighLineTestHelpers 6 | 7 | subject { described_class.new(config: config) } 8 | 9 | let(:config) do 10 | instance_double( 11 | Configuration, 12 | download_strategy: "delay_metadata", 13 | download_strategy_modified?: download_strategy_modified 14 | ) 15 | end 16 | let!(:highline_streams) { prepare_highline } 17 | let(:input) { highline_streams[0] } 18 | let(:output) { highline_streams[1] } 19 | let(:download_strategy_chooser) do 20 | instance_double(Setup::GlobalOptions::DownloadStrategyChooser, run: nil) 21 | end 22 | let(:download_strategy_modified) { false } 23 | 24 | before do 25 | allow(Kernel).to receive(:system) 26 | allow(Setup::GlobalOptions::DownloadStrategyChooser). 27 | to receive(:new) { download_strategy_chooser } 28 | end 29 | 30 | it "clears the screen" do 31 | subject.run 32 | 33 | expect(Kernel).to have_received(:system).with("clear") 34 | end 35 | 36 | it "shows the menu" do 37 | subject.run 38 | 39 | expect(output.string).to match(/Global Options/) 40 | end 41 | 42 | it "shows the current download strategy" do 43 | subject.run 44 | 45 | expect(output.string).to match(/currently: 'delay writing metadata'/) 46 | end 47 | 48 | context "when the download strategy has been modified" do 49 | let(:download_strategy_modified) { true } 50 | 51 | it "shows a modified indicator" do 52 | subject.run 53 | 54 | expect(output.string).to match(/currently: 'delay writing metadata'\) \*/) 55 | end 56 | end 57 | 58 | it "accepts choices" do 59 | allow(input).to receive(:gets).and_return("1\n", "q\n") 60 | 61 | subject.run 62 | 63 | expect(download_strategy_chooser).to have_received(:run) 64 | end 65 | end 66 | end 67 | -------------------------------------------------------------------------------- /spec/unit/migrator_spec.rb: -------------------------------------------------------------------------------- 1 | require "imap/backup/migrator" 2 | 3 | require "imap/backup/account/folder" 4 | require "imap/backup/serializer" 5 | require "imap/backup/serializer/message" 6 | 7 | module Imap::Backup 8 | RSpec.describe Migrator do 9 | subject { described_class.new(serializer, folder, reset: reset) } 10 | 11 | let(:serializer) { instance_double(Serializer, uids: [1]) } 12 | let(:folder) do 13 | instance_double( 14 | Account::Folder, 15 | append: nil, clear: nil, create: nil, name: "name", uids: folder_uids 16 | ) 17 | end 18 | let(:folder_uids) { [] } 19 | let(:reset) { false } 20 | let(:message) do 21 | instance_double( 22 | Imap::Backup::Serializer::Message, 23 | uid: 33, 24 | body: "foo", 25 | flags: [:MyFlag] 26 | ) 27 | end 28 | let(:body) { "body" } 29 | 30 | before do 31 | allow(serializer).to receive(:each_message) do 32 | [message].enum_for(:each) 33 | end 34 | end 35 | 36 | it "creates the folder" do 37 | subject.run 38 | 39 | expect(folder).to have_received(:create) 40 | end 41 | 42 | it "uploads messages" do 43 | subject.run 44 | 45 | expect(folder).to have_received(:append).with(message) 46 | end 47 | 48 | context "when the folder is not empty" do 49 | let(:folder_uids) { [99] } 50 | 51 | it "works normally" do 52 | expect { subject.run }.to_not raise_error 53 | end 54 | 55 | context "when `reset` is true" do 56 | let(:reset) { true } 57 | 58 | it "clears the folder" do 59 | subject.run 60 | 61 | expect(folder).to have_received(:clear) 62 | end 63 | end 64 | end 65 | 66 | context "when the upload fails" do 67 | before do 68 | allow(folder).to receive(:append).and_raise(RuntimeError) 69 | end 70 | 71 | it "continues to work" do 72 | expect { subject.run }.to_not raise_error 73 | end 74 | end 75 | end 76 | end 77 | -------------------------------------------------------------------------------- /.github/workflows/publish-image.yaml: -------------------------------------------------------------------------------- 1 | name: Build and publish the production container image to the GitHub Registry 2 | 3 | # Run this workflow when new releases are created 4 | on: 5 | release: 6 | types: [published] 7 | 8 | env: 9 | IMAGE_NAME: imap-backup 10 | REGISTRY_USER: ${{ github.actor }} 11 | REGISTRY_PASSWORD: ${{ github.token }} 12 | REGISTRY: ghcr.io/${{ github.repository_owner }} 13 | 14 | jobs: 15 | build-and-push-image: 16 | runs-on: ubuntu-latest 17 | # Sets the permissions granted to the `GITHUB_TOKEN` for the actions in this job. 18 | permissions: 19 | contents: read 20 | packages: write 21 | steps: 22 | - name: Checkout repository 23 | uses: actions/checkout@v4 24 | 25 | - name: Get the latest tag 26 | run: | 27 | echo "GIT_TAG=$(git describe --tags `git rev-list --tags --max-count=1`)" >> $GITHUB_ENV 28 | 29 | - name: Build Image 30 | # https://github.com/marketplace/actions/buildah-build 31 | id: build-image 32 | uses: redhat-actions/buildah-build@v2 33 | with: 34 | image: ${{ env.IMAGE_NAME }} 35 | tags: latest ${{ github.sha }} ${{ env.GIT_TAG }} 36 | extra-args: | 37 | --ignorefile ./container/.containerignore 38 | containerfiles: | 39 | ./container/Containerfile 40 | labels: | 41 | ${{ env.IMAGE_NAME }}:latest 42 | 43 | # Check that imap-backup runs 44 | - name: Test Image 45 | run: podman run ${{ env.IMAGE_NAME }}:latest imap-backup help | grep 'Commands:' 46 | 47 | - name: Publish Image 48 | # https://github.com/marketplace/actions/push-to-registry 49 | id: push-to-registry 50 | uses: redhat-actions/push-to-registry@v2 51 | with: 52 | image: ${{ steps.build-image.outputs.image }} 53 | tags: ${{ steps.build-image.outputs.tags }} 54 | registry: ${{ env.REGISTRY }} 55 | username: ${{ env.REGISTRY_USER }} 56 | password: ${{ env.REGISTRY_PASSWORD }} 57 | -------------------------------------------------------------------------------- /spec/features/utils/ignore_history_spec.rb: -------------------------------------------------------------------------------- 1 | require "features/helper" 2 | 3 | RSpec.describe "imap-backup utils ignore-history", :container, type: :aruba do 4 | include_context "message-fixtures" 5 | 6 | let(:account_config) { test_server_connection_parameters.merge(folders: [folder]) } 7 | let(:email) { account_config[:username] } 8 | let(:folder) { "my_folder" } 9 | let(:config_options) { {accounts: [account_config]} } 10 | let(:expected_mbox_content) do 11 | <<~MESSAGE 12 | From fake@email.com 13 | From: fake@email.com 14 | Subject: Message 1 not backed up 15 | Skipped 1 16 | 17 | MESSAGE 18 | end 19 | 20 | let!(:setup) do 21 | test_server.warn_about_non_default_folders 22 | test_server.delete_folder folder 23 | test_server.create_folder folder 24 | test_server.send_email folder, **message_one 25 | create_config(**config_options) 26 | end 27 | 28 | after do 29 | test_server.delete_folder folder 30 | test_server.disconnect 31 | end 32 | 33 | it "fills the .imap file with dummy data" do 34 | run_command_and_stop "imap-backup utils ignore-history #{email}" 35 | 36 | content = imap_parsed(email, folder) 37 | 38 | expect(content[:messages].count).to eq(1) 39 | end 40 | 41 | it "fills the .mbox file with dummy data" do 42 | run_command_and_stop "imap-backup utils ignore-history #{email}" 43 | 44 | content = mbox_content(email, folder) 45 | 46 | expect(content).to eq(expected_mbox_content) 47 | end 48 | 49 | context "when a config path is supplied" do 50 | let(:custom_config_path) { File.join(File.expand_path("~/.imap-backup"), "foo.json") } 51 | let(:config_options) { super().merge(path: custom_config_path) } 52 | 53 | it "creates the required dummy messages" do 54 | run_command_and_stop( 55 | "imap-backup utils ignore-history #{email} --config #{custom_config_path}" 56 | ) 57 | 58 | content = imap_parsed(email, folder, configuration_path: custom_config_path) 59 | 60 | expect(content[:messages].count).to eq(1) 61 | end 62 | end 63 | end 64 | -------------------------------------------------------------------------------- /lib/imap/backup/cli/backup.rb: -------------------------------------------------------------------------------- 1 | require "net/imap" 2 | require "thor" 3 | 4 | require "imap/backup/account/backup" 5 | require "imap/backup/cli/helpers" 6 | require "imap/backup/logger" 7 | 8 | module Imap; end 9 | 10 | module Imap::Backup 11 | class CLI < Thor; end 12 | 13 | # Runs backups of configured accounts 14 | class CLI::Backup < Thor 15 | include Thor::Actions 16 | include CLI::Helpers 17 | 18 | def initialize(options) 19 | super([]) 20 | @options = options 21 | end 22 | 23 | # @!method run 24 | # @return [void] 25 | no_commands do 26 | def run 27 | Logger.logger.debug "Loading configuration" 28 | config = load_config(**options) 29 | exit_code = nil 30 | accounts = requested_accounts(config) 31 | # Filter to only include accounts available for backup 32 | accounts = accounts.select(&:available_for_backup?) 33 | if accounts.none? 34 | Logger.logger.warn "No matching accounts found to backup" 35 | return 36 | end 37 | Logger.logger.debug "Starting backup of #{accounts.count} accounts" 38 | accounts.each do |account| 39 | backup = Account::Backup.new(account: account, refresh: refresh) 40 | backup.run 41 | rescue StandardError => e 42 | exit_code ||= choose_exit_code(e) 43 | message = <<~ERROR 44 | Backup for account '#{account.username}' failed with error #{e} 45 | #{e.backtrace.join("\n")} 46 | ERROR 47 | Logger.logger.error message 48 | next 49 | end 50 | Logger.logger.debug "Backup complete" 51 | exit(exit_code) if exit_code 52 | end 53 | end 54 | 55 | private 56 | 57 | attr_reader :options 58 | 59 | def refresh 60 | options.key?(:refresh) ? !!options[:refresh] : false 61 | end 62 | 63 | def choose_exit_code(exception) 64 | case exception 65 | when Net::IMAP::NoResponseError, Errno::ECONNREFUSED 66 | 111 67 | else 68 | 1 69 | end 70 | end 71 | end 72 | end 73 | -------------------------------------------------------------------------------- /lib/imap/backup/cli/local/check.rb: -------------------------------------------------------------------------------- 1 | require "thor" 2 | 3 | require "imap/backup/account/serialized_folders" 4 | require "imap/backup/cli/helpers" 5 | require "imap/backup/serializer/integrity_checker" 6 | 7 | module Imap; end 8 | 9 | module Imap::Backup 10 | class CLI; end 11 | class CLI::Local < Thor; end 12 | 13 | # Runs integrity check on local backups 14 | class CLI::Local::Check 15 | include CLI::Helpers 16 | 17 | def initialize(options) 18 | @options = options 19 | end 20 | 21 | # Runs the check 22 | # @return [void] 23 | def run 24 | results = requested_accounts(config).map do |account| 25 | serialized_folders = Account::SerializedFolders.new(account: account) 26 | folder_results = serialized_folders.map do |serializer, _folder| 27 | serializer.check_integrity! 28 | {name: serializer.folder, result: "OK"} 29 | rescue Serializer::FolderIntegrityError => e 30 | message = e.to_s 31 | if options[:delete_corrupt] 32 | serializer.delete 33 | message << " and has been deleted" 34 | end 35 | 36 | { 37 | name: serializer.folder, 38 | result: message 39 | } 40 | end 41 | {account: account.username, folders: folder_results} 42 | end 43 | 44 | case options[:format] 45 | when "json" 46 | print_check_results_as_json(results) 47 | else 48 | print_check_results_as_text(results) 49 | end 50 | end 51 | 52 | private 53 | 54 | attr_reader :options 55 | 56 | def print_check_results_as_json(results) 57 | Kernel.puts results.to_json 58 | end 59 | 60 | def print_check_results_as_text(results) 61 | results.each do |account_results| 62 | Kernel.puts "Account: #{account_results[:account]}" 63 | account_results[:folders].each do |folder_results| 64 | Kernel.puts "\t#{folder_results[:name]}: #{folder_results[:result]}" 65 | end 66 | end 67 | end 68 | 69 | def config 70 | @config ||= load_config(**options) 71 | end 72 | end 73 | end 74 | -------------------------------------------------------------------------------- /spec/features/single/backup_spec.rb: -------------------------------------------------------------------------------- 1 | require "features/helper" 2 | 3 | RSpec.describe "imap-backup single backup", :container, type: :aruba do 4 | include_context "message-fixtures" 5 | 6 | let(:folder) { "single-backup" } 7 | let(:messages_as_mbox) do 8 | to_mbox_entry(**message_one) + to_mbox_entry(**message_two) 9 | end 10 | let(:account) { test_server_connection_parameters } 11 | let(:connection_options) { account[:connection_options].to_json } 12 | let(:command) do 13 | "imap-backup single backup " \ 14 | "--email #{account[:username]} " \ 15 | "--password #{account[:password]} " \ 16 | "--server #{account[:server]} " \ 17 | "--path #{account[:local_path]} " \ 18 | "--connection-options '#{connection_options}'" 19 | end 20 | 21 | before do 22 | test_server.warn_about_non_default_folders 23 | test_server.create_folder folder 24 | test_server.send_email folder, **message_one 25 | test_server.send_email folder, **message_two 26 | end 27 | 28 | after do 29 | test_server.delete_folder folder 30 | test_server.disconnect 31 | end 32 | 33 | it "downloads messages" do 34 | run_command_and_stop command 35 | 36 | actual = mbox_content(account[:username], folder, local_path: account[:local_path]) 37 | expect(actual).to eq(messages_as_mbox) 38 | end 39 | 40 | context "in mirror mode" do 41 | let(:imap_path) { File.join(account[:local_path], "Foo.imap") } 42 | let(:mbox_path) { File.join(account[:local_path], "Foo.mbox") } 43 | let(:command) { "#{super()} --mirror" } 44 | 45 | before do 46 | create_directory account[:local_path] 47 | File.write(imap_path, "existing imap") 48 | File.write(mbox_path, "existing mbox") 49 | end 50 | 51 | context "with folders that are not being backed up" do 52 | it "deletes .imap files" do 53 | run_command_and_stop command 54 | 55 | expect(File.exist?(imap_path)).to be false 56 | end 57 | 58 | it "deletes .mbox files" do 59 | run_command_and_stop command 60 | 61 | expect(File.exist?(mbox_path)).to be false 62 | end 63 | end 64 | end 65 | end 66 | -------------------------------------------------------------------------------- /ARCHITECTURE.md: -------------------------------------------------------------------------------- 1 | # Command Line 2 | 3 | `imap-backup` is a command line application. 4 | 5 | It is written in Ruby. 6 | 7 | Commands are processed by Thor modules - 8 | under the CLI namespace. 9 | 10 | # Setup 11 | 12 | The setup program uses the highline gem to 13 | present its menu-driven interface. 14 | 15 | # Serialization 16 | 17 | In an attempt to use a standard format, 18 | the program saves emails on disk in mboxrd 19 | files. 20 | The format was chosen for two reasons: 21 | mboxrd does not suffer from the problems related to 22 | serialising 'From ' headers and 23 | the Thunderbird email client uses this format. 24 | 25 | # Backup Strategies 26 | 27 | The backup system saves a metadata file alongside 28 | the mboxrd file. 29 | This file contains information about each email: 30 | its length and offset in the mboxrd file. 31 | If this file is rewritten when each new message is downloaded, 32 | the backup slows down progressively as the mailbox grows. 33 | To avoid this problem, the default strategy is to only write 34 | metadata at the end of the download for each folder. 35 | 36 | # Mirroring 37 | 38 | If an account is mirrored to another server, 39 | the emails on each server are different. 40 | In order to know which email on the mirror relates to 41 | which email on the source account, 42 | a map file is created. 43 | 44 | # Rubocop 45 | 46 | The project's code style is guaranteed by Rubocop rules. 47 | 48 | # Tests 49 | 50 | Tests use RSpec. There are three types: 51 | unit, integration ("feature") and performance. 52 | 53 | The performance tests are not for normal use - 54 | they are extremely slow, taking many hours to complete. 55 | 56 | The intention is to have almost complete coverage 57 | on two levels - unit and integration. 58 | 59 | The integration tests run the application using the 60 | aruba gem. 61 | 62 | These tests make connections to two containers 63 | running dovecot and postfix. This allows simulation 64 | of mirroring and migrations. 65 | 66 | # Documentation 67 | 68 | This project has two READMEs: 69 | 70 | * The {file:.github/README.md GitHub README} the end-user documentation which appears 71 | on GitHub, 72 | * {file:README.md} - the developer documentation. 73 | -------------------------------------------------------------------------------- /spec/unit/setup/backup_path_spec.rb: -------------------------------------------------------------------------------- 1 | require "imap/backup/setup/backup_path" 2 | 3 | module Imap::Backup 4 | RSpec.describe Setup::BackupPath do 5 | include HighLineTestHelpers 6 | 7 | subject { described_class.new(account: account, config: config) } 8 | 9 | let!(:highline_streams) { prepare_highline } 10 | let(:stdin) { highline_streams[0] } 11 | let(:stdout) { highline_streams[1] } 12 | let(:account) do 13 | instance_double( 14 | Account, 15 | username: "username@example.com", 16 | local_path: "/backup/path" 17 | ) 18 | end 19 | let(:account1) do 20 | instance_double(Account, username: "account1", local_path: other_existing_path) 21 | end 22 | let(:other_existing_path) { "/other/existing/path" } 23 | let(:accounts) { [account, account1] } 24 | let(:config) { instance_double(Configuration, accounts: accounts, path: "/config/path") } 25 | let(:new_backup_path) { "/new/path" } 26 | 27 | before do 28 | allow(Kernel).to receive(:puts) 29 | allow(account).to receive(:"local_path=") 30 | allow(Setup.highline).to receive(:get_response_line_mode) { new_backup_path } 31 | end 32 | 33 | context "with valid input" do 34 | it "asks for input" do 35 | subject.run 36 | 37 | expect(stdout.string).to match(%r(backup directory: \|/backup/path)) 38 | end 39 | 40 | it "updates the path" do 41 | subject.run 42 | 43 | expect(account).to have_received(:"local_path=").with(new_backup_path) 44 | end 45 | end 46 | 47 | context "when the path is used by other backups" do 48 | before do 49 | allow(Setup.highline).to receive(:get_response_line_mode). 50 | and_return(other_existing_path, new_backup_path) 51 | allow(Setup.highline).to receive(:say) 52 | 53 | subject.run 54 | end 55 | 56 | it "fails validation" do 57 | expect(Setup.highline).to have_received(:say).with("Choose a different directory ") 58 | end 59 | 60 | it "accepts a valid response" do 61 | expect(account).to have_received(:"local_path=").with(new_backup_path) 62 | end 63 | end 64 | end 65 | end 66 | -------------------------------------------------------------------------------- /dev/README.md: -------------------------------------------------------------------------------- 1 | # Containerized Rubies 2 | 3 | This directory contains some files that allow experimenting with 4 | any desired Ruby version locally. 5 | 6 | This is especially useful for older, deprecated Ruby versions 7 | which are often difficult to install due to openssl 8 | compatibility problems. 9 | 10 | The supplied `dev/ruby-compose.yml` starts the same 11 | two IMAP servers that are run under development and CI 12 | alongside a container with your chosen Ruby version. 13 | 14 | This container has the project root available as the `/app` 15 | directory so that you can run tests and edit code. 16 | 17 | # Start Containers 18 | 19 | Do the following from the project's root directory: 20 | 21 | ```sh 22 | export RUBY_VERSION=[VERSION] 23 | podman-compose --file dev/ruby-compose.yml build 24 | podman-compose --file dev/ruby-compose.yml up -d 25 | podman attach imap-backup 26 | ``` 27 | 28 | ...and stop the server afterwards: 29 | 30 | ```sh 31 | podman-compose --file dev/ruby-compose.yml down 32 | ``` 33 | 34 | # Setup Project 35 | 36 | It's best to delete any `Gemfile.lock` you may have 37 | in order to get gem versions which 38 | are compatible with the Ruby version you are using. 39 | 40 | ```sh 41 | rm Gemfile.lock 42 | bundle install 43 | ``` 44 | 45 | As the BUNDLE_BINSTUBS environment variable is set, 46 | we get a version of imap-backup that can be invoked 47 | without prepending `bundle exec`. 48 | 49 | The `PATH` environment variable includes `/app/bin/stubs`, 50 | so you can invoke imap-backup directly 51 | 52 | ```sh 53 | imap-backup help 54 | ``` 55 | 56 | # Run tests 57 | 58 | ```sh 59 | rake 60 | ``` 61 | 62 | # Connect to the Test IMAP Server 63 | 64 | An example file `dev/config.json` is supplied. 65 | 66 | The following should produce a list of folders 67 | 68 | ```sh 69 | imap-backup remote folders address@example.com -c dev/config.json 70 | ``` 71 | 72 | You can use the test helpers to interact with the test IMAP servers: 73 | 74 | ```sh 75 | $ pry 76 | > require "rspec" 77 | > require_relative "spec/features/support/30_email_server_helpers" 78 | > include EmailServerHelpers 79 | > test_server.send_email("INBOX", uid: 123, from: "address@example.org", subject: "Test 1", body: "body 1\nHi") 80 | ``` 81 | -------------------------------------------------------------------------------- /spec/features/stats_spec.rb: -------------------------------------------------------------------------------- 1 | require "features/helper" 2 | 3 | RSpec.describe "imap-backup stats", :container, type: :aruba do 4 | include_context "message-fixtures" 5 | 6 | let(:account) { test_server_connection_parameters } 7 | let(:folder) { "stats-stuff" } 8 | let(:command) { "imap-backup stats #{account[:username]}" } 9 | let(:config_options) { {accounts: [account]} } 10 | let!(:setup) do 11 | test_server.warn_about_non_default_folders 12 | test_server.create_folder folder 13 | test_server.send_email folder, **message_one 14 | test_server.disconnect 15 | create_config(**config_options) 16 | end 17 | 18 | after do 19 | test_server.delete_folder folder 20 | test_server.disconnect 21 | end 22 | 23 | it "lists messages to be backed up" do 24 | run_command_and_stop command 25 | 26 | expect(last_command_started).to have_output(/stats-stuff\s+\|\s+1\|\s+0\|\s+0/) 27 | end 28 | 29 | context "when JSON is requested" do 30 | let(:command) { "imap-backup stats #{account[:username]} --format json" } 31 | 32 | it "produces JSON" do 33 | run_command_and_stop command 34 | 35 | expect(last_command_started). 36 | to have_output(/\{"folder":"stats-stuff","remote":1,"both":0,"local":0\}/) 37 | end 38 | end 39 | 40 | context "when a config path is supplied" do 41 | let(:account) { other_server_connection_parameters } 42 | let(:custom_config_path) { File.join(File.expand_path("~/.imap-backup"), "foo.json") } 43 | let(:config_options) { {path: custom_config_path, accounts: [account]} } 44 | let(:command) do 45 | "imap-backup stats #{account[:username]} --config #{custom_config_path} --quiet" 46 | end 47 | let(:setup) do 48 | other_server.create_folder "other_public.ciao" 49 | other_server.send_email "other_public.ciao", **message_one 50 | other_server.disconnect 51 | create_config(**config_options) 52 | end 53 | 54 | after do 55 | other_server.delete_folder "other_public.ciao" 56 | other_server.disconnect 57 | end 58 | 59 | it "lists statistics" do 60 | run_command_and_stop command 61 | 62 | expect(last_command_started).to have_output(/other_public\.ciao\s+\|\s+1\|\s+0\|\s+0/) 63 | end 64 | end 65 | end 66 | -------------------------------------------------------------------------------- /lib/imap/backup/cli/options.rb: -------------------------------------------------------------------------------- 1 | require "thor" 2 | 3 | module Imap; end 4 | 5 | module Imap::Backup 6 | class CLI < Thor; end 7 | 8 | # Defines option methods for CLI classes 9 | class CLI::Options 10 | attr_reader :base 11 | 12 | # Options common to many commands 13 | OPTIONS = [ 14 | { 15 | name: "accounts", 16 | parameters: { 17 | type: :string, aliases: ["-a"], 18 | desc: "a comma-separated list of accounts (defaults to all configured accounts)" 19 | } 20 | }, 21 | { 22 | name: "config", 23 | parameters: { 24 | type: :string, aliases: ["-c"], 25 | desc: "supply the configuration file path (default: ~/.imap-backup/config.json)" 26 | } 27 | }, 28 | { 29 | name: "format", 30 | parameters: { 31 | type: :string, desc: "the output type, 'text' for plain text or 'json'", aliases: ["-f"] 32 | } 33 | }, 34 | { 35 | name: "quiet", 36 | parameters: { 37 | type: :boolean, desc: "silence all output", aliases: ["-q"] 38 | } 39 | }, 40 | { 41 | name: "refresh", 42 | parameters: { 43 | type: :boolean, aliases: ["-r"], 44 | desc: "in the default 'keep all emails' mode, " \ 45 | "updates flags for messages that are already downloaded" 46 | } 47 | }, 48 | { 49 | name: "verbose", 50 | parameters: { 51 | type: :boolean, aliases: ["-v"], repeatable: true, 52 | desc: "increase the amount of logging. " \ 53 | "Without this option, the program gives minimal output. " \ 54 | "Using this option once gives more detailed output. " \ 55 | "Whereas, using this option twice also shows all IMAP network calls" 56 | } 57 | } 58 | ].freeze 59 | 60 | def initialize(base:) 61 | @base = base 62 | end 63 | 64 | def define_options 65 | OPTIONS.each do |option| 66 | base.singleton_class.class_eval do 67 | define_method("#{option[:name]}_option") do 68 | method_option(option[:name], **option[:parameters]) 69 | end 70 | end 71 | end 72 | end 73 | end 74 | end 75 | -------------------------------------------------------------------------------- /spec/unit/account/backup_folders_spec.rb: -------------------------------------------------------------------------------- 1 | require "imap/backup/account/backup_folders" 2 | 3 | require "imap/backup/account" 4 | require "imap/backup/client/default" 5 | 6 | module Imap::Backup 7 | RSpec.describe Account::BackupFolders do 8 | subject { described_class.new(client: client, account: account) } 9 | 10 | let(:account) do 11 | instance_double( 12 | Account, 13 | folders: account_folders, 14 | folder_blacklist: folder_blacklist 15 | ) 16 | end 17 | let(:client) { instance_double(Client::Default, list: %w(foo bar baz)) } 18 | let(:account_folders) { ["foo"] } 19 | let(:folder_blacklist) { false } 20 | let(:result) { subject.each } 21 | 22 | it "returns a folder for each configured folder" do 23 | expect(subject.map(&:name)).to eq(%w(foo)) 24 | end 25 | 26 | it "returns Account::Folders" do 27 | expect(result.first).to be_a(Account::Folder) 28 | end 29 | 30 | it "sets the client" do 31 | expect(result.first.client).to eq(client) 32 | end 33 | 34 | context "when no folders are configured" do 35 | let(:account_folders) { nil } 36 | 37 | it "returns all online folders" do 38 | expect(result.map(&:name)).to eq(%w(foo bar baz)) 39 | end 40 | end 41 | 42 | context "when the configured folders are an empty list" do 43 | let(:account_folders) { [] } 44 | 45 | it "returns all online folders" do 46 | expect(result.map(&:name)).to eq(%w(foo bar baz)) 47 | end 48 | end 49 | 50 | context "when the folder_blacklist flag is set" do 51 | let(:folder_blacklist) { true } 52 | 53 | it "returns account folders except the configured folders" do 54 | expect(result.map(&:name)).to eq(%w(bar baz)) 55 | end 56 | 57 | context "when no folders are configured" do 58 | let(:account_folders) { nil } 59 | 60 | it "returns all online folders" do 61 | expect(result.map(&:name)).to eq(%w(foo bar baz)) 62 | end 63 | end 64 | 65 | context "when the configured folders are an empty list" do 66 | let(:account_folders) { [] } 67 | 68 | it "returns all online folders" do 69 | expect(result.map(&:name)).to eq(%w(foo bar baz)) 70 | end 71 | end 72 | end 73 | end 74 | end 75 | -------------------------------------------------------------------------------- /docs/howto/migrate-server-keep-address.md: -------------------------------------------------------------------------------- 1 | 4 | # Migrate to a new e-mail server while keeping your existing address 5 | 6 | While switching e-mail provider (from provider `A` to `B`), 7 | you might want to keep the same address (`mymail@domain.com`), 8 | and copy all your existing e-mails to your new server `B`. 9 | `imap-backup` can do that too! 10 | 11 | It is best to use [`imap-backup migrate`](/docs/commands/migrate.md) 12 | and not [`imap-backup restore`](/docs/commands/restore.md) here because 13 | `migrate` simply copies emails to folders with the same name as the ones 14 | they were downloaded from, while `restore` changes the names of restored 15 | folders if folders with the same name already exist on the destination server. 16 | 17 | 1. Backup your e-mails: use [`imap-backup setup`](/docs/commands/setup.md) 18 | to setup connection to your old provider `A`, 19 | then launch [`imap-backup backup`](/docs/commands/backup.md). 20 | 1. Actually switch your e-mail service provider (update your DNS MX and all that...). 21 | 1. As both the source and the destination have the same address, 22 | you need to manually rename your old account first: 23 | 24 | 1. Modify your configuration file manually 25 | (i.e. not via `imap-backup setup`) and 26 | rename your account to `mymail-old@domain.com`: 27 | 28 | ```diff 29 | "accounts": [ 30 | { 31 | - "username": "mymail@domain.com", 32 | + "username": "mymail-old@domain.com", 33 | "password": "...", 34 | - "local_path": "/some/path/.imap-backup/mymail_domain.com", 35 | + "local_path": "/some/path/.imap-backup/mymail-old_domain.com", 36 | "folders": [...], 37 | "server": "..." 38 | } 39 | ``` 40 | 41 | 1. Rename the backup directory from `mymail_domain.com` 42 | to `mymail-old_domain.com`. 43 | 44 | 1. Set up a new account giving access to the new provider `B` 45 | using `imap-backup setup`. 46 | 1. Now you can use `imap-backup migrate`, optionally adapting 47 | [delimiters and prefixes configuration](/docs/delimiters-and-prefixes.md) 48 | if need be: 49 | 50 | imap-backup migrate mymail-old@domain.com mymail@domain.com [options] 51 | -------------------------------------------------------------------------------- /spec/unit/cli/remote_spec.rb: -------------------------------------------------------------------------------- 1 | require "imap/backup/cli/remote" 2 | 3 | require "ostruct" 4 | require "imap/backup/client/default" 5 | require "imap/backup/configuration" 6 | require "support/shared_examples/an_action_that_handles_logger_options" 7 | 8 | module Imap::Backup 9 | RSpec.describe CLI::Remote do 10 | let(:account) do 11 | instance_double(Account, client: client, namespaces: namespaces, username: "user") 12 | end 13 | let(:client) { instance_double(Client::Default, list: %w(foo)) } 14 | let(:config) { instance_double(Configuration, accounts: [account]) } 15 | let(:namespaces) do 16 | OpenStruct.new( 17 | personal: [OpenStruct.new(prefix: "x", delim: "/")], 18 | other: [OpenStruct.new(prefix: "x", delim: "/")], 19 | shared: [OpenStruct.new(prefix: "x", delim: "/")] 20 | ) 21 | end 22 | 23 | before do 24 | allow(Configuration).to receive(:exist?) { true } 25 | allow(Configuration).to receive(:new) { config } 26 | allow(Kernel).to receive(:puts) 27 | end 28 | 29 | describe "#folders" do 30 | it_behaves_like( 31 | "an action that requires an existing configuration", 32 | action: ->(subject) { subject.folders("email") } 33 | ) 34 | 35 | it "prints names of emails to be backed up" do 36 | subject.folders(account.username) 37 | 38 | expect(Kernel).to have_received(:puts).with('"foo"') 39 | end 40 | 41 | it_behaves_like( 42 | "an action that handles Logger options", 43 | action: ->(subject, options) do 44 | subject.invoke(:folders, ["user"], options) 45 | end 46 | ) 47 | end 48 | 49 | describe "#namespaces" do 50 | it_behaves_like( 51 | "an action that requires an existing configuration", 52 | action: ->(subject) { subject.namespaces("email") } 53 | ) 54 | 55 | it "prints namespaces with prefixes and delimiters" do 56 | subject.invoke(:namespaces, [account.username], format: "json") 57 | 58 | expect(Kernel).to have_received(:puts).with(%r({"personal":{"prefix":"x","delim":"/"})) 59 | end 60 | 61 | it_behaves_like( 62 | "an action that handles Logger options", 63 | action: ->(subject, options) do 64 | subject.invoke(:namespaces, ["user"], options) 65 | end 66 | ) 67 | end 68 | end 69 | end 70 | -------------------------------------------------------------------------------- /lib/imap/backup/account/serialized_folders.rb: -------------------------------------------------------------------------------- 1 | require "pathname" 2 | 3 | require "imap/backup/account/folder" 4 | require "imap/backup/account/folder_ensurer" 5 | require "imap/backup/serializer" 6 | 7 | module Imap; end 8 | 9 | module Imap::Backup 10 | class Account; end 11 | 12 | # Enumerates over an account's backed-up folders 13 | class Account::SerializedFolders 14 | include Enumerable 15 | 16 | def initialize(account:) 17 | @account = account 18 | end 19 | 20 | # Runs the enumeration over local serializers and remote folders 21 | # @yieldparam serializer [Serializer] the folder's serializer 22 | # @yieldparam folder [Account::Folder] the online folder 23 | # @return [void] 24 | def each(&block) 25 | return enum_for(:each) if !block 26 | 27 | glob.each do |path| 28 | name = path.relative_path_from(base).to_s[0..-6] 29 | serializer = Serializer.new(account.local_path, name) 30 | folder = Account::Folder.new(account.client, name) 31 | block.call(serializer, folder) 32 | end 33 | end 34 | 35 | # Runs the enumeration over each local serializer 36 | # @yieldparam serializer [Serializer] the folder's serializer 37 | # @return [void] 38 | def each_key(&block) 39 | return enum_for(:each_key) if !block 40 | 41 | glob.each do |path| 42 | name = path.relative_path_from(base).to_s[0..-6] 43 | serializer = Serializer.new(account.local_path, name) 44 | block.call(serializer) 45 | end 46 | end 47 | 48 | # Runs the enumeration over each remote folder 49 | # @yieldparam folder [Account::Folder] the online folder 50 | # @return [void] 51 | def each_value(&block) 52 | return enum_for(:each_value) if !block 53 | 54 | glob.each do |path| 55 | name = path.relative_path_from(base).to_s[0..-6] 56 | folder = Account::Folder.new(account.client, name) 57 | block.call(folder) 58 | end 59 | end 60 | 61 | private 62 | 63 | attr_reader :account 64 | 65 | def base 66 | @base ||= Pathname.new(account.local_path) 67 | end 68 | 69 | def glob 70 | @glob ||= begin 71 | Account::FolderEnsurer.new(account: account).run 72 | 73 | pattern = File.join(account.local_path, "**", "*.imap") 74 | Pathname.glob(pattern) 75 | end 76 | end 77 | end 78 | end 79 | -------------------------------------------------------------------------------- /lib/imap/backup/logger.rb: -------------------------------------------------------------------------------- 1 | require "net/imap" 2 | require "logger" 3 | require "singleton" 4 | 5 | require "imap/backup/text/sanitizer" 6 | 7 | module Imap; end 8 | 9 | module Imap::Backup 10 | # Wraps the standard logger, providing configuration and sanitization 11 | class Logger 12 | include Singleton 13 | 14 | # @return [Imap::Backup::Logger] the singleton instance of this class 15 | def self.logger 16 | Logger.instance.logger 17 | end 18 | 19 | # @param options [Hash] command-line options 20 | # @option options [Boolean] :quiet (false) if true, no output will be written 21 | # @option options [Array] :verbose ([]) counts how many `--verbose` 22 | # parameters were passed (and, potentially subtracts the number of 23 | # `--no-verbose` parameters). 24 | # If the result is 0, does normal info-level logging, 25 | # If the result is 1, does debug logging, 26 | # If the result is 2, does debug logging and client-server debug logging. 27 | # This option is overridden by the `:verbose` option. 28 | # 29 | # @return [Hash] the options without the :quiet and :verbose keys 30 | def self.setup_logging(options = {}) 31 | copy = options.clone 32 | quiet = copy.delete(:quiet) 33 | verbose = copy.delete(:verbose) || [] 34 | verbose_count = count(verbose) 35 | level = 36 | case 37 | when quiet 38 | ::Logger::Severity::UNKNOWN 39 | when verbose_count >= 2 40 | ::Logger::Severity::DEBUG 41 | else 42 | ::Logger::Severity::INFO 43 | end 44 | logger.level = level 45 | 46 | Net::IMAP.debug = (verbose_count >= 3) 47 | 48 | copy 49 | end 50 | 51 | # Wraps a block, filtering output to standard error, 52 | # hidng passwords and outputs the results to standard out 53 | # @return [void] 54 | def self.sanitize_stderr(&block) 55 | sanitizer = Text::Sanitizer.new($stdout) 56 | previous_stderr = $stderr 57 | $stderr = sanitizer 58 | block.call 59 | ensure 60 | sanitizer.flush 61 | $stderr = previous_stderr 62 | end 63 | 64 | # @private 65 | def self.count(verbose) 66 | verbose.reduce(1) { |acc, v| acc + (v ? 1 : -1) } 67 | end 68 | 69 | # @return [Logger] the configured Logger 70 | attr_reader :logger 71 | 72 | def initialize 73 | @logger = ::Logger.new($stdout) 74 | $stdout.sync = true 75 | end 76 | end 77 | end 78 | -------------------------------------------------------------------------------- /lib/imap/backup/uploader.rb: -------------------------------------------------------------------------------- 1 | require "imap/backup/account/folder" 2 | require "imap/backup/logger" 3 | require "imap/backup/serializer" 4 | 5 | module Imap; end 6 | 7 | module Imap::Backup 8 | # Uploads a backed-up folder 9 | class Uploader 10 | # @param folder [Account::Folder] an online folder 11 | # @param serializer [Serializer] a local folder backup 12 | def initialize(folder, serializer) 13 | @folder = folder 14 | @serializer = serializer 15 | end 16 | 17 | # Uploads messages that are present in the backup, but not in the online folder 18 | # @return [void] 19 | def run 20 | if folder.uids.any? 21 | rename_serialized_folder 22 | else 23 | folder.create 24 | serializer.force_uid_validity(folder.uid_validity) 25 | end 26 | 27 | return if count.zero? 28 | 29 | Logger.logger.debug "[#{folder.name}] #{count} to restore" 30 | serializer.each_message(missing_uids).with_index do |message, i| 31 | upload_message message, i + 1 32 | end 33 | end 34 | 35 | private 36 | 37 | attr_reader :folder 38 | attr_reader :serializer 39 | 40 | def upload_message(message, index) 41 | return if message.nil? 42 | 43 | log_prefix = "[#{folder.name}] uid: #{message.uid} (#{index}/#{count}) -" 44 | Logger.logger.debug( 45 | "#{log_prefix} #{message.body.size} bytes" 46 | ) 47 | 48 | begin 49 | new_uid = folder.append(message) 50 | serializer.update_uid(message.uid, new_uid) 51 | rescue StandardError => e 52 | Logger.logger.warn "#{log_prefix} append error: #{e}" 53 | end 54 | end 55 | 56 | def count 57 | @count ||= missing_uids.count 58 | end 59 | 60 | def missing_uids 61 | serializer.uids - folder.uids 62 | end 63 | 64 | def rename_serialized_folder 65 | Logger.logger.debug( 66 | "There's already a '#{folder.name}' folder with emails" 67 | ) 68 | 69 | # Rename the local folder to a unique name 70 | new_name = serializer.apply_uid_validity(folder.uid_validity) 71 | 72 | return if !new_name 73 | 74 | # Restore the renamed folder 75 | Logger.logger.debug( 76 | "Backup '#{serializer.folder}' renamed and restored to '#{new_name}'" 77 | ) 78 | @folder = Account::Folder.new(folder.client, new_name) 79 | folder.create 80 | @serializer = Serializer.new(serializer.path, new_name) 81 | serializer.force_uid_validity(@folder.uid_validity) 82 | end 83 | end 84 | end 85 | -------------------------------------------------------------------------------- /spec/unit/cli/restore_spec.rb: -------------------------------------------------------------------------------- 1 | require "imap/backup/cli/restore" 2 | 3 | require "imap/backup/account" 4 | require "imap/backup/configuration" 5 | 6 | module Imap::Backup 7 | RSpec.describe CLI::Restore do 8 | subject { described_class.new(email, options) } 9 | 10 | let(:email) { "email" } 11 | let(:options) { {} } 12 | let(:account) { instance_double(Account, username: email) } 13 | let(:config) { instance_double(Configuration, accounts: [account]) } 14 | let(:restore) { instance_double(Account::Restore, run: nil) } 15 | 16 | before do 17 | allow(Configuration).to receive(:exist?) { true } 18 | allow(Configuration).to receive(:new) { config } 19 | allow(Account::Restore).to receive(:new) { restore } 20 | end 21 | 22 | it_behaves_like( 23 | "an action that requires an existing configuration", 24 | action: lambda(&:run) 25 | ) 26 | 27 | it "runs restore on the account" do 28 | subject.run 29 | 30 | expect(restore).to have_received(:run) 31 | end 32 | 33 | context "when options are provided" do 34 | let(:options) { {delimiter: "/", prefix: "CIAO"} } 35 | 36 | it "passes them to the restore" do 37 | subject.run 38 | 39 | expect(Account::Restore).to have_received(:new). 40 | with(hash_including(delimiter: "/", prefix: "CIAO")) 41 | end 42 | end 43 | 44 | context "when neither an email nor a list of account names is provided" do 45 | let(:email) { nil } 46 | let(:options) { {} } 47 | 48 | before do 49 | allow(subject).to receive(:requested_accounts) { [account] } 50 | end 51 | 52 | it "runs restore on each account" do 53 | subject.run 54 | 55 | expect(restore).to have_received(:run) 56 | end 57 | end 58 | 59 | context "when an email and a list of account names is provided" do 60 | let(:email) { "email" } 61 | let(:options) { {accounts: "email2"} } 62 | 63 | it "fails" do 64 | expect do 65 | subject.run 66 | end.to raise_error(RuntimeError, /Missing EMAIL parameter/) 67 | end 68 | end 69 | 70 | context "when just a list of account names is provided" do 71 | let(:email) { nil } 72 | let(:options) { {accounts: "email2"} } 73 | 74 | before do 75 | allow(subject).to receive(:requested_accounts) { [account] } 76 | end 77 | 78 | it "runs restore on each account" do 79 | subject.run 80 | 81 | expect(restore).to have_received(:run) 82 | end 83 | end 84 | end 85 | end 86 | -------------------------------------------------------------------------------- /spec/unit/logger_spec.rb: -------------------------------------------------------------------------------- 1 | require "imap/backup/logger" 2 | 3 | module Imap::Backup 4 | RSpec.describe Logger do 5 | describe ".setup_logging" do 6 | around do |example| 7 | logger_previous = described_class.logger.level 8 | net_imap_previous = Net::IMAP.debug 9 | described_class.logger.level = 42 10 | Net::IMAP.debug = 42 11 | example.run 12 | Net::IMAP.debug = net_imap_previous 13 | described_class.logger.level = logger_previous 14 | end 15 | 16 | let(:options) { {ciao: true} } 17 | let!(:result) { described_class.setup_logging(options) } 18 | 19 | it "sets logger level to info" do 20 | expect(described_class.logger.level).to eq(::Logger::Severity::INFO) 21 | end 22 | 23 | it "unsets the Net::IMAP debug flag" do 24 | expect(Net::IMAP.debug).to be false 25 | end 26 | 27 | it "returns options" do 28 | expect(result).to eq({ciao: true}) 29 | end 30 | 31 | context "when logger-related options are passed" do 32 | let(:options) { {ciao: true, quiet: true, verbose: [true]} } 33 | 34 | it "excludes them and returns other options" do 35 | expect(result).to eq({ciao: true}) 36 | end 37 | end 38 | 39 | context "when one verbose flag is passed" do 40 | let(:options) { {verbose: [true]} } 41 | 42 | it "sets logger level to debug" do 43 | expect(described_class.logger.level).to eq(::Logger::Severity::DEBUG) 44 | end 45 | end 46 | 47 | context "when two verbose flags are passed" do 48 | let(:options) { {verbose: [true, true]} } 49 | 50 | it "sets the Net::IMAP debug flag" do 51 | expect(Net::IMAP.debug).to be true 52 | end 53 | end 54 | 55 | context "when quiet is passed" do 56 | let(:options) { {quiet: true} } 57 | 58 | it "sets logger level to unknown" do 59 | expect(described_class.logger.level).to eq(::Logger::Severity::UNKNOWN) 60 | end 61 | 62 | it "unsets the Net::IMAP debug flag" do 63 | expect(Net::IMAP.debug).to be false 64 | end 65 | end 66 | 67 | context "when quiet and verbose are passed" do 68 | let(:options) { {quiet: true, verbose: [true]} } 69 | 70 | it "sets logger level to unknown" do 71 | expect(described_class.logger.level).to eq(::Logger::Severity::UNKNOWN) 72 | end 73 | 74 | it "unsets the Net::IMAP debug flag" do 75 | expect(Net::IMAP.debug).to be false 76 | end 77 | end 78 | end 79 | end 80 | end 81 | -------------------------------------------------------------------------------- /docs/commands/setup.md: -------------------------------------------------------------------------------- 1 | 4 | # Setup 5 | 6 | ```sh 7 | imap-backup setup 8 | ``` 9 | 10 | This command starts the interactive, menu-driven setup tool. 11 | 12 | By default, the tool saves a configuration file in `~/.imap-backup/config.json`. 13 | 14 | ## Custom Configuration Path 15 | 16 | You can override the location where the file is accessed and stored with the `--config` parameter: 17 | 18 | ```sh 19 | imap-backup setup --config /home/me/.local/imap-backup/config.json 20 | ``` 21 | 22 | In this case, it is up to you to create the directory for the configuration file. 23 | 24 | # Account Setup 25 | 26 | ## `modify server` 27 | 28 | For GMail accounts, use `imap.gmail.com` as the 'server' setting. 29 | 30 | ## `modify connection options` 31 | 32 | You can override the parameters passed to `Net::IMAP` with `modify connection options`. 33 | 34 | Connection options must be entered as JSON. 35 | 36 | See the Ruby Standard Library documentation for `Net::IMAP` of details of 37 | [supported parameters](https://ruby-doc.org/stdlib-3.1.2/libdoc/net-imap/rdoc/Net/IMAP.html#method-c-new). 38 | 39 | Specifically, if you are using a self-signed certificate and get SSL errors, e.g. 40 | `certificate verify failed`, you can choose to not verify the TLS connection. 41 | 42 | For example: 43 | 44 | ![Entering connection options as JSON](../images/entering-connection-options-as-json.png "Entering connection options as JSON") 45 | 46 | ## `toggle folder inclusion mode (whitelist/blacklist)` 47 | 48 | This setting, combined with the following `folders` setting, 49 | govern which folders are backed up. 50 | 51 | If you choose `whitelist`, then *only* the selected `folders` 52 | will be backed up. 53 | 54 | If you choose `blacklist`, all folders *except* those selected 55 | will be backed up. 56 | 57 | ## `choose folders` 58 | 59 | By default, without a list of folders, all folders are backed up. 60 | 61 | You can change this behaviour by choosing specific folders. 62 | 63 | ## `modify multi-fetch size` 64 | 65 | By default, one email is downloaded and backed up at a time. 66 | 67 | If your email server supports faster fetching, 68 | you can set the multi-fetch size to a larger numbe 69 | to fetch more emails at a time. 70 | 71 | ## `fix changes to unread flags during download` 72 | 73 | Certain mail servers mark emails as `Read` when `imap-backup` fetches 74 | them. Activating this setting will cause `imap-backup` 75 | apply a workaround, where is checks flags *before* fetching 76 | emails and then re-applies them after the fetch. 77 | 78 | # The Configuration File 79 | 80 | [More information about the configuration file is available in the specific documentation](../files/config.md). 81 | -------------------------------------------------------------------------------- /spec/unit/serializer/message_spec.rb: -------------------------------------------------------------------------------- 1 | require "imap/backup/serializer/message" 2 | require "imap/backup/email/mboxrd/message" 3 | require "imap/backup/serializer/mbox" 4 | 5 | module Imap::Backup 6 | RSpec.describe Serializer::Message do 7 | subject { described_class.new(**parameters) } 8 | 9 | let(:parameters) do 10 | {uid: uid, offset: offset, length: length, mbox: mbox, flags: flags} 11 | end 12 | let(:uid) { 42 } 13 | let(:offset) { 13 } 14 | let(:length) { 23 } 15 | let(:mbox) { instance_double(Serializer::Mbox) } 16 | let(:flags) { %w(foo bar) } 17 | let(:raw) { "raw message" } 18 | let(:message) { instance_double(Email::Mboxrd::Message) } 19 | let(:date) { Date.today } 20 | 21 | before do 22 | allow(Email::Mboxrd::Message).to receive(:from_serialized) { message } 23 | allow(mbox).to receive(:read).with(offset, length) { raw } 24 | allow(message).to receive(:supplied_body) { "supplied_body" } 25 | allow(message).to receive(:imap_body) { "imap_body" } 26 | allow(message).to receive(:date) { date } 27 | allow(message).to receive(:subject) { subject } 28 | end 29 | 30 | describe "#flags" do 31 | let(:parameters) do 32 | {uid: uid, offset: offset, length: length, mbox: mbox} 33 | end 34 | 35 | it "defaults to an empty Array" do 36 | expect(subject.flags).to eq([]) 37 | end 38 | end 39 | 40 | describe "#to_h" do 41 | let(:result) { subject.to_h } 42 | 43 | it "returns the uid" do 44 | expect(result[:uid]).to eq(uid) 45 | end 46 | 47 | it "returns the offset" do 48 | expect(result[:offset]).to eq(offset) 49 | end 50 | 51 | it "returns the length" do 52 | expect(result[:length]).to eq(length) 53 | end 54 | 55 | it "returns the flags" do 56 | expect(result[:flags]).to eq(flags) 57 | end 58 | end 59 | 60 | describe "#message" do 61 | let(:result) { subject.message } 62 | 63 | it "returns the message from the mbox" do 64 | expect(result).to eq(message) 65 | end 66 | end 67 | 68 | describe "#body" do 69 | it "returns the message body" do 70 | expect(subject.body).to eq("supplied_body") 71 | end 72 | end 73 | 74 | describe "#imap_body" do 75 | it "returns the message body" do 76 | expect(subject.imap_body).to eq("imap_body") 77 | end 78 | end 79 | 80 | describe "#date" do 81 | it "returns the message date" do 82 | expect(subject.date).to eq(date) 83 | end 84 | end 85 | 86 | describe "#subject" do 87 | it "returns the message subject" do 88 | expect(subject.subject).to eq(subject) 89 | end 90 | end 91 | end 92 | end 93 | -------------------------------------------------------------------------------- /lib/imap/backup/cli/stats.rb: -------------------------------------------------------------------------------- 1 | require "imap/backup/account/backup_folders" 2 | require "imap/backup/serializer" 3 | 4 | module Imap; end 5 | 6 | module Imap::Backup 7 | # Prints various statistics about an account and its backup 8 | class CLI::Stats < Thor 9 | include Thor::Actions 10 | include CLI::Helpers 11 | 12 | def initialize(email, options) 13 | super([]) 14 | @email = email 15 | @options = options 16 | end 17 | 18 | # @!method run 19 | # @return [void] 20 | no_commands do 21 | def run 22 | case options[:format] 23 | when "json" 24 | Kernel.puts stats.to_json 25 | else 26 | format_text stats 27 | end 28 | end 29 | end 30 | 31 | private 32 | 33 | TEXT_COLUMNS = [ 34 | {name: :folder, width: 20, alignment: :left}, 35 | {name: :remote, width: 8, alignment: :right}, 36 | {name: :both, width: 8, alignment: :right}, 37 | {name: :local, width: 8, alignment: :right} 38 | ].freeze 39 | ALIGNMENT_FORMAT_SYMBOL = {left: "-", right: " "}.freeze 40 | 41 | attr_reader :email 42 | attr_reader :options 43 | 44 | def stats 45 | Logger.logger.debug("[Stats] loading configuration") 46 | config = load_config(**options) 47 | account = account(config, email) 48 | 49 | backup_folders = Account::BackupFolders.new( 50 | client: account.client, account: account 51 | ) 52 | backup_folders.map do |folder| 53 | next if !folder.exist? 54 | 55 | serializer = Serializer.new(account.local_path, folder.name) 56 | local_uids = serializer.uids 57 | Logger.logger.debug("[Stats] fetching email list for '#{folder.name}'") 58 | remote_uids = folder.uids 59 | { 60 | folder: folder.name, 61 | remote: (remote_uids - local_uids).count, 62 | both: (serializer.uids & folder.uids).count, 63 | local: (local_uids - remote_uids).count 64 | } 65 | end.compact 66 | end 67 | 68 | def format_text(stats) 69 | Kernel.puts text_header 70 | 71 | stats.each do |stat| 72 | columns = TEXT_COLUMNS.map do |column| 73 | symbol = ALIGNMENT_FORMAT_SYMBOL[column[:alignment]] 74 | count = stat[column[:name]] 75 | format("%#{symbol}#{column[:width]}s", count) 76 | end.join("|") 77 | 78 | Kernel.puts columns 79 | end 80 | end 81 | 82 | def text_header 83 | titles = TEXT_COLUMNS.map do |column| 84 | format("%-#{column[:width]}s", column[:name]) 85 | end.join("|") 86 | 87 | underline = TEXT_COLUMNS.map do |column| 88 | "-" * column[:width] 89 | end.join("|") 90 | 91 | "#{titles}\n#{underline}" 92 | end 93 | end 94 | end 95 | -------------------------------------------------------------------------------- /lib/imap/backup/cli/migrate.rb: -------------------------------------------------------------------------------- 1 | module Imap; end 2 | 3 | module Imap::Backup 4 | # Processes parameters to run the `migrate` command via command-line parameters 5 | module CLI::Migrate 6 | include Thor::Actions 7 | include CLI::Helpers 8 | 9 | LONG_DESCRIPTION = <<~DESC.freeze 10 | This command is deprecated and will be removed in a future version. 11 | Use 'copy' instead. 12 | 13 | All emails which have been backed up for the "source account" (SOURCE_EMAIL) are 14 | uploaded to the "destination account" (DESTINATION_EMAIL). 15 | 16 | Some configuration may be necessary, as follows: 17 | 18 | #{CLI::Helpers::NAMESPACE_CONFIGURATION_DESCRIPTION} 19 | 20 | Finally, if you want to delete existing emails in destination folders, 21 | use the `--reset` option. In this case, all existing emails are 22 | deleted before uploading the migrated emails. 23 | DESC 24 | 25 | def self.included(base) 26 | base.class_eval do 27 | desc( 28 | "migrate SOURCE_EMAIL DESTINATION_EMAIL [OPTIONS]", 29 | "(Deprecated) Uploads backed-up emails from account SOURCE_EMAIL " \ 30 | "to account DESTINATION_EMAIL" 31 | ) 32 | long_desc LONG_DESCRIPTION 33 | config_option 34 | quiet_option 35 | verbose_option 36 | method_option( 37 | "automatic-namespaces", 38 | type: :boolean, 39 | desc: "automatically choose delimiters and prefixes" 40 | ) 41 | method_option( 42 | "destination-delimiter", 43 | type: :string, 44 | desc: "the delimiter for destination folder names" 45 | ) 46 | method_option( 47 | "destination-prefix", 48 | type: :string, 49 | desc: "the prefix (namespace) to add to destination folder names", 50 | aliases: ["-d"] 51 | ) 52 | method_option( 53 | "reset", 54 | type: :boolean, 55 | desc: "DANGER! This option deletes all messages from destination " \ 56 | "folders before uploading", 57 | aliases: ["-r"] 58 | ) 59 | method_option( 60 | "source-delimiter", 61 | type: :string, 62 | desc: "the delimiter for source folder names" 63 | ) 64 | method_option( 65 | "source-prefix", 66 | type: :string, 67 | desc: "the prefix (namespace) to strip from source folder names", 68 | aliases: ["-s"] 69 | ) 70 | # Migrates emails from one account to another 71 | # @return [void] 72 | def migrate(source_email, destination_email) 73 | non_logging_options = Imap::Backup::Logger.setup_logging(options) 74 | CLI::Transfer.new(:migrate, source_email, destination_email, non_logging_options).run 75 | end 76 | end 77 | end 78 | end 79 | end 80 | -------------------------------------------------------------------------------- /lib/imap/backup/account/folder_backup.rb: -------------------------------------------------------------------------------- 1 | require "imap/backup/serializer/delayed_metadata_serializer" 2 | require "imap/backup/downloader" 3 | require "imap/backup/flag_refresher" 4 | require "imap/backup/local_only_message_deleter" 5 | require "imap/backup/logger" 6 | require "imap/backup/serializer" 7 | 8 | module Imap; end 9 | 10 | module Imap::Backup 11 | class Account; end 12 | 13 | # Implements backup for a single folder 14 | class Account::FolderBackup 15 | def initialize(account:, folder:, refresh: false) 16 | @account = account 17 | @folder = folder 18 | @refresh = refresh 19 | end 20 | 21 | # Runs the backup 22 | # @raise [RuntimeError] if the configured download strategy is incorrect 23 | # @return [void] 24 | def run 25 | Logger.logger.debug "Running backup for folder '#{folder.name}'" 26 | 27 | folder_ok = folder_ok? 28 | return if !folder_ok 29 | 30 | serializer.apply_uid_validity(folder.uid_validity) 31 | 32 | serializer.transaction do 33 | downloader.run 34 | FlagRefresher.new(folder, serializer).run if account.mirror_mode || refresh 35 | end 36 | # After the transaction the serializer will have any appended messages 37 | # so we can check differences between the server and the local backup 38 | LocalOnlyMessageDeleter.new(folder, raw_serializer).run if account.mirror_mode 39 | Logger.logger.debug "Backup for folder '#{folder.name}' complete" 40 | end 41 | 42 | private 43 | 44 | attr_reader :account 45 | attr_reader :folder 46 | attr_reader :refresh 47 | 48 | def folder_ok? 49 | begin 50 | if !folder.exist? 51 | Logger.logger.info "Skipping backup for folder '#{folder.name}' as it does not exist" 52 | return false 53 | end 54 | rescue Encoding::UndefinedConversionError 55 | message = "Skipping backup for '#{folder.name}' " \ 56 | "as it's name is not UTF-7 encoded correctly" 57 | Logger.logger.info message 58 | return false 59 | end 60 | 61 | true 62 | end 63 | 64 | def downloader 65 | @downloader ||= Downloader.new( 66 | folder, 67 | serializer, 68 | multi_fetch_size: account.multi_fetch_size, 69 | reset_seen_flags_after_fetch: account.reset_seen_flags_after_fetch 70 | ) 71 | end 72 | 73 | def serializer 74 | @serializer ||= 75 | case account.download_strategy 76 | when "direct" 77 | raw_serializer 78 | when "delay_metadata" 79 | Serializer::DelayedMetadataSerializer.new(serializer: raw_serializer) 80 | else 81 | raise "Unknown download strategy '#{account.download_strategy}'" 82 | end 83 | end 84 | 85 | def raw_serializer 86 | @raw_serializer ||= Serializer.new(account.local_path, folder.name) 87 | end 88 | end 89 | end 90 | -------------------------------------------------------------------------------- /spec/unit/serializer/integrity_checker_spec.rb: -------------------------------------------------------------------------------- 1 | require "imap/backup/serializer/integrity_checker" 2 | require "imap/backup/serializer/imap" 3 | require "imap/backup/serializer/mbox" 4 | require "imap/backup/serializer/message" 5 | 6 | module Imap::Backup 7 | RSpec.describe Serializer::IntegrityChecker do 8 | subject { described_class.new(imap: imap, mbox: mbox) } 9 | 10 | let(:imap) do 11 | instance_double(Serializer::Imap, valid?: imap_valid, messages: messages, pathname: "imap") 12 | end 13 | let(:imap_valid) { true } 14 | let(:messages) { [message1] } 15 | let(:message1) do 16 | instance_double(Serializer::Message, offset: 0, length: body1.length, uid: "uid") 17 | end 18 | let(:mbox) do 19 | instance_double( 20 | Serializer::Mbox, exist?: mbox_exists, length: mbox_length, read: body1, pathname: "mbox" 21 | ) 22 | end 23 | let(:mbox_exists) { true } 24 | let(:mbox_length) { body1.length } 25 | let(:body1) { "From #{'a' * 95}" } 26 | 27 | it "returns nil" do 28 | expect(subject.run).to be_nil 29 | end 30 | 31 | context "when the folder is empty" do 32 | let(:messages) { [] } 33 | let(:body1) { "" } 34 | 35 | it "returns nil" do 36 | expect(subject.run).to be_nil 37 | end 38 | 39 | context "when the mbox is not empty" do 40 | let(:body1) { "Foo" } 41 | 42 | it "fails" do 43 | expect do 44 | subject.run 45 | end.to raise_error(Serializer::FolderIntegrityError, /not empty/) 46 | end 47 | end 48 | end 49 | 50 | context "when the imap offsets are out of order" do 51 | let(:messages) { [message2, message1] } 52 | let(:message2) { instance_double(Serializer::Message, offset: body1.length, length: 99) } 53 | 54 | it "fails" do 55 | expect do 56 | subject.run 57 | end.to raise_error(Serializer::FolderIntegrityError, /out of order/) 58 | end 59 | end 60 | 61 | context "when the mbox is shorter than expected" do 62 | let(:mbox_length) { body1.length - 1 } 63 | 64 | it "fails" do 65 | expect do 66 | subject.run 67 | end.to raise_error(Serializer::FolderIntegrityError, /shorter than indicated/) 68 | end 69 | end 70 | 71 | context "when the mbox is longer than expected" do 72 | let(:mbox_length) { body1.length + 1 } 73 | 74 | it "fails" do 75 | expect do 76 | subject.run 77 | end.to raise_error(Serializer::FolderIntegrityError, /longer than indicated/) 78 | end 79 | end 80 | 81 | context "when messages do not start at the indicated offsets" do 82 | before do 83 | allow(mbox).to receive(:read) { "Wrong text" } 84 | end 85 | 86 | it "fails" do 87 | expect do 88 | subject.run 89 | end.to raise_error(Serializer::FolderIntegrityError, /not found/) 90 | end 91 | end 92 | end 93 | end 94 | -------------------------------------------------------------------------------- /contrib/import-thunderbird-folder: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | 3 | # This script is an example of how to import messages from a Thunderbird 4 | # folder into imap-backup. It is not meant to be a general-purpose 5 | # Thunderbird importer, but rather a starting point for writing your own. 6 | # Please adapt it to your specific needs. 7 | 8 | require "bundler/inline" 9 | 10 | gemfile do 11 | source "https://rubygems.org" 12 | 13 | gem "imap-backup" 14 | gem "optparse" 15 | gem "thunderbird", "~> 0.5.0" 16 | end 17 | 18 | require "imap/backup/logger" 19 | require "imap/backup/configuration" 20 | require "imap/backup/serializer" 21 | require "thunderbird/mbox" 22 | 23 | class Options 24 | attr_accessor :email 25 | attr_accessor :config_path 26 | attr_accessor :folder 27 | attr_accessor :mbox_path 28 | attr_accessor :verbose 29 | attr_accessor :quiet 30 | 31 | def parse! 32 | OptionParser.new do |opts| 33 | opts.banner = <<~BANNER 34 | Usage: #{$PROGRAM_NAME} [options]" 35 | 36 | Import email messages from a Thunderbird folder into imap-backup. 37 | 38 | BANNER 39 | 40 | opts.on("--config=CONFIG", "The path to an existing (or new) imap-backup config file") do |v| 41 | self.config_path = v 42 | end 43 | opts.on("--email=EMAIL", "The email address configured in imap-backup") do |v| 44 | self.email = v 45 | end 46 | opts.on("--folder=FOLDER", "The folder name to import into") do |v| 47 | self.folder = v 48 | end 49 | opts.on("--mbox=MBOX_PATH", "The path to a Thunderbird folder") do |v| 50 | self.mbox_path = v 51 | end 52 | opts.on("-q", "--quiet", "Do not print any output") do 53 | self.quiet = true 54 | end 55 | opts.on("-v", "--[no-]verbose", "Run verbosely") do |v| 56 | self.verbose = v 57 | end 58 | end.parse! 59 | 60 | raise "Please supply a --config PATH option" if !config_path 61 | raise "Please supply a --email EMAIL option" if !email 62 | raise "Please supply a --folder FOLDER option" if !folder 63 | raise "Please supply a --mbox PATH option" if !mbox_path 64 | end 65 | 66 | def for_logging 67 | {verbose: [verbose], quiet: quiet} 68 | end 69 | end 70 | 71 | options = Options.new.tap(&:parse!) 72 | 73 | Imap::Backup::Logger.setup_logging(options.for_logging) 74 | 75 | config = Imap::Backup::Configuration.new(path: options.config_path) 76 | 77 | account = config.accounts.find { |a| a.username == options.email } 78 | raise "No account found for email address '#{options.email}'" if account.nil? 79 | 80 | mbox = Thunderbird::Mbox.new(path: options.mbox_path) 81 | 82 | serializer = Imap::Backup::Serializer.new(account.local_path, options.folder) 83 | serializer.force_uid_validity(mbox.uid_validity) 84 | 85 | mbox.each do |id, message| 86 | uid = id.to_i 87 | next if serializer.uids.include?(uid) 88 | 89 | # Remove Thunderbird mbox "From" line 90 | message.sub!(/^From[\s\r\n]*/m, "") 91 | serializer.append(id, message, []) 92 | end 93 | -------------------------------------------------------------------------------- /spec/unit/cli/backup_spec.rb: -------------------------------------------------------------------------------- 1 | require "imap/backup/cli/backup" 2 | 3 | require "imap/backup/account" 4 | require "imap/backup/configuration" 5 | 6 | module Imap::Backup 7 | RSpec.describe CLI::Backup do 8 | subject { described_class.new({}) } 9 | 10 | let(:account) do 11 | instance_double(Account, username: "me@example.com", available_for_backup?: true) 12 | end 13 | let(:backup) { instance_double(Account::Backup, "backup", run: nil) } 14 | 15 | before do 16 | allow(Configuration).to receive(:exist?) { true } 17 | allow(Account::Backup).to receive(:new) { backup } 18 | allow(subject).to receive(:requested_accounts) { [account] } 19 | end 20 | 21 | it_behaves_like( 22 | "an action that requires an existing configuration", 23 | action: lambda(&:run) 24 | ) 25 | 26 | it "runs the backup for each connection" do 27 | subject.run 28 | 29 | expect(backup).to have_received(:run) 30 | end 31 | 32 | context "when one connection fails" do 33 | let(:account2) { instance_double(Account, "account2", available_for_backup?: true) } 34 | 35 | before do 36 | outcomes = [-> { raise "Foo" }, -> { true }] 37 | allow(backup).to receive(:run) { outcomes.shift.call } 38 | 39 | allow(subject).to receive(:requested_accounts) { [account, account2] } 40 | end 41 | 42 | it "runs other backups" do 43 | # rubocop:disable Lint/SuppressedException 44 | begin 45 | subject.run 46 | rescue SystemExit 47 | end 48 | # rubocop:enable Lint/SuppressedException 49 | 50 | expect(backup).to have_received(:run).twice 51 | end 52 | 53 | it "exits with an error" do 54 | expect do 55 | subject.run 56 | end.to raise_exception(SystemExit) 57 | end 58 | end 59 | 60 | context "when accounts have different statuses" do 61 | let(:active_account) do 62 | instance_double(Account, username: "active@example.com", available_for_backup?: true) 63 | end 64 | let(:archived_account) do 65 | instance_double(Account, username: "archived@example.com", available_for_backup?: false) 66 | end 67 | let(:offline_account) do 68 | instance_double(Account, username: "offline@example.com", available_for_backup?: false) 69 | end 70 | 71 | before do 72 | allow(subject).to receive(:requested_accounts) { 73 | [active_account, archived_account, offline_account] 74 | } 75 | end 76 | 77 | it "only runs backup for accounts available for backup" do 78 | subject.run 79 | 80 | expect(Account::Backup).to have_received(:new).with(account: active_account, refresh: false) 81 | expect(Account::Backup).not_to have_received(:new). 82 | with(account: archived_account, refresh: anything) 83 | expect(Account::Backup).not_to have_received(:new). 84 | with(account: offline_account, refresh: anything) 85 | end 86 | end 87 | end 88 | end 89 | -------------------------------------------------------------------------------- /lib/imap/backup/cli/mirror.rb: -------------------------------------------------------------------------------- 1 | module Imap; end 2 | 3 | module Imap::Backup 4 | # Processes parameters to run the `mirror` command via command-line parameters 5 | module CLI::Mirror 6 | include Thor::Actions 7 | include CLI::Helpers 8 | 9 | LONG_DESCRIPTION = <<~DESC.freeze 10 | This command is deprecated and will be removed in a future version. 11 | Use 'copy' instead. 12 | 13 | This command updates the DESTINATION_EMAIL account's folders to have the same contents 14 | as those on the SOURCE_EMAIL account. 15 | 16 | If a folder list is configured for the SOURCE_EMAIL account, 17 | only the folders indicated by the setting are copied. 18 | 19 | First, it runs the download of the SOURCE_EMAIL account. 20 | If the SOURCE_EMAIL account is **not** configured to be in 'mirror' mode, 21 | a warning is printed. 22 | 23 | When the mirror command is used, for each folder that is processed, 24 | a new file is created alongside the normal backup files (.imap and .mbox) 25 | This file has a '.mirror' extension. This file contains a mapping of 26 | the known UIDs on the source account to those on the destination account. 27 | 28 | Some configuration may be necessary, as follows: 29 | 30 | #{CLI::Helpers::NAMESPACE_CONFIGURATION_DESCRIPTION} 31 | DESC 32 | 33 | def self.included(base) 34 | base.class_eval do 35 | desc( 36 | "mirror SOURCE_EMAIL DESTINATION_EMAIL [OPTIONS]", 37 | "(Deprecated) Keeps the DESTINATION_EMAIL account aligned with the SOURCE_EMAIL account" 38 | ) 39 | long_desc LONG_DESCRIPTION 40 | config_option 41 | quiet_option 42 | verbose_option 43 | method_option( 44 | "automatic-namespaces", 45 | type: :boolean, 46 | desc: "automatically choose delimiters and prefixes" 47 | ) 48 | method_option( 49 | "destination-delimiter", 50 | type: :string, 51 | desc: "the delimiter for destination folder names" 52 | ) 53 | method_option( 54 | "destination-prefix", 55 | type: :string, 56 | desc: "the prefix (namespace) to add to destination folder names", 57 | aliases: ["-d"] 58 | ) 59 | method_option( 60 | "source-delimiter", 61 | type: :string, 62 | desc: "the delimiter for source folder names" 63 | ) 64 | method_option( 65 | "source-prefix", 66 | type: :string, 67 | desc: "the prefix (namespace) to strip from source folder names", 68 | aliases: ["-s"] 69 | ) 70 | # Keeps one email account in line with another 71 | # @return [void] 72 | def mirror(source_email, destination_email) 73 | non_logging_options = Imap::Backup::Logger.setup_logging(options) 74 | CLI::Transfer.new(:mirror, source_email, destination_email, non_logging_options).run 75 | end 76 | end 77 | end 78 | end 79 | end 80 | -------------------------------------------------------------------------------- /spec/features/utils/export_to_thunderbird_spec.rb: -------------------------------------------------------------------------------- 1 | require "features/helper" 2 | 3 | RSpec.describe "imap-backup utils export-to-thunderbird", type: :aruba do 4 | include_context "message-fixtures" 5 | 6 | let(:account_config) { test_server_connection_parameters.merge(folders: [folder]) } 7 | let(:email) { account_config[:username] } 8 | let(:folder) { "Foo" } 9 | let(:config_options) { {accounts: [account_config]} } 10 | let(:root_path) { File.expand_path("~/.thunderbird") } 11 | let(:write_thunderbird_profiles_ini) do 12 | FileUtils.mkdir_p root_path 13 | path = File.join(root_path, "profiles.ini") 14 | content = <<~PROFILES 15 | [Install0] 16 | Name=default 17 | Default=#{profile_path} 18 | 19 | [Profile1] 20 | IsRelative=1 21 | Path=#{profile_path} 22 | PROFILES 23 | File.write(path, content) 24 | end 25 | let(:create_local_folders) { create_directory local_folders_path } 26 | let(:write_serialized_folder) do 27 | create_local_folder email: email, folder: folder, uid_validity: 1 28 | append_local email: email, folder: folder, body: "Email content" 29 | end 30 | let(:profile_path) { "Profiles/qioxtndq.default" } 31 | let(:local_folders_path) { File.join(root_path, profile_path, "Mail/Local Folders") } 32 | let(:folder_path) do 33 | File.join(local_folders_path, "imap-backup.sbd", "#{email}.sbd", folder) 34 | end 35 | let!(:setup) do 36 | create_config(**config_options) 37 | write_thunderbird_profiles_ini 38 | create_local_folders 39 | write_serialized_folder 40 | end 41 | 42 | it "exports emails" do 43 | run_command_and_stop "imap-backup utils export-to-thunderbird #{email}" 44 | 45 | content = File.read(folder_path) 46 | expect(content).to include("Email content") 47 | end 48 | 49 | context "when Thunderbird is not installed" do 50 | let(:setup) {} 51 | 52 | it "fails" do 53 | run_command "imap-backup utils export-to-thunderbird #{email}" 54 | last_command_started.stop 55 | 56 | expect(last_command_started).to_not have_exit_status(0) 57 | end 58 | end 59 | 60 | context "when a config path is supplied" do 61 | let(:custom_config_path) { File.join(File.expand_path("~/.imap-backup"), "foo.json") } 62 | let(:config_options) { super().merge(path: custom_config_path) } 63 | let(:write_serialized_folder) do 64 | create_local_folder( 65 | email: email, 66 | folder: folder, 67 | uid_validity: 1, 68 | configuration_path: custom_config_path 69 | ) 70 | append_local( 71 | email: email, 72 | folder: folder, 73 | body: "Email content", 74 | configuration_path: custom_config_path 75 | ) 76 | end 77 | 78 | it "exports emails" do 79 | run_command_and_stop \ 80 | "imap-backup utils export-to-thunderbird " \ 81 | "#{email} " \ 82 | "-c #{custom_config_path}" 83 | 84 | content = File.read(folder_path) 85 | expect(content).to include("Email content") 86 | end 87 | end 88 | end 89 | -------------------------------------------------------------------------------- /spec/performance/backup_spec.rb: -------------------------------------------------------------------------------- 1 | require "json" 2 | require "features/helper" 3 | require "imap/backup/configuration" 4 | 5 | # rubocop:disable RSpec/BeforeAfterAll 6 | 7 | RSpec.describe "imap-backup backup performance", :container, :performance, type: :aruba do 8 | # Use exponentially-spaced values so we get an even plot on a logarithmic scale 9 | counts = 0.upto(12).map { |p| (Math::E ** p).round } 10 | runs = 4 11 | results = [] 12 | 13 | before(:all) do 14 | test_server.folders.each do |folder| 15 | next if !folder.start_with?("bulk-") 16 | 17 | test_server.delete_folder folder 18 | end 19 | counts.each do |count| 20 | folder = "bulk-#{count}" 21 | test_server.create_folder folder 22 | message = {from: "address@example.org", subject: "Test 1", body: "body 1\nHi"} 23 | test_server.send_multiple_emails folder, count: count, batch: 1000, **message 24 | end 25 | end 26 | 27 | counts.each do |message_count| 28 | context "with #{message_count} emails" do 29 | count_runs = {count: message_count} 30 | Imap::Backup::Configuration::DOWNLOAD_STRATEGIES.each do |strategy| 31 | context "with #{strategy[:key]} download strategy" do 32 | 1.upto(runs) do |run| 33 | context "with run #{run}" do 34 | let(:account_config) do 35 | test_server_connection_parameters.merge( 36 | folders: [folder], 37 | multi_fetch_size: multi_fetch_size 38 | ) 39 | end 40 | let(:multi_fetch_size) { 25 } 41 | let(:folder) { "bulk-#{message_count}" } 42 | let(:config_options) do 43 | {accounts: [account_config], download_strategy: strategy[:key]} 44 | end 45 | let(:t_start_run) { Time.now } 46 | let(:t_finish_run) { Time.now } 47 | 48 | before do 49 | create_config(**config_options) 50 | end 51 | 52 | after do 53 | test_server.disconnect 54 | end 55 | 56 | specify "time" do 57 | t_start_run 58 | run_command_and_stop "imap-backup backup" 59 | t_finish_run 60 | time_taken = t_finish_run - t_start_run 61 | count_runs[strategy[:key]] ||= [] 62 | count_runs[strategy[:key]] << time_taken 63 | email = account_config[:username] 64 | metadata = imap_parsed(email, folder) 65 | expect(metadata[:messages].count).to eq(message_count) 66 | end 67 | end 68 | end 69 | end 70 | end 71 | results << count_runs 72 | end 73 | end 74 | 75 | after(:all) do 76 | test_server.folders.each do |folder| 77 | next if !folder.start_with?("bulk-") 78 | 79 | test_server.delete_folder folder 80 | end 81 | test_server.disconnect 82 | puts results.to_json 83 | end 84 | end 85 | 86 | # rubocop:enable RSpec/BeforeAfterAll 87 | -------------------------------------------------------------------------------- /lib/imap/backup/setup/global_options/download_strategy_chooser.rb: -------------------------------------------------------------------------------- 1 | require "imap/backup/configuration" 2 | 3 | module Imap; end 4 | module Imap::Backup; end 5 | class Imap::Backup::Setup; end 6 | 7 | class Imap::Backup::Setup::GlobalOptions 8 | # Allows changing the globally configured download strategy 9 | class DownloadStrategyChooser 10 | # @param config [Configuration] the application configuration 11 | def initialize(config:) 12 | @config = config 13 | end 14 | 15 | # Shows the menu 16 | # @return [void] 17 | def run 18 | catch :done do 19 | loop do 20 | Kernel.system("clear") 21 | create_menu 22 | end 23 | end 24 | end 25 | 26 | private 27 | 28 | attr_reader :config 29 | 30 | def create_menu 31 | strategies = Imap::Backup::Configuration::DOWNLOAD_STRATEGIES 32 | highline.choose do |menu| 33 | menu.header = "Choose a Download Strategy" 34 | 35 | strategies.each do |s| 36 | current = s[:key] == config.download_strategy ? " <- current" : "" 37 | topic = "#{s[:description]}#{current}" 38 | menu.choice(topic) do 39 | config.download_strategy = s[:key] 40 | end 41 | end 42 | show_help menu 43 | menu.choice("(q) return to main menu") { throw :done } 44 | menu.hidden("quit") { throw :done } 45 | end 46 | end 47 | 48 | def show_help(menu) 49 | menu.choice("help") do 50 | Kernel.puts <<~HELP 51 | This setting changes how often data is written to disk during backups. 52 | 53 | imap-backup uses two files per folder, a .mbox file with the actual 54 | messages and a .imap file with metadata like message lengths and their 55 | offsets within the .mbox file. 56 | 57 | # write straight to disk 58 | 59 | With this setting, each message and its metadata are written to disk 60 | as they are downloaded. 61 | 62 | This choice uses least memory and so is suitable for backing up onto 63 | devices with limited memory, like Raspberry Pis. 64 | 65 | # delay writing metadata 66 | 67 | This is the default setting. 68 | 69 | Here, messages (which are potentially very large) are appended to the 70 | .mbox file as they are received, but the metadata is only written to 71 | the .imap file once all the folder's messages have been downloaded. 72 | 73 | This choice uses a little more memory than the previous setting, but 74 | is **much** faster for large folders (potentially >30 times for 75 | folders with >100k messages) and is less wearing on the disk. 76 | 77 | # Other Performance Settings 78 | 79 | Another configuration which affects backup performance is the 80 | `multi_fetch_size` account-level setting. 81 | 82 | HELP 83 | highline.ask "Press a key " 84 | end 85 | end 86 | 87 | def highline 88 | Imap::Backup::Setup.highline 89 | end 90 | end 91 | end 92 | --------------------------------------------------------------------------------