├── .travis.yml ├── .gitignore ├── lib ├── mysql2psql │ ├── writer.rb │ ├── config.rb │ ├── version.rb │ ├── errors.rb │ ├── postgres_db_writer.rb │ ├── config_base.rb │ ├── converter.rb │ ├── postgres_file_writer.rb │ ├── postgres_writer.rb │ ├── connection.rb │ └── mysql_reader.rb └── mysqltopostgres.rb ├── Gemfile ├── test ├── fixtures │ ├── seed_integration_tests.sql │ └── config_all_options.yml ├── integration │ ├── postgres_db_writer_base_test.rb │ ├── convert_to_db_test.rb │ ├── converter_test.rb │ ├── mysql_reader_base_test.rb │ ├── mysql_reader_test.rb │ └── convert_to_file_test.rb ├── lib │ ├── ext_test_unit.rb │ └── test_helper.rb └── units │ ├── postgres_file_writer_test.rb │ ├── config_test.rb │ └── config_base_test.rb ├── bin └── mysqltopostgres ├── MIT-LICENSE ├── config └── default.database.yml ├── README.md ├── Rakefile └── mysqltopostgres.gemspec /.travis.yml: -------------------------------------------------------------------------------- 1 | language: ruby 2 | rvm: 3 | - 1.9.3 -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | test/fixtures/test*.sql 2 | config/database.yml 3 | pkg 4 | .rvmrc 5 | .bundle 6 | *.sql 7 | Gemfile.lock 8 | -------------------------------------------------------------------------------- /lib/mysql2psql/writer.rb: -------------------------------------------------------------------------------- 1 | class Mysql2psql 2 | class Writer 3 | def inload 4 | fail "Method 'inload' needs to be overridden..." 5 | end 6 | end 7 | end 8 | -------------------------------------------------------------------------------- /lib/mysql2psql/config.rb: -------------------------------------------------------------------------------- 1 | require 'mysql2psql/config_base' 2 | 3 | class Mysql2psql 4 | class Config < ConfigBase 5 | def initialize(yaml) 6 | super(yaml) 7 | end 8 | end 9 | end 10 | -------------------------------------------------------------------------------- /lib/mysql2psql/version.rb: -------------------------------------------------------------------------------- 1 | class Mysql2psql 2 | module Version 3 | MAJOR = 0 4 | MINOR = 2 5 | PATCH = 0 6 | 7 | STRING = [MAJOR, MINOR, PATCH].compact.join('.') 8 | end 9 | end 10 | -------------------------------------------------------------------------------- /lib/mysql2psql/errors.rb: -------------------------------------------------------------------------------- 1 | 2 | class Mysql2psql 3 | class GeneralError < StandardError 4 | end 5 | 6 | class ConfigurationError < StandardError 7 | end 8 | class UninitializedValueError < ConfigurationError 9 | end 10 | class ConfigurationFileNotFound < ConfigurationError 11 | end 12 | class ConfigurationFileInitialized < ConfigurationError 13 | end 14 | end 15 | -------------------------------------------------------------------------------- /Gemfile: -------------------------------------------------------------------------------- 1 | ruby '1.9.3' 2 | source 'https://rubygems.org' 3 | 4 | gem 'rake', '~> 10.3' 5 | gem 'mysql-pr', '~> 2.9' 6 | gem 'postgres-pr', '~> 0.6' 7 | 8 | platforms :jruby do 9 | gem 'activerecord' 10 | gem 'jdbc-postgres' 11 | gem 'activerecord-jdbc-adapter' 12 | gem 'activerecord-jdbcpostgresql-adapter' 13 | end 14 | 15 | platforms :mri_19 do 16 | gem 'pg', '~> 0.17' 17 | end 18 | 19 | gem 'test-unit' 20 | 21 | group :test do 22 | gem 'jeweler', '~> 2.0' 23 | end 24 | -------------------------------------------------------------------------------- /test/fixtures/seed_integration_tests.sql: -------------------------------------------------------------------------------- 1 | -- seed data for integration tests 2 | 3 | DROP TABLE IF EXISTS numeric_types_basics; 4 | CREATE TABLE numeric_types_basics ( 5 | id int, 6 | f_tinyint TINYINT, 7 | f_smallint SMALLINT, 8 | f_mediumint MEDIUMINT, 9 | f_int INT, 10 | f_integer INTEGER, 11 | f_bigint BIGINT, 12 | f_real REAL, 13 | f_double DOUBLE, 14 | f_float FLOAT, 15 | f_decimal DECIMAL, 16 | f_numeric NUMERIC 17 | ); 18 | 19 | INSERT INTO numeric_types_basics VALUES 20 | (1,1,1,1,1,1,1,1,1,1,1,1), 21 | (2,2,2,2,2,2,2,2,2,2,2,2), 22 | (23,23,23,23,23,23,23,23,23,23,23,23); 23 | 24 | 25 | -------------------------------------------------------------------------------- /lib/mysql2psql/postgres_db_writer.rb: -------------------------------------------------------------------------------- 1 | require 'mysql2psql/postgres_writer' 2 | require 'mysql2psql/connection' 3 | 4 | class Mysql2psql 5 | class PostgresDbWriter < PostgresFileWriter 6 | attr_reader :connection, :filename 7 | 8 | def initialize(filename, options) 9 | # note that the superclass opens and truncates filename for writing 10 | super(filename) 11 | @filename = filename 12 | @connection = Connection.new(options) 13 | end 14 | 15 | def inload(path = filename) 16 | connection.load_file(path) 17 | end 18 | 19 | def clear_schema 20 | connection.clear_schema 21 | end 22 | end 23 | end 24 | -------------------------------------------------------------------------------- /test/integration/postgres_db_writer_base_test.rb: -------------------------------------------------------------------------------- 1 | require 'test_helper' 2 | 3 | require 'mysql2psql/postgres_db_writer' 4 | 5 | class PostgresDbWriterBaseTest < Test::Unit::TestCase 6 | class << self 7 | def startup 8 | seed_test_database 9 | @@options = get_test_config_by_label(:localmysql_to_db_convert_nothing) 10 | end 11 | 12 | def shutdown 13 | delete_files_for_test_config(@@options) 14 | end 15 | end 16 | def setup 17 | end 18 | 19 | def teardown 20 | end 21 | 22 | def options 23 | @@options 24 | end 25 | 26 | def test_pg_connection 27 | assert_nothing_raised do 28 | reader = Mysql2psql::PostgresDbWriter.new(options) 29 | end 30 | end 31 | end 32 | -------------------------------------------------------------------------------- /test/integration/convert_to_db_test.rb: -------------------------------------------------------------------------------- 1 | require 'test_helper' 2 | 3 | require 'mysqltopostgres' 4 | 5 | class ConvertToDbTest < Test::Unit::TestCase 6 | class << self 7 | def startup 8 | seed_test_database 9 | @@options = get_test_config_by_label(:localmysql_to_db_convert_all) 10 | @@mysql2psql = Mysql2psql.new([@@options.filepath]) 11 | @@mysql2psql.convert 12 | @@mysql2psql.writer.open 13 | end 14 | 15 | def shutdown 16 | @@mysql2psql.writer.close 17 | delete_files_for_test_config(@@options) 18 | end 19 | end 20 | def setup 21 | end 22 | 23 | def teardown 24 | end 25 | 26 | def test_table_creation 27 | assert_true @@mysql2psql.writer.exists?('numeric_types_basics') 28 | end 29 | end 30 | -------------------------------------------------------------------------------- /test/lib/ext_test_unit.rb: -------------------------------------------------------------------------------- 1 | module Test::Unit 2 | class TestCase 3 | def self.must(name, &block) 4 | test_name = "test_#{name.gsub(/\s+/, '_')}".to_sym 5 | defined = instance_method(test_name) rescue false 6 | fail "#{test_name} is already defined in #{self}" if defined 7 | if block_given? 8 | define_method(test_name, &block) 9 | else 10 | define_method(test_name) do 11 | flunk "No implementation provided for #{name}" 12 | end 13 | end 14 | end 15 | end 16 | end 17 | 18 | module Test::Unit::Assertions 19 | def assert_false(object, message = '') 20 | assert_equal(false, object, message) 21 | end 22 | 23 | def assert_true(object, message = '') 24 | assert_equal(true, object, message) 25 | end 26 | end 27 | -------------------------------------------------------------------------------- /test/units/postgres_file_writer_test.rb: -------------------------------------------------------------------------------- 1 | require 'test_helper' 2 | 3 | require 'mysqltopostgres' 4 | 5 | class PostgresFileWriterTest < Test::Unit::TestCase 6 | attr_accessor :destfile 7 | def setup 8 | f = Tempfile.new('mysql2psql_test_destfile') 9 | @destfile = f.path 10 | f.close! 11 | rescue => e 12 | raise StandardError.new('Failed to initialize integration test db. See README for setup requirements.') 13 | end 14 | 15 | def teardown 16 | File.delete(destfile) if File.exist?(destfile) 17 | end 18 | 19 | def test_basic_write 20 | writer = Mysql2psql::PostgresFileWriter.new(destfile) 21 | writer.close 22 | content = IO.read(destfile) 23 | assert_not_nil content.match("SET client_encoding = 'UTF8'") 24 | assert_nil content.match('unobtanium') 25 | end 26 | end 27 | -------------------------------------------------------------------------------- /test/integration/converter_test.rb: -------------------------------------------------------------------------------- 1 | require 'test_helper' 2 | 3 | require 'mysql2psql/converter' 4 | 5 | class ConverterTest < Test::Unit::TestCase 6 | class << self 7 | def startup 8 | seed_test_database 9 | @@options = get_test_config_by_label(:localmysql_to_file_convert_nothing) 10 | end 11 | 12 | def shutdown 13 | delete_files_for_test_config(@@options) 14 | end 15 | end 16 | def setup 17 | end 18 | 19 | def teardown 20 | end 21 | 22 | def options 23 | @@options 24 | end 25 | 26 | def test_new_converter 27 | assert_nothing_raised do 28 | reader = get_test_reader(options) 29 | writer = get_test_file_writer(options) 30 | converter = Mysql2psql::Converter.new(reader, writer, options) 31 | assert_equal 0, converter.convert 32 | end 33 | end 34 | end 35 | -------------------------------------------------------------------------------- /bin/mysqltopostgres: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | $LOAD_PATH.unshift File.join(File.dirname(__FILE__), '..', 'lib') 3 | 4 | require 'rubygems' 5 | require 'bundler/setup' 6 | 7 | require 'mysqltopostgres' 8 | 9 | CONFIG_FILE = File.expand_path(File.expand_path(File.dirname(__FILE__)) + '/../config/database.yml') 10 | 11 | if FileTest.exist?(CONFIG_FILE) || (ARGV.length > 0 && FileTest.exist?(File.expand_path(ARGV[0]))) 12 | 13 | if ARGV.length > 0 14 | file = ARGV[0] 15 | else 16 | file = CONFIG_FILE 17 | end 18 | 19 | db_yaml = YAML.load_file file 20 | 21 | if db_yaml.key?('mysql2psql') 22 | # puts db_yaml["mysql2psql"].to_s 23 | Mysql2psql.new(db_yaml['mysql2psql']).convert 24 | else 25 | # Oh Noes! There is no key in the hash... 26 | fail "'#{file}' does not contain a configuration directive for mysql -> postgres" 27 | end 28 | 29 | else 30 | fail "'#{file}' does not exist" 31 | end 32 | -------------------------------------------------------------------------------- /lib/mysql2psql/config_base.rb: -------------------------------------------------------------------------------- 1 | require 'yaml' 2 | require 'mysql2psql/errors' 3 | 4 | class Mysql2psql 5 | class ConfigBase 6 | attr_reader :config, :filepath 7 | 8 | def initialize(yaml) 9 | @filepath = nil 10 | @config = yaml # YAML::load(File.read(filepath)) 11 | end 12 | 13 | def [](key) 14 | send(key) 15 | end 16 | 17 | def method_missing(name, *args) 18 | token = name.to_s 19 | default = args.length > 0 ? args[0] : '' 20 | must_be_defined = default == :none 21 | case token 22 | when /mysql/i 23 | key = token.sub(/^mysql/, '') 24 | value = config['mysql'][key] 25 | when /dest/i 26 | key = token.sub(/^dest/, '') 27 | value = config['destination'][key] 28 | when /only_tables/i 29 | value = config['tables'] 30 | else 31 | value = config[token] 32 | end 33 | value.nil? ? ( must_be_defined ? (fail Mysql2psql::UninitializedValueError.new("no value and no default for #{name}")) : default) : value 34 | end 35 | end 36 | end 37 | -------------------------------------------------------------------------------- /test/fixtures/config_all_options.yml: -------------------------------------------------------------------------------- 1 | mysql: 2 | hostname: localhost 3 | port: 3306 4 | socket: /tmp/mysql.sock 5 | username: somename 6 | password: secretpassword 7 | database: somename 8 | 9 | destination: 10 | # if file is given, output goes to file, else postgres 11 | file: somefile 12 | postgres: 13 | hostname: localhost 14 | port: 5432 15 | username: somename 16 | password: secretpassword 17 | database: somename 18 | 19 | # if tables is given, only the listed tables will be converted. leave empty to convert all tables. 20 | tables: 21 | - table1 22 | - table2 23 | - table3 24 | - table4 25 | 26 | # if exclude_tables is given, exclude the listed tables from the conversion. 27 | exclude_tables: 28 | - table5 29 | - table6 30 | 31 | # if suppress_data is true, only the schema definition will be exported/migrated, and not the data 32 | suppress_data: true 33 | 34 | # if suppress_ddl is true, only the data will be exported/imported, and not the schema 35 | suppress_ddl: false 36 | 37 | # if force_truncate is true, forces a table truncate before table loading 38 | force_truncate: false 39 | -------------------------------------------------------------------------------- /test/integration/mysql_reader_base_test.rb: -------------------------------------------------------------------------------- 1 | require 'test_helper' 2 | 3 | require 'mysql2psql/mysql_reader' 4 | 5 | class MysqlReaderBaseTest < Test::Unit::TestCase 6 | class << self 7 | def startup 8 | seed_test_database 9 | @@options = get_test_config_by_label(:localmysql_to_file_convert_nothing) 10 | end 11 | 12 | def shutdown 13 | delete_files_for_test_config(@@options) 14 | end 15 | end 16 | def setup 17 | end 18 | 19 | def teardown 20 | end 21 | 22 | def options 23 | @@options 24 | end 25 | 26 | def test_mysql_connection 27 | assert_nothing_raised do 28 | reader = Mysql2psql::MysqlReader.new(options) 29 | end 30 | end 31 | 32 | def test_mysql_reconnect 33 | assert_nothing_raised do 34 | reader = Mysql2psql::MysqlReader.new(options) 35 | reader.reconnect 36 | end 37 | end 38 | 39 | def test_mysql_connection_without_port 40 | assert_nothing_raised do 41 | options.mysqlport = '' 42 | options.mysqlsocket = '' 43 | reader = Mysql2psql::MysqlReader.new(options) 44 | end 45 | end 46 | end 47 | -------------------------------------------------------------------------------- /test/units/config_test.rb: -------------------------------------------------------------------------------- 1 | require 'test_helper' 2 | 3 | require 'mysql2psql/config' 4 | 5 | class ConfigTest < Test::Unit::TestCase 6 | attr_reader :configfile_new, :configfile_all_opts, :configfile_not_found 7 | def setup 8 | @configfile_all_opts = "#{File.dirname(__FILE__)}/../fixtures/config_all_options.yml" 9 | @configfile_not_found = "#{File.dirname(__FILE__)}/../fixtures/config_not_found.yml.do_not_create_this_file" 10 | @configfile_new = get_temp_file('mysql2psql_test_config') 11 | end 12 | 13 | def teardown 14 | File.delete(configfile_new) if File.exist?(configfile_new) 15 | end 16 | 17 | def test_config_loaded 18 | value = Mysql2psql::Config.new(configfile_all_opts, false) 19 | assert_not_nil value 20 | end 21 | 22 | def test_config_file_not_found 23 | assert_raise(Mysql2psql::ConfigurationFileNotFound) do 24 | value = Mysql2psql::Config.new(configfile_not_found, false) 25 | end 26 | end 27 | 28 | def test_initialize_new_config_file 29 | assert_raise(Mysql2psql::ConfigurationFileInitialized) do 30 | value = Mysql2psql::Config.new(configfile_new, true) 31 | end 32 | end 33 | end 34 | -------------------------------------------------------------------------------- /MIT-LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2012-2013 Regents of the University of Minnesota 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining 4 | a copy of this software and associated documentation files (the 5 | "Software"), to deal in the Software without restriction, including 6 | without limitation the rights to use, copy, modify, merge, publish, 7 | distribute, sublicense, and/or sell copies of the Software, and to 8 | permit persons to whom the Software is furnished to do so, subject to 9 | the following conditions: 10 | 11 | The above copyright notice and this permission notice shall be 12 | included in all copies or substantial portions of the Software. 13 | 14 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 15 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 16 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 17 | NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE 18 | LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 19 | OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 20 | WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 21 | -------------------------------------------------------------------------------- /test/units/config_base_test.rb: -------------------------------------------------------------------------------- 1 | require 'test_helper' 2 | require 'mysqltopostgres' 3 | 4 | # 5 | # 6 | class ConfigBaseTest < Test::Unit::TestCase 7 | attr_reader :config, :configfilepath 8 | def setup 9 | @configfilepath = "#{File.dirname(__FILE__)}/../fixtures/config_all_options.yml" 10 | @config = Mysql2psql::ConfigBase.new(configfilepath) 11 | end 12 | 13 | def teardown 14 | @config = nil 15 | end 16 | 17 | def test_config_loaded 18 | assert_not_nil config.config 19 | assert_equal configfilepath, config.filepath 20 | end 21 | 22 | def test_uninitialized_error_when_not_found_and_no_default 23 | assert_raises(Mysql2psql::UninitializedValueError) do 24 | value = @config.not_found(:none) 25 | end 26 | end 27 | 28 | def test_default_when_not_found 29 | expected = 'defaultvalue' 30 | value = @config.not_found(expected) 31 | assert_equal expected, value 32 | end 33 | 34 | def test_mysql_hostname 35 | value = @config.mysqlhostname 36 | assert_equal 'localhost', value 37 | end 38 | 39 | def test_mysql_hostname_array_access 40 | value = @config[:mysqlhostname] 41 | assert_equal 'localhost', value 42 | end 43 | 44 | def test_dest_file 45 | value = @config.destfile 46 | assert_equal 'somefile', value 47 | end 48 | end 49 | -------------------------------------------------------------------------------- /test/integration/mysql_reader_test.rb: -------------------------------------------------------------------------------- 1 | require 'test_helper' 2 | 3 | class MysqlReaderTest < Test::Unit::TestCase 4 | class << self 5 | def startup 6 | seed_test_database 7 | @@options = get_test_config_by_label(:localmysql_to_file_convert_nothing) 8 | @@reader = get_test_reader(@@options) 9 | end 10 | 11 | def shutdown 12 | delete_files_for_test_config(@@options) 13 | end 14 | end 15 | def setup 16 | end 17 | 18 | def teardown 19 | end 20 | 21 | def reader 22 | @@reader 23 | end 24 | 25 | def test_db_connection 26 | assert_nothing_raised do 27 | reader.mysql.ping 28 | end 29 | end 30 | 31 | def test_tables_collection 32 | values = reader.tables.select { |t| t.name == 'numeric_types_basics' } 33 | assert_true values.length == 1 34 | assert_equal 'numeric_types_basics', values[0].name 35 | end 36 | 37 | def test_paginated_read 38 | expected_rows = 3 39 | page_size = 2 40 | expected_pages = (1.0 * expected_rows / page_size).ceil 41 | 42 | row_count = my_row_count = 0 43 | table = reader.tables.select { |t| t.name == 'numeric_types_basics' }[0] 44 | reader.paginated_read(table, page_size) do |_row, counter| 45 | row_count = counter 46 | my_row_count += 1 47 | end 48 | assert_equal expected_rows, row_count 49 | assert_equal expected_rows, my_row_count 50 | end 51 | end 52 | -------------------------------------------------------------------------------- /lib/mysqltopostgres.rb: -------------------------------------------------------------------------------- 1 | 2 | if RUBY_PLATFORM == 'java' 3 | require 'active_record' 4 | require 'postgres-pr/postgres-compat' 5 | else 6 | require 'pg' 7 | require 'pg_ext' 8 | require 'pg/exceptions' 9 | require 'pg/constants' 10 | require 'pg/connection' 11 | require 'pg/result' 12 | end 13 | 14 | require 'mysql2psql/errors' 15 | require 'mysql2psql/version' 16 | require 'mysql2psql/config' 17 | require 'mysql2psql/converter' 18 | require 'mysql2psql/mysql_reader' 19 | require 'mysql2psql/writer' 20 | require 'mysql2psql/postgres_writer' 21 | require 'mysql2psql/postgres_file_writer.rb' 22 | require 'mysql2psql/postgres_db_writer.rb' 23 | 24 | class Mysql2psql 25 | attr_reader :options, :reader, :writer 26 | 27 | def initialize(yaml) 28 | @options = Config.new(yaml) 29 | end 30 | 31 | def send_file_to_postgres(path) 32 | connection = Connection.new(options) 33 | connection.load_file(path) 34 | end 35 | 36 | def convert 37 | @reader = MysqlReader.new(options) 38 | 39 | tag = Time.new.to_s.gsub(/((\-)|( )|(:))+/, '') 40 | 41 | path = './' 42 | 43 | unless options.config['dump_file_directory'].nil? 44 | path = options.config['dump_file_directory'] 45 | end 46 | 47 | filename = File.expand_path(File.join(path, tag + '_output.sql')) 48 | 49 | @writer = PostgresDbWriter.new(filename, options) 50 | 51 | Converter.new(reader, writer, options).convert 52 | 53 | if options.config['remove_dump_file'] 54 | File.delete filename if File.exist?(filename) 55 | end 56 | end 57 | end 58 | -------------------------------------------------------------------------------- /config/default.database.yml: -------------------------------------------------------------------------------- 1 | default: &default 2 | adapter: jdbcpostgresql 3 | encoding: unicode 4 | pool: 4 5 | username: terrapotamus 6 | password: default 7 | host: 127.0.0.1 8 | template: template_gis 9 | gis_schema_name: gis_tmp 10 | 11 | development: &development 12 | <<: *default 13 | database: default_development 14 | 15 | test: &test 16 | <<: *default 17 | database: default_test 18 | 19 | production: &production 20 | <<: *default 21 | database: default_production 22 | 23 | mysql_data_source: &pii 24 | hostname: localhost 25 | port: 3306 26 | username: username 27 | password: default 28 | database: awesome_possum 29 | 30 | mysql2psql: 31 | mysql: 32 | <<: *pii 33 | 34 | destination: 35 | production: 36 | <<: *production 37 | test: 38 | <<: *test 39 | development: 40 | <<: *development 41 | 42 | tables: 43 | - countries 44 | - samples 45 | - universes 46 | - variable_groups 47 | - variables 48 | - sample_variables 49 | 50 | # If suppress_data is true, only the schema definition will be exported/migrated, and not the data 51 | suppress_data: false 52 | 53 | # If suppress_ddl is true, only the data will be exported/imported, and not the schema 54 | suppress_ddl: true 55 | 56 | # If force_truncate is true, forces a table truncate before table loading 57 | force_truncate: false 58 | 59 | preserve_order: true 60 | 61 | remove_dump_file: true 62 | 63 | dump_file_directory: /tmp 64 | 65 | report_status: json # false, json, xml 66 | 67 | # If clear_schema is true, the public schema will be recreated before conversion 68 | # The import will fail if both clear_schema and suppress_ddl are true. 69 | clear_schema: false 70 | -------------------------------------------------------------------------------- /lib/mysql2psql/converter.rb: -------------------------------------------------------------------------------- 1 | class Mysql2psql 2 | class Converter 3 | attr_reader :reader, :writer, :options 4 | attr_reader :exclude_tables, :only_tables, :suppress_data, :suppress_ddl, :force_truncate, :preserve_order, :clear_schema 5 | 6 | def initialize(reader, writer, options) 7 | @reader = reader 8 | @writer = writer 9 | @options = options 10 | @exclude_tables = options.exclude_tables([]) 11 | @only_tables = options.only_tables(nil) 12 | @suppress_data = options.suppress_data(false) 13 | @suppress_ddl = options.suppress_ddl(false) 14 | @force_truncate = options.force_truncate(false) 15 | @preserve_order = options.preserve_order(false) 16 | @clear_schema = options.clear_schema(false) 17 | end 18 | 19 | def convert 20 | tables = reader.tables 21 | .reject { |table| @exclude_tables.include?(table.name) } 22 | .select { |table| @only_tables ? @only_tables.include?(table.name) : true } 23 | 24 | if @preserve_order 25 | 26 | reordered_tables = [] 27 | 28 | @only_tables.each do |only_table| 29 | idx = tables.index { |table| table.name == only_table } 30 | reordered_tables << tables[idx] 31 | end 32 | 33 | tables = reordered_tables 34 | 35 | end 36 | 37 | tables.each do |table| 38 | writer.write_table(table) 39 | end unless @suppress_ddl 40 | 41 | # tables.each do |table| 42 | # writer.truncate(table) if force_truncate && suppress_ddl 43 | # writer.write_contents(table, reader) 44 | # end unless @suppress_data 45 | 46 | unless @suppress_data 47 | 48 | tables.each do |table| 49 | writer.truncate(table) if force_truncate && suppress_ddl 50 | end 51 | 52 | tables.each do |table| 53 | writer.write_contents(table, reader) 54 | end 55 | 56 | end 57 | 58 | tables.each do |table| 59 | writer.write_indexes(table) 60 | end unless @suppress_ddl 61 | tables.each do |table| 62 | writer.write_constraints(table) 63 | end unless @suppress_ddl 64 | 65 | writer.close 66 | 67 | if @clear_schema 68 | writer.clear_schema 69 | end 70 | 71 | writer.inload 72 | 73 | 0 74 | end 75 | end 76 | end 77 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # mysql-to-postgres - MySQL to PostgreSQL Data Translation 2 | 3 | MRI or jruby supported. 4 | 5 | With a bit of a modified rails database.yml configuration, you can integrate mysql-to-postgres into a project. 6 | 7 | ## Installation 8 | 9 | ### Via RubyGems 10 | 11 | ```sh 12 | gem install mysqltopostgres 13 | ``` 14 | 15 | ### From source 16 | 17 | ```sh 18 | git clone https://github.com/maxlapshin/mysql2postgres.git 19 | cd mysql2postgres 20 | bundle install 21 | gem build mysqltopostgres.gemspec 22 | sudo gem install mysqltopostgres-0.2.19.gem 23 | ``` 24 | 25 | ## Sample Configuration 26 | 27 | Configuration is written in [YAML format](http://www.yaml.org/ "YAML Ain't Markup Language") 28 | and passed as the first argument on the command line. 29 | 30 | ```yaml 31 | default: &default 32 | adapter: jdbcpostgresql 33 | encoding: unicode 34 | pool: 4 35 | username: terrapotamus 36 | password: default 37 | host: 127.0.0.1 38 | 39 | development: &development 40 | <<: *default 41 | database: default_development 42 | 43 | test: &test 44 | <<: *default 45 | database: default_test 46 | 47 | production: &production 48 | <<: *default 49 | database: default_production 50 | 51 | mysql_data_source: &pii 52 | hostname: localhost 53 | port: 3306 54 | socket: /tmp/mysqld.sock 55 | username: username 56 | password: default 57 | database: awesome_possum 58 | 59 | mysql2psql: 60 | mysql: 61 | <<: *pii 62 | 63 | destination: 64 | production: 65 | <<: *production 66 | test: 67 | <<: *test 68 | development: 69 | <<: *development 70 | 71 | tables: 72 | - countries 73 | - samples 74 | - universes 75 | - variable_groups 76 | - variables 77 | - sample_variables 78 | 79 | # If suppress_data is true, only the schema definition will be exported/migrated, and not the data 80 | suppress_data: false 81 | 82 | # If suppress_ddl is true, only the data will be exported/imported, and not the schema 83 | suppress_ddl: true 84 | 85 | # If force_truncate is true, forces a table truncate before table loading 86 | force_truncate: false 87 | 88 | preserve_order: true 89 | 90 | remove_dump_file: true 91 | 92 | dump_file_directory: /tmp 93 | 94 | report_status: json # false, json, xml 95 | 96 | # If clear_schema is true, the public schema will be recreated before conversion 97 | # The import will fail if both clear_schema and suppress_ddl are true. 98 | clear_schema: false 99 | ``` 100 | 101 | Please note that the MySQL connection will be using socket in case the host is not defined (`nil`) or it is `'localhost'`. 102 | -------------------------------------------------------------------------------- /test/integration/convert_to_file_test.rb: -------------------------------------------------------------------------------- 1 | require 'test_helper' 2 | 3 | require 'mysqltopostgres' 4 | 5 | class ConvertToFileTest < Test::Unit::TestCase 6 | class << self 7 | def startup 8 | seed_test_database 9 | @@options = get_test_config_by_label(:localmysql_to_file_convert_all) 10 | @@mysql2psql = Mysql2psql.new([@@options.filepath]) 11 | @@mysql2psql.convert 12 | @@content = IO.read(@@mysql2psql.options.destfile) 13 | end 14 | 15 | def shutdown 16 | delete_files_for_test_config(@@options) 17 | end 18 | end 19 | def setup 20 | end 21 | 22 | def teardown 23 | end 24 | 25 | def content 26 | @@content 27 | end 28 | 29 | def test_table_creation 30 | assert_not_nil content.match('DROP TABLE IF EXISTS "numeric_types_basics" CASCADE') 31 | assert_not_nil content.match(/CREATE TABLE "numeric_types_basics"/) 32 | end 33 | 34 | def test_basic_numerics_tinyint 35 | assert_not_nil Regexp.new('CREATE TABLE "numeric_types_basics".*"f_tinyint" smallint,.*\)', Regexp::MULTILINE).match(content) 36 | end 37 | 38 | def test_basic_numerics_smallint 39 | assert_not_nil Regexp.new('CREATE TABLE "numeric_types_basics".*"f_smallint" integer,.*\)', Regexp::MULTILINE).match(content) 40 | end 41 | 42 | def test_basic_numerics_mediumint 43 | assert_not_nil Regexp.new('CREATE TABLE "numeric_types_basics".*"f_mediumint" integer,.*\)', Regexp::MULTILINE).match(content) 44 | end 45 | 46 | def test_basic_numerics_int 47 | assert_not_nil Regexp.new('CREATE TABLE "numeric_types_basics".*"f_int" integer,.*\)', Regexp::MULTILINE).match(content) 48 | end 49 | 50 | def test_basic_numerics_integer 51 | assert_not_nil Regexp.new('CREATE TABLE "numeric_types_basics".*"f_integer" integer,.*\)', Regexp::MULTILINE).match(content) 52 | end 53 | 54 | def test_basic_numerics_bigint 55 | assert_not_nil Regexp.new('CREATE TABLE "numeric_types_basics".*"f_bigint" bigint,.*\)', Regexp::MULTILINE).match(content) 56 | end 57 | 58 | def test_basic_numerics_real 59 | assert_not_nil Regexp.new('CREATE TABLE "numeric_types_basics".*"f_real" double precision,.*\)', Regexp::MULTILINE).match(content) 60 | end 61 | 62 | def test_basic_numerics_double 63 | assert_not_nil Regexp.new('CREATE TABLE "numeric_types_basics".*"f_double" double precision,.*\)', Regexp::MULTILINE).match(content) 64 | end 65 | 66 | def test_basic_numerics_float 67 | assert_not_nil Regexp.new('CREATE TABLE "numeric_types_basics".*"f_float" double precision,.*\)', Regexp::MULTILINE).match(content) 68 | end 69 | 70 | def test_basic_numerics_decimal 71 | assert_not_nil Regexp.new('CREATE TABLE "numeric_types_basics".*"f_decimal" numeric\(10, 0\),.*\)', Regexp::MULTILINE).match(content) 72 | end 73 | 74 | def test_basic_numerics_numeric 75 | assert_not_nil Regexp.new('CREATE TABLE "numeric_types_basics".*"f_numeric" numeric\(10, 0\)[\w\n]*\)', Regexp::MULTILINE).match(content) 76 | end 77 | end 78 | -------------------------------------------------------------------------------- /Rakefile: -------------------------------------------------------------------------------- 1 | require 'rubygems' 2 | require 'rake' 3 | 4 | $LOAD_PATH.unshift('lib') 5 | require 'mysql2psql/version' 6 | 7 | begin 8 | require 'jeweler' 9 | Jeweler::Tasks.new do |gem| 10 | gem.name = 'mysql2psql' 11 | gem.version = Mysql2psql::Version::STRING 12 | gem.summary = %(Tool for converting mysql database to postgresql) 13 | gem.description = %{It can create postgresql dump from mysql database or directly load data from mysql to 14 | postgresql (at about 100 000 records per minute). Translates most data types and indexes.} 15 | gem.email = 'gallagher.paul@gmail.com' 16 | gem.homepage = 'http://github.com/tardate/mysql2postgresql' 17 | gem.authors = [ 18 | 'Max Lapshin ', 19 | 'Anton Ageev ', 20 | 'Samuel Tribehou ', 21 | 'Marco Nenciarini ', 22 | 'James Nobis ', 23 | 'quel ', 24 | 'Holger Amann ', 25 | 'Maxim Dobriakov ', 26 | 'Michael Kimsal ', 27 | 'Jacob Coby ', 28 | 'Neszt Tibor ', 29 | 'Miroslav Kratochvil ', 30 | 'Paul Gallagher ' 31 | ] 32 | gem.add_dependency 'mysql', '= 2.8.1' 33 | gem.add_dependency 'pg', '= 0.9.0' 34 | gem.add_development_dependency 'test-unit', '>= 2.1.1' 35 | # gem is a Gem::Specification... see http://www.rubygems.org/read/chapter/20 for additional settings 36 | end 37 | Jeweler::GemcutterTasks.new 38 | rescue LoadError 39 | puts 'Jeweler (or a dependency) not available. Install it with: gem install jeweler' 40 | end 41 | 42 | require 'rake/testtask' 43 | namespace :test do 44 | Rake::TestTask.new(:units) do |test| 45 | test.libs << 'lib' << 'test/lib' 46 | test.pattern = 'test/units/*test.rb' 47 | test.verbose = true 48 | end 49 | 50 | Rake::TestTask.new(:integration) do |test| 51 | test.libs << 'lib' << 'test/lib' 52 | test.pattern = 'test/integration/*test.rb' 53 | test.verbose = true 54 | end 55 | end 56 | 57 | desc 'Run all tests' 58 | task :test do 59 | Rake::Task['test:units'].invoke 60 | Rake::Task['test:integration'].invoke 61 | end 62 | 63 | begin 64 | require 'rcov/rcovtask' 65 | Rcov::RcovTask.new do |test| 66 | test.libs << 'test' 67 | test.pattern = 'test/**/*test.rb' 68 | test.verbose = true 69 | end 70 | rescue LoadError 71 | task :rcov do 72 | abort 'RCov is not available. In order to run rcov, you must: sudo gem install spicycode-rcov' 73 | end 74 | end 75 | 76 | task default: :test 77 | 78 | require 'rdoc/task' 79 | Rake::RDocTask.new do |rdoc| 80 | version = Mysql2psql::Version::STRING 81 | 82 | rdoc.rdoc_dir = 'rdoc' 83 | rdoc.title = "mysql2psql #{version}" 84 | rdoc.rdoc_files.include('README*') 85 | rdoc.rdoc_files.include('lib/**/*.rb') 86 | end 87 | -------------------------------------------------------------------------------- /test/lib/test_helper.rb: -------------------------------------------------------------------------------- 1 | require 'rubygems' 2 | begin 3 | gem 'test-unit' 4 | require 'test/unit' 5 | rescue LoadError 6 | # assume using stdlib Test:Unit 7 | require 'test/unit' 8 | end 9 | 10 | require 'ext_test_unit' 11 | 12 | def seed_test_database 13 | options = get_test_config_by_label(:localmysql_to_file_convert_nothing) 14 | seedfilepath = "#{File.dirname(__FILE__)}/../fixtures/seed_integration_tests.sql" 15 | rc = system("mysql -u#{options.mysqlusername} #{options.mysqldatabase} < #{seedfilepath}") 16 | fail StandardError unless rc 17 | return true 18 | rescue 19 | raise StandardError.new('Failed to seed integration test db. See README for setup requirements.') 20 | ensure 21 | delete_files_for_test_config(options) 22 | end 23 | 24 | def get_test_reader(options) 25 | require 'mysql2psql/mysql_reader' 26 | Mysql2psql::MysqlReader.new(options) 27 | rescue 28 | raise StandardError.new('Failed to initialize integration test db. See README for setup requirements.') 29 | end 30 | 31 | def get_test_file_writer(options) 32 | require 'mysql2psql/postgres_file_writer' 33 | Mysql2psql::PostgresFileWriter.new(options.destfile) 34 | rescue => e 35 | puts e.inspect 36 | raise StandardError.new("Failed to initialize file writer from #{options.inspect}. See README for setup requirements.") 37 | end 38 | 39 | def get_test_converter(options) 40 | require 'mysql2psql/converter' 41 | reader = get_test_reader(options) 42 | writer = get_test_file_writer(options) 43 | Mysql2psql::Converter.new(reader, writer, options) 44 | rescue 45 | raise StandardError.new("Failed to initialize converter from #{options.inspect}. See README for setup requirements.") 46 | end 47 | 48 | def get_temp_file(basename) 49 | require 'tempfile' 50 | f = Tempfile.new(basename) 51 | path = f.path 52 | f.close! 53 | path 54 | end 55 | 56 | def get_new_test_config(to_file = true, include_tables = [], exclude_tables = [], suppress_data = false, suppress_ddl = false, force_truncate = false) 57 | require 'mysql2psql/config' 58 | require 'mysql2psql/config_base' 59 | to_filename = to_file ? get_temp_file('mysql2psql_tmp_output') : nil 60 | configtext = Mysql2psql::Config.template(to_filename, include_tables, exclude_tables, suppress_data, suppress_ddl, force_truncate) 61 | configfile = get_temp_file('mysql2psql_tmp_config') 62 | File.open(configfile, 'w:UTF-8') { |f| f.write(configtext) } 63 | Mysql2psql::ConfigBase.new(configfile) 64 | rescue 65 | raise StandardError.new("Failed to initialize options from #{configfile}. See README for setup requirements.") 66 | end 67 | 68 | def get_test_config_by_label(name) 69 | case name 70 | when :localmysql_to_file_convert_nothing 71 | get_new_test_config(true, ['unobtainium'], ['kryptonite'], true, true, false) 72 | when :localmysql_to_file_convert_all 73 | get_new_test_config(true, [], [], false, false, false) 74 | when :localmysql_to_db_convert_all 75 | get_new_test_config(false, [], [], false, false, false) 76 | when :localmysql_to_db_convert_nothing 77 | get_new_test_config(false, ['unobtainium'], ['kryptonite'], true, true, false) 78 | else 79 | fail StandardError.new("Invalid label: #{name}") 80 | end 81 | end 82 | 83 | def delete_files_for_test_config(config) 84 | File.delete(config.destfile) if File.exist?(config.destfile) 85 | File.delete(config.filepath) if File.exist?(config.filepath) 86 | rescue 87 | end 88 | -------------------------------------------------------------------------------- /mysqltopostgres.gemspec: -------------------------------------------------------------------------------- 1 | 2 | Gem::Specification.new do |s| 3 | s.name = 'mysqltopostgres' 4 | s.version = '0.2.19' 5 | s.licenses = ['MIT'] 6 | 7 | s.authors = ['Max Lapshin ', 'Anton Ageev ', 'Samuel Tribehou ', 'Marco Nenciarini ', 'James Nobis ', 'quel ', 'Holger Amann ', 'Maxim Dobriakov ', 'Michael Kimsal ', 'Jacob Coby ', 'Neszt Tibor ', 'Miroslav Kratochvil ', 'Paul Gallagher ', 'Alex C Jokela ', 'Peter Clark '] 8 | s.date = '2012-09-21' 9 | s.default_executable = 'mysqltopostgres' 10 | s.description = 'Translates MySQL -> PostgreSQL' 11 | s.email = 'ajokela@umn.edu' 12 | s.executables = ['mysqltopostgres'] 13 | 14 | s.files = [ 15 | '.gitignore', 16 | 'MIT-LICENSE', 17 | 'README.md', 18 | 'Rakefile', 19 | 'bin/mysqltopostgres', 20 | 'lib/mysqltopostgres.rb', 21 | 'lib/mysql2psql/config.rb', 22 | 'lib/mysql2psql/config_base.rb', 23 | 'lib/mysql2psql/converter.rb', 24 | 'lib/mysql2psql/connection.rb', 25 | 'lib/mysql2psql/errors.rb', 26 | 'lib/mysql2psql/mysql_reader.rb', 27 | 'lib/mysql2psql/postgres_db_writer.rb', 28 | 'lib/mysql2psql/postgres_file_writer.rb', 29 | 'lib/mysql2psql/postgres_db_writer.rb', 30 | 'lib/mysql2psql/postgres_writer.rb', 31 | 'lib/mysql2psql/version.rb', 32 | 'lib/mysql2psql/writer.rb', 33 | 'mysqltopostgres.gemspec', 34 | 'test/fixtures/config_all_options.yml', 35 | 'test/fixtures/seed_integration_tests.sql', 36 | 'test/integration/convert_to_db_test.rb', 37 | 'test/integration/convert_to_file_test.rb', 38 | 'test/integration/converter_test.rb', 39 | 'test/integration/mysql_reader_base_test.rb', 40 | 'test/integration/mysql_reader_test.rb', 41 | 'test/integration/postgres_db_writer_base_test.rb', 42 | 'test/lib/ext_test_unit.rb', 43 | 'test/lib/test_helper.rb', 44 | 'test/units/config_base_test.rb', 45 | 'test/units/config_test.rb', 46 | 'test/units/postgres_file_writer_test.rb' 47 | ] 48 | s.homepage = 'https://github.com/ajokela/mysqltopostgres' 49 | s.rdoc_options = ['--charset=UTF-8'] 50 | s.require_paths = ['lib'] 51 | s.rubygems_version = '1.3.7' 52 | s.summary = 'Tool for converting mysql database to postgresql' 53 | s.test_files = [ 54 | 'test/integration/convert_to_db_test.rb', 55 | 'test/integration/convert_to_file_test.rb', 56 | 'test/integration/converter_test.rb', 57 | 'test/integration/mysql_reader_base_test.rb', 58 | 'test/integration/mysql_reader_test.rb', 59 | 'test/integration/postgres_db_writer_base_test.rb', 60 | 'test/lib/ext_test_unit.rb', 61 | 'test/lib/test_helper.rb', 62 | 'test/units/config_base_test.rb', 63 | 'test/units/config_test.rb', 64 | 'test/units/postgres_file_writer_test.rb' 65 | ] 66 | 67 | s.add_dependency('mysql-pr', ['>= 2.9.10']) 68 | s.add_dependency('postgres-pr', ['= 0.6.3']) 69 | s.add_dependency('activerecord', ['>= 3.2.6']) 70 | s.add_dependency('test-unit', ['>= 2.1.1']) 71 | 72 | if RUBY_PLATFORM == 'java' 73 | s.add_dependency('activerecord-jdbc-adapter', ['>= 1.2.2']) 74 | s.add_dependency('activerecord-jdbcpostgresql-adapter', ['>= 1.2.2']) 75 | s.add_dependency('activerecord-jdbcsqlite3-adapter', ['>= 1.2.2']) 76 | end 77 | 78 | end 79 | -------------------------------------------------------------------------------- /lib/mysql2psql/postgres_file_writer.rb: -------------------------------------------------------------------------------- 1 | require 'mysql2psql/postgres_writer' 2 | 3 | class Mysql2psql 4 | class PostgresFileWriter < PostgresWriter 5 | def initialize(file) 6 | @f = File.open(file, 'w+:UTF-8') 7 | @f << <<-EOF 8 | -- MySQL 2 PostgreSQL dump\n 9 | SET client_encoding = 'UTF8'; 10 | SET standard_conforming_strings = off; 11 | SET check_function_bodies = false; 12 | SET client_min_messages = warning; 13 | 14 | EOF 15 | end 16 | 17 | def truncate(table) 18 | serial_key = nil 19 | maxval = nil 20 | 21 | table.columns.map do |column| 22 | if column[:auto_increment] 23 | serial_key = column[:name] 24 | maxval = column[:maxval].to_i < 1 ? 1 : column[:maxval] + 1 25 | end 26 | end 27 | 28 | @f << <<-EOF 29 | -- TRUNCATE #{table.name}; 30 | TRUNCATE #{PGconn.quote_ident(table.name)} CASCADE; 31 | 32 | EOF 33 | if serial_key 34 | @f << <<-EOF 35 | SELECT pg_catalog.setval(pg_get_serial_sequence('#{table.name}', '#{serial_key}'), #{maxval}, true); 36 | EOF 37 | end 38 | end 39 | 40 | def write_table(table) 41 | primary_keys = [] 42 | serial_key = nil 43 | maxval = nil 44 | 45 | columns = table.columns.map do |column| 46 | if column[:auto_increment] 47 | serial_key = column[:name] 48 | maxval = column[:maxval].to_i < 1 ? 1 : column[:maxval] + 1 49 | end 50 | if column[:primary_key] 51 | primary_keys << column[:name] 52 | end 53 | ' ' + column_description(column) 54 | end.join(",\n") 55 | 56 | if serial_key 57 | 58 | @f << <<-EOF 59 | -- 60 | -- Name: #{table.name}_#{serial_key}_seq; Type: SEQUENCE; Schema: public 61 | -- 62 | 63 | DROP SEQUENCE IF EXISTS #{table.name}_#{serial_key}_seq CASCADE; 64 | 65 | CREATE SEQUENCE #{table.name}_#{serial_key}_seq 66 | INCREMENT BY 1 67 | NO MAXVALUE 68 | NO MINVALUE 69 | CACHE 1; 70 | 71 | 72 | SELECT pg_catalog.setval('#{table.name}_#{serial_key}_seq', #{maxval}, true); 73 | 74 | EOF 75 | end 76 | 77 | @f << <<-EOF 78 | -- Table: #{table.name} 79 | 80 | -- DROP TABLE #{table.name}; 81 | DROP TABLE IF EXISTS #{PGconn.quote_ident(table.name)} CASCADE; 82 | 83 | CREATE TABLE #{PGconn.quote_ident(table.name)} ( 84 | EOF 85 | 86 | @f << columns 87 | 88 | if primary_index = table.indexes.find { |index| index[:primary] } 89 | @f << ",\n CONSTRAINT #{table.name}_pkey PRIMARY KEY(#{primary_index[:columns].map { |col| PGconn.quote_ident(col) }.join(', ')})" 90 | end 91 | 92 | @f << <<-EOF 93 | \n) 94 | WITHOUT OIDS; 95 | EOF 96 | 97 | table.indexes.each do |index| 98 | next if index[:primary] 99 | unique = index[:unique] ? 'UNIQUE ' : nil 100 | @f << <<-EOF 101 | DROP INDEX IF EXISTS #{PGconn.quote_ident(index[:name])} CASCADE; 102 | CREATE #{unique}INDEX #{PGconn.quote_ident(index[:name])} ON #{PGconn.quote_ident(table.name)} (#{index[:columns].map { |col| PGconn.quote_ident(col) }.join(', ')}); 103 | EOF 104 | end 105 | end 106 | 107 | def write_indexes(_table) 108 | end 109 | 110 | def write_constraints(table) 111 | table.foreign_keys.each do |key| 112 | @f << "ALTER TABLE #{PGconn.quote_ident(table.name)} ADD FOREIGN KEY (#{key[:column].map { |c|PGconn.quote_ident(c) }.join(', ')}) REFERENCES #{PGconn.quote_ident(key[:ref_table])}(#{key[:ref_column].map { |c|PGconn.quote_ident(c) }.join(', ')}) ON UPDATE #{key[:on_update]} ON DELETE #{key[:on_delete]};\n" 113 | end 114 | end 115 | 116 | def write_contents(table, reader) 117 | @f << <<-EOF 118 | -- 119 | -- Data for Name: #{table.name}; Type: TABLE DATA; Schema: public 120 | -- 121 | 122 | COPY "#{table.name}" (#{table.columns.map { |column| PGconn.quote_ident(column[:name]) }.join(', ')}) FROM stdin; 123 | EOF 124 | 125 | reader.paginated_read(table, 1000) do |row, _counter| 126 | line = [] 127 | process_row(table, row) 128 | @f << row.join("\t") + "\n" 129 | end 130 | @f << "\\.\n\n" 131 | # @f << "VACUUM FULL ANALYZE #{PGconn.quote_ident(table.name)};\n\n" 132 | end 133 | 134 | def close 135 | @f.close 136 | end 137 | end 138 | end 139 | -------------------------------------------------------------------------------- /lib/mysql2psql/postgres_writer.rb: -------------------------------------------------------------------------------- 1 | require 'mysql2psql/writer' 2 | 3 | class Mysql2psql 4 | class PostgresWriter < Writer 5 | def column_description(column) 6 | "#{PGconn.quote_ident(column[:name])} #{column_type_info(column)}" 7 | end 8 | 9 | def column_type(column) 10 | column_type_info(column).split(' ').first 11 | end 12 | 13 | def column_type_info(column) 14 | if column[:auto_increment] 15 | return "integer DEFAULT nextval('#{column[:table_name]}_#{column[:name]}_seq'::regclass) NOT NULL" 16 | end 17 | 18 | default = column[:default] ? " DEFAULT #{column[:default].nil? ? 'NULL' : "'" + PGconn.escape(column[:default]) + "'"}" : nil 19 | null = column[:null] ? '' : ' NOT NULL' 20 | type = 21 | case column[:type] 22 | 23 | # String types 24 | when 'char' 25 | default = default + '::char' if default 26 | "character(#{column[:length]})" 27 | when 'varchar' 28 | default = default + '::character varying' if default 29 | # puts "VARCHAR: #{column.inspect}" 30 | "character varying(#{column[:length]})" 31 | 32 | # Integer and numeric types 33 | when 'integer' 34 | default = " DEFAULT #{column[:default].nil? ? 'NULL' : column[:default].to_i}" if default 35 | 'integer' 36 | when 'bigint' 37 | default = " DEFAULT #{column[:default].nil? ? 'NULL' : column[:default].to_i}" if default 38 | 'bigint' 39 | when 'tinyint' 40 | default = " DEFAULT #{column[:default].nil? ? 'NULL' : column[:default].to_i}" if default 41 | 'smallint' 42 | 43 | when 'boolean' 44 | default = " DEFAULT #{column[:default].to_i == 1 ? 'true' : 'false'}" if default 45 | 'boolean' 46 | when 'float' 47 | default = " DEFAULT #{column[:default].nil? ? 'NULL' : column[:default].to_f}" if default 48 | 'real' 49 | when 'float unsigned' 50 | default = " DEFAULT #{column[:default].nil? ? 'NULL' : column[:default].to_f}" if default 51 | 'real' 52 | when 'decimal' 53 | default = " DEFAULT #{column[:default].nil? ? 'NULL' : column[:default]}" if default 54 | "numeric(#{column[:length] || 10}, #{column[:decimals] || 0})" 55 | 56 | when 'double precision' 57 | default = " DEFAULT #{column[:default].nil? ? 'NULL' : column[:default]}" if default 58 | 'double precision' 59 | 60 | # Mysql datetime fields 61 | when 'datetime' 62 | default = nil 63 | 'timestamp without time zone' 64 | when 'date' 65 | default = nil 66 | 'date' 67 | when 'timestamp' 68 | default = ' DEFAULT CURRENT_TIMESTAMP' if column[:default] == 'CURRENT_TIMESTAMP' 69 | default = " DEFAULT '1970-01-01 00:00'" if column[:default] == '0000-00-00 00:00' 70 | default = " DEFAULT '1970-01-01 00:00:00'" if column[:default] == '0000-00-00 00:00:00' 71 | 'timestamp without time zone' 72 | when 'time' 73 | default = ' DEFAULT NOW()' if default 74 | 'time without time zone' 75 | 76 | when 'tinyblob' 77 | 'bytea' 78 | when 'mediumblob' 79 | 'bytea' 80 | when 'longblob' 81 | 'bytea' 82 | when 'blob' 83 | 'bytea' 84 | when 'varbinary' 85 | 'bytea' 86 | when 'tinytext' 87 | 'text' 88 | when 'mediumtext' 89 | 'text' 90 | when 'longtext' 91 | 'text' 92 | when 'text' 93 | 'text' 94 | when /^enum/ 95 | default = default + '::character varying' if default 96 | enum = column[:type].gsub(/enum|\(|\)/, '') 97 | max_enum_size = enum.split(',').map { |check| check.size - 2 }.sort[-1] 98 | "character varying(#{max_enum_size}) check( #{column[:name]} in (#{enum}))" 99 | else 100 | puts "Unknown #{column.inspect}" 101 | column[:type].inspect 102 | return '' 103 | end 104 | "#{type}#{default}#{null}" 105 | end 106 | 107 | def process_row(table, row) 108 | table.columns.each_with_index do |column, index| 109 | 110 | if column[:type] == 'time' 111 | row[index] = '%02d:%02d:%02d' % [row[index].hour, row[index].minute, row[index].second] 112 | end 113 | 114 | if row[index].is_a?(MysqlPR::Time) 115 | row[index] = row[index].to_s.gsub('0000-00-00 00:00', '1970-01-01 00:00') 116 | row[index] = row[index].to_s.gsub('0000-00-00 00:00:00', '1970-01-01 00:00:00') 117 | end 118 | 119 | if column_type(column) == 'boolean' 120 | row[index] = row[index] == 1 ? 't' : row[index] == 0 ? 'f' : row[index] 121 | end 122 | 123 | if row[index].is_a?(String) 124 | if column_type(column) == 'bytea' 125 | row[index] = PGconn.escape_bytea(row[index]) 126 | else 127 | row[index] = row[index].gsub(/\\/, '\\\\\\').gsub(/\n/, '\n').gsub(/\t/, '\t').gsub(/\r/, '\r').gsub(/\0/, '') 128 | end 129 | end 130 | 131 | row[index] = '\N' unless row[index] 132 | end 133 | end 134 | 135 | def truncate(_table) 136 | end 137 | end 138 | end 139 | -------------------------------------------------------------------------------- /lib/mysql2psql/connection.rb: -------------------------------------------------------------------------------- 1 | 2 | class Mysql2psql 3 | class Connection 4 | attr_reader :conn, :adapter, :hostname, :login, :password, :database, :schema, :port, :environment, :jruby, :copy_manager, :stream, :is_copying 5 | 6 | def initialize(options) 7 | # Rails-centric stuffs 8 | 9 | @environment = ENV['RAILS_ENV'].nil? ? 'development' : ENV['RAILS_ENV'] 10 | 11 | if options.key?('config') && options['config'].key?('destination') && options['config']['destination'].key?(environment) 12 | 13 | pg_options = Config.new(YAML.load(options['config']['destination'][environment].to_yaml)) 14 | @hostname, @login, @password, @database, @port = pg_options.host('localhost'), pg_options.username, pg_options.password, pg_options.database, pg_options.port(5432).to_s 15 | @database, @schema = database.split(':') 16 | 17 | @adapter = pg_options.adapter('jdbcpostgresql') 18 | 19 | else 20 | fail 'Unable to locate PostgreSQL destination environment in the configuration file' 21 | end 22 | 23 | if RUBY_PLATFORM == 'java' 24 | @jruby = true 25 | 26 | ActiveRecord::Base.establish_connection( 27 | adapter: adapter, 28 | database: database, 29 | username: login, 30 | password: password, 31 | host: hostname, 32 | port: port) 33 | 34 | @conn = ActiveRecord::Base.connection_pool.checkout 35 | 36 | unless conn.nil? 37 | raw_connection = conn.raw_connection 38 | @copy_manager = org.postgresql.copy.CopyManager.new(raw_connection.connection) 39 | else 40 | raise_nil_connection 41 | end 42 | 43 | else 44 | @jruby = false 45 | 46 | @conn = PG.connect(dbname: database, user: login, password: password, host: hostname, port: port) 47 | 48 | if conn.nil? 49 | raise_nil_connection 50 | end 51 | 52 | end 53 | 54 | @is_copying = false 55 | @current_statement = '' 56 | end 57 | 58 | # ensure that the copy is completed, in case we hadn't seen a '\.' in the data stream. 59 | def flush 60 | if jruby 61 | stream.end_copy if @is_copying 62 | else 63 | conn.put_copy_end 64 | end 65 | rescue => e 66 | $stderr.puts e 67 | ensure 68 | @is_copying = false 69 | end 70 | 71 | def execute(sql) 72 | if sql.match(/^COPY /) && !is_copying 73 | # sql.chomp! # cHomp! cHomp! 74 | 75 | if jruby 76 | @stream = copy_manager.copy_in(sql) 77 | else 78 | conn.exec(sql) 79 | end 80 | 81 | @is_copying = true 82 | 83 | elsif sql.match(/^(ALTER|CREATE|DROP|SELECT|SET|TRUNCATE) /) && !is_copying 84 | 85 | @current_statement = sql 86 | 87 | else 88 | 89 | if is_copying 90 | 91 | if sql.chomp == '\.' || sql.chomp.match(/^$/) 92 | 93 | flush 94 | 95 | else 96 | 97 | if jruby 98 | 99 | begin 100 | row = sql.to_java_bytes 101 | stream.write_to_copy(row, 0, row.length) 102 | 103 | rescue => e 104 | 105 | stream.cancel_copy 106 | @is_copying = false 107 | $stderr.puts e 108 | 109 | raise e 110 | end 111 | 112 | else 113 | 114 | begin 115 | 116 | until conn.put_copy_data(sql) 117 | $stderr.puts ' waiting for connection to be writable...' 118 | sleep 0.1 119 | end 120 | 121 | rescue => e 122 | @is_copying = false 123 | $stderr.puts e 124 | raise e 125 | end 126 | end 127 | end 128 | elsif @current_statement.length > 0 129 | @current_statement << ' ' 130 | @current_statement << sql 131 | else 132 | # maybe a comment line? 133 | end 134 | end 135 | 136 | if @current_statement.match(/;$/) 137 | run_statement(@current_statement) 138 | @current_statement = '' 139 | end 140 | end 141 | 142 | # we're done talking to the database, so close the connection cleanly. 143 | def finish 144 | if jruby 145 | ActiveRecord::Base.connection_pool.checkin(@conn) if @conn 146 | else 147 | @conn.finish if @conn 148 | end 149 | end 150 | 151 | # given a file containing psql syntax at path, pipe it down to the database. 152 | def load_file(path) 153 | if @conn 154 | File.open(path, 'r:UTF-8') do |file| 155 | file.each_line do |line| 156 | execute(line) 157 | end 158 | flush 159 | end 160 | finish 161 | else 162 | raise_nil_connection 163 | end 164 | end 165 | 166 | def clear_schema 167 | statements = ['DROP SCHEMA PUBLIC CASCADE', 'CREATE SCHEMA PUBLIC'] 168 | statements.each do |statement| 169 | run_statement(statement) 170 | end 171 | end 172 | 173 | def raise_nil_connection 174 | fail 'No Connection' 175 | end 176 | 177 | private 178 | 179 | def run_statement(statement) 180 | method = jruby ? :execute : :exec 181 | @conn.send(method, statement) 182 | end 183 | end 184 | end 185 | -------------------------------------------------------------------------------- /lib/mysql2psql/mysql_reader.rb: -------------------------------------------------------------------------------- 1 | require 'rubygems' 2 | require 'bundler/setup' 3 | 4 | require 'mysql-pr' 5 | require 'csv' 6 | 7 | class Mysql2psql 8 | class MysqlReader 9 | class Field 10 | end 11 | 12 | class Table 13 | attr_reader :name 14 | 15 | def initialize(reader, name) 16 | @reader = reader 17 | @name = name 18 | end 19 | 20 | @@types = %w(tiny enum decimal short long float double null timestamp longlong int24 date time datetime year set blob string var_string char).reduce({}) do |list, type| 21 | list[eval("::MysqlPR::Field::TYPE_#{type.upcase}")] = type 22 | list 23 | end 24 | 25 | @@types[246] = 'decimal' 26 | 27 | def columns 28 | @columns ||= load_columns 29 | end 30 | 31 | def convert_type(type) 32 | case type 33 | when /int.* unsigned/ 34 | 'bigint' 35 | when /bigint/ 36 | 'bigint' 37 | when 'bit(1)' 38 | 'boolean' 39 | when 'tinyint(1)' 40 | 'boolean' 41 | when /tinyint/ 42 | 'tinyint' 43 | when /int/ 44 | 'integer' 45 | when /varchar/ 46 | 'varchar' 47 | when /char/ 48 | 'char' 49 | when /decimal/ 50 | 'decimal' 51 | when /(float|double)/ 52 | 'double precision' 53 | else 54 | type 55 | end 56 | end 57 | 58 | def load_columns 59 | @reader.reconnect 60 | result = @reader.mysql.list_fields(name) 61 | mysql_flags = ::MysqlPR::Field.constants.select { |c| c =~ /FLAG/ } 62 | fields = [] 63 | @reader.mysql.query("EXPLAIN `#{name}`") do |res| 64 | while field = res.fetch_row 65 | length = field[1][/\((\d+)\)/, 1] if field[1] =~ /\((\d+)\)/ 66 | length = field[1][/\((\d+),(\d+)\)/, 1] if field[1] =~ /\((\d+),(\d+)\)/ 67 | desc = { 68 | name: field[0], 69 | table_name: name, 70 | type: convert_type(field[1]), 71 | length: length && length.to_i, 72 | decimals: field[1][/\((\d+),(\d+)\)/, 2], 73 | null: field[2] == 'YES', 74 | primary_key: field[3] == 'PRI', 75 | auto_increment: field[5] == 'auto_increment' 76 | } 77 | desc[:default] = field[4] unless field[4].nil? 78 | fields << desc 79 | end 80 | end 81 | 82 | fields.select { |field| field[:auto_increment] }.each do |field| 83 | @reader.mysql.query("SELECT max(`#{field[:name]}`) FROM `#{name}`") do |res| 84 | field[:maxval] = res.fetch_row[0].to_i 85 | end 86 | end 87 | fields 88 | end 89 | 90 | def indexes 91 | load_indexes unless @indexes 92 | @indexes 93 | end 94 | 95 | def foreign_keys 96 | load_indexes unless @foreign_keys 97 | @foreign_keys 98 | end 99 | 100 | def load_indexes 101 | @indexes = [] 102 | @foreign_keys = [] 103 | 104 | @reader.mysql.query("SHOW CREATE TABLE `#{name}`") do |result| 105 | explain = result.fetch_row[1] 106 | explain.split(/\n/).each do |line| 107 | next unless line =~ / KEY / 108 | index = {} 109 | if match_data = /CONSTRAINT `(\w+)` FOREIGN KEY \((.*?)\) REFERENCES `(\w+)` \((.*?)\)(.*)/.match(line) 110 | index[:name] = 'fk_' + name + '_' + match_data[1] 111 | index[:column] = match_data[2].gsub!('`', '').split(', ') 112 | index[:ref_table] = match_data[3] 113 | index[:ref_column] = match_data[4].gsub!('`', '').split(', ') 114 | 115 | the_rest = match_data[5] 116 | 117 | if match_data = /ON DELETE (SET NULL|SET DEFAULT|RESTRICT|NO ACTION|CASCADE)/.match(the_rest) 118 | index[:on_delete] = match_data[1] 119 | else 120 | index[:on_delete] ||= 'RESTRICT' 121 | end 122 | 123 | if match_data = /ON UPDATE (SET NULL|SET DEFAULT|RESTRICT|NO ACTION|CASCADE)/.match(the_rest) 124 | index[:on_update] = match_data[1] 125 | else 126 | index[:on_update] ||= 'RESTRICT' 127 | end 128 | 129 | @foreign_keys << index 130 | elsif match_data = /KEY `(\w+)` \((.*)\)/.match(line) 131 | index[:name] = 'idx_' + name + '_' + match_data[1] 132 | index[:columns] = match_data[2].split(',').map { |col| col[/`(\w+)`/, 1] } 133 | index[:unique] = true if line =~ /UNIQUE/ 134 | @indexes << index 135 | elsif match_data = /PRIMARY KEY .*\((.*)\)/.match(line) 136 | index[:primary] = true 137 | index[:columns] = match_data[1].split(',').map { |col| col.strip.gsub(/`/, '') } 138 | @indexes << index 139 | end 140 | end 141 | end 142 | end 143 | 144 | def count_rows 145 | @reader.mysql.query("SELECT COUNT(*) FROM `#{name}`") do |res| 146 | return res.fetch_row[0].to_i 147 | end 148 | end 149 | 150 | def has_id? 151 | !!columns.find { |col| col[:name] == 'id' } 152 | end 153 | 154 | def count_for_pager 155 | query = has_id? ? 'MAX(id)' : 'COUNT(*)' 156 | @reader.mysql.query("SELECT #{query} FROM `#{name}`") do |res| 157 | return res.fetch_row[0].to_i 158 | end 159 | end 160 | 161 | def query_for_pager 162 | query = has_id? ? 'WHERE id >= ? AND id < ?' : 'LIMIT ?,?' 163 | "SELECT #{columns.map { |c| '`' + c[:name] + '`' }.join(', ')} FROM `#{name}` #{query}" 164 | end 165 | end 166 | 167 | def connect 168 | @mysql = ::MysqlPR.connect(@host, @user, @passwd, @db, @port, @sock) 169 | @mysql.charset = ::MysqlPR::Charset.by_number 192 # utf8_unicode_ci :: http://rubydoc.info/gems/mysql-pr/MysqlPR/Charset 170 | @mysql.query('SET NAMES utf8') 171 | 172 | # MySQL raises QueryCacheDisabled exception when disabling query cache and query cache is already disabled so permit silently 173 | begin 174 | @mysql.query('SET SESSION query_cache_type = OFF') 175 | rescue MysqlPR::ServerError::QueryCacheDisabled 176 | end 177 | end 178 | 179 | def reconnect 180 | @mysql.close rescue false 181 | connect 182 | end 183 | 184 | def initialize(options) 185 | @host, @user, @passwd, @db, @port, @sock, @flag = 186 | options.mysqlhost('localhost'), options.mysqlusername, 187 | options.mysqlpassword, options.mysqldatabase, 188 | options.mysqlport, options.mysqlsocket 189 | @port = 3306 if @port == '' # for things like Amazon's RDS you don't have a port and connect fails with "" for a value 190 | @sock = nil if @sock == '' 191 | @flag = nil if @flag == '' 192 | connect 193 | end 194 | 195 | attr_reader :mysql 196 | 197 | def tables 198 | @tables ||= @mysql.list_tables.map { |table| Table.new(self, table) } 199 | end 200 | 201 | def paginated_read(table, page_size) 202 | count = table.count_for_pager 203 | return if count < 1 204 | statement = @mysql.prepare(table.query_for_pager) 205 | counter = 0 206 | 0.upto((count + page_size) / page_size) do |i| 207 | statement.execute(i * page_size, table.has_id? ? (i + 1) * page_size : page_size) 208 | while row = statement.fetch 209 | counter += 1 210 | yield(row, counter) 211 | end 212 | end 213 | counter 214 | end 215 | end 216 | end 217 | --------------------------------------------------------------------------------