├── .travis.yml ├── docs ├── changes.rst ├── api │ ├── mysql2pgsql │ │ ├── index.rst │ │ ├── mysql_reader.rst │ │ ├── postgres_db_writer.rst │ │ ├── postgres_file_writer.rst │ │ └── postgres_writer.rst │ └── index.rst ├── index.rst ├── Makefile ├── make.bat └── conf.py ├── requirements.txt ├── mysql2pgsql ├── __init__.py ├── lib │ ├── errors.py │ ├── config.py │ ├── converter.py │ ├── __init__.py │ ├── postgres_file_writer.py │ ├── postgres_db_writer.py │ ├── mysql_reader.py │ └── postgres_writer.py └── mysql2pgsql.py ├── tests ├── test_db_writer.py ├── test_file_writer.py ├── test_converter.py ├── __init__.py ├── mysql2pgsql-test.yml ├── mysql2pgsql-test.yml.example ├── test_mysql2pgsql.py ├── test_config.py ├── test_reader.py └── test_writer.py ├── .hgtags ├── .gitignore ├── MANIFEST.in ├── CHANGES ├── LICENSE ├── bin └── py-mysql2pgsql ├── setup.py └── README.rst /.travis.yml: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /docs/changes.rst: -------------------------------------------------------------------------------- 1 | .. include:: ../CHANGES 2 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | mysql-python 2 | psycopg2 3 | pyyaml 4 | termcolor 5 | argparse 6 | pytz 7 | -------------------------------------------------------------------------------- /mysql2pgsql/__init__.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | from .mysql2pgsql import Mysql2Pgsql 3 | 4 | __version__ = '0.1.6' 5 | 6 | -------------------------------------------------------------------------------- /tests/test_db_writer.py: -------------------------------------------------------------------------------- 1 | from __future__ import with_statement 2 | import sys 3 | import os 4 | import unittest 5 | 6 | sys.path.append(os.path.abspath('../')) 7 | -------------------------------------------------------------------------------- /docs/api/mysql2pgsql/index.rst: -------------------------------------------------------------------------------- 1 | :mod:`mysql2pgsql` 2 | ================== 3 | 4 | .. automodule:: mysql2pgsql.mysql2pgsql 5 | :members: 6 | :undoc-members: 7 | -------------------------------------------------------------------------------- /tests/test_file_writer.py: -------------------------------------------------------------------------------- 1 | from __future__ import with_statement 2 | import sys 3 | import os 4 | import unittest 5 | 6 | sys.path.append(os.path.abspath('../')) 7 | -------------------------------------------------------------------------------- /docs/api/mysql2pgsql/mysql_reader.rst: -------------------------------------------------------------------------------- 1 | :mod:`mysql_reader` 2 | =================== 3 | 4 | .. automodule:: mysql2pgsql.lib.mysql_reader 5 | :members: 6 | :undoc-members: 7 | -------------------------------------------------------------------------------- /docs/api/mysql2pgsql/postgres_db_writer.rst: -------------------------------------------------------------------------------- 1 | :mod:`postgres_db_writer` 2 | =========================== 3 | 4 | .. automodule:: mysql2pgsql.lib.postgres_db_writer 5 | :members: 6 | -------------------------------------------------------------------------------- /docs/api/mysql2pgsql/postgres_file_writer.rst: -------------------------------------------------------------------------------- 1 | :mod:`postgres_file_writer` 2 | =========================== 3 | 4 | .. automodule:: mysql2pgsql.lib.postgres_file_writer 5 | :members: 6 | -------------------------------------------------------------------------------- /docs/api/mysql2pgsql/postgres_writer.rst: -------------------------------------------------------------------------------- 1 | :mod:`postgres_writer` 2 | =========================== 3 | 4 | .. automodule:: mysql2pgsql.lib.postgres_writer 5 | :members: 6 | :undoc-members: 7 | -------------------------------------------------------------------------------- /.hgtags: -------------------------------------------------------------------------------- 1 | 9562da6c0edbfea3eded674148927cff5f4344ae v0.1.5 2 | 823ebc36922b5a77c15fafe64760a3b42e6efd19 v0.1.4 3 | dc204ec378e5f55b23b1cde87b2fc17347af8c98 v0.1.3 4 | 32ff796dc95c9f990c2be0f69d18786ffce0f825 v0.1.6 5 | -------------------------------------------------------------------------------- /docs/api/index.rst: -------------------------------------------------------------------------------- 1 | API Documentation 2 | ================= 3 | 4 | Just in case you're interested in digging into the internals 5 | 6 | .. toctree:: 7 | :glob: 8 | :maxdepth: 2 9 | 10 | mysql2pgsql/* 11 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | .coverage 3 | *.pyc 4 | *.pyo 5 | env 6 | env* 7 | dist 8 | build 9 | *.egg 10 | *.egg-info 11 | MANIFEST 12 | \#* 13 | .\#* 14 | *~ 15 | *.egg-info 16 | mysql2pgsql.yml 17 | *.yaml 18 | docs/_build 19 | tests/reports 20 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include LICENSE CHANGES README.rst 2 | recursive-include tests * 3 | recursive-include docs * 4 | recursive-exclude * *.pyc 5 | recursive-exclude * *.pyo 6 | recursive-exclude * *.yml 7 | recursive-exclude * *.DS_Store 8 | recursive-exclude * *.coverage 9 | 10 | prune tests/reports 11 | -------------------------------------------------------------------------------- /mysql2pgsql/lib/errors.py: -------------------------------------------------------------------------------- 1 | class GeneralException(Exception): pass 2 | 3 | 4 | class ConfigurationException(Exception): pass 5 | 6 | 7 | class UninitializedValueError(GeneralException): pass 8 | 9 | 10 | class ConfigurationFileNotFound(ConfigurationException): pass 11 | 12 | 13 | class ConfigurationFileInitialized(ConfigurationException): pass 14 | -------------------------------------------------------------------------------- /docs/index.rst: -------------------------------------------------------------------------------- 1 | .. py-mysql2pgsql documentation master file, created by 2 | sphinx-quickstart on Tue Aug 9 12:17:31 2011. 3 | You can adapt this file completely to your liking, but it should at least 4 | contain the root `toctree` directive. 5 | 6 | 7 | 8 | 9 | .. include:: ../README.rst 10 | 11 | 12 | 13 | Contents: 14 | 15 | .. toctree:: 16 | :maxdepth: 2 17 | 18 | api/index 19 | changes 20 | 21 | Indices and tables 22 | ================== 23 | 24 | * :ref:`genindex` 25 | * :ref:`modindex` 26 | * :ref:`search` 27 | -------------------------------------------------------------------------------- /tests/test_converter.py: -------------------------------------------------------------------------------- 1 | from __future__ import with_statement, absolute_import 2 | import os 3 | import sys 4 | import re 5 | 6 | from . import WithReader 7 | 8 | sys.path.append(os.path.abspath('../')) 9 | 10 | from mysql2pgsql.lib.postgres_writer import PostgresWriter 11 | from mysql2pgsql.lib.converter import Converter 12 | 13 | class TestConverter(WithReader): 14 | def setUp(self): 15 | super(self.__class__, self).setUp() 16 | mock_writer = type('MockWriter', (PostgresWriter, ), {'close': lambda s: None, 17 | 'write_contents': lambda s, t, r: None}) 18 | self.writer = mock_writer() 19 | 20 | def test_converter(self): 21 | Converter(self.reader, self.writer, {}, True).convert() 22 | Converter(self.reader, self.writer, {'force_truncate':True, 'supress_ddl': True}, True).convert() 23 | 24 | -------------------------------------------------------------------------------- /CHANGES: -------------------------------------------------------------------------------- 1 | py-mysql2pgsql Changelog 2 | ======================== 3 | 4 | 5 | Version 0.1.6 6 | ------------- 7 | 8 | - Feature: use -t to convert date fields to UTC timezone. 9 | - Data types detection corrected 10 | - Missed changes from timezone option support 11 | - Support NULL as timestamp default 12 | - Corrected enum chomping on braces and commas 13 | - Decode mysql table names so they can be internalized now 14 | 15 | 16 | Version 0.1.5 17 | ------------- 18 | 19 | Support for Python 2.6 removed. 20 | 21 | 22 | Version 0.1.2 23 | ------------- 24 | 25 | Finally had a chance to test on Windows running Python 2.7 26 | 27 | - Fixed import issue with termcolor import on Windows installs 28 | - Updating documentation to help any potential windows users. 29 | 30 | 31 | Version 0.1.1 32 | ------------- 33 | 34 | - Fixed improperly escaped backslashes `\` in `copy` data 35 | - Updated documentation 36 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import os 3 | import unittest 4 | 5 | sys.path.append(os.path.abspath('../')) 6 | 7 | from mysql2pgsql.lib.config import Config 8 | from mysql2pgsql.lib.mysql_reader import MysqlReader 9 | from mysql2pgsql.lib.postgres_writer import PostgresWriter 10 | from mysql2pgsql.lib.errors import ConfigurationFileNotFound 11 | 12 | class WithReader(unittest.TestCase): 13 | def setUp(self): 14 | try: 15 | self.config_file = os.path.join(os.path.dirname(__file__), 'mysql2pgsql-test.yml') 16 | self.config = Config(self.config_file, False) 17 | except ConfigurationFileNotFound: 18 | print("In order to run this test you must create the file %s" % config) 19 | sys.exit(-1) 20 | 21 | self.reader = MysqlReader(self.config.options['mysql']) 22 | 23 | def tearDown(self): 24 | try: 25 | self.reader.close() 26 | except AttributeError: 27 | pass 28 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2011 Philip Southam 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining 4 | a copy of this software and associated documentation files (the 5 | "Software"), to deal in the Software without restriction, including 6 | without limitation the rights to use, copy, modify, merge, publish, 7 | distribute, sublicense, and/or sell copies of the Software, and to 8 | permit persons to whom the Software is furnished to do so, subject to 9 | the following conditions: 10 | 11 | The above copyright notice and this permission notice shall be 12 | included in all copies or substantial portions of the Software. 13 | 14 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 15 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 16 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 17 | NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE 18 | LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 19 | OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 20 | WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 21 | -------------------------------------------------------------------------------- /tests/mysql2pgsql-test.yml: -------------------------------------------------------------------------------- 1 | 2 | # if a socket is specified we will use that 3 | # if tcp is chosen you can use compression 4 | mysql: 5 | hostname: 127.0.0.1 6 | port: 3306 7 | socket: 8 | username: root 9 | password: 10 | database: mysql2pgsql 11 | compress: false 12 | destination: 13 | # if file is given, output goes to file, else postgres 14 | file: 15 | postgres: 16 | hostname: 127.0.0.1 17 | port: 5432 18 | username: postgres 19 | password: 20 | database: mysql2pgsql 21 | 22 | # if tables is given, only the listed tables will be converted. leave empty to convert all tables. 23 | #only_tables: 24 | #- table1 25 | #- table2 26 | # if exclude_tables is given, exclude the listed tables from the conversion. 27 | #exclude_tables: 28 | #- table3 29 | #- table4 30 | 31 | # if supress_data is true, only the schema definition will be exported/migrated, and not the data 32 | supress_data: false 33 | 34 | # if supress_ddl is true, only the data will be exported/imported, and not the schema 35 | supress_ddl: false 36 | 37 | # if force_truncate is true, forces a table truncate before table loading 38 | force_truncate: false 39 | -------------------------------------------------------------------------------- /tests/mysql2pgsql-test.yml.example: -------------------------------------------------------------------------------- 1 | # if a socket is specified we will use that 2 | # if tcp is chosen you can use compression 3 | mysql: 4 | hostname: localhost 5 | port: 3306 6 | username: root 7 | password: 8 | database: mysql2pgsql_test 9 | compress: true 10 | destination: 11 | # if file is given, output goes to file, else postgres 12 | file: #/tmp/testing.sql 13 | postgres: 14 | hostname: localhost 15 | port: 5432 16 | username: postgres 17 | password: 18 | database: mysql2pgsql_test 19 | 20 | # if tables is given, only the listed tables will be converted. leave empty to convert all tables. 21 | only_tables: 22 | - type_conversion_test_1 23 | - type_conversion_test_2 24 | #- tmp_youtube_playlists 25 | #- table2 26 | # if exclude_tables is given, exclude the listed tables from the conversion. 27 | #exclude_tables: 28 | #- table3 29 | #- table4 30 | 31 | # if supress_data is true, only the schema definition will be exported/migrated, and not the data 32 | supress_data: false 33 | 34 | # if supress_ddl is true, only the data will be exported/imported, and not the schema 35 | supress_ddl: false 36 | 37 | # if force_truncate is true, forces a table truncate before table loading 38 | force_truncate: false 39 | -------------------------------------------------------------------------------- /bin/py-mysql2pgsql: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import os 3 | import sys 4 | import argparse 5 | import mysql2pgsql 6 | from mysql2pgsql.lib.errors import ConfigurationFileInitialized 7 | 8 | if __name__ == '__main__': 9 | description = 'Tool for migrating/converting data from mysql to postgresql.' 10 | epilog = 'https://github.com/philipsoutham/py-mysql2pgsql' 11 | 12 | parser = argparse.ArgumentParser( 13 | description=description, 14 | epilog=epilog) 15 | parser.add_argument( 16 | '-v', '--verbose', 17 | action='store_true', 18 | help='Show progress of data migration.' 19 | ) 20 | parser.add_argument( 21 | '-f', '--file', 22 | default='mysql2pgsql.yml', 23 | help='Location of configuration file (default: %(default)s). If none exists at that path, one will be created for you.', 24 | ) 25 | parser.add_argument( 26 | '-V', '--version', 27 | action='store_true', 28 | help='Print version and exit.' 29 | ) 30 | options = parser.parse_args() 31 | 32 | if options.version: 33 | # Someone wants to know the version, print and exit 34 | print(mysql2pgsql.__version__) 35 | sys.exit(0) 36 | 37 | try: 38 | mysql2pgsql.Mysql2Pgsql(options).convert() 39 | except ConfigurationFileInitialized: 40 | sys.exit(-1) 41 | -------------------------------------------------------------------------------- /tests/test_mysql2pgsql.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import os 3 | import unittest 4 | import tempfile 5 | 6 | sys.path.append(os.path.abspath('../')) 7 | 8 | from mysql2pgsql import Mysql2Pgsql 9 | from mysql2pgsql.lib.errors import ConfigurationFileInitialized 10 | class TestFullBoat(unittest.TestCase): 11 | def setUp(self): 12 | mock_options = type('MockOptions', (), {'file': os.path.join(os.path.dirname(__file__), 'mysql2pgsql-test.yml'), 13 | 'verbose': False}) 14 | mock_missing_options = type('MockMissingOptions', (), {'file': os.path.join(os.path.dirname(__file__), 'mysql2pgsql-missing.yml'), 15 | 'verbose': False}) 16 | self.options = mock_options() 17 | self.missing_options = mock_missing_options() 18 | 19 | def test_mysql2pgsql(self): 20 | m = Mysql2Pgsql(self.options) 21 | m._get_file = lambda f: tempfile.NamedTemporaryFile() 22 | m.convert() 23 | 24 | m.file_options['destination']['file'] = None 25 | 26 | m.convert() 27 | 28 | def test_missing_config(self): 29 | self.assertRaises(ConfigurationFileInitialized, Mysql2Pgsql, self.missing_options) 30 | 31 | 32 | def tearDown(self): 33 | if os.path.exists(self.missing_options.file): 34 | os.remove(self.missing_options.file) 35 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | from setuptools import setup 4 | 5 | install_requires = [ 6 | 'mysql-python>=1.2.3', 7 | 'psycopg2>=2.4.2', 8 | 'pyyaml>=3.10.0', 9 | 'pytz', 10 | ] 11 | 12 | if os.name == 'posix': 13 | install_requires.append('termcolor>=1.1.0') 14 | 15 | version = sys.version_info[:2] 16 | 17 | if version < (2,7) or (3,0) <= version <= (3,1): 18 | install_requires += ['argparse'] 19 | 20 | setup( 21 | name='py-mysql2pgsql', 22 | version='0.1.6', 23 | description='Tool for migrating/converting from mysql to postgresql.', 24 | long_description=open('README.rst').read(), 25 | license='MIT License', 26 | author='Philip Southam', 27 | author_email='philipsoutham@gmail.com', 28 | url='https://github.com/philipsoutham/py-mysql2pgsql', 29 | zip_safe=False, 30 | packages=['mysql2pgsql', 'mysql2pgsql.lib'], 31 | scripts=['bin/py-mysql2pgsql'], 32 | platforms='any', 33 | install_requires=install_requires, 34 | classifiers=[ 35 | 'License :: OSI Approved :: MIT License', 36 | 'Development Status :: 3 - Alpha', 37 | 'Environment :: Console', 38 | 'Intended Audience :: System Administrators', 39 | 'Intended Audience :: Developers', 40 | 'Natural Language :: English', 41 | 'Operating System :: OS Independent', 42 | 'Programming Language :: Python :: 2.7', 43 | 'Topic :: Database', 44 | 'Topic :: Utilities' 45 | ], 46 | keywords = 'mysql postgres postgresql pgsql psql migration', 47 | ) 48 | -------------------------------------------------------------------------------- /mysql2pgsql/mysql2pgsql.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | import codecs 4 | 5 | from .lib import print_red 6 | from .lib.mysql_reader import MysqlReader 7 | from .lib.postgres_file_writer import PostgresFileWriter 8 | from .lib.postgres_db_writer import PostgresDbWriter 9 | from .lib.converter import Converter 10 | from .lib.config import Config 11 | from .lib.errors import ConfigurationFileInitialized 12 | 13 | 14 | class Mysql2Pgsql(object): 15 | def __init__(self, options): 16 | self.run_options = options 17 | try: 18 | self.file_options = Config(options.file, True).options 19 | except ConfigurationFileInitialized, e: 20 | print_red(e.message) 21 | raise e 22 | 23 | def convert(self): 24 | reader = MysqlReader(self.file_options['mysql']) 25 | 26 | if self.file_options['destination']['file']: 27 | writer = PostgresFileWriter(self._get_file(self.file_options['destination']['file']), 28 | self.run_options.verbose, 29 | index_prefix=self.file_options.get("index_prefix"), 30 | tz=self.file_options.get('timezone')) 31 | else: 32 | writer = PostgresDbWriter(self.file_options['destination']['postgres'], 33 | self.run_options.verbose, 34 | index_prefix=self.file_options.get("index_prefix"), 35 | tz=self.file_options.get('timezone')) 36 | 37 | Converter(reader, writer, self.file_options, self.run_options.verbose).convert() 38 | 39 | def _get_file(self, file_path): 40 | return codecs.open(file_path, 'wb', 'utf-8') 41 | -------------------------------------------------------------------------------- /tests/test_config.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import os 3 | import unittest 4 | import tempfile 5 | 6 | sys.path.append(os.path.abspath('../')) 7 | 8 | from mysql2pgsql.lib.config import Config, CONFIG_TEMPLATE 9 | from mysql2pgsql.lib.errors import ConfigurationFileInitialized,\ 10 | ConfigurationFileNotFound 11 | 12 | class TestMissingConfig(unittest.TestCase): 13 | def setUp(self): 14 | self.temp_file_1 = tempfile.NamedTemporaryFile().name 15 | temp_file_2 = tempfile.NamedTemporaryFile() 16 | self.temp_file_2 = temp_file_2.name 17 | temp_file_2.close() 18 | 19 | def test_create_new_file(self): 20 | self.assertRaises(ConfigurationFileInitialized, Config, self.temp_file_1, True) 21 | self.assertEqual(CONFIG_TEMPLATE, open(self.temp_file_1).read()) 22 | 23 | def test_dont_create_new_file(self): 24 | self.assertRaises(ConfigurationFileNotFound, Config, self.temp_file_2, False) 25 | 26 | 27 | class TestDefaultConfig(unittest.TestCase): 28 | def setUp(self): 29 | self.temp_file = tempfile.NamedTemporaryFile(delete=False) 30 | self.temp_file.write(CONFIG_TEMPLATE) 31 | self.temp_file.close() 32 | 33 | def tearDown(self): 34 | os.remove(self.temp_file.name) 35 | 36 | def test_config(self): 37 | c = Config(self.temp_file.name) 38 | assert c 39 | 40 | options = c.options 41 | assert options 42 | self.assertIsInstance(options, dict) 43 | 44 | assert 'mysql' in options 45 | assert 'hostname' in options['mysql'] 46 | assert 'destination' in options 47 | assert 'file' in options['destination'] 48 | assert 'postgres' in options['destination'] 49 | assert 'supress_data' in options 50 | assert 'supress_ddl' in options 51 | assert 'force_truncate' in options 52 | assert 'only_tables' not in options 53 | assert 'exclude_tables' not in options 54 | -------------------------------------------------------------------------------- /mysql2pgsql/lib/config.py: -------------------------------------------------------------------------------- 1 | from __future__ import with_statement, absolute_import 2 | 3 | import os.path 4 | 5 | from yaml import load 6 | 7 | try: 8 | from yaml import CLoader as Loader, CDumper as Dumper 9 | except ImportError: 10 | from yaml import Loader, Dumper 11 | 12 | from .errors import ConfigurationFileInitialized,\ 13 | ConfigurationFileNotFound 14 | 15 | 16 | class ConfigBase(object): 17 | def __init__(self, config_file_path): 18 | self.options = load(open(config_file_path)) 19 | 20 | 21 | class Config(ConfigBase): 22 | def __init__(self, config_file_path, generate_if_not_found=True): 23 | if not os.path.isfile(config_file_path): 24 | if generate_if_not_found: 25 | self.reset_configfile(config_file_path) 26 | if os.path.isfile(config_file_path): 27 | raise ConfigurationFileInitialized("""No configuration file found. 28 | A new file has been initialized at: %s 29 | Please review the configuration and retry...""" % config_file_path) 30 | else: 31 | raise ConfigurationFileNotFound("cannot load config file %s" % config_file_path) 32 | 33 | super(Config, self).__init__(config_file_path) 34 | 35 | def reset_configfile(self, file_path): 36 | with open(file_path, 'w') as f: 37 | f.write(CONFIG_TEMPLATE) 38 | 39 | CONFIG_TEMPLATE = """ 40 | # a socket connection will be selected if a 'socket' is specified 41 | # also 'localhost' is a special 'hostname' for MySQL that overrides the 'port' option 42 | # and forces it to use a local socket connection 43 | # if tcp is chosen, you can use compression 44 | 45 | mysql: 46 | hostname: localhost 47 | port: 3306 48 | socket: /tmp/mysql.sock 49 | username: mysql2psql 50 | password: 51 | database: mysql2psql_test 52 | compress: false 53 | destination: 54 | # if file is given, output goes to file, else postgres 55 | file: 56 | postgres: 57 | hostname: localhost 58 | port: 5432 59 | username: mysql2psql 60 | password: 61 | database: mysql2psql_test 62 | 63 | # if tables is given, only the listed tables will be converted. leave empty to convert all tables. 64 | #only_tables: 65 | #- table1 66 | #- table2 67 | # if exclude_tables is given, exclude the listed tables from the conversion. 68 | #exclude_tables: 69 | #- table3 70 | #- table4 71 | 72 | # if supress_data is true, only the schema definition will be exported/migrated, and not the data 73 | supress_data: false 74 | 75 | # if supress_ddl is true, only the data will be exported/imported, and not the schema 76 | supress_ddl: false 77 | 78 | # if force_truncate is true, forces a table truncate before table loading 79 | force_truncate: false 80 | 81 | # if timezone is true, forces to append/convert to UTC tzinfo mysql data 82 | timezone: false 83 | 84 | # if index_prefix is given, indexes will be created whith a name prefixed with index_prefix 85 | index_prefix: 86 | 87 | """ 88 | -------------------------------------------------------------------------------- /mysql2pgsql/lib/converter.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | from . import print_start_table 4 | 5 | 6 | class Converter(object): 7 | def __init__(self, reader, writer, file_options, verbose=False): 8 | self.verbose = verbose 9 | self.reader = reader 10 | self.writer = writer 11 | self.file_options = file_options 12 | self.exclude_tables = file_options.get('exclude_tables', []) 13 | self.only_tables = file_options.get('only_tables', []) 14 | self.supress_ddl = file_options.get('supress_ddl', None) 15 | self.supress_data = file_options.get('supress_data', None) 16 | self.force_truncate = file_options.get('force_truncate', None) 17 | self.index_prefix = file_options.get('index_prefix', u"") 18 | 19 | def convert(self): 20 | if self.verbose: 21 | print_start_table('>>>>>>>>>> STARTING <<<<<<<<<<\n\n') 22 | 23 | tables = [t for t in (t for t in self.reader.tables if t.name not in self.exclude_tables) if not self.only_tables or t.name in self.only_tables] 24 | if self.only_tables: 25 | tables.sort(key=lambda t: self.only_tables.index(t.name)) 26 | 27 | if not self.supress_ddl: 28 | if self.verbose: 29 | print_start_table('START CREATING TABLES') 30 | 31 | for table in tables: 32 | self.writer.write_table(table) 33 | 34 | if self.verbose: 35 | print_start_table('DONE CREATING TABLES') 36 | 37 | if self.force_truncate and self.supress_ddl: 38 | if self.verbose: 39 | print_start_table('START TRUNCATING TABLES') 40 | 41 | for table in tables: 42 | self.writer.truncate(table) 43 | 44 | if self.verbose: 45 | print_start_table('DONE TRUNCATING TABLES') 46 | 47 | if not self.supress_data: 48 | if self.verbose: 49 | print_start_table('START WRITING TABLE DATA') 50 | 51 | for table in tables: 52 | self.writer.write_contents(table, self.reader) 53 | 54 | if self.verbose: 55 | print_start_table('DONE WRITING TABLE DATA') 56 | 57 | if not self.supress_ddl: 58 | if self.verbose: 59 | print_start_table('START CREATING INDEXES, CONSTRAINTS, AND TRIGGERS') 60 | 61 | for table in tables: 62 | self.writer.write_indexes(table) 63 | 64 | for table in tables: 65 | self.writer.write_constraints(table) 66 | 67 | for table in tables: 68 | self.writer.write_triggers(table) 69 | 70 | if self.verbose: 71 | print_start_table('DONE CREATING INDEXES, CONSTRAINTS, AND TRIGGERS') 72 | 73 | if self.verbose: 74 | print_start_table('\n\n>>>>>>>>>> FINISHED <<<<<<<<<<') 75 | 76 | self.writer.close() 77 | -------------------------------------------------------------------------------- /mysql2pgsql/lib/__init__.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | import sys 4 | from functools import wraps 5 | 6 | from .mysql_reader import MysqlReader 7 | try: 8 | from termcolor import cprint 9 | except ImportError: 10 | pass 11 | 12 | 13 | def print_row_progress(val): 14 | try: 15 | cprint(' %s' % val, 'cyan', end=' ') 16 | except NameError: 17 | print(' %s' % val), 18 | sys.stdout.flush() 19 | 20 | 21 | def print_start_table(val): 22 | try: 23 | cprint(val, 'magenta') 24 | except NameError: 25 | print(val) 26 | 27 | 28 | def print_table_actions(val): 29 | try: 30 | cprint(' %s' % val, 'green') 31 | except NameError: 32 | print(' %s' % val) 33 | 34 | 35 | def find_first(items, func): 36 | return next((item for item in items if func(item)), None) 37 | 38 | 39 | def print_red(val): 40 | try: 41 | cprint(val, 'red') 42 | except NameError: 43 | print(val) 44 | 45 | 46 | def status_logger(f): 47 | start_template = 'START - %s' 48 | finish_template = 'FINISH - %s' 49 | truncate_template = 'TRUNCATING TABLE %s' 50 | create_template = 'CREATING TABLE %s' 51 | constraints_template = 'ADDING CONSTRAINTS ON %s' 52 | write_contents_template = 'WRITING DATA TO %s' 53 | index_template = 'ADDING INDEXES TO %s' 54 | trigger_template = 'ADDING TRIGGERS TO %s' 55 | statuses = { 56 | 'truncate': { 57 | 'start': start_template % truncate_template, 58 | 'finish': finish_template % truncate_template 59 | }, 60 | 'write_table': { 61 | 'start': start_template % create_template, 62 | 'finish': finish_template % create_template, 63 | }, 64 | 'write_constraints': { 65 | 'start': start_template % constraints_template, 66 | 'finish': finish_template % constraints_template, 67 | }, 68 | 'write_contents': { 69 | 'start': start_template % write_contents_template, 70 | 'finish': finish_template % write_contents_template, 71 | }, 72 | 'write_indexes': { 73 | 'start': start_template % index_template, 74 | 'finish': finish_template % index_template, 75 | }, 76 | 'write_triggers': { 77 | 'start': start_template % trigger_template, 78 | 'finish': finish_template % trigger_template, 79 | }, 80 | } 81 | 82 | @wraps(f) 83 | def decorated_function(*args, **kwargs): 84 | if getattr(args[0], 'verbose', False): 85 | if 'table' in kwargs: 86 | table = kwargs['table'] 87 | else: 88 | table = find_first(list(args) + kwargs.values(), lambda c: c.__class__ is MysqlReader.Table) 89 | assert table 90 | print_table_actions(statuses[f.func_name]['start'] % table.name) 91 | ret = f(*args, **kwargs) 92 | print_table_actions(statuses[f.func_name]['finish'] % table.name) 93 | return ret 94 | else: 95 | return f(*args, **kwargs) 96 | return decorated_function 97 | -------------------------------------------------------------------------------- /tests/test_reader.py: -------------------------------------------------------------------------------- 1 | from __future__ import with_statement 2 | import sys 3 | import os 4 | import unittest 5 | 6 | from contextlib import closing 7 | 8 | import MySQLdb 9 | 10 | sys.path.append(os.path.abspath('../')) 11 | 12 | from mysql2pgsql.lib.config import Config 13 | from mysql2pgsql.lib.mysql_reader import MysqlReader 14 | from mysql2pgsql.lib.errors import ConfigurationFileNotFound 15 | 16 | class TestMysqlReader(unittest.TestCase): 17 | def setUp(self): 18 | try: 19 | self.config_file = os.path.join(os.path.dirname(__file__), 'mysql2pgsql-test.yml') 20 | config = Config(self.config_file, False) 21 | except ConfigurationFileNotFound: 22 | print("In order to run this test you must create the file %s" % config) 23 | sys.exit(-1) 24 | self.options = config.options['mysql'] 25 | 26 | self.args = { 27 | 'user': self.options.get('username', 'root'), 28 | 'db': self.options['database'], 29 | 'use_unicode': True, 30 | 'charset': 'utf8', 31 | } 32 | 33 | if self.options.get('password', None): 34 | self.args['passwd'] = self.options.get('password', None) 35 | 36 | if self.options.get('socket', None): 37 | self.args['unix_socket'] = self.options['socket'] 38 | else: 39 | self.args['host'] = self.options.get('hostname', 'localhost') 40 | self.args['port'] = self.options.get('port', 3306) 41 | self.args['compress'] = self.options.get('compress', True) 42 | 43 | with open(os.path.join(os.path.dirname(__file__), 'schema.sql')) as sql: 44 | self.sql = sql.read() 45 | with closing(MySQLdb.connect(**self.args)) as conn: 46 | with closing(conn.cursor()) as cur: 47 | for cmd in self.sql.split('-- SPLIT'): 48 | cur.execute(cmd) 49 | conn.commit() 50 | self.reader = MysqlReader(self.options) 51 | self.type_to_pos = { 52 | 'text': (21, 22), 53 | 'float': (83, 84, 85, 86, 87, 88, 89, 90), 54 | 'numeric': (75, 76, 77, 78), 55 | 'datetime': (113, 114, 115, 116, 117, 118), 56 | 'char': (9, 10, 11, 12), 57 | 'boolean': (49, 50), 58 | "enum('small','medium','large')": (1, 2, 3, 4), 59 | 'bit(8)': (37, 38, 39, 40), 60 | 'mediumblob': (27, 28), 61 | 'mediumtext': (19, 20), 62 | 'blob': (29, 30), 63 | "set('a','b','c','d','e')": (5, 6, 7, 8), 64 | 'varchar': (13, 14, 15, 16), 65 | 'timestamp': (125, 126, 127, 128, 129, 130), 66 | 'binary(3)': (33, 34), 67 | 'varbinary(50)': (35, 36), 68 | 'date': (107, 108, 109, 110, 111, 112), 69 | 'integer': (0, 51, 52, 53, 54, 59, 60, 61, 62, 63, 64, 65, 66, 71, 72, 73, 74), 70 | 'double precision': (91, 92, 93, 94, 95, 96, 97, 98), 71 | 'tinytext': (17, 18), 72 | 'decimal': (99, 100, 101, 102, 103, 104, 105, 106, 136, 137, 138, 139, 140, 141, 142, 143), 73 | 'longtext': (23, 24), 74 | 'tinyint': (41, 42, 43, 44, 45, 46, 47, 48, 55, 56, 57, 58, 131, 132, 133, 134, 135), 75 | 'bigint': (67, 68, 69, 70, 79, 80, 81, 82), 76 | 'time': (119, 120, 121, 122, 123, 124), 77 | 'tinyblob': (25, 26), 78 | 'longblob': (31, 32) 79 | } 80 | 81 | def tearDown(self): 82 | self.reader.close() 83 | ''' 84 | with closing(MySQLdb.connect(**self.args)) as conn: 85 | with closing(conn.cursor()) as cur: 86 | for cmd in self.sql.split('-- SPLIT')[:2]: 87 | cur.execute(cmd) 88 | conn.commit() 89 | ''' 90 | 91 | def test_tables(self): 92 | table_list = list(self.reader.tables) 93 | assert table_list 94 | assert len(table_list) == 2 95 | 96 | def test_columns(self): 97 | for table in self.reader.tables: 98 | columns = table.columns 99 | if table.name == 'type_conversion_test_1': 100 | for k, v in self.type_to_pos.iteritems(): 101 | assert all(columns[i]['type'] == k for i in v) 102 | 103 | 104 | 105 | def test_indexes(self): 106 | for table in self.reader.tables: 107 | assert table.indexes 108 | 109 | def test_constraints(self): 110 | assert list(self.reader.tables)[1].foreign_keys 111 | 112 | -------------------------------------------------------------------------------- /tests/test_writer.py: -------------------------------------------------------------------------------- 1 | from __future__ import with_statement, absolute_import 2 | import os 3 | import sys 4 | import re 5 | import tempfile 6 | import unittest 7 | 8 | from . import WithReader 9 | 10 | sys.path.append(os.path.abspath('../')) 11 | 12 | from mysql2pgsql.lib.postgres_writer import PostgresWriter 13 | from mysql2pgsql.lib.postgres_file_writer import PostgresFileWriter 14 | from mysql2pgsql.lib.postgres_db_writer import PostgresDbWriter 15 | 16 | def squeeze(val): 17 | return re.sub(r"[\x00-\x20]+", " ", val).strip() 18 | 19 | class WithTables(WithReader): 20 | def setUp(self): 21 | super(WithTables, self).setUp() 22 | self.table1 = next((t for t in self.reader.tables if t.name == 'type_conversion_test_1'), None) 23 | self.table2 = next((t for t in self.reader.tables if t.name == 'type_conversion_test_2'), None) 24 | assert self.table1 25 | assert self.table2 26 | 27 | 28 | class TestPostgresWriter(WithTables): 29 | def setUp(self): 30 | super(self.__class__, self).setUp() 31 | self.writer = PostgresWriter() 32 | assert self.writer 33 | 34 | def test_truncate(self): 35 | trunc_cmds = self.writer.truncate(self.table1) 36 | assert len(trunc_cmds) == 2 37 | trunc_stmt, reset_seq = trunc_cmds 38 | assert squeeze(trunc_stmt) == 'TRUNCATE "%s" CASCADE;' % self.table1.name 39 | if reset_seq: 40 | self.assertRegexpMatches(squeeze(reset_seq), 41 | "^SELECT pg_catalog.setval\(pg_get_serial_sequence\('%s', 'id'\), \d+, true\);$" % self.table1.name) 42 | 43 | def test_write_table(self): 44 | write_table_cmds = self.writer.write_table(self.table1) 45 | assert len(write_table_cmds) == 2 46 | table_cmds, seq_cmds = write_table_cmds 47 | assert len(table_cmds) == 2 48 | assert squeeze(table_cmds[0]) == 'DROP TABLE IF EXISTS "%s" CASCADE;' % self.table1.name 49 | assert 'CREATE TABLE "%s"' % self.table1.name in table_cmds[1] 50 | # assert self.assertRegexpMatches(squeeze(table_cmds[1]), 51 | # '^CREATE TABLE "%s" \(.*\) WITHOUT OIDS;$' % self.table1.name) 52 | 53 | if seq_cmds: 54 | assert len(seq_cmds) == 3 55 | self.assertRegexpMatches(squeeze(seq_cmds[0]), 56 | '^DROP SEQUENCE IF EXISTS %s_([^\s]+)_seq CASCADE;$' % self.table1.name) 57 | self.assertRegexpMatches(squeeze(seq_cmds[1]), 58 | '^CREATE SEQUENCE %s_([^\s]+)_seq INCREMENT BY 1 NO MAXVALUE NO MINVALUE CACHE 1;$' % self.table1.name) 59 | self.assertRegexpMatches(squeeze(seq_cmds[2]), 60 | "^SELECT pg_catalog.setval\('%s_([^\s]+)_seq', \d+, true\);$" % self.table1.name) 61 | 62 | def test_write_indexex(self): 63 | index_cmds = self.writer.write_indexes(self.table1) 64 | assert len(index_cmds) == 9 65 | 66 | def test_write_constraints(self): 67 | constraint_cmds = self.writer.write_constraints(self.table2) 68 | assert constraint_cmds 69 | 70 | 71 | class WithOutput(WithTables): 72 | 73 | def setUp(self): 74 | super(WithOutput, self).setUp() 75 | 76 | def tearDown(self): 77 | super(WithOutput, self).tearDown() 78 | 79 | 80 | 81 | class TestPostgresFileWriter(WithOutput): 82 | def setUp(self): 83 | super(self.__class__, self).setUp() 84 | self.outfile = tempfile.NamedTemporaryFile() 85 | self.writer = PostgresFileWriter(self.outfile) 86 | 87 | def tearDown(self): 88 | super(self.__class__, self).tearDown() 89 | self.writer.close() 90 | 91 | def test_truncate(self): 92 | self.writer.truncate(self.table1) 93 | 94 | def test_write_table(self): 95 | self.writer.write_table(self.table1) 96 | 97 | def test_write_indexes(self): 98 | self.writer.write_indexes(self.table1) 99 | 100 | def test_write_constraints(self): 101 | self.writer.write_constraints(self.table2) 102 | 103 | def test_write_contents(self): 104 | self.writer.write_contents(self.table1, self.reader) 105 | 106 | 107 | class TestPostgresDbWriter(WithOutput): 108 | def setUp(self): 109 | super(self.__class__, self).setUp() 110 | self.writer = PostgresDbWriter(self.config.options['destination']['postgres'], True) 111 | def tearDown(self): 112 | super(self.__class__, self).tearDown() 113 | self.writer.close() 114 | 115 | def test_truncate(self): 116 | self.writer.truncate(self.table1) 117 | 118 | def test_write_table_indexes_and_constraints(self): 119 | self.writer.write_table(table=self.table1) 120 | self.writer.write_indexes(self.table1) 121 | self.writer.write_constraints(self.table2) 122 | 123 | def test_write_contents(self): 124 | self.writer.write_contents(self.table1, self.reader) 125 | -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line. 5 | SPHINXOPTS = 6 | SPHINXBUILD = sphinx-build 7 | PAPER = 8 | BUILDDIR = _build 9 | 10 | # Internal variables. 11 | PAPEROPT_a4 = -D latex_paper_size=a4 12 | PAPEROPT_letter = -D latex_paper_size=letter 13 | ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . 14 | 15 | .PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest 16 | 17 | help: 18 | @echo "Please use \`make ' where is one of" 19 | @echo " html to make standalone HTML files" 20 | @echo " dirhtml to make HTML files named index.html in directories" 21 | @echo " singlehtml to make a single large HTML file" 22 | @echo " pickle to make pickle files" 23 | @echo " json to make JSON files" 24 | @echo " htmlhelp to make HTML files and a HTML help project" 25 | @echo " qthelp to make HTML files and a qthelp project" 26 | @echo " devhelp to make HTML files and a Devhelp project" 27 | @echo " epub to make an epub" 28 | @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" 29 | @echo " latexpdf to make LaTeX files and run them through pdflatex" 30 | @echo " text to make text files" 31 | @echo " man to make manual pages" 32 | @echo " changes to make an overview of all changed/added/deprecated items" 33 | @echo " linkcheck to check all external links for integrity" 34 | @echo " doctest to run all doctests embedded in the documentation (if enabled)" 35 | 36 | clean: 37 | -rm -rf $(BUILDDIR)/* 38 | 39 | html: 40 | $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html 41 | @echo 42 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." 43 | 44 | dirhtml: 45 | $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml 46 | @echo 47 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." 48 | 49 | singlehtml: 50 | $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml 51 | @echo 52 | @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." 53 | 54 | pickle: 55 | $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle 56 | @echo 57 | @echo "Build finished; now you can process the pickle files." 58 | 59 | json: 60 | $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json 61 | @echo 62 | @echo "Build finished; now you can process the JSON files." 63 | 64 | htmlhelp: 65 | $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp 66 | @echo 67 | @echo "Build finished; now you can run HTML Help Workshop with the" \ 68 | ".hhp project file in $(BUILDDIR)/htmlhelp." 69 | 70 | qthelp: 71 | $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp 72 | @echo 73 | @echo "Build finished; now you can run "qcollectiongenerator" with the" \ 74 | ".qhcp project file in $(BUILDDIR)/qthelp, like this:" 75 | @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/py-mysql2pgsql.qhcp" 76 | @echo "To view the help file:" 77 | @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/py-mysql2pgsql.qhc" 78 | 79 | devhelp: 80 | $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp 81 | @echo 82 | @echo "Build finished." 83 | @echo "To view the help file:" 84 | @echo "# mkdir -p $$HOME/.local/share/devhelp/py-mysql2pgsql" 85 | @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/py-mysql2pgsql" 86 | @echo "# devhelp" 87 | 88 | epub: 89 | $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub 90 | @echo 91 | @echo "Build finished. The epub file is in $(BUILDDIR)/epub." 92 | 93 | latex: 94 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 95 | @echo 96 | @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." 97 | @echo "Run \`make' in that directory to run these through (pdf)latex" \ 98 | "(use \`make latexpdf' here to do that automatically)." 99 | 100 | latexpdf: 101 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 102 | @echo "Running LaTeX files through pdflatex..." 103 | make -C $(BUILDDIR)/latex all-pdf 104 | @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." 105 | 106 | text: 107 | $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text 108 | @echo 109 | @echo "Build finished. The text files are in $(BUILDDIR)/text." 110 | 111 | man: 112 | $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man 113 | @echo 114 | @echo "Build finished. The manual pages are in $(BUILDDIR)/man." 115 | 116 | changes: 117 | $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes 118 | @echo 119 | @echo "The overview file is in $(BUILDDIR)/changes." 120 | 121 | linkcheck: 122 | $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck 123 | @echo 124 | @echo "Link check complete; look for any errors in the above output " \ 125 | "or in $(BUILDDIR)/linkcheck/output.txt." 126 | 127 | doctest: 128 | $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest 129 | @echo "Testing of doctests in the sources finished, look at the " \ 130 | "results in $(BUILDDIR)/doctest/output.txt." 131 | -------------------------------------------------------------------------------- /docs/make.bat: -------------------------------------------------------------------------------- 1 | @ECHO OFF 2 | 3 | REM Command file for Sphinx documentation 4 | 5 | if "%SPHINXBUILD%" == "" ( 6 | set SPHINXBUILD=sphinx-build 7 | ) 8 | set BUILDDIR=_build 9 | set ALLSPHINXOPTS=-d %BUILDDIR%/doctrees %SPHINXOPTS% . 10 | if NOT "%PAPER%" == "" ( 11 | set ALLSPHINXOPTS=-D latex_paper_size=%PAPER% %ALLSPHINXOPTS% 12 | ) 13 | 14 | if "%1" == "" goto help 15 | 16 | if "%1" == "help" ( 17 | :help 18 | echo.Please use `make ^` where ^ is one of 19 | echo. html to make standalone HTML files 20 | echo. dirhtml to make HTML files named index.html in directories 21 | echo. singlehtml to make a single large HTML file 22 | echo. pickle to make pickle files 23 | echo. json to make JSON files 24 | echo. htmlhelp to make HTML files and a HTML help project 25 | echo. qthelp to make HTML files and a qthelp project 26 | echo. devhelp to make HTML files and a Devhelp project 27 | echo. epub to make an epub 28 | echo. latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter 29 | echo. text to make text files 30 | echo. man to make manual pages 31 | echo. changes to make an overview over all changed/added/deprecated items 32 | echo. linkcheck to check all external links for integrity 33 | echo. doctest to run all doctests embedded in the documentation if enabled 34 | goto end 35 | ) 36 | 37 | if "%1" == "clean" ( 38 | for /d %%i in (%BUILDDIR%\*) do rmdir /q /s %%i 39 | del /q /s %BUILDDIR%\* 40 | goto end 41 | ) 42 | 43 | if "%1" == "html" ( 44 | %SPHINXBUILD% -b html %ALLSPHINXOPTS% %BUILDDIR%/html 45 | if errorlevel 1 exit /b 1 46 | echo. 47 | echo.Build finished. The HTML pages are in %BUILDDIR%/html. 48 | goto end 49 | ) 50 | 51 | if "%1" == "dirhtml" ( 52 | %SPHINXBUILD% -b dirhtml %ALLSPHINXOPTS% %BUILDDIR%/dirhtml 53 | if errorlevel 1 exit /b 1 54 | echo. 55 | echo.Build finished. The HTML pages are in %BUILDDIR%/dirhtml. 56 | goto end 57 | ) 58 | 59 | if "%1" == "singlehtml" ( 60 | %SPHINXBUILD% -b singlehtml %ALLSPHINXOPTS% %BUILDDIR%/singlehtml 61 | if errorlevel 1 exit /b 1 62 | echo. 63 | echo.Build finished. The HTML pages are in %BUILDDIR%/singlehtml. 64 | goto end 65 | ) 66 | 67 | if "%1" == "pickle" ( 68 | %SPHINXBUILD% -b pickle %ALLSPHINXOPTS% %BUILDDIR%/pickle 69 | if errorlevel 1 exit /b 1 70 | echo. 71 | echo.Build finished; now you can process the pickle files. 72 | goto end 73 | ) 74 | 75 | if "%1" == "json" ( 76 | %SPHINXBUILD% -b json %ALLSPHINXOPTS% %BUILDDIR%/json 77 | if errorlevel 1 exit /b 1 78 | echo. 79 | echo.Build finished; now you can process the JSON files. 80 | goto end 81 | ) 82 | 83 | if "%1" == "htmlhelp" ( 84 | %SPHINXBUILD% -b htmlhelp %ALLSPHINXOPTS% %BUILDDIR%/htmlhelp 85 | if errorlevel 1 exit /b 1 86 | echo. 87 | echo.Build finished; now you can run HTML Help Workshop with the ^ 88 | .hhp project file in %BUILDDIR%/htmlhelp. 89 | goto end 90 | ) 91 | 92 | if "%1" == "qthelp" ( 93 | %SPHINXBUILD% -b qthelp %ALLSPHINXOPTS% %BUILDDIR%/qthelp 94 | if errorlevel 1 exit /b 1 95 | echo. 96 | echo.Build finished; now you can run "qcollectiongenerator" with the ^ 97 | .qhcp project file in %BUILDDIR%/qthelp, like this: 98 | echo.^> qcollectiongenerator %BUILDDIR%\qthelp\py-mysql2pgsql.qhcp 99 | echo.To view the help file: 100 | echo.^> assistant -collectionFile %BUILDDIR%\qthelp\py-mysql2pgsql.ghc 101 | goto end 102 | ) 103 | 104 | if "%1" == "devhelp" ( 105 | %SPHINXBUILD% -b devhelp %ALLSPHINXOPTS% %BUILDDIR%/devhelp 106 | if errorlevel 1 exit /b 1 107 | echo. 108 | echo.Build finished. 109 | goto end 110 | ) 111 | 112 | if "%1" == "epub" ( 113 | %SPHINXBUILD% -b epub %ALLSPHINXOPTS% %BUILDDIR%/epub 114 | if errorlevel 1 exit /b 1 115 | echo. 116 | echo.Build finished. The epub file is in %BUILDDIR%/epub. 117 | goto end 118 | ) 119 | 120 | if "%1" == "latex" ( 121 | %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex 122 | if errorlevel 1 exit /b 1 123 | echo. 124 | echo.Build finished; the LaTeX files are in %BUILDDIR%/latex. 125 | goto end 126 | ) 127 | 128 | if "%1" == "text" ( 129 | %SPHINXBUILD% -b text %ALLSPHINXOPTS% %BUILDDIR%/text 130 | if errorlevel 1 exit /b 1 131 | echo. 132 | echo.Build finished. The text files are in %BUILDDIR%/text. 133 | goto end 134 | ) 135 | 136 | if "%1" == "man" ( 137 | %SPHINXBUILD% -b man %ALLSPHINXOPTS% %BUILDDIR%/man 138 | if errorlevel 1 exit /b 1 139 | echo. 140 | echo.Build finished. The manual pages are in %BUILDDIR%/man. 141 | goto end 142 | ) 143 | 144 | if "%1" == "changes" ( 145 | %SPHINXBUILD% -b changes %ALLSPHINXOPTS% %BUILDDIR%/changes 146 | if errorlevel 1 exit /b 1 147 | echo. 148 | echo.The overview file is in %BUILDDIR%/changes. 149 | goto end 150 | ) 151 | 152 | if "%1" == "linkcheck" ( 153 | %SPHINXBUILD% -b linkcheck %ALLSPHINXOPTS% %BUILDDIR%/linkcheck 154 | if errorlevel 1 exit /b 1 155 | echo. 156 | echo.Link check complete; look for any errors in the above output ^ 157 | or in %BUILDDIR%/linkcheck/output.txt. 158 | goto end 159 | ) 160 | 161 | if "%1" == "doctest" ( 162 | %SPHINXBUILD% -b doctest %ALLSPHINXOPTS% %BUILDDIR%/doctest 163 | if errorlevel 1 exit /b 1 164 | echo. 165 | echo.Testing of doctests in the sources finished, look at the ^ 166 | results in %BUILDDIR%/doctest/output.txt. 167 | goto end 168 | ) 169 | 170 | :end 171 | -------------------------------------------------------------------------------- /mysql2pgsql/lib/postgres_file_writer.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | import time 4 | 5 | 6 | from .postgres_writer import PostgresWriter 7 | 8 | from . import print_row_progress, status_logger 9 | 10 | 11 | class PostgresFileWriter(PostgresWriter): 12 | """Class used to ouput the PostgreSQL 13 | compatable DDL and/or data to the specified 14 | output :py:obj:`file` from a MySQL server. 15 | 16 | :Parameters: 17 | - `output_file`: the output :py:obj:`file` to send the DDL and/or data 18 | - `verbose`: whether or not to log progress to :py:obj:`stdout` 19 | 20 | """ 21 | verbose = None 22 | 23 | def __init__(self, output_file, verbose=False, *args, **kwargs): 24 | super(PostgresFileWriter, self).__init__(*args, **kwargs) 25 | self.verbose = verbose 26 | self.f = output_file 27 | self.f.write(""" 28 | -- MySQL 2 PostgreSQL dump\n 29 | SET client_encoding = 'UTF8'; 30 | SET standard_conforming_strings = off; 31 | SET check_function_bodies = false; 32 | SET client_min_messages = warning; 33 | """) 34 | 35 | @status_logger 36 | def truncate(self, table): 37 | """Write DDL to truncate the specified `table` 38 | 39 | :Parameters: 40 | - `table`: an instance of a :py:class:`mysql2pgsql.lib.mysql_reader.MysqlReader.Table` object that represents the table to read/write. 41 | 42 | Returns None 43 | """ 44 | truncate_sql, serial_key_sql = super(PostgresFileWriter, self).truncate(table) 45 | self.f.write(""" 46 | -- TRUNCATE %(table_name)s; 47 | %(truncate_sql)s 48 | """ % {'table_name': table.name, 'truncate_sql': truncate_sql}) 49 | 50 | if serial_key_sql: 51 | self.f.write(""" 52 | %(serial_key_sql)s 53 | """ % { 54 | 'serial_key_sql': serial_key_sql}) 55 | 56 | @status_logger 57 | def write_table(self, table): 58 | """Write DDL to create the specified `table`. 59 | 60 | :Parameters: 61 | - `table`: an instance of a :py:class:`mysql2pgsql.lib.mysql_reader.MysqlReader.Table` object that represents the table to read/write. 62 | 63 | Returns None 64 | """ 65 | table_sql, serial_key_sql = super(PostgresFileWriter, self).write_table(table) 66 | if serial_key_sql: 67 | self.f.write(""" 68 | %(serial_key_sql)s 69 | """ % { 70 | 'serial_key_sql': '\n'.join(serial_key_sql) 71 | }) 72 | 73 | self.f.write(""" 74 | -- Table: %(table_name)s 75 | %(table_sql)s 76 | """ % { 77 | 'table_name': table.name, 78 | 'table_sql': '\n'.join(table_sql), 79 | }) 80 | 81 | @status_logger 82 | def write_indexes(self, table): 83 | """Write DDL of `table` indexes to the output file 84 | 85 | :Parameters: 86 | - `table`: an instance of a :py:class:`mysql2pgsql.lib.mysql_reader.MysqlReader.Table` object that represents the table to read/write. 87 | 88 | Returns None 89 | """ 90 | self.f.write('\n'.join(super(PostgresFileWriter, self).write_indexes(table))) 91 | 92 | @status_logger 93 | def write_constraints(self, table): 94 | """Write DDL of `table` constraints to the output file 95 | 96 | :Parameters: 97 | - `table`: an instance of a :py:class:`mysql2pgsql.lib.mysql_reader.MysqlReader.Table` object that represents the table to read/write. 98 | 99 | Returns None 100 | """ 101 | self.f.write('\n'.join(super(PostgresFileWriter, self).write_constraints(table))) 102 | 103 | @status_logger 104 | def write_triggers(self, table): 105 | """Write TRIGGERs existing on `table` to the output file 106 | 107 | :Parameters: 108 | - `table`: an instance of a :py:class:`mysql2pgsql.lib.mysql_reader.MysqlReader.Table` object that represents the table to read/write. 109 | 110 | Returns None 111 | """ 112 | self.f.write('\n'.join(super(PostgresFileWriter, self).write_triggers(table))) 113 | 114 | @status_logger 115 | def write_contents(self, table, reader): 116 | """Write the data contents of `table` to the output file. 117 | 118 | :Parameters: 119 | - `table`: an instance of a :py:class:`mysql2pgsql.lib.mysql_reader.MysqlReader.Table` object that represents the table to read/write. 120 | - `reader`: an instance of a :py:class:`mysql2pgsql.lib.mysql_reader.MysqlReader` object that allows reading from the data source. 121 | 122 | Returns None 123 | """ 124 | # start variable optimiztions 125 | pr = self.process_row 126 | f_write = self.f.write 127 | verbose = self.verbose 128 | # end variable optimiztions 129 | 130 | f_write(""" 131 | -- 132 | -- Data for Name: %(table_name)s; Type: TABLE DATA; 133 | -- 134 | 135 | COPY "%(table_name)s" (%(column_names)s) FROM stdin; 136 | """ % { 137 | 'table_name': table.name, 138 | 'column_names': ', '.join(('"%s"' % col['name']) for col in table.columns)}) 139 | if verbose: 140 | tt = time.time 141 | start_time = tt() 142 | prev_val_len = 0 143 | prev_row_count = 0 144 | for i, row in enumerate(reader.read(table), 1): 145 | row = list(row) 146 | pr(table, row) 147 | try: 148 | f_write(u'%s\n' % (u'\t'.join(row))) 149 | except UnicodeDecodeError: 150 | f_write(u'%s\n' % (u'\t'.join(r.decode('utf-8') for r in row))) 151 | if verbose: 152 | if (i % 20000) == 0: 153 | now = tt() 154 | elapsed = now - start_time 155 | val = '%.2f rows/sec [%s] ' % ((i - prev_row_count) / elapsed, i) 156 | print_row_progress('%s%s' % (("\b" * prev_val_len), val)) 157 | prev_val_len = len(val) + 3 158 | start_time = now 159 | prev_row_count = i 160 | 161 | f_write("\\.\n\n") 162 | if verbose: 163 | print('') 164 | 165 | def close(self): 166 | """Closes the output :py:obj:`file`""" 167 | self.f.close() 168 | -------------------------------------------------------------------------------- /docs/conf.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # py-mysql2pgsql documentation build configuration file, created by 4 | # sphinx-quickstart on Tue Aug 9 12:17:31 2011. 5 | # 6 | # This file is execfile()d with the current directory set to its containing dir. 7 | # 8 | # Note that not all possible configuration values are present in this 9 | # autogenerated file. 10 | # 11 | # All configuration values have a default; values that are commented out 12 | # serve to show the default. 13 | 14 | import sys, os 15 | 16 | # If extensions (or modules to document with autodoc) are in another directory, 17 | # add these directories to sys.path here. If the directory is relative to the 18 | # documentation root, use os.path.abspath to make it absolute, like shown here. 19 | #sys.path.insert(0, os.path.abspath('.')) 20 | 21 | # -- General configuration ----------------------------------------------------- 22 | 23 | # If your documentation needs a minimal Sphinx version, state it here. 24 | #needs_sphinx = '1.0' 25 | 26 | # Add any Sphinx extension module names here, as strings. They can be extensions 27 | # coming with Sphinx (named 'sphinx.ext.*') or your custom ones. 28 | extensions = ['sphinx.ext.autodoc', 'sphinx.ext.todo'] 29 | 30 | # Add any paths that contain templates here, relative to this directory. 31 | templates_path = ['_templates'] 32 | 33 | # The suffix of source filenames. 34 | source_suffix = '.rst' 35 | 36 | # The encoding of source files. 37 | #source_encoding = 'utf-8-sig' 38 | 39 | # The master toctree document. 40 | master_doc = 'index' 41 | 42 | # General information about the project. 43 | project = u'py-mysql2pgsql' 44 | copyright = u'2011, Philip Southam' 45 | 46 | # The version info for the project you're documenting, acts as replacement for 47 | # |version| and |release|, also used in various other places throughout the 48 | # built documents. 49 | #try: 50 | # import mysql2pgsql 51 | #except ImportError: 52 | sys.path.append(os.path.abspath('../')) 53 | # The full version, including alpha/beta/rc tags. 54 | from mysql2pgsql import __version__ as release 55 | # The short X.Y version. 56 | version = release 57 | 58 | 59 | 60 | # The language for content autogenerated by Sphinx. Refer to documentation 61 | # for a list of supported languages. 62 | #language = None 63 | 64 | # There are two options for replacing |today|: either, you set today to some 65 | # non-false value, then it is used: 66 | #today = '' 67 | # Else, today_fmt is used as the format for a strftime call. 68 | #today_fmt = '%B %d, %Y' 69 | 70 | # List of patterns, relative to source directory, that match files and 71 | # directories to ignore when looking for source files. 72 | exclude_patterns = ['_build'] 73 | 74 | # The reST default role (used for this markup: `text`) to use for all documents. 75 | #default_role = None 76 | 77 | # If true, '()' will be appended to :func: etc. cross-reference text. 78 | #add_function_parentheses = True 79 | 80 | # If true, the current module name will be prepended to all description 81 | # unit titles (such as .. function::). 82 | #add_module_names = True 83 | 84 | # If true, sectionauthor and moduleauthor directives will be shown in the 85 | # output. They are ignored by default. 86 | #show_authors = False 87 | 88 | # The name of the Pygments (syntax highlighting) style to use. 89 | pygments_style = 'sphinx' 90 | 91 | # A list of ignored prefixes for module index sorting. 92 | #modindex_common_prefix = [] 93 | 94 | 95 | # -- Options for HTML output --------------------------------------------------- 96 | 97 | # The theme to use for HTML and HTML Help pages. See the documentation for 98 | # a list of builtin themes. 99 | html_theme = 'default' 100 | 101 | # Theme options are theme-specific and customize the look and feel of a theme 102 | # further. For a list of options available for each theme, see the 103 | # documentation. 104 | #html_theme_options = {} 105 | 106 | # Add any paths that contain custom themes here, relative to this directory. 107 | #html_theme_path = [] 108 | 109 | # The name for this set of Sphinx documents. If None, it defaults to 110 | # " v documentation". 111 | #html_title = None 112 | 113 | # A shorter title for the navigation bar. Default is the same as html_title. 114 | #html_short_title = None 115 | 116 | # The name of an image file (relative to this directory) to place at the top 117 | # of the sidebar. 118 | #html_logo = None 119 | 120 | # The name of an image file (within the static path) to use as favicon of the 121 | # docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 122 | # pixels large. 123 | #html_favicon = None 124 | 125 | # Add any paths that contain custom static files (such as style sheets) here, 126 | # relative to this directory. They are copied after the builtin static files, 127 | # so a file named "default.css" will overwrite the builtin "default.css". 128 | html_static_path = ['_static'] 129 | 130 | # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, 131 | # using the given strftime format. 132 | #html_last_updated_fmt = '%b %d, %Y' 133 | 134 | # If true, SmartyPants will be used to convert quotes and dashes to 135 | # typographically correct entities. 136 | #html_use_smartypants = True 137 | 138 | # Custom sidebar templates, maps document names to template names. 139 | #html_sidebars = {} 140 | 141 | # Additional templates that should be rendered to pages, maps page names to 142 | # template names. 143 | #html_additional_pages = {} 144 | 145 | # If false, no module index is generated. 146 | #html_domain_indices = True 147 | 148 | # If false, no index is generated. 149 | #html_use_index = True 150 | 151 | # If true, the index is split into individual pages for each letter. 152 | #html_split_index = False 153 | 154 | # If true, links to the reST sources are added to the pages. 155 | #html_show_sourcelink = True 156 | 157 | # If true, "Created using Sphinx" is shown in the HTML footer. Default is True. 158 | #html_show_sphinx = True 159 | 160 | # If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. 161 | #html_show_copyright = True 162 | 163 | # If true, an OpenSearch description file will be output, and all pages will 164 | # contain a tag referring to it. The value of this option must be the 165 | # base URL from which the finished HTML is served. 166 | #html_use_opensearch = '' 167 | 168 | # This is the file name suffix for HTML files (e.g. ".xhtml"). 169 | #html_file_suffix = None 170 | 171 | # Output file base name for HTML help builder. 172 | htmlhelp_basename = 'py-mysql2pgsqldoc' 173 | 174 | 175 | # -- Options for LaTeX output -------------------------------------------------- 176 | 177 | # The paper size ('letter' or 'a4'). 178 | #latex_paper_size = 'letter' 179 | 180 | # The font size ('10pt', '11pt' or '12pt'). 181 | #latex_font_size = '10pt' 182 | 183 | # Grouping the document tree into LaTeX files. List of tuples 184 | # (source start file, target name, title, author, documentclass [howto/manual]). 185 | latex_documents = [ 186 | ('index', 'py-mysql2pgsql.tex', u'py-mysql2pgsql Documentation', 187 | u'Philip Southam', 'manual'), 188 | ] 189 | 190 | # The name of an image file (relative to this directory) to place at the top of 191 | # the title page. 192 | #latex_logo = None 193 | 194 | # For "manual" documents, if this is true, then toplevel headings are parts, 195 | # not chapters. 196 | #latex_use_parts = False 197 | 198 | # If true, show page references after internal links. 199 | #latex_show_pagerefs = False 200 | 201 | # If true, show URL addresses after external links. 202 | #latex_show_urls = False 203 | 204 | # Additional stuff for the LaTeX preamble. 205 | #latex_preamble = '' 206 | 207 | # Documents to append as an appendix to all manuals. 208 | #latex_appendices = [] 209 | 210 | # If false, no module index is generated. 211 | #latex_domain_indices = True 212 | 213 | 214 | # -- Options for manual page output -------------------------------------------- 215 | 216 | # One entry per manual page. List of tuples 217 | # (source start file, name, description, authors, manual section). 218 | man_pages = [ 219 | ('index', 'py-mysql2pgsql', u'py-mysql2pgsql Documentation', 220 | [u'Philip Southam'], 1) 221 | ] 222 | -------------------------------------------------------------------------------- /mysql2pgsql/lib/postgres_db_writer.py: -------------------------------------------------------------------------------- 1 | from __future__ import with_statement, absolute_import 2 | 3 | import time 4 | from contextlib import closing 5 | 6 | import psycopg2 7 | 8 | from . import print_row_progress, status_logger 9 | from .postgres_writer import PostgresWriter 10 | 11 | 12 | class PostgresDbWriter(PostgresWriter): 13 | """Class used to stream DDL and/or data 14 | from a MySQL server to a PostgreSQL. 15 | 16 | :Parameters: 17 | - `db_options`: :py:obj:`dict` containing connection specific variables 18 | - `verbose`: whether or not to log progress to :py:obj:`stdout` 19 | 20 | """ 21 | class FileObjFaker(object): 22 | """A file-like class to support streaming 23 | table data directly to :py:meth:`pscopg2.copy_from`. 24 | 25 | :Parameters: 26 | - `table`: an instance of a :py:class:`mysql2pgsql.lib.mysql_reader.MysqlReader.Table` object that represents the table to read/write. 27 | - `data`: 28 | - `processor`: 29 | - `verbose`: whether or not to log progress to :py:obj:`stdout` 30 | """ 31 | def __init__(self, table, data, processor, verbose=False): 32 | self.data = iter(data) 33 | self.table = table 34 | self.processor = processor 35 | self.verbose = verbose 36 | 37 | if verbose: 38 | self.idx = 1 39 | self.start_time = time.time() 40 | self.prev_val_len = 0 41 | self.prev_idx = 0 42 | 43 | def readline(self, *args, **kwargs): 44 | try: 45 | row = list(self.data.next()) 46 | except StopIteration: 47 | if self.verbose: 48 | print('') 49 | return '' 50 | else: 51 | self.processor(self.table, row) 52 | try: 53 | return '%s\n' % ('\t'.join(row)) 54 | except UnicodeDecodeError: 55 | return '%s\n' % ('\t'.join(r.decode('utf8') for r in row)) 56 | finally: 57 | if self.verbose: 58 | if (self.idx % 20000) == 0: 59 | now = time.time() 60 | elapsed = now - self.start_time 61 | val = '%.2f rows/sec [%s] ' % ((self.idx - self.prev_idx) / elapsed, self.idx) 62 | print_row_progress('%s%s' % (("\b" * self.prev_val_len), val)), 63 | self.prev_val_len = len(val) + 3 64 | self.start_time = now 65 | self.prev_idx = self.idx + 0 66 | self.idx += 1 67 | 68 | def read(self, *args, **kwargs): 69 | return self.readline(*args, **kwargs) 70 | 71 | def __init__(self, db_options, verbose=False, *args, **kwargs): 72 | super(PostgresDbWriter, self).__init__(*args, **kwargs) 73 | self.verbose = verbose 74 | self.db_options = { 75 | 'host': str(db_options['hostname']), 76 | 'port': db_options.get('port', 5432), 77 | 'database': str(db_options['database']), 78 | 'password': str(db_options.get('password', None)) or '', 79 | 'user': str(db_options['username']), 80 | } 81 | if ':' in str(db_options['database']): 82 | self.db_options['database'], self.schema = self.db_options['database'].split(':') 83 | else: 84 | self.schema = None 85 | 86 | self.open() 87 | 88 | def open(self): 89 | self.conn = psycopg2.connect(**self.db_options) 90 | with closing(self.conn.cursor()) as cur: 91 | if self.schema: 92 | cur.execute('SET search_path TO %s' % self.schema) 93 | cur.execute('SET client_encoding = \'UTF8\'') 94 | if self.conn.server_version >= 80200: 95 | cur.execute('SET standard_conforming_strings = off') 96 | cur.execute('SET check_function_bodies = false') 97 | cur.execute('SET client_min_messages = warning') 98 | 99 | def query(self, sql, args=(), one=False): 100 | with closing(self.conn.cursor()) as cur: 101 | cur.execute(sql, args) 102 | return cur.fetchone() if one else cur 103 | 104 | def execute(self, sql, args=(), many=False): 105 | with closing(self.conn.cursor()) as cur: 106 | if many: 107 | cur.executemany(sql, args) 108 | else: 109 | cur.execute(sql, args) 110 | self.conn.commit() 111 | 112 | def copy_from(self, file_obj, table_name, columns): 113 | with closing(self.conn.cursor()) as cur: 114 | cur.copy_from(file_obj, 115 | table=table_name, 116 | columns=columns 117 | ) 118 | 119 | self.conn.commit() 120 | 121 | def close(self): 122 | """Closes connection to the PostgreSQL server""" 123 | self.conn.close() 124 | 125 | def exists(self, relname): 126 | rc = self.query('SELECT COUNT(!) FROM pg_class WHERE relname = %s', (relname, ), one=True) 127 | return rc and int(rc[0]) == 1 128 | 129 | @status_logger 130 | def truncate(self, table): 131 | """Send DDL to truncate the specified `table` 132 | 133 | :Parameters: 134 | - `table`: an instance of a :py:class:`mysql2pgsql.lib.mysql_reader.MysqlReader.Table` object that represents the table to read/write. 135 | 136 | Returns None 137 | """ 138 | truncate_sql, serial_key_sql = super(PostgresDbWriter, self).truncate(table) 139 | self.execute(truncate_sql) 140 | if serial_key_sql: 141 | self.execute(serial_key_sql) 142 | 143 | @status_logger 144 | def write_table(self, table): 145 | """Send DDL to create the specified `table` 146 | 147 | :Parameters: 148 | - `table`: an instance of a :py:class:`mysql2pgsql.lib.mysql_reader.MysqlReader.Table` object that represents the table to read/write. 149 | 150 | Returns None 151 | """ 152 | table_sql, serial_key_sql = super(PostgresDbWriter, self).write_table(table) 153 | for sql in serial_key_sql + table_sql: 154 | self.execute(sql) 155 | 156 | @status_logger 157 | def write_indexes(self, table): 158 | """Send DDL to create the specified `table` indexes 159 | 160 | :Parameters: 161 | - `table`: an instance of a :py:class:`mysql2pgsql.lib.mysql_reader.MysqlReader.Table` object that represents the table to read/write. 162 | 163 | Returns None 164 | """ 165 | index_sql = super(PostgresDbWriter, self).write_indexes(table) 166 | for sql in index_sql: 167 | self.execute(sql) 168 | 169 | @status_logger 170 | def write_triggers(self, table): 171 | """Send DDL to create the specified `table` triggers 172 | 173 | :Parameters: 174 | - `table`: an instance of a :py:class:`mysql2pgsql.lib.mysql_reader.MysqlReader.Table` object that represents the table to read/write. 175 | 176 | Returns None 177 | """ 178 | index_sql = super(PostgresDbWriter, self).write_triggers(table) 179 | for sql in index_sql: 180 | self.execute(sql) 181 | 182 | @status_logger 183 | def write_constraints(self, table): 184 | """Send DDL to create the specified `table` constraints 185 | 186 | :Parameters: 187 | - `table`: an instance of a :py:class:`mysql2pgsql.lib.mysql_reader.MysqlReader.Table` object that represents the table to read/write. 188 | 189 | Returns None 190 | """ 191 | constraint_sql = super(PostgresDbWriter, self).write_constraints(table) 192 | for sql in constraint_sql: 193 | self.execute(sql) 194 | 195 | @status_logger 196 | def write_contents(self, table, reader): 197 | """Write the contents of `table` 198 | 199 | :Parameters: 200 | - `table`: an instance of a :py:class:`mysql2pgsql.lib.mysql_reader.MysqlReader.Table` object that represents the table to read/write. 201 | - `reader`: an instance of a :py:class:`mysql2pgsql.lib.mysql_reader.MysqlReader` object that allows reading from the data source. 202 | 203 | Returns None 204 | """ 205 | f = self.FileObjFaker(table, reader.read(table), self.process_row, self.verbose) 206 | self.copy_from(f, '"%s"' % table.name, ['"%s"' % c['name'] for c in table.columns]) 207 | -------------------------------------------------------------------------------- /mysql2pgsql/lib/mysql_reader.py: -------------------------------------------------------------------------------- 1 | from __future__ import with_statement, absolute_import 2 | 3 | import re 4 | from contextlib import closing 5 | 6 | import MySQLdb 7 | import MySQLdb.cursors 8 | 9 | 10 | re_column_length = re.compile(r'\((\d+)\)') 11 | re_column_precision = re.compile(r'\((\d+),(\d+)\)') 12 | re_key_1 = re.compile(r'CONSTRAINT `(\w+)` FOREIGN KEY \(`(\w+)`\) REFERENCES `(\w+)` \(`(\w+)`\)') 13 | re_key_2 = re.compile(r'KEY `(\w+)` \((.*)\)') 14 | re_key_3 = re.compile(r'PRIMARY KEY +\((.*)\)') 15 | 16 | 17 | class DB: 18 | """ 19 | Class that wraps MySQLdb functions that auto reconnects 20 | thus (hopefully) preventing the frustrating 21 | "server has gone away" error. Also adds helpful 22 | helper functions. 23 | """ 24 | conn = None 25 | 26 | def __init__(self, options): 27 | args = { 28 | 'user': str(options.get('username', 'root')), 29 | 'db': options['database'], 30 | 'use_unicode': True, 31 | 'charset': 'utf8', 32 | } 33 | 34 | if options.get('password', None): 35 | args['passwd'] = str(options.get('password', None)) 36 | 37 | if options.get('socket', None): 38 | args['unix_socket'] = str(options['socket']) 39 | else: 40 | args['host'] = str(options.get('hostname', 'localhost')) 41 | args['port'] = options.get('port', 3306) 42 | args['compress'] = options.get('compress', True) 43 | 44 | self.options = args 45 | 46 | def connect(self): 47 | self.conn = MySQLdb.connect(**self.options) 48 | 49 | def close(self): 50 | self.conn.close() 51 | 52 | def cursor(self, cursorclass=MySQLdb.cursors.Cursor): 53 | try: 54 | return self.conn.cursor(cursorclass) 55 | except (AttributeError, MySQLdb.OperationalError): 56 | self.connect() 57 | return self.conn.cursor(cursorclass) 58 | 59 | def list_tables(self): 60 | return self.query('SHOW TABLES;') 61 | 62 | def query(self, sql, args=(), one=False, large=False): 63 | return self.query_one(sql, args) if one\ 64 | else self.query_many(sql, args, large) 65 | 66 | def query_one(self, sql, args): 67 | with closing(self.cursor()) as cur: 68 | cur.execute(sql, args) 69 | return cur.fetchone() 70 | 71 | def query_many(self, sql, args, large): 72 | with closing(self.cursor(MySQLdb.cursors.SSCursor if large else MySQLdb.cursors.Cursor)) as cur: 73 | cur.execute(sql, args) 74 | for row in cur: 75 | yield row 76 | 77 | 78 | class MysqlReader(object): 79 | 80 | class Table(object): 81 | def __init__(self, reader, name): 82 | self.reader = reader 83 | self._name = name 84 | self._indexes = [] 85 | self._foreign_keys = [] 86 | self._triggers = [] 87 | self._columns = self._load_columns() 88 | self._comment = self._load_table_comment() 89 | self._load_indexes() 90 | self._load_triggers() 91 | 92 | def _convert_type(self, data_type): 93 | """Normalize MySQL `data_type`""" 94 | if data_type.startswith('varchar'): 95 | return 'varchar' 96 | elif data_type.startswith('char'): 97 | return 'char' 98 | elif data_type in ('bit(1)', 'tinyint(1)', 'tinyint(1) unsigned'): 99 | return 'boolean' 100 | elif re.search(r'^smallint.* unsigned', data_type) or data_type.startswith('mediumint'): 101 | return 'integer' 102 | elif data_type.startswith('smallint'): 103 | return 'tinyint' 104 | elif data_type.startswith('tinyint') or data_type.startswith('year('): 105 | return 'tinyint' 106 | elif data_type.startswith('bigint') and 'unsigned' in data_type: 107 | return 'numeric' 108 | elif re.search(r'^int.* unsigned', data_type) or \ 109 | (data_type.startswith('bigint') and 'unsigned' not in data_type): 110 | return 'bigint' 111 | elif data_type.startswith('int'): 112 | return 'integer' 113 | elif data_type.startswith('float'): 114 | return 'float' 115 | elif data_type.startswith('decimal'): 116 | return 'decimal' 117 | elif data_type.startswith('double'): 118 | return 'double precision' 119 | else: 120 | return data_type 121 | 122 | def _load_columns(self): 123 | fields = [] 124 | for row in self.reader.db.query('SHOW FULL COLUMNS FROM `%s`' % self.name): 125 | res = () 126 | for field in row: 127 | if type(field) == unicode: 128 | res += field.encode('utf8'), 129 | else: 130 | res += field, 131 | length_match = re_column_length.search(res[1]) 132 | precision_match = re_column_precision.search(res[1]) 133 | length = length_match.group(1) if length_match else \ 134 | precision_match.group(1) if precision_match else None 135 | name = res[0] 136 | comment = res[8] 137 | field_type = self._convert_type(res[1]) 138 | desc = { 139 | 'name': name, 140 | 'table_name': self.name, 141 | 'type': field_type, 142 | 'length': int(length) if length else None, 143 | 'decimals': precision_match.group(2) if precision_match else None, 144 | 'null': res[3] == 'YES' or field_type.startswith('enum') or field_type in ('date', 'datetime', 'timestamp'), 145 | 'primary_key': res[4] == 'PRI', 146 | 'auto_increment': res[6] == 'auto_increment', 147 | 'default': res[5] if not res[5] == 'NULL' else None, 148 | 'comment': comment, 149 | 'select': '`%s`' % name if not field_type.startswith('enum') else 150 | 'CASE `%(name)s` WHEN "" THEN NULL ELSE `%(name)s` END' % {'name': name}, 151 | } 152 | fields.append(desc) 153 | 154 | for field in (f for f in fields if f['auto_increment']): 155 | res = self.reader.db.query('SELECT MAX(`%s`) FROM `%s`;' % (field['name'], self.name), one=True) 156 | field['maxval'] = int(res[0]) if res[0] else 0 157 | 158 | return fields 159 | 160 | def _load_table_comment(self): 161 | table_status = self.reader.db.query('SHOW TABLE STATUS WHERE Name="%s"' % self.name, one=True) 162 | comment = table_status[17] 163 | return comment 164 | 165 | 166 | def _load_indexes(self): 167 | explain = self.reader.db.query('SHOW CREATE TABLE `%s`' % self.name, one=True) 168 | explain = explain[1] 169 | for line in explain.split('\n'): 170 | if ' KEY ' not in line: 171 | continue 172 | index = {} 173 | match_data = re_key_1.search(line) 174 | if match_data: 175 | index['name'] = match_data.group(1) 176 | index['column'] = match_data.group(2) 177 | index['ref_table'] = match_data.group(3) 178 | index['ref_column'] = match_data.group(4) 179 | self._foreign_keys.append(index) 180 | continue 181 | match_data = re_key_2.search(line) 182 | if match_data: 183 | index['name'] = match_data.group(1) 184 | index['columns'] = [re.search(r'`(\w+)`', col).group(1) for col in match_data.group(2).split(',')] 185 | index['unique'] = 'UNIQUE' in line 186 | self._indexes.append(index) 187 | continue 188 | match_data = re_key_3.search(line) 189 | if match_data: 190 | index['primary'] = True 191 | index['columns'] = [re.sub(r'\(\d+\)', '', col.replace('`', '')) for col in match_data.group(1).split(',')] 192 | self._indexes.append(index) 193 | continue 194 | 195 | def _load_triggers(self): 196 | explain = self.reader.db.query('SHOW TRIGGERS WHERE `table` = \'%s\'' % self.name) 197 | for row in explain: 198 | if type(row) is tuple: 199 | trigger = {} 200 | trigger['name'] = row[0] 201 | trigger['event'] = row[1] 202 | trigger['statement'] = row[3] 203 | trigger['timing'] = row[4] 204 | 205 | trigger['statement'] = re.sub('^BEGIN', '', trigger['statement']) 206 | trigger['statement'] = re.sub('^END', '', trigger['statement'], flags=re.MULTILINE) 207 | trigger['statement'] = re.sub('`', '', trigger['statement']) 208 | 209 | self._triggers.append(trigger) 210 | 211 | @property 212 | def name(self): 213 | return self._name 214 | 215 | @property 216 | def columns(self): 217 | return self._columns 218 | 219 | @property 220 | def comment(self): 221 | return self._comment 222 | 223 | @property 224 | def indexes(self): 225 | return self._indexes 226 | 227 | @property 228 | def foreign_keys(self): 229 | return self._foreign_keys 230 | 231 | @property 232 | def triggers(self): 233 | return self._triggers 234 | 235 | @property 236 | def query_for(self): 237 | return 'SELECT %(column_names)s FROM `%(table_name)s`' % { 238 | 'table_name': self.name, 239 | 'column_names': ', '. join(c['select'] for c in self.columns)} 240 | 241 | def __init__(self, options): 242 | self.db = DB(options) 243 | 244 | @property 245 | def tables(self): 246 | return (self.Table(self, t[0]) for t in self.db.list_tables()) 247 | 248 | def read(self, table): 249 | return self.db.query(table.query_for, large=True) 250 | 251 | def close(self): 252 | self.db.close() 253 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | ======================================================================================== 2 | py-mysql2pgsql - A tool for migrating/converting/exporting data from MySQL to PostgreSQL 3 | ======================================================================================== 4 | 5 | This tool allows you to take data from an MySQL server (only tested on 6 | 5.x) and write a PostgresSQL compatible (8.2 or higher) dump file or pipe it directly 7 | into your running PostgreSQL server (8.2 or higher). 8 | 9 | .. attention:: 10 | Currently there is no support for importing `spatial data from MySQL 11 | `_. 12 | 13 | 14 | Installation: 15 | ============= 16 | 17 | If you're like me you don't like random stuff polluting your python 18 | install. Might I suggest installing this in an virtualenv? 19 | 20 | :: 21 | 22 | > virtualenv --no-site-packages ~/envs/py-mysql2pgsql 23 | > source ~/envs/py-mysql2pgsql/bin/activate 24 | 25 | 26 | Requirements: 27 | ------------- 28 | 29 | * `Python 2.7 `_ 30 | * `MySQL-python `_ 31 | * `psycopg2 `_ 32 | * `PyYAML `_ 33 | * `termcolor `_ (unless you're installing on windows) 34 | * `pytz `_ 35 | 36 | 37 | On Windows 38 | ---------- 39 | 40 | I have only done limited testing on this platform using Python 41 | 2.7. Here are the driver dependencies for windows, install these 42 | before attempting to install py-mysql2pgsql or it will fail. 43 | 44 | * `psycopg2 for Windows `_ 45 | * `MySQL-python for Windows `_ 46 | 47 | 48 | 49 | From PyPI: 50 | ---------- 51 | 52 | All dependencies **should** be automatically installed when installing 53 | the app the following ways 54 | 55 | :: 56 | 57 | > pip install py-mysql2pgsql 58 | 59 | 60 | From source: 61 | ------------ 62 | 63 | :: 64 | 65 | > git clone git://github.com/philipsoutham/py-mysql2pgsql.git 66 | > cd py-mysql2pgsql 67 | > python setup.py install 68 | 69 | 70 | Usage: 71 | ====== 72 | 73 | Looking for help? 74 | 75 | :: 76 | 77 | > py-mysql2pgsql -h 78 | usage: py-mysql2pgsql [-h] [-v] [-f FILE] 79 | 80 | Tool for migrating/converting data from mysql to postgresql. 81 | 82 | optional arguments: 83 | -h, --help show this help message and exit 84 | -v, --verbose Show progress of data migration. 85 | -f FILE, --file FILE Location of configuration file (default: 86 | mysql2pgsql.yml). If none exists at that path, 87 | one will be created for you. 88 | 89 | 90 | Don't worry if this is your first time, it'll be gentle. 91 | 92 | :: 93 | 94 | > py-mysql2pgsql 95 | No configuration file found. 96 | A new file has been initialized at: mysql2pgsql.yml 97 | Please review the configuration and retry... 98 | 99 | As the output suggests, a file was created at mysql2pgsql.yml for you 100 | to edit. For the impatient, here is what the file contains. 101 | 102 | :: 103 | 104 | # a socket connection will be selected if a 'socket' is specified 105 | # also 'localhost' is a special 'hostname' for MySQL that overrides the 'port' option 106 | # and forces it to use a local socket connection 107 | # if tcp is chosen, you can use compression 108 | 109 | mysql: 110 | hostname: localhost 111 | port: 3306 112 | socket: /tmp/mysql.sock 113 | username: mysql2psql 114 | password: 115 | database: mysql2psql_test 116 | compress: false 117 | destination: 118 | # if file is given, output goes to file, else postgres 119 | file: 120 | postgres: 121 | hostname: localhost 122 | port: 5432 123 | username: mysql2psql 124 | password: 125 | database: mysql2psql_test 126 | 127 | # if only_tables is given, only the listed tables will be converted. leave empty to convert all tables. 128 | #only_tables: 129 | #- table1 130 | #- table2 131 | # if exclude_tables is given, exclude the listed tables from the conversion. 132 | #exclude_tables: 133 | #- table3 134 | #- table4 135 | 136 | # if supress_data is true, only the schema definition will be exported/migrated, and not the data 137 | supress_data: false 138 | 139 | # if supress_ddl is true, only the data will be exported/imported, and not the schema 140 | supress_ddl: false 141 | 142 | # if force_truncate is true, forces a table truncate before table loading 143 | force_truncate: false 144 | 145 | # if timezone is true, forces to append/convert to UTC tzinfo mysql data 146 | timezone: false 147 | 148 | # if index_prefix is given, indexes will be created whith a name prefixed with index_prefix 149 | index_prefix: 150 | 151 | Pretty self explanatory right? A couple things to note, first if 152 | `destination -> file` is populated all output will be dumped to the 153 | specified location regardless of what is contained in `destination -> 154 | postgres`. So if you want to dump directly to your server make sure 155 | the `file` value is blank. 156 | 157 | Say you have a MySQL db with many, many tables, but you're only 158 | interested in exporting a subset of those table, no problem. Add only 159 | the tables you want to include in `only_tables` or tables that you 160 | don't want exported to `exclude_tables`. 161 | 162 | Other items of interest may be to skip moving the data and just create 163 | the schema or vice versa. To skip the data and only create the schema 164 | set `supress_data` to `true`. To migrate only data and not recreate the 165 | tables set `supress_ddl` to `true`; if there's existing data that you 166 | want to drop before importing set `force_truncate` to 167 | `true`. `force_truncate` is not necessary when `supress_ddl` is set to 168 | `false`. 169 | 170 | Note that when migrating, it's sometimes possible to knock your 171 | sequences out of whack. When this happens, you may get IntegrityErrors 172 | about your primary keys saying things like, "duplicate key value violates 173 | unique constraint." See `this page `_ for a fix 174 | 175 | Due to different naming conventions in mysql an postgresql, there is a chance 176 | that the tool generates index names that collide with table names. This can 177 | be circumvented by setting index_prefix. 178 | 179 | One last thing, the `--verbose` flag. Without it the tool will just go 180 | on it's merry way without bothering you with any output until it's 181 | done. With it you'll get a play-by-play summary of what's going 182 | on. Here's an example. 183 | 184 | :: 185 | 186 | > py-mysql2pgsql -v -f mysql2pgsql.yml 187 | START PROCESSING table_one 188 | START - CREATING TABLE table_one 189 | FINISH - CREATING TABLE table_one 190 | START - WRITING DATA TO table_one 191 | 24812.02 rows/sec [20000] 192 | FINISH - WRITING DATA TO table_one 193 | START - ADDING INDEXES TO table_one 194 | FINISH - ADDING INDEXES TO table_one 195 | START - ADDING CONSTRAINTS ON table_one 196 | FINISH - ADDING CONSTRAINTS ON table_one 197 | FINISHED PROCESSING table_one 198 | 199 | START PROCESSING table_two 200 | START - CREATING TABLE table_two 201 | FINISH - CREATING TABLE table_two 202 | START - WRITING DATA TO table_two 203 | 204 | FINISH - WRITING DATA TO table_two 205 | START - ADDING INDEXES TO table_two 206 | FINISH - ADDING INDEXES TO table_two 207 | START - ADDING CONSTRAINTS ON table_two 208 | FINISH - ADDING CONSTRAINTS ON table_two 209 | FINISHED PROCESSING table_two 210 | 211 | 212 | Data Type Conversion Legend 213 | =========================== 214 | 215 | Since there is not a one-to-one mapping between MySQL and 216 | PostgreSQL data types, listed below are the conversions that are applied. I've 217 | taken some liberties with some, others should come as no surprise. 218 | 219 | ==================== =========================================== 220 | MySQL PostgreSQL 221 | ==================== =========================================== 222 | char character 223 | varchar character varying 224 | tinytext text 225 | mediumtext text 226 | text text 227 | longtext text 228 | tinyblob bytea 229 | mediumblob bytea 230 | blob bytea 231 | longblob bytea 232 | binary bytea 233 | varbinary bytea 234 | bit bit varying 235 | tinyint smallint 236 | tinyint unsigned smallint 237 | smallint smallint 238 | smallint unsigned integer 239 | mediumint integer 240 | mediumint unsigned integer 241 | int integer 242 | int unsigned bigint 243 | bigint bigint 244 | bigint unsigned numeric 245 | float real 246 | float unsigned real 247 | double double precision 248 | double unsigned double precision 249 | decimal numeric 250 | decimal unsigned numeric 251 | numeric numeric 252 | numeric unsigned numeric 253 | date date 254 | datetime timestamp without time zone 255 | time time without time zone 256 | timestamp timestamp without time zone 257 | year smallint 258 | enum character varying (with `check` constraint) 259 | set ARRAY[]::text[] 260 | ==================== =========================================== 261 | 262 | 263 | Conversion caveats: 264 | =================== 265 | 266 | Not just any valid MySQL database schema can be simply converted to the 267 | PostgreSQL. So when you end with a different database schema please note that: 268 | 269 | * Most MySQL versions don't enforce `NOT NULL` constraint on `date` and `enum` 270 | fields. Because of that `NOT NULL` is skipped for this types. Here's an 271 | excuse for the dates: ``_. 272 | 273 | About: 274 | ====== 275 | 276 | I ported much of this from an existing project written in Ruby by Max 277 | Lapshin over at ``_. I 278 | found that it worked fine for most things, but for migrating large tables 279 | with millions of rows it started to break down. This motivated me to 280 | write *py-mysql2pgsql* which uses a server side cursor, so there is no "paging" 281 | which means there is no slow down while working it's way through a 282 | large dataset. 283 | -------------------------------------------------------------------------------- /mysql2pgsql/lib/postgres_writer.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | import re 4 | from cStringIO import StringIO 5 | from datetime import date, datetime, timedelta 6 | 7 | from psycopg2.extensions import AsIs, Binary, QuotedString 8 | from pytz import timezone 9 | 10 | 11 | class PostgresWriter(object): 12 | """Base class for :py:class:`mysql2pgsql.lib.postgres_file_writer.PostgresFileWriter` 13 | and :py:class:`mysql2pgsql.lib.postgres_db_writer.PostgresDbWriter`. 14 | """ 15 | 16 | def __init__(self, index_prefix, tz=False): 17 | self.column_types = {} 18 | self.index_prefix = index_prefix if index_prefix else '' 19 | if tz: 20 | self.tz = timezone('UTC') 21 | self.tz_offset = '+00:00' 22 | else: 23 | self.tz = None 24 | self.tz_offset = '' 25 | 26 | def column_description(self, column): 27 | return '"%s" %s' % (column['name'], self.column_type_info(column)) 28 | 29 | def column_type(self, column): 30 | hash_key = hash(frozenset(column.items())) 31 | self.column_types[hash_key] = self.column_type_info(column).split(" ")[0] 32 | return self.column_types[hash_key] 33 | 34 | def column_type_info(self, column): 35 | """ 36 | """ 37 | null = "" if column['null'] else " NOT NULL" 38 | 39 | def get_type(column): 40 | """This in conjunction with :py:class:`mysql2pgsql.lib.mysql_reader.MysqlReader._convert_type` 41 | determines the PostgreSQL data type. In my opinion this is way too fugly, will need 42 | to refactor one day. 43 | """ 44 | t = lambda v: not v == None 45 | default = (' DEFAULT %s' % QuotedString(column['default']).getquoted()) if t(column['default']) else None 46 | 47 | if column['type'] == 'char': 48 | default = ('%s::char' % default) if t(default) else None 49 | return default, 'character(%s)' % column['length'] 50 | elif column['type'] == 'varchar': 51 | default = ('%s::character varying' % default) if t(default) else None 52 | return default, 'character varying(%s)' % column['length'] 53 | elif column['type'] == 'integer': 54 | default = (" DEFAULT %s" % (column['default'] if t(column['default']) else 'NULL')) if t(default) else None 55 | return default, 'integer' 56 | elif column['type'] == 'bigint': 57 | default = (" DEFAULT %s" % (column['default'] if t(column['default']) else 'NULL')) if t(default) else None 58 | return default, 'bigint' 59 | elif column['type'] == 'tinyint': 60 | default = (" DEFAULT %s" % (column['default'] if t(column['default']) else 'NULL')) if t(default) else None 61 | return default, 'smallint' 62 | elif column['type'] == 'boolean': 63 | default = (" DEFAULT %s" % ('true' if int(column['default']) == 1 else 'false')) if t(default) else None 64 | return default, 'boolean' 65 | elif column['type'] == 'float': 66 | default = (" DEFAULT %s" % (column['default'] if t(column['default']) else 'NULL')) if t(default) else None 67 | return default, 'real' 68 | elif column['type'] == 'float unsigned': 69 | default = (" DEFAULT %s" % (column['default'] if t(column['default']) else 'NULL')) if t(default) else None 70 | return default, 'real' 71 | elif column['type'] in ('numeric', 'decimal'): 72 | default = (" DEFAULT %s" % (column['default'] if t(column['default']) else 'NULL')) if t(default) else None 73 | return default, 'numeric(%s, %s)' % (column['length'] or 20, column['decimals'] or 0) 74 | elif column['type'] == 'double precision': 75 | default = (" DEFAULT %s" % (column['default'] if t(column['default']) else 'NULL')) if t(default) else None 76 | return default, 'double precision' 77 | elif column['type'] == 'datetime' or column['type'].startswith('datetime('): 78 | default = None 79 | if self.tz: 80 | return default, 'timestamp with time zone' 81 | else: 82 | return default, 'timestamp without time zone' 83 | elif column['type'] == 'date': 84 | default = None 85 | return default, 'date' 86 | elif column['type'] == 'timestamp': 87 | if column['default'] == None: 88 | default = None 89 | elif "current_timestamp()" in column['default']: 90 | default = ' DEFAULT CURRENT_TIMESTAMP' 91 | elif "CURRENT_TIMESTAMP" in column['default']: 92 | default = ' DEFAULT CURRENT_TIMESTAMP' 93 | elif "0000-00-00 00:00" in column['default']: 94 | if self.tz: 95 | default = " DEFAULT '1970-01-01T00:00:00.000000%s'" % self.tz_offset 96 | elif "0000-00-00 00:00:00" in column['default']: 97 | default = " DEFAULT '1970-01-01 00:00:00'" 98 | else: 99 | default = " DEFAULT '1970-01-01 00:00'" 100 | if self.tz: 101 | return default, 'timestamp with time zone' 102 | else: 103 | return default, 'timestamp without time zone' 104 | elif column['type'] == 'time' or column['type'].startswith('time('): 105 | default = " DEFAULT NOW()" if t(default) else None 106 | if self.tz: 107 | return default, 'time with time zone' 108 | else: 109 | return default, 'time without time zone' 110 | elif column['type'] in ('blob', 'binary', 'longblob', 'mediumblob', 'tinyblob', 'varbinary'): 111 | return default, 'bytea' 112 | elif column['type'].startswith('binary(') or column['type'].startswith('varbinary('): 113 | return default, 'bytea' 114 | elif column['type'] in ('tinytext', 'mediumtext', 'longtext', 'text'): 115 | return default, 'text' 116 | elif column['type'].startswith('enum'): 117 | default = (' %s::character varying' % default) if t(default) else None 118 | enum = re.sub(r'^enum\(|\)$', '', column['type']) 119 | # TODO: will work for "'.',',',''''" but will fail for "'.'',','.'" 120 | max_enum_size = max([len(e.replace("''", "'")) for e in enum.split("','")]) 121 | return default, ' character varying(%s) check("%s" in (%s))' % (max_enum_size, column['name'], enum) 122 | elif column['type'].startswith('bit('): 123 | return ' DEFAULT %s' % column['default'].upper() if column['default'] else column['default'], 'varbit(%s)' % re.search(r'\((\d+)\)', column['type']).group(1) 124 | elif column['type'].startswith('set('): 125 | if default: 126 | default = ' DEFAULT ARRAY[%s]::text[]' % ','.join(QuotedString( 127 | v).getquoted() for v in re.search(r"'(.*)'", default).group(1).split(',')) 128 | return default, 'text[]' 129 | else: 130 | raise Exception('unknown %s' % column['type']) 131 | 132 | default, column_type = get_type(column) 133 | 134 | if column.get('auto_increment', None): 135 | return '%s DEFAULT nextval(\'"%s_%s_seq"\'::regclass) NOT NULL' % ( 136 | column_type, column['table_name'], column['name']) 137 | 138 | return '%s%s%s' % (column_type, (default if not default == None else ''), null) 139 | 140 | def table_comments(self, table): 141 | comments = [] 142 | if table.comment: 143 | comments.append('COMMENT ON TABLE %s is %s;' % (table.name, QuotedString(table.comment).getquoted())) 144 | for column in table.columns: 145 | if column['comment']: 146 | comments.append('COMMENT ON COLUMN %s.%s is %s;' % (table.name, column['name'], QuotedString(column['comment']).getquoted())) 147 | return comments 148 | 149 | def process_row(self, table, row): 150 | """Examines row data from MySQL and alters 151 | the values when necessary to be compatible with 152 | sending to PostgreSQL via the copy command 153 | """ 154 | for index, column in enumerate(table.columns): 155 | hash_key = hash(frozenset(column.items())) 156 | column_type = self.column_types[hash_key] if hash_key in self.column_types else self.column_type(column) 157 | if row[index] == None and ('timestamp' not in column_type or not column['default']): 158 | row[index] = '\N' 159 | elif row[index] == None and column['default']: 160 | if self.tz: 161 | row[index] = '1970-01-01T00:00:00.000000' + self.tz_offset 162 | else: 163 | row[index] = '1970-01-01 00:00:00' 164 | elif 'bit' in column_type: 165 | row[index] = bin(ord(row[index]))[2:] 166 | elif isinstance(row[index], (str, unicode, basestring)): 167 | if column_type == 'bytea': 168 | row[index] = Binary(row[index]).getquoted()[1:-8] if row[index] else row[index] 169 | elif 'text[' in column_type: 170 | row[index] = '{%s}' % ','.join('"%s"' % v.replace('"', r'\"') for v in row[index].split(',')) 171 | else: 172 | row[index] = row[index].replace('\\', r'\\').replace('\n', r'\n').replace( 173 | '\t', r'\t').replace('\r', r'\r').replace('\0', '') 174 | elif column_type == 'boolean': 175 | # We got here because you used a tinyint(1), if you didn't want a bool, don't use that type 176 | row[index] = 't' if row[index] not in (None, 0) else 'f' if row[index] == 0 else row[index] 177 | elif isinstance(row[index], (date, datetime)): 178 | if isinstance(row[index], datetime) and self.tz: 179 | try: 180 | if row[index].tzinfo: 181 | row[index] = row[index].astimezone(self.tz).isoformat() 182 | else: 183 | row[index] = datetime(*row[index].timetuple()[:6], tzinfo=self.tz).isoformat() 184 | except Exception as e: 185 | print e.message 186 | else: 187 | row[index] = row[index].isoformat() 188 | elif isinstance(row[index], timedelta): 189 | row[index] = datetime.utcfromtimestamp(_get_total_seconds(row[index])).time().isoformat() 190 | else: 191 | row[index] = AsIs(row[index]).getquoted() 192 | 193 | def table_attributes(self, table): 194 | primary_keys = [] 195 | serial_key = None 196 | maxval = None 197 | columns = StringIO() 198 | 199 | for column in table.columns: 200 | if column['auto_increment']: 201 | serial_key = column['name'] 202 | maxval = 1 if column['maxval'] < 1 else column['maxval'] + 1 203 | if column['primary_key']: 204 | primary_keys.append(column['name']) 205 | columns.write(' %s,\n' % self.column_description(column)) 206 | return primary_keys, serial_key, maxval, columns.getvalue()[:-2] 207 | 208 | def truncate(self, table): 209 | serial_key = None 210 | maxval = None 211 | 212 | for column in table.columns: 213 | if column['auto_increment']: 214 | serial_key = column['name'] 215 | maxval = 1 if column['maxval'] < 1 else column['maxval'] + 1 216 | 217 | truncate_sql = 'TRUNCATE "%s" CASCADE;' % table.name 218 | serial_key_sql = None 219 | 220 | if serial_key: 221 | serial_key_sql = "SELECT pg_catalog.setval(pg_get_serial_sequence(%(table_name)s, %(serial_key)s), %(maxval)s, true);" % { 222 | 'table_name': QuotedString('"%s"' % table.name).getquoted(), 223 | 'serial_key': QuotedString(serial_key).getquoted(), 224 | 'maxval': maxval} 225 | 226 | return (truncate_sql, serial_key_sql) 227 | 228 | def write_table(self, table): 229 | primary_keys, serial_key, maxval, columns = self.table_attributes(table) 230 | serial_key_sql = [] 231 | table_sql = [] 232 | if serial_key: 233 | serial_key_seq = '%s_%s_seq' % (table.name, serial_key) 234 | serial_key_sql.append('DROP SEQUENCE IF EXISTS "%s" CASCADE;' % serial_key_seq) 235 | serial_key_sql.append("""CREATE SEQUENCE "%s" INCREMENT BY 1 236 | NO MAXVALUE NO MINVALUE CACHE 1;""" % serial_key_seq) 237 | serial_key_sql.append('SELECT pg_catalog.setval(\'"%s"\', %s, true);' % (serial_key_seq, maxval)) 238 | 239 | table_sql.append('DROP TABLE IF EXISTS "%s" CASCADE;' % table.name) 240 | table_sql.append('CREATE TABLE "%s" (\n%s\n)\nWITHOUT OIDS;' % (table.name.encode('utf8'), columns)) 241 | table_sql.extend(self.table_comments(table)) 242 | return (table_sql, serial_key_sql) 243 | 244 | def write_indexes(self, table): 245 | index_sql = [] 246 | primary_index = [idx for idx in table.indexes if idx.get('primary', None)] 247 | index_prefix = self.index_prefix 248 | if primary_index: 249 | index_sql.append('ALTER TABLE "%(table_name)s" ADD CONSTRAINT "%(index_name)s_pkey" PRIMARY KEY(%(column_names)s);' % { 250 | 'table_name': table.name, 251 | 'index_name': '%s%s_%s' % (index_prefix, table.name, 252 | '_'.join(primary_index[0]['columns'])), 253 | 'column_names': ', '.join('"%s"' % col for col in primary_index[0]['columns']), 254 | }) 255 | for index in table.indexes: 256 | if 'primary' in index: 257 | continue 258 | unique = 'UNIQUE ' if index.get('unique', None) else '' 259 | index_name = '%s%s_%s' % (index_prefix, table.name, '_'.join(index['columns'])) 260 | index_sql.append('DROP INDEX IF EXISTS "%s" CASCADE;' % index_name) 261 | index_sql.append('CREATE %(unique)sINDEX "%(index_name)s" ON "%(table_name)s" (%(column_names)s);' % { 262 | 'unique': unique, 263 | 'index_name': index_name, 264 | 'table_name': table.name, 265 | 'column_names': ', '.join('"%s"' % col for col in index['columns']), 266 | }) 267 | 268 | return index_sql 269 | 270 | def write_constraints(self, table): 271 | constraint_sql = [] 272 | for key in table.foreign_keys: 273 | constraint_sql.append("""ALTER TABLE "%(table_name)s" ADD FOREIGN KEY ("%(column_name)s") 274 | REFERENCES "%(ref_table_name)s"(%(ref_column_name)s);""" % { 275 | 'table_name': table.name, 276 | 'column_name': key['column'], 277 | 'ref_table_name': key['ref_table'], 278 | 'ref_column_name': key['ref_column']}) 279 | return constraint_sql 280 | 281 | def write_triggers(self, table): 282 | trigger_sql = [] 283 | for key in table.triggers: 284 | trigger_sql.append("""CREATE OR REPLACE FUNCTION %(fn_trigger_name)s RETURNS TRIGGER AS $%(trigger_name)s$ 285 | BEGIN 286 | %(trigger_statement)s 287 | RETURN NULL; 288 | END; 289 | $%(trigger_name)s$ LANGUAGE plpgsql;""" % { 290 | 'table_name': table.name, 291 | 'trigger_time': key['timing'], 292 | 'trigger_event': key['event'], 293 | 'trigger_name': key['name'], 294 | 'fn_trigger_name': 'fn_' + key['name'] + '()', 295 | 'trigger_statement': key['statement']}) 296 | 297 | trigger_sql.append("""CREATE TRIGGER %(trigger_name)s %(trigger_time)s %(trigger_event)s ON %(table_name)s 298 | FOR EACH ROW 299 | EXECUTE PROCEDURE fn_%(trigger_name)s();""" % { 300 | 'table_name': table.name, 301 | 'trigger_time': key['timing'], 302 | 'trigger_event': key['event'], 303 | 'trigger_name': key['name']}) 304 | 305 | return trigger_sql 306 | 307 | def close(self): 308 | raise NotImplementedError 309 | 310 | def write_contents(self, table, reader): 311 | raise NotImplementedError 312 | 313 | # Original fix for Py2.6: https://github.com/mozilla/mozdownload/issues/73 314 | def _get_total_seconds(dt): 315 | # Keep backward compatibility with Python 2.6 which doesn't have this method 316 | if hasattr(datetime, 'total_seconds'): 317 | return dt.total_seconds() 318 | else: 319 | return (dt.microseconds + (dt.seconds + dt.days * 24 * 3600) * 10**6) / 10**6 320 | --------------------------------------------------------------------------------