├── tests ├── __init__.py ├── func │ └── __init__.py ├── unit │ ├── __init__.py │ ├── test_types_fallback.py │ ├── test_create_view_reconnect_and_errors.py │ ├── test_collation_sqlglot_augmented.py │ ├── test_views_build_paths_extra.py │ ├── test_cli_views_flag.py │ ├── test_views_create_view.py │ ├── test_create_and_transfer_reconnect.py │ ├── test_defaults_sqlglot_enhanced.py │ ├── test_build_create_table_sql_sqlglot_identifiers.py │ ├── test_indices_prefix_and_uniqueness.py │ ├── test_types_sqlglot_augmented.py │ ├── test_types_and_defaults_extra.py │ ├── test_cli_error_paths.py │ ├── test_views_sqlglot.py │ ├── test_click_utils.py │ ├── test_sqlite_utils.py │ ├── test_debug_info.py │ └── test_mysql_utils.py ├── faker_providers.py ├── database.py ├── factories.py ├── models.py └── conftest.py ├── src └── mysql_to_sqlite3 │ ├── py.typed │ ├── __init__.py │ ├── mysql_utils.py │ ├── sqlite_utils.py │ ├── click_utils.py │ ├── types.py │ ├── debug_info.py │ └── cli.py ├── docs ├── requirements.txt ├── modules.rst ├── Makefile ├── make.bat ├── conf.py ├── mysql_to_sqlite3.rst ├── index.rst └── README.rst ├── .github ├── FUNDING.yml ├── dependabot.yml ├── ISSUE_TEMPLATE │ ├── feature_request.md │ └── bug_report.md ├── workflows │ ├── docs.yml │ ├── docker.yml │ ├── publish.yml │ └── test.yml ├── pull_request_template.md └── copilot-instructions.md ├── .flake8 ├── Dockerfile ├── requirements_dev.txt ├── LICENSE ├── SECURITY.md ├── .gitignore ├── tox.ini ├── CONTRIBUTING.md ├── pyproject.toml ├── CODE-OF-CONDUCT.md ├── README.md ├── .bandit.yml └── CHANGELOG.md /tests/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/func/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/unit/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /src/mysql_to_sqlite3/py.typed: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /docs/requirements.txt: -------------------------------------------------------------------------------- 1 | Sphinx==8.2.3 -------------------------------------------------------------------------------- /.github/FUNDING.yml: -------------------------------------------------------------------------------- 1 | github: techouse 2 | custom: [ "https://paypal.me/ktusar" ] 3 | -------------------------------------------------------------------------------- /.flake8: -------------------------------------------------------------------------------- 1 | [flake8] 2 | ignore = I100,I201,I202,D203,D401,W503,E203,F401,F403,C901,E501 3 | exclude = tests 4 | max-line-length = 88 -------------------------------------------------------------------------------- /docs/modules.rst: -------------------------------------------------------------------------------- 1 | mysql_to_sqlite3 2 | ================ 3 | 4 | .. toctree:: 5 | :maxdepth: 4 6 | 7 | mysql_to_sqlite3 8 | -------------------------------------------------------------------------------- /src/mysql_to_sqlite3/__init__.py: -------------------------------------------------------------------------------- 1 | """Utility to transfer data from MySQL to SQLite 3.""" 2 | 3 | __version__ = "2.5.5" 4 | 5 | from .transporter import MySQLtoSQLite 6 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.14-alpine 2 | 3 | LABEL maintainer="https://github.com/techouse" 4 | 5 | RUN pip install --no-cache-dir --upgrade pip && \ 6 | pip install --no-cache-dir mysql-to-sqlite3 7 | 8 | ENTRYPOINT ["mysql2sqlite"] -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | - package-ecosystem: "pip" 4 | directory: "/" 5 | schedule: 6 | interval: "weekly" 7 | - package-ecosystem: "github-actions" 8 | directory: "/" 9 | schedule: 10 | interval: "weekly" -------------------------------------------------------------------------------- /requirements_dev.txt: -------------------------------------------------------------------------------- 1 | click>=8.1.3 2 | docker>=6.1.3 3 | factory-boy 4 | Faker>=18.10.0 5 | mysql-connector-python>=9.0.0 6 | mysqlclient>=2.1.1 7 | pytest>=7.3.1 8 | pytest-cov 9 | pytest-mock 10 | pytest-timeout 11 | pytimeparse2 12 | python-dateutil>=2.9.0.post0 13 | types-python-dateutil 14 | python-slugify>=7.0.0 15 | types-python-slugify 16 | simplejson>=3.19.1 17 | types-simplejson 18 | sqlalchemy>=2.0.0 19 | sqlalchemy-utils 20 | sqlglot>=27.27.0 21 | types-sqlalchemy-utils 22 | tox 23 | tqdm>=4.65.0 24 | types-tqdm 25 | packaging 26 | tabulate 27 | types-tabulate 28 | typing-extensions; python_version < "3.11" 29 | requests 30 | types-requests 31 | mypy>=1.3.0 32 | -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Minimal makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line, and also 5 | # from the environment for the first two. 6 | SPHINXOPTS ?= 7 | SPHINXBUILD ?= sphinx-build 8 | SOURCEDIR = . 9 | BUILDDIR = _build 10 | 11 | # Put it first so that "make" without argument is like "make help". 12 | help: 13 | @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 14 | 15 | .PHONY: help Makefile 16 | 17 | # Catch-all target: route all unknown targets to Sphinx using the new 18 | # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). 19 | %: Makefile 20 | @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 21 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature_request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Feature request 3 | about: Suggest an idea for this project 4 | title: '' 5 | labels: enhancement 6 | assignees: techouse 7 | 8 | --- 9 | 10 | **Is your feature request related to a problem? Please describe.** 11 | A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] 12 | 13 | **Describe the solution you'd like** 14 | A clear and concise description of what you want to happen. 15 | 16 | **Describe alternatives you've considered** 17 | A clear and concise description of any alternative solutions or features you've considered. 18 | 19 | **Additional context** 20 | Add any other context or screenshots about the feature request here. 21 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/bug_report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug report 3 | about: Create a report to help us improve 4 | title: '' 5 | labels: bug 6 | assignees: techouse 7 | 8 | --- 9 | 10 | **Describe the bug** 11 | A clear and concise description of what the bug is. 12 | 13 | **Expected behaviour** 14 | What you expected. 15 | 16 | **Actual result** 17 | What happened instead. 18 | 19 | **System Information** 20 | 21 | ```bash 22 | $ mysql2sqlite --version 23 | ``` 24 | 25 | ``` 26 | 27 | ``` 28 | 29 | This command is only available on v1.3.2 and greater. Otherwise, please provide some basic information about your system (Python version, operating system, etc.). 30 | 31 | **Additional context** 32 | Add any other context about the problem here. 33 | 34 | In case of errors please run the same command with `--debug`. This option is only available on v1.4.12 or greater. 35 | -------------------------------------------------------------------------------- /tests/faker_providers.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | from typing import Optional 3 | 4 | from faker.providers import BaseProvider, date_time 5 | from faker.typing import DateParseType 6 | 7 | 8 | class DateTimeProviders(BaseProvider): 9 | def time_object_without_microseconds(self, end_datetime: Optional[DateParseType] = None) -> datetime.time: 10 | return date_time.Provider(self.generator).time_object(end_datetime).replace(microsecond=0) 11 | 12 | def date_time_this_century_without_microseconds( 13 | self, 14 | before_now: bool = True, 15 | after_now: bool = False, 16 | tzinfo: Optional[datetime.tzinfo] = None, 17 | ) -> datetime.datetime: 18 | return ( 19 | date_time.Provider(self.generator) 20 | .date_time_this_century(before_now, after_now, tzinfo) 21 | .replace(microsecond=0) 22 | ) 23 | -------------------------------------------------------------------------------- /docs/make.bat: -------------------------------------------------------------------------------- 1 | @ECHO OFF 2 | 3 | pushd %~dp0 4 | 5 | REM Command file for Sphinx documentation 6 | 7 | if "%SPHINXBUILD%" == "" ( 8 | set SPHINXBUILD=sphinx-build 9 | ) 10 | set SOURCEDIR=. 11 | set BUILDDIR=_build 12 | 13 | %SPHINXBUILD% >NUL 2>NUL 14 | if errorlevel 9009 ( 15 | echo. 16 | echo.The 'sphinx-build' command was not found. Make sure you have Sphinx 17 | echo.installed, then set the SPHINXBUILD environment variable to point 18 | echo.to the full path of the 'sphinx-build' executable. Alternatively you 19 | echo.may add the Sphinx directory to PATH. 20 | echo. 21 | echo.If you don't have Sphinx installed, grab it from 22 | echo.https://www.sphinx-doc.org/ 23 | exit /b 1 24 | ) 25 | 26 | if "%1" == "" goto help 27 | 28 | %SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% 29 | goto end 30 | 31 | :help 32 | %SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% 33 | 34 | :end 35 | popd 36 | -------------------------------------------------------------------------------- /.github/workflows/docs.yml: -------------------------------------------------------------------------------- 1 | name: Docs 2 | 3 | on: 4 | workflow_dispatch: 5 | workflow_call: 6 | 7 | permissions: 8 | contents: write 9 | 10 | concurrency: 11 | group: docs-${{ github.ref }} 12 | cancel-in-progress: false 13 | 14 | jobs: 15 | docs: 16 | name: "Docs" 17 | runs-on: ubuntu-latest 18 | steps: 19 | - uses: actions/checkout@v6 20 | - uses: actions/setup-python@v6 21 | with: 22 | python-version: "3.x" 23 | cache: "pip" 24 | - name: Install package 25 | run: pip install -e . 26 | - name: Install Sphinx dependencies 27 | working-directory: docs 28 | run: pip install -r requirements.txt 29 | - name: Sphinx build 30 | working-directory: docs 31 | run: sphinx-build . _build 32 | - name: Deploy to GitHub Pages 33 | uses: peaceiris/actions-gh-pages@v4 34 | with: 35 | publish_branch: gh-pages 36 | github_token: ${{ secrets.GITHUB_TOKEN }} 37 | publish_dir: docs/_build/ 38 | force_orphan: true -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2025 Klemen Tusar 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /tests/unit/test_types_fallback.py: -------------------------------------------------------------------------------- 1 | import importlib.util 2 | import sys 3 | import types as pytypes 4 | 5 | from typing_extensions import TypedDict as ExtensionsTypedDict 6 | 7 | 8 | def test_types_module_uses_typing_extensions_when_typed_dict_missing() -> None: 9 | """Reload the types module without typing.TypedDict to exercise the fallback branch.""" 10 | import mysql_to_sqlite3.types as original_module 11 | 12 | module_path = original_module.__file__ 13 | assert module_path is not None 14 | 15 | # Swap in a stripped-down typing module that lacks TypedDict. 16 | real_typing = sys.modules["typing"] 17 | fake_typing = pytypes.ModuleType("typing") 18 | fake_typing.__dict__.update({k: v for k, v in real_typing.__dict__.items() if k != "TypedDict"}) 19 | sys.modules["typing"] = fake_typing 20 | 21 | try: 22 | spec = importlib.util.spec_from_file_location("mysql_to_sqlite3.types_fallback", module_path) 23 | assert spec and spec.loader 24 | module = importlib.util.module_from_spec(spec) 25 | spec.loader.exec_module(module) 26 | finally: 27 | sys.modules["typing"] = real_typing 28 | sys.modules.pop("mysql_to_sqlite3.types_fallback", None) 29 | 30 | assert module.TypedDict is ExtensionsTypedDict 31 | -------------------------------------------------------------------------------- /src/mysql_to_sqlite3/mysql_utils.py: -------------------------------------------------------------------------------- 1 | """Miscellaneous MySQL utilities.""" 2 | 3 | import typing as t 4 | 5 | from mysql.connector import CharacterSet 6 | from mysql.connector.charsets import MYSQL_CHARACTER_SETS 7 | 8 | 9 | CHARSET_INTRODUCERS: t.Tuple[str, ...] = tuple( 10 | f"_{charset[0]}" for charset in MYSQL_CHARACTER_SETS if charset is not None 11 | ) 12 | 13 | 14 | class CharSet(t.NamedTuple): 15 | """MySQL character set as a named tuple.""" 16 | 17 | id: int 18 | charset: str 19 | collation: str 20 | 21 | 22 | def mysql_supported_character_sets(charset: t.Optional[str] = None) -> t.Iterator[CharSet]: 23 | """Get supported MySQL character sets.""" 24 | index: int 25 | info: t.Optional[t.Tuple[str, str, bool]] 26 | if charset is not None: 27 | for index, info in enumerate(MYSQL_CHARACTER_SETS): 28 | if info is not None: 29 | try: 30 | if info[0] == charset: 31 | yield CharSet(index, charset, info[1]) 32 | except KeyError: 33 | continue 34 | else: 35 | for charset in CharacterSet().get_supported(): 36 | for index, info in enumerate(MYSQL_CHARACTER_SETS): 37 | if info is not None: 38 | try: 39 | yield CharSet(index, charset, info[1]) 40 | except KeyError: 41 | continue 42 | -------------------------------------------------------------------------------- /.github/pull_request_template.md: -------------------------------------------------------------------------------- 1 | # Pull Request Template 2 | 3 | ## Description 4 | 5 | Please include a summary of the change and which issue is fixed. Please also include relevant motivation and context. 6 | List any dependencies that are required for this change. 7 | 8 | Fixes # (issue) 9 | 10 | ## Type of change 11 | 12 | Please delete options that are not relevant. 13 | 14 | - [ ] Bug fix (non-breaking change which fixes an issue) 15 | - [ ] New feature (non-breaking change which adds functionality) 16 | - [ ] Breaking change (fix or feature that would cause existing functionality to not work as expected) 17 | - [ ] This change requires a documentation update 18 | 19 | ## How Has This Been Tested? 20 | 21 | Please describe the tests that you ran to verify your changes. Provide instructions so we can reproduce. Please also 22 | list any relevant details for your test configuration 23 | 24 | - [ ] Test A 25 | - [ ] Test B 26 | 27 | ## Checklist: 28 | 29 | - [ ] My code follows the style guidelines of this project 30 | - [ ] I have performed a self-review of my own code 31 | - [ ] I have commented my code, particularly in hard-to-understand areas 32 | - [ ] I have made corresponding changes to the documentation 33 | - [ ] My changes generate no new warnings 34 | - [ ] I have added tests that prove my fix is effective or that my feature works 35 | - [ ] New and existing unit tests pass locally with my changes 36 | - [ ] Any dependent changes have been merged and published in downstream modules -------------------------------------------------------------------------------- /tests/database.py: -------------------------------------------------------------------------------- 1 | import typing as t 2 | from datetime import datetime, timedelta 3 | from decimal import Decimal 4 | 5 | import simplejson as json 6 | from sqlalchemy import create_engine 7 | from sqlalchemy.engine import Engine 8 | from sqlalchemy.orm import sessionmaker 9 | from sqlalchemy_utils import create_database, database_exists 10 | 11 | from .models import Base 12 | 13 | 14 | class Database: 15 | engine: Engine 16 | Session: sessionmaker 17 | 18 | def __init__(self, database_uri): 19 | self.Session = sessionmaker() 20 | self.engine = create_engine(database_uri, json_serializer=self.dumps, json_deserializer=json.loads) 21 | if not database_exists(self.engine.url): 22 | create_database(self.engine.url) 23 | self._create_db_tables() 24 | self.Session.configure(bind=self.engine) 25 | 26 | def _create_db_tables(self) -> None: 27 | Base.metadata.create_all(self.engine) 28 | 29 | @classmethod 30 | def dumps(cls, data: t.Any) -> str: 31 | return json.dumps(data, default=cls.json_serializer) 32 | 33 | @staticmethod 34 | def json_serializer(data: t.Any) -> t.Optional[str]: 35 | if isinstance(data, datetime): 36 | return data.isoformat() 37 | if isinstance(data, Decimal): 38 | return str(data) 39 | if isinstance(data, timedelta): 40 | hours, remainder = divmod(data.total_seconds(), 3600) 41 | minutes, seconds = divmod(remainder, 60) 42 | return "{:02}:{:02}:{:02}".format(int(hours), int(minutes), int(seconds)) 43 | return None 44 | -------------------------------------------------------------------------------- /docs/conf.py: -------------------------------------------------------------------------------- 1 | # Configuration file for the Sphinx documentation builder. 2 | # 3 | # For the full list of built-in configuration values, see the documentation: 4 | # https://www.sphinx-doc.org/en/master/usage/configuration.html 5 | import os 6 | import re 7 | import sys 8 | from pathlib import Path 9 | 10 | 11 | sys.path.insert(0, os.path.abspath("../src")) 12 | 13 | _ROOT = Path(__file__).resolve().parents[1] 14 | _ver_file = _ROOT / "src" / "mysql_to_sqlite3" / "__init__.py" 15 | _m = re.search(r'^__version__\s*=\s*[\'"]([^\'"]+)[\'"]', _ver_file.read_text(encoding="utf-8"), re.M) 16 | __version__ = _m.group(1) if _m else "0+unknown" 17 | 18 | 19 | # -- Project information ----------------------------------------------------- 20 | # https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information 21 | 22 | project = "mysql-to-sqlite3" 23 | copyright = "%Y, Klemen Tusar" 24 | author = "Klemen Tusar" 25 | release = __version__ 26 | 27 | # -- General configuration --------------------------------------------------- 28 | # https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration 29 | 30 | extensions = ["sphinx.ext.autodoc", "sphinx.ext.viewcode", "sphinx.ext.napoleon"] 31 | 32 | napoleon_google_docstring = True 33 | napoleon_include_init_with_doc = True 34 | 35 | templates_path = ["_templates"] 36 | exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"] 37 | 38 | 39 | # -- Options for HTML output ------------------------------------------------- 40 | # https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output 41 | 42 | html_theme = "alabaster" 43 | html_static_path = ["_static"] 44 | -------------------------------------------------------------------------------- /docs/mysql_to_sqlite3.rst: -------------------------------------------------------------------------------- 1 | mysql\_to\_sqlite3 package 2 | ========================== 3 | 4 | Submodules 5 | ---------- 6 | 7 | mysql\_to\_sqlite3.cli module 8 | ----------------------------- 9 | 10 | .. automodule:: mysql_to_sqlite3.cli 11 | :members: 12 | :undoc-members: 13 | :show-inheritance: 14 | 15 | mysql\_to\_sqlite3.click\_utils module 16 | -------------------------------------- 17 | 18 | .. automodule:: mysql_to_sqlite3.click_utils 19 | :members: 20 | :undoc-members: 21 | :show-inheritance: 22 | 23 | mysql\_to\_sqlite3.debug\_info module 24 | ------------------------------------- 25 | 26 | .. automodule:: mysql_to_sqlite3.debug_info 27 | :members: 28 | :undoc-members: 29 | :show-inheritance: 30 | 31 | mysql\_to\_sqlite3.mysql\_utils module 32 | -------------------------------------- 33 | 34 | .. automodule:: mysql_to_sqlite3.mysql_utils 35 | :members: 36 | :undoc-members: 37 | :show-inheritance: 38 | 39 | mysql\_to\_sqlite3.sqlite\_utils module 40 | --------------------------------------- 41 | 42 | .. automodule:: mysql_to_sqlite3.sqlite_utils 43 | :members: 44 | :undoc-members: 45 | :show-inheritance: 46 | 47 | mysql\_to\_sqlite3.transporter module 48 | ------------------------------------- 49 | 50 | .. automodule:: mysql_to_sqlite3.transporter 51 | :members: 52 | :undoc-members: 53 | :show-inheritance: 54 | 55 | mysql\_to\_sqlite3.types module 56 | ------------------------------- 57 | 58 | .. automodule:: mysql_to_sqlite3.types 59 | :members: 60 | :undoc-members: 61 | :show-inheritance: 62 | 63 | Module contents 64 | --------------- 65 | 66 | .. automodule:: mysql_to_sqlite3 67 | :members: 68 | :undoc-members: 69 | :show-inheritance: 70 | -------------------------------------------------------------------------------- /tests/unit/test_create_view_reconnect_and_errors.py: -------------------------------------------------------------------------------- 1 | import sqlite3 2 | from unittest.mock import MagicMock, patch 3 | 4 | import mysql.connector 5 | import pytest 6 | from mysql.connector import errorcode 7 | 8 | from mysql_to_sqlite3.transporter import MySQLtoSQLite 9 | 10 | 11 | def test_create_view_reconnect_on_server_lost_then_success() -> None: 12 | with patch.object(MySQLtoSQLite, "__init__", return_value=None): 13 | inst = MySQLtoSQLite() # type: ignore[call-arg] 14 | 15 | inst._mysql = MagicMock() 16 | inst._sqlite = MagicMock() 17 | inst._sqlite_cur = MagicMock() 18 | inst._logger = MagicMock() 19 | 20 | # First build fails with CR_SERVER_LOST, second returns valid SQL 21 | err = mysql.connector.Error(msg="lost", errno=errorcode.CR_SERVER_LOST) 22 | inst._build_create_view_sql = MagicMock(side_effect=[err, 'CREATE VIEW "v" AS SELECT 1;']) 23 | 24 | inst._create_view("v") 25 | 26 | inst._mysql.reconnect.assert_called_once() 27 | inst._sqlite_cur.execute.assert_called_once() 28 | inst._sqlite.commit.assert_called_once() 29 | 30 | 31 | def test_create_view_sqlite_error_is_logged_and_raised() -> None: 32 | with patch.object(MySQLtoSQLite, "__init__", return_value=None): 33 | inst = MySQLtoSQLite() # type: ignore[call-arg] 34 | 35 | inst._mysql = MagicMock() 36 | inst._sqlite = MagicMock() 37 | inst._sqlite_cur = MagicMock() 38 | inst._logger = MagicMock() 39 | 40 | inst._build_create_view_sql = MagicMock(return_value='CREATE VIEW "v" AS SELECT 1;') 41 | inst._sqlite_cur.execute.side_effect = sqlite3.Error("broken") 42 | 43 | with pytest.raises(sqlite3.Error): 44 | inst._create_view("v") 45 | 46 | inst._logger.error.assert_called() 47 | -------------------------------------------------------------------------------- /SECURITY.md: -------------------------------------------------------------------------------- 1 | # Security Policy 2 | 3 | ## Supported Versions 4 | 5 | | Version | Supported | 6 | |---------|--------------------| 7 | | 2.1.x | :white_check_mark: | 8 | | 2.0.x | :x: | 9 | | 1.x.x | :x: | 10 | 11 | ## Reporting a Vulnerability 12 | 13 | We take the security of our software seriously. If you believe you have found a security vulnerability, please report it 14 | to us as described below. 15 | 16 | **DO NOT CREATE A GITHUB ISSUE** reporting the vulnerability. 17 | 18 | Instead, send an email to [techouse@gmail.com](mailto:techouse@gmail.com). 19 | 20 | In the report, please include the following: 21 | 22 | - Your name and affiliation (if any). 23 | - A description of the technical details of the vulnerabilities. It is very important to let us know how we can 24 | reproduce your findings. 25 | - An explanation who can exploit this vulnerability, and what they gain when doing so -- write an attack scenario. This 26 | will help us evaluate your submission quickly, especially if it is a complex or creative vulnerability. 27 | - Whether this vulnerability is public or known to third parties. If it is, please provide details. 28 | 29 | If you don’t get an acknowledgment from us or have heard nothing from us in a week, please contact us again. 30 | 31 | We will send a response indicating the next steps in handling your report. We will keep you informed about the progress 32 | towards a fix and full announcement. 33 | 34 | We will not disclose your identity to the public without your permission. We strive to credit researchers in our 35 | advisories when we release a fix, but only after getting your permission. 36 | 37 | We appreciate your efforts to responsibly disclose your findings, and will make every effort to acknowledge your 38 | contributions. -------------------------------------------------------------------------------- /.github/workflows/docker.yml: -------------------------------------------------------------------------------- 1 | name: Publish Docker image 2 | 3 | on: 4 | workflow_call: 5 | defaults: 6 | run: 7 | shell: bash 8 | 9 | concurrency: 10 | group: docker-${{ github.ref }} 11 | cancel-in-progress: false 12 | 13 | jobs: 14 | push_to_registry: 15 | name: Push Docker image to Docker Hub 16 | runs-on: ubuntu-latest 17 | permissions: 18 | packages: write 19 | contents: read 20 | environment: 21 | name: docker 22 | url: https://hub.docker.com/r/${{ vars.DOCKERHUB_REPOSITORY }} 23 | steps: 24 | - name: Check out the repo 25 | uses: actions/checkout@v6 26 | - name: Set up QEMU 27 | uses: docker/setup-qemu-action@v3 28 | - name: Set up Docker Buildx 29 | uses: docker/setup-buildx-action@v3 30 | - name: Docker meta 31 | id: meta 32 | uses: docker/metadata-action@v5 33 | with: 34 | images: | 35 | ${{ vars.DOCKERHUB_REPOSITORY }} 36 | ghcr.io/${{ github.repository }} 37 | tags: | 38 | type=ref,event=branch 39 | type=ref,event=pr 40 | type=semver,pattern={{version}} 41 | type=semver,pattern={{major}}.{{minor}}.{{patch}} 42 | - name: Log in to Docker Hub 43 | uses: docker/login-action@v3 44 | with: 45 | username: ${{ secrets.DOCKERHUB_USERNAME }} 46 | password: ${{ secrets.DOCKERHUB_TOKEN }} 47 | - name: Log in to the Container registry 48 | uses: docker/login-action@v3 49 | with: 50 | registry: ghcr.io 51 | username: ${{ github.actor }} 52 | password: ${{ secrets.GITHUB_TOKEN }} 53 | - name: Build and push 54 | uses: docker/build-push-action@v6 55 | with: 56 | context: . 57 | platforms: linux/amd64,linux/arm64 58 | push: true 59 | tags: ${{ steps.meta.outputs.tags }} 60 | labels: ${{ steps.meta.outputs.labels }} 61 | -------------------------------------------------------------------------------- /tests/unit/test_collation_sqlglot_augmented.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from mysql_to_sqlite3.sqlite_utils import CollatingSequences 4 | from mysql_to_sqlite3.transporter import MySQLtoSQLite 5 | 6 | 7 | class TestCollationSqlglotAugmented: 8 | @pytest.mark.parametrize( 9 | "mysql_type", 10 | [ 11 | "char varying(12)", 12 | "CHARACTER VARYING(12)", 13 | ], 14 | ) 15 | def test_collation_applied_for_char_varying_synonyms(self, mysql_type: str) -> None: 16 | out = MySQLtoSQLite._data_type_collation_sequence(collation=CollatingSequences.NOCASE, column_type=mysql_type) 17 | assert out == f"COLLATE {CollatingSequences.NOCASE}" 18 | 19 | def test_collation_applied_for_national_character_varying(self) -> None: 20 | out = MySQLtoSQLite._data_type_collation_sequence( 21 | collation=CollatingSequences.NOCASE, column_type="national character varying(15)" 22 | ) 23 | assert out == f"COLLATE {CollatingSequences.NOCASE}" 24 | 25 | def test_no_collation_for_json(self) -> None: 26 | # Regardless of case or synonym handling, JSON should not have collation applied 27 | assert ( 28 | MySQLtoSQLite._data_type_collation_sequence(collation=CollatingSequences.NOCASE, column_type="json") == "" 29 | ) 30 | 31 | def test_no_collation_when_binary_collation(self) -> None: 32 | # BINARY collation disables COLLATE clause entirely 33 | assert ( 34 | MySQLtoSQLite._data_type_collation_sequence(collation=CollatingSequences.BINARY, column_type="VARCHAR(10)") 35 | == "" 36 | ) 37 | 38 | @pytest.mark.parametrize( 39 | "numeric_synonym", 40 | [ 41 | "double precision", 42 | "FIXED(10,2)", 43 | ], 44 | ) 45 | def test_no_collation_for_numeric_synonyms(self, numeric_synonym: str) -> None: 46 | assert ( 47 | MySQLtoSQLite._data_type_collation_sequence( 48 | collation=CollatingSequences.NOCASE, column_type=numeric_synonym 49 | ) 50 | == "" 51 | ) 52 | -------------------------------------------------------------------------------- /tests/unit/test_views_build_paths_extra.py: -------------------------------------------------------------------------------- 1 | import sqlite3 2 | from unittest.mock import MagicMock, patch 3 | 4 | import pytest 5 | 6 | from mysql_to_sqlite3.transporter import MySQLtoSQLite 7 | 8 | 9 | def _inst_with_mysql_dict(): 10 | with patch.object(MySQLtoSQLite, "__init__", return_value=None): 11 | inst = MySQLtoSQLite() # type: ignore[call-arg] 12 | inst._mysql_cur_dict = MagicMock() 13 | inst._mysql_cur = MagicMock() 14 | inst._mysql_database = "db" 15 | return inst 16 | 17 | 18 | def test_build_create_view_sql_information_schema_bytes_decode_failure_falls_back( 19 | monkeypatch: pytest.MonkeyPatch, 20 | ) -> None: 21 | inst = _inst_with_mysql_dict() 22 | 23 | # information_schema returns bytes that fail to decode in UTF-8 24 | bad_bytes = b"\xff\xfe\xfa" 25 | inst._mysql_cur_dict.fetchone.return_value = {"definition": bad_bytes} 26 | 27 | captured = {} 28 | 29 | def fake_converter(*, view_select_sql: str, view_name: str) -> str: 30 | captured["view_select_sql"] = view_select_sql 31 | captured["view_name"] = view_name 32 | return f'CREATE VIEW IF NOT EXISTS "{view_name}" AS SELECT 1;' 33 | 34 | monkeypatch.setattr(MySQLtoSQLite, "_mysql_viewdef_to_sqlite", staticmethod(fake_converter)) 35 | 36 | sql = inst._build_create_view_sql("v_strange") 37 | 38 | # Converter was invoked with the string representation of the undecodable bytes 39 | assert captured["view_name"] == "v_strange" 40 | assert isinstance(captured["view_select_sql"], str) 41 | # And a CREATE VIEW statement was produced 42 | assert sql.startswith('CREATE VIEW IF NOT EXISTS "v_strange" AS') 43 | assert sql.strip().endswith(";") 44 | 45 | 46 | def test_build_create_view_sql_raises_when_no_definition_available() -> None: 47 | inst = _inst_with_mysql_dict() 48 | 49 | # information_schema path -> None 50 | inst._mysql_cur_dict.fetchone.return_value = None 51 | # SHOW CREATE VIEW returns None 52 | inst._mysql_cur.fetchone.return_value = None 53 | 54 | with pytest.raises(sqlite3.Error): 55 | inst._build_create_view_sql("missing_view") 56 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | pip-wheel-metadata/ 24 | share/python-wheels/ 25 | *.egg-info/ 26 | .installed.cfg 27 | *.egg 28 | MANIFEST 29 | 30 | # PyInstaller 31 | # Usually these files are written by a python script from a template 32 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 33 | *.manifest 34 | *.spec 35 | 36 | # Installer logs 37 | pip-log.txt 38 | pip-delete-this-directory.txt 39 | 40 | # Unit test / coverage reports 41 | htmlcov/ 42 | .tox/ 43 | .nox/ 44 | .coverage 45 | .coverage.* 46 | .cache 47 | nosetests.xml 48 | coverage.xml 49 | *.cover 50 | .hypothesis/ 51 | .pytest_cache/ 52 | 53 | # Translations 54 | *.mo 55 | *.pot 56 | 57 | # Django stuff: 58 | *.log 59 | local_settings.py 60 | db.sqlite3 61 | 62 | # Flask stuff: 63 | instance/ 64 | .webassets-cache 65 | 66 | # Scrapy stuff: 67 | .scrapy 68 | 69 | # Sphinx documentation 70 | docs/_build/ 71 | 72 | # PyBuilder 73 | target/ 74 | 75 | # Jupyter Notebook 76 | .ipynb_checkpoints 77 | 78 | # IPython 79 | profile_default/ 80 | ipython_config.py 81 | 82 | # pyenv 83 | .python-version 84 | 85 | # celery beat schedule file 86 | celerybeat-schedule 87 | 88 | # SageMath parsed files 89 | *.sage.py 90 | 91 | # Environments 92 | .env 93 | .venv 94 | env/ 95 | venv/ 96 | ENV/ 97 | env.bak/ 98 | venv.bak/ 99 | 100 | # Spyder project settings 101 | .spyderproject 102 | .spyproject 103 | 104 | # Rope project settings 105 | .ropeproject 106 | 107 | # mkdocs documentation 108 | /site 109 | 110 | # mypy 111 | .mypy_cache/ 112 | .dmypy.json 113 | dmypy.json 114 | 115 | # Pyre type checker 116 | .pyre/ 117 | 118 | # IDE specific 119 | .idea 120 | 121 | # macOS specific 122 | .DS_Store 123 | 124 | # Potential leftovers 125 | tests/db_credentials.json 126 | log.txt 127 | -------------------------------------------------------------------------------- /tests/unit/test_cli_views_flag.py: -------------------------------------------------------------------------------- 1 | import typing as t 2 | 3 | import pytest 4 | from click.testing import CliRunner 5 | 6 | from mysql_to_sqlite3.cli import cli as mysql2sqlite 7 | 8 | 9 | class TestCLIViewsFlag: 10 | def test_mysql_views_as_tables_flag_is_threaded(self, monkeypatch: pytest.MonkeyPatch) -> None: 11 | """Ensure --mysql-views-as-tables reaches MySQLtoSQLite as views_as_views=False (legacy materialization).""" 12 | received_kwargs: t.Dict[str, t.Any] = {} 13 | 14 | class FakeConverter: 15 | def __init__(self, **kwargs: t.Any) -> None: 16 | received_kwargs.update(kwargs) 17 | 18 | def transfer(self) -> None: # pragma: no cover - nothing to do 19 | return None 20 | 21 | # Patch the converter used by the CLI 22 | monkeypatch.setattr("mysql_to_sqlite3.cli.MySQLtoSQLite", FakeConverter) 23 | 24 | runner = CliRunner() 25 | result = runner.invoke( 26 | mysql2sqlite, 27 | [ 28 | "-f", 29 | "out.sqlite3", 30 | "-d", 31 | "db", 32 | "-u", 33 | "user", 34 | "--mysql-views-as-tables", 35 | ], 36 | ) 37 | assert result.exit_code == 0 38 | assert received_kwargs.get("views_as_views") is False 39 | 40 | def test_mysql_views_as_tables_short_flag_is_threaded(self, monkeypatch: pytest.MonkeyPatch) -> None: 41 | """Ensure -T (short for --mysql-views-as-tables) reaches MySQLtoSQLite as views_as_views=False.""" 42 | received_kwargs: t.Dict[str, t.Any] = {} 43 | 44 | class FakeConverter: 45 | def __init__(self, **kwargs: t.Any) -> None: 46 | received_kwargs.update(kwargs) 47 | 48 | def transfer(self) -> None: # pragma: no cover - nothing to do 49 | return None 50 | 51 | # Patch the converter used by the CLI 52 | monkeypatch.setattr("mysql_to_sqlite3.cli.MySQLtoSQLite", FakeConverter) 53 | 54 | runner = CliRunner() 55 | result = runner.invoke( 56 | mysql2sqlite, 57 | [ 58 | "-f", 59 | "out.sqlite3", 60 | "-d", 61 | "db", 62 | "-u", 63 | "user", 64 | "-T", 65 | ], 66 | ) 67 | assert result.exit_code == 0 68 | assert received_kwargs.get("views_as_views") is False 69 | -------------------------------------------------------------------------------- /src/mysql_to_sqlite3/sqlite_utils.py: -------------------------------------------------------------------------------- 1 | """SQLite adapters and converters for unsupported data types.""" 2 | 3 | import sqlite3 4 | import typing as t 5 | from datetime import date, timedelta 6 | from decimal import Decimal 7 | 8 | from dateutil.parser import ParserError 9 | from dateutil.parser import parse as dateutil_parse 10 | from pytimeparse2 import parse 11 | 12 | 13 | def adapt_decimal(value: t.Any) -> str: 14 | """Convert decimal.Decimal to string.""" 15 | return str(value) 16 | 17 | 18 | def convert_decimal(value: t.Any) -> Decimal: 19 | """Convert string to decimal.Decimal.""" 20 | return Decimal(value) 21 | 22 | 23 | def adapt_timedelta(value: t.Any) -> str: 24 | """Convert datetime.timedelta to %H:%M:%S string.""" 25 | hours, remainder = divmod(value.total_seconds(), 3600) 26 | minutes, seconds = divmod(remainder, 60) 27 | return "{:02}:{:02}:{:02}".format(int(hours), int(minutes), int(seconds)) 28 | 29 | 30 | def convert_timedelta(value: t.Any) -> timedelta: 31 | """Convert %H:%M:%S string to datetime.timedelta.""" 32 | return timedelta(seconds=parse(value)) 33 | 34 | 35 | def encode_data_for_sqlite(value: t.Any) -> t.Any: 36 | """Fix encoding bytes.""" 37 | if isinstance(value, bytes): 38 | try: 39 | return value.decode() 40 | except (UnicodeDecodeError, AttributeError): 41 | return sqlite3.Binary(value) 42 | elif isinstance(value, str): 43 | return value 44 | else: 45 | try: 46 | return sqlite3.Binary(value) 47 | except TypeError: 48 | return value 49 | 50 | 51 | class CollatingSequences: 52 | """Taken from https://www.sqlite.org/datatype3.html#collating_sequences.""" 53 | 54 | BINARY: str = "BINARY" 55 | NOCASE: str = "NOCASE" 56 | RTRIM: str = "RTRIM" 57 | 58 | 59 | def convert_date(value: t.Union[str, bytes]) -> date: 60 | """Handle SQLite date conversion.""" 61 | try: 62 | return dateutil_parse(value.decode() if isinstance(value, bytes) else value).date() 63 | except ParserError as err: 64 | raise ValueError(f"DATE field contains {err}") # pylint: disable=W0707 65 | 66 | 67 | Integer_Types: t.Set[str] = { 68 | "INTEGER", 69 | "INTEGER UNSIGNED", 70 | "INT", 71 | "INT UNSIGNED", 72 | "BIGINT", 73 | "BIGINT UNSIGNED", 74 | "MEDIUMINT", 75 | "MEDIUMINT UNSIGNED", 76 | "SMALLINT", 77 | "SMALLINT UNSIGNED", 78 | "TINYINT", 79 | "TINYINT UNSIGNED", 80 | "NUMERIC", 81 | } 82 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | isolated_build = true 3 | envlist = 4 | python3.9, 5 | python3.10, 6 | python3.11, 7 | python3.12, 8 | python3.13, 9 | python3.14, 10 | black, 11 | flake8, 12 | linters 13 | skip_missing_interpreters = true 14 | 15 | [gh-actions] 16 | python = 17 | 3.9: python3.9 18 | 3.10: python3.10 19 | 3.11: python3.11 20 | 3.12: python3.12 21 | 3.13: python3.13 22 | 3.14: python3.14 23 | 24 | [testenv] 25 | passenv = 26 | LANG 27 | LEGACY_DB 28 | deps = 29 | -rrequirements_dev.txt 30 | commands = pytest -v --cov=src/mysql_to_sqlite3 --cov-report=xml 31 | 32 | [testenv:black] 33 | basepython = python3 34 | skip_install = true 35 | deps = 36 | black 37 | commands = black src/mysql_to_sqlite3 tests/ 38 | 39 | [testenv:isort] 40 | basepython = python3 41 | skip_install = true 42 | deps = 43 | isort 44 | commands = 45 | isort --check-only --diff . 46 | 47 | [testenv:flake8] 48 | basepython = python3 49 | skip_install = true 50 | deps = 51 | flake8 52 | flake8-colors 53 | flake8-docstrings 54 | flake8-import-order 55 | flake8-typing-imports 56 | pep8-naming 57 | commands = 58 | flake8 src/mysql_to_sqlite3 59 | 60 | [testenv:pylint] 61 | basepython = python3 62 | skip_install = true 63 | deps = 64 | pylint 65 | -rrequirements_dev.txt 66 | disable = C0209,C0301,C0411,R,W0107,W0622 67 | commands = 68 | pylint --rcfile=tox.ini src/mysql_to_sqlite3 69 | 70 | [testenv:bandit] 71 | basepython = python3 72 | skip_install = true 73 | deps = 74 | bandit 75 | commands = 76 | bandit -r src/mysql_to_sqlite3 -c .bandit.yml 77 | 78 | [testenv:mypy] 79 | basepython = python3 80 | skip_install = true 81 | deps = 82 | mypy>=1.3.0 83 | -rrequirements_dev.txt 84 | commands = 85 | mypy src/mysql_to_sqlite3 86 | 87 | [testenv:linters] 88 | basepython = python3 89 | skip_install = true 90 | deps = 91 | {[testenv:black]deps} 92 | {[testenv:isort]deps} 93 | {[testenv:flake8]deps} 94 | {[testenv:pylint]deps} 95 | {[testenv:bandit]deps} 96 | {[testenv:mypy]deps} 97 | commands = 98 | {[testenv:black]commands} 99 | {[testenv:isort]commands} 100 | {[testenv:flake8]commands} 101 | {[testenv:pylint]commands} 102 | {[testenv:bandit]commands} 103 | {[testenv:mypy]commands} 104 | 105 | [flake8] 106 | ignore = I100,I201,I202,D203,D401,W503,E203,F401,F403,C901,E501 107 | exclude = 108 | *__init__.py 109 | *__version__.py 110 | .tox 111 | max-complexity = 10 112 | max-line-length = 88 113 | import-order-style = pycharm 114 | application-import-names = flake8 115 | 116 | [pylint] 117 | disable = C0209,C0301,C0411,R,W0107,W0622,C0103,C0302 -------------------------------------------------------------------------------- /tests/unit/test_views_create_view.py: -------------------------------------------------------------------------------- 1 | import typing as t 2 | from unittest.mock import MagicMock, patch 3 | 4 | import mysql.connector 5 | from mysql.connector import errorcode 6 | 7 | from mysql_to_sqlite3.transporter import MySQLtoSQLite 8 | 9 | 10 | def test_show_create_view_fallback_handles_newline_and_backticks(monkeypatch: "t.Any") -> None: 11 | """ 12 | Force the SHOW CREATE VIEW fallback path and verify: 13 | - The executed SQL escapes backticks in the view name. 14 | - The regex extracts the SELECT when it follows "AS\n" (across newline). 15 | - The extracted SELECT (without trailing semicolon) is passed to _mysql_viewdef_to_sqlite. 16 | """ 17 | with patch.object(MySQLtoSQLite, "__init__", return_value=None): 18 | instance = MySQLtoSQLite() # type: ignore[call-arg] 19 | 20 | # Make information_schema path return None so fallback is used 21 | instance._mysql_cur_dict = MagicMock() 22 | instance._mysql_cur_dict.execute.return_value = None 23 | instance._mysql_cur_dict.fetchone.return_value = None 24 | 25 | # Prepare SHOW CREATE VIEW return value with AS followed by newline 26 | create_stmt = ( 27 | "CREATE ALGORITHM=UNDEFINED DEFINER=`user`@`%` SQL SECURITY DEFINER " "VIEW `we``ird` AS\nSELECT 1 AS `x`;" 28 | ) 29 | executed_sql: t.List[str] = [] 30 | 31 | def capture_execute(sql: str) -> None: 32 | executed_sql.append(sql) 33 | 34 | instance._mysql_cur = MagicMock() 35 | instance._mysql_cur.execute.side_effect = capture_execute 36 | instance._mysql_cur.fetchone.return_value = ("we`ird", create_stmt) 37 | 38 | # Capture the definition passed to _mysql_viewdef_to_sqlite and return a dummy SQL 39 | captured: t.Dict[str, str] = {} 40 | 41 | def fake_mysql_viewdef_to_sqlite(*, view_select_sql: str, view_name: str) -> str: 42 | captured["select"] = view_select_sql 43 | captured["view_name"] = view_name 44 | return 'CREATE VIEW IF NOT EXISTS "dummy" AS SELECT 1;' 45 | 46 | monkeypatch.setattr(MySQLtoSQLite, "_mysql_viewdef_to_sqlite", staticmethod(fake_mysql_viewdef_to_sqlite)) 47 | 48 | instance._mysql_database = "db" 49 | 50 | # Build the SQL (triggers fallback path) 51 | sql = instance._build_create_view_sql("we`ird") 52 | 53 | # Assert backticks in the view name were escaped in the SHOW CREATE VIEW statement 54 | assert executed_sql and executed_sql[0] == "SHOW CREATE VIEW `we``ird`" 55 | 56 | # The resulting SQL is our fake output 57 | assert sql.startswith('CREATE VIEW IF NOT EXISTS "dummy" AS') 58 | 59 | # Ensure the extracted SELECT excludes the trailing semicolon and spans newlines 60 | assert captured["select"] == "SELECT 1 AS `x`" 61 | # Check view_name was threaded unchanged to the converter 62 | assert captured["view_name"] == "we`ird" 63 | -------------------------------------------------------------------------------- /src/mysql_to_sqlite3/click_utils.py: -------------------------------------------------------------------------------- 1 | """Click utilities.""" 2 | 3 | import typing as t 4 | 5 | import click 6 | 7 | 8 | class OptionEatAll(click.Option): 9 | """Taken from https://stackoverflow.com/questions/48391777/nargs-equivalent-for-options-in-click#answer-48394004.""" # noqa: ignore=E501 pylint: disable=C0301 10 | 11 | def __init__(self, *args, **kwargs): 12 | """Override.""" 13 | self.save_other_options = kwargs.pop("save_other_options", True) 14 | nargs = kwargs.pop("nargs", -1) 15 | if nargs != -1: 16 | raise ValueError(f"nargs, if set, must be -1 not {nargs}") 17 | super(OptionEatAll, self).__init__(*args, **kwargs) 18 | self._previous_parser_process = None 19 | self._eat_all_parser = None 20 | 21 | def add_to_parser(self, parser, ctx) -> None: 22 | """Override.""" 23 | 24 | def parser_process(value, state): 25 | # method to hook to the parser.process 26 | done = False 27 | value = [value] 28 | if self.save_other_options: 29 | # grab everything up to the next option 30 | while state.rargs and not done: 31 | for prefix in self._eat_all_parser.prefixes: 32 | if state.rargs[0].startswith(prefix): 33 | done = True 34 | if not done: 35 | value.append(state.rargs.pop(0)) 36 | else: 37 | # grab everything remaining 38 | value += state.rargs 39 | state.rargs[:] = [] 40 | value = tuple(value) 41 | 42 | # call the actual process 43 | self._previous_parser_process(value, state) 44 | 45 | retval = super(OptionEatAll, self).add_to_parser(parser, ctx) # pylint: disable=E1111 46 | for name in self.opts: 47 | # pylint: disable=W0212 48 | our_parser = parser._long_opt.get(name) or parser._short_opt.get(name) 49 | if our_parser: 50 | self._eat_all_parser = our_parser 51 | self._previous_parser_process = our_parser.process 52 | our_parser.process = parser_process 53 | break 54 | return retval 55 | 56 | 57 | def prompt_password(ctx: click.core.Context, param: t.Any, use_password: bool): # pylint: disable=W0613 58 | """Prompt for password.""" 59 | if use_password: 60 | mysql_password = ctx.params.get("mysql_password") 61 | if not mysql_password: 62 | mysql_password = click.prompt("MySQL password", hide_input=True) 63 | 64 | return mysql_password 65 | 66 | 67 | def validate_positive_integer(ctx: click.core.Context, param: t.Any, value: int): # pylint: disable=W0613 68 | """Allow only positive integers and 0.""" 69 | if value < 0: 70 | raise click.BadParameter("Should be a positive integer or 0.") 71 | return value 72 | -------------------------------------------------------------------------------- /tests/unit/test_create_and_transfer_reconnect.py: -------------------------------------------------------------------------------- 1 | import sqlite3 2 | from unittest.mock import MagicMock, patch 3 | 4 | import mysql.connector 5 | import pytest 6 | from mysql.connector import errorcode 7 | 8 | from mysql_to_sqlite3.transporter import MySQLtoSQLite 9 | 10 | 11 | def test_create_table_reconnect_on_server_lost_then_success() -> None: 12 | with patch.object(MySQLtoSQLite, "__init__", return_value=None): 13 | inst = MySQLtoSQLite() # type: ignore[call-arg] 14 | 15 | # Patch dependencies 16 | inst._mysql = MagicMock() 17 | inst._sqlite = MagicMock() 18 | inst._sqlite_cur = MagicMock() 19 | inst._logger = MagicMock() 20 | 21 | # First call to build SQL raises CR_SERVER_LOST; second returns a valid SQL 22 | err = mysql.connector.Error(msg="lost", errno=errorcode.CR_SERVER_LOST) 23 | 24 | inst._build_create_table_sql = MagicMock(side_effect=[err, 'CREATE TABLE IF NOT EXISTS "t" ("id" INTEGER);']) 25 | 26 | inst._create_table("t") 27 | 28 | # Reconnect should have been attempted once 29 | inst._mysql.reconnect.assert_called_once() 30 | # executescript should have been called once with the returned SQL 31 | inst._sqlite_cur.executescript.assert_called_once() 32 | inst._sqlite.commit.assert_called_once() 33 | 34 | 35 | def test_create_table_sqlite_error_is_logged_and_raised() -> None: 36 | with patch.object(MySQLtoSQLite, "__init__", return_value=None): 37 | inst = MySQLtoSQLite() # type: ignore[call-arg] 38 | 39 | inst._mysql = MagicMock() 40 | inst._sqlite = MagicMock() 41 | inst._sqlite_cur = MagicMock() 42 | inst._logger = MagicMock() 43 | 44 | inst._build_create_table_sql = MagicMock(return_value='CREATE TABLE "t" ("id" INTEGER);') 45 | inst._sqlite_cur.executescript.side_effect = sqlite3.Error("broken") 46 | 47 | with pytest.raises(sqlite3.Error): 48 | inst._create_table("t") 49 | 50 | inst._logger.error.assert_called() 51 | 52 | 53 | def test_transfer_table_data_reconnect_on_server_lost_then_success() -> None: 54 | with patch.object(MySQLtoSQLite, "__init__", return_value=None): 55 | inst = MySQLtoSQLite() # type: ignore[call-arg] 56 | 57 | inst._mysql = MagicMock() 58 | inst._mysql_cur = MagicMock() 59 | inst._sqlite = MagicMock() 60 | inst._sqlite_cur = MagicMock() 61 | inst._logger = MagicMock() 62 | inst._quiet = True 63 | inst._chunk_size = None 64 | 65 | # First fetchall raises CR_SERVER_LOST; second returns rows 66 | err = mysql.connector.Error(msg="lost", errno=errorcode.CR_SERVER_LOST) 67 | inst._mysql_cur.fetchall.side_effect = [err, [(1,), (2,)]] 68 | 69 | inst._sqlite_cur.executemany = MagicMock() 70 | 71 | inst._transfer_table_data(table_name="t", sql="INSERT INTO t VALUES (?)", total_records=2) 72 | 73 | inst._mysql.reconnect.assert_called_once() 74 | inst._sqlite_cur.executemany.assert_called_once() 75 | inst._sqlite.commit.assert_called_once() 76 | -------------------------------------------------------------------------------- /tests/unit/test_defaults_sqlglot_enhanced.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from mysql_to_sqlite3.transporter import MySQLtoSQLite 4 | 5 | 6 | class TestDefaultsSqlglotEnhanced: 7 | @pytest.mark.parametrize( 8 | "expr,expected", 9 | [ 10 | ("CURRENT_TIME", "DEFAULT CURRENT_TIME"), 11 | ("CURRENT_DATE", "DEFAULT CURRENT_DATE"), 12 | ("CURRENT_TIMESTAMP", "DEFAULT CURRENT_TIMESTAMP"), 13 | ], 14 | ) 15 | def test_current_tokens_passthrough(self, expr: str, expected: str) -> None: 16 | assert MySQLtoSQLite._translate_default_from_mysql_to_sqlite(expr, column_extra="DEFAULT_GENERATED") == expected 17 | 18 | def test_null_literal_generated(self) -> None: 19 | assert ( 20 | MySQLtoSQLite._translate_default_from_mysql_to_sqlite("NULL", column_extra="DEFAULT_GENERATED") 21 | == "DEFAULT NULL" 22 | ) 23 | 24 | @pytest.mark.parametrize( 25 | "expr,boolean_type,expected", 26 | [ 27 | ("true", "BOOLEAN", {"DEFAULT(TRUE)", "DEFAULT '1'"}), 28 | ("false", "BOOLEAN", {"DEFAULT(FALSE)", "DEFAULT '0'"}), 29 | ("true", "INTEGER", {"DEFAULT '1'"}), 30 | ("false", "INTEGER", {"DEFAULT '0'"}), 31 | ], 32 | ) 33 | def test_boolean_tokens_generated(self, expr: str, boolean_type: str, expected: set) -> None: 34 | out = MySQLtoSQLite._translate_default_from_mysql_to_sqlite( 35 | expr, column_type=boolean_type, column_extra="DEFAULT_GENERATED" 36 | ) 37 | assert out in expected 38 | 39 | def test_parenthesized_string_literal_generated(self) -> None: 40 | out = MySQLtoSQLite._translate_default_from_mysql_to_sqlite("('abc')", column_extra="DEFAULT_GENERATED") 41 | # Either DEFAULT 'abc' or DEFAULT ('abc') depending on normalization 42 | assert out in {"DEFAULT 'abc'", "DEFAULT ('abc')"} 43 | 44 | def test_parenthesized_numeric_literal_generated(self) -> None: 45 | out = MySQLtoSQLite._translate_default_from_mysql_to_sqlite("(42)", column_extra="DEFAULT_GENERATED") 46 | assert out in {"DEFAULT 42", "DEFAULT (42)"} 47 | 48 | def test_constant_arithmetic_expression_generated(self) -> None: 49 | out = MySQLtoSQLite._translate_default_from_mysql_to_sqlite("1+2*3", column_extra="DEFAULT_GENERATED") 50 | # sqlglot formats with spaces for sqlite dialect 51 | assert out in {"DEFAULT 1 + 2 * 3", "DEFAULT (1 + 2 * 3)"} 52 | 53 | def test_hex_blob_literal_generated(self) -> None: 54 | out = MySQLtoSQLite._translate_default_from_mysql_to_sqlite("x'41'", column_extra="DEFAULT_GENERATED") 55 | # Should recognize as blob literal and keep as-is 56 | assert out.upper() == "DEFAULT X'41'" 57 | 58 | def test_plain_string_escaping_single_quote(self) -> None: 59 | out = MySQLtoSQLite._translate_default_from_mysql_to_sqlite("O'Reilly") 60 | assert out == "DEFAULT 'O''Reilly'" 61 | -------------------------------------------------------------------------------- /src/mysql_to_sqlite3/types.py: -------------------------------------------------------------------------------- 1 | """Types for mysql-to-sqlite3.""" 2 | 3 | import os 4 | import typing as t 5 | from logging import Logger 6 | from sqlite3 import Connection, Cursor 7 | 8 | from mysql.connector.abstracts import MySQLConnectionAbstract 9 | from mysql.connector.cursor import MySQLCursorDict, MySQLCursorPrepared, MySQLCursorRaw 10 | 11 | 12 | try: 13 | # Python 3.11+ 14 | from typing import TypedDict # type: ignore[attr-defined] 15 | except ImportError: 16 | # Python < 3.11 17 | from typing_extensions import TypedDict 18 | 19 | 20 | class MySQLtoSQLiteParams(TypedDict): 21 | """MySQLtoSQLite parameters.""" 22 | 23 | buffered: t.Optional[bool] 24 | chunk: t.Optional[int] 25 | collation: t.Optional[str] 26 | exclude_mysql_tables: t.Optional[t.Sequence[str]] 27 | json_as_text: t.Optional[bool] 28 | limit_rows: t.Optional[int] 29 | log_file: t.Optional[t.Union[str, "os.PathLike[t.Any]"]] 30 | mysql_database: str 31 | mysql_host: str 32 | mysql_password: t.Optional[t.Union[str, bool]] 33 | mysql_port: int 34 | mysql_charset: t.Optional[str] 35 | mysql_collation: t.Optional[str] 36 | mysql_ssl_disabled: t.Optional[bool] 37 | mysql_tables: t.Optional[t.Sequence[str]] 38 | mysql_user: str 39 | prefix_indices: t.Optional[bool] 40 | quiet: t.Optional[bool] 41 | sqlite_file: t.Union[str, "os.PathLike[t.Any]"] 42 | sqlite_strict: t.Optional[bool] 43 | vacuum: t.Optional[bool] 44 | without_tables: t.Optional[bool] 45 | without_data: t.Optional[bool] 46 | without_foreign_keys: t.Optional[bool] 47 | views_as_views: t.Optional[bool] 48 | 49 | 50 | class MySQLtoSQLiteAttributes: 51 | """MySQLtoSQLite attributes.""" 52 | 53 | _buffered: bool 54 | _chunk_size: t.Optional[int] 55 | _collation: str 56 | _current_chunk_number: int 57 | _exclude_mysql_tables: t.Sequence[str] 58 | _json_as_text: bool 59 | _limit_rows: int 60 | _logger: Logger 61 | _mysql: MySQLConnectionAbstract 62 | _mysql_cur: MySQLCursorRaw 63 | _mysql_cur_dict: MySQLCursorDict 64 | _mysql_cur_prepared: MySQLCursorPrepared 65 | _mysql_database: str 66 | _mysql_host: str 67 | _mysql_password: t.Optional[str] 68 | _mysql_port: int 69 | _mysql_charset: str 70 | _mysql_collation: str 71 | _mysql_ssl_disabled: bool 72 | _mysql_tables: t.Sequence[str] 73 | _mysql_user: str 74 | _prefix_indices: bool 75 | _quiet: bool 76 | _sqlite: Connection 77 | _sqlite_cur: Cursor 78 | _sqlite_file: t.Union[str, "os.PathLike[t.Any]"] 79 | _sqlite_strict: bool 80 | _without_tables: bool 81 | _sqlite_json1_extension_enabled: bool 82 | _vacuum: bool 83 | _without_data: bool 84 | _without_foreign_keys: bool 85 | _views_as_views: bool 86 | # Tracking of SQLite index names and counters to ensure uniqueness when prefixing is disabled 87 | _seen_sqlite_index_names: t.Set[str] 88 | _sqlite_index_name_counters: t.Dict[str, int] 89 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing 2 | 3 | I greatly appreciate your interest in reading this message, as this project requires volunteer developers to assist 4 | in developing and maintaining it. 5 | 6 | Before making any changes to this repository, please first discuss the proposed modifications with the repository owners 7 | through an issue, email, or any other appropriate communication channel. 8 | 9 | Please be aware that a [code of conduct](CODE-OF-CONDUCT.md) is in place, and should be adhered to during all 10 | interactions related to the project. 11 | 12 | ## Python version support 13 | 14 | Ensuring backward compatibility is an imperative requirement. 15 | 16 | Currently, the tool supports Python versions 3.9, 3.10, 3.11, 3.12, 3.13, and 3.14. 17 | 18 | ## MySQL version support 19 | 20 | This tool is intended to fully support MySQL versions 5.5, 5.6, 5.7, and 8.0, including major forks like MariaDB. 21 | We should prioritize and be dedicated to maintaining compatibility with these versions for a smooth user experience. 22 | 23 | ## Testing 24 | 25 | As this project/tool involves the critical process of transferring data between different database types, it is of 26 | utmost importance to ensure thorough testing. Please remember to write tests for any new code you create, utilizing the 27 | [pytest](https://docs.pytest.org/en/latest/) framework for all test cases. 28 | 29 | ### Running the test suite 30 | 31 | In order to run the test suite run these commands using a Docker MySQL image. 32 | 33 | **Requires a running Docker instance!** 34 | 35 | ```bash 36 | git clone https://github.com/techouse/mysql-to-sqlite3 37 | cd mysql-to-sqlite3 38 | python3 -m venv env 39 | source env/bin/activate 40 | pip install -e . 41 | pip install -r requirements_dev.txt 42 | tox 43 | ``` 44 | 45 | ## Submitting changes 46 | 47 | To contribute to this project, please submit a 48 | new [pull request](https://github.com/techouse/mysql-to-sqlite3/pull/new/master) and provide a clear list of your 49 | modifications. For guidance on creating pull requests, you can refer 50 | to [this resource](http://help.github.com/pull-requests/). 51 | 52 | When sending a pull request, we highly appreciate the inclusion of [pytest](https://docs.pytest.org/en/latest/) tests, 53 | as we strive to enhance our test coverage. Following our coding conventions is essential, and it would be ideal if you 54 | ensure that each commit focuses on a single feature. 55 | 56 | For commits, please write clear log messages. While concise one-line messages are suitable for small changes, more 57 | substantial modifications should follow a format similar to the example below: 58 | 59 | ```bash 60 | git commit -m "A brief summary of the commit 61 | > 62 | > A paragraph describing what changed and its impact." 63 | ``` 64 | 65 | ## Coding standards 66 | 67 | It is essential to prioritize code readability and conciseness. To achieve this, we recommend 68 | using [Black](https://github.com/psf/black) for code formatting. 69 | 70 | Once your work is deemed complete, it is advisable to run the following command: 71 | 72 | ```bash 73 | tox -e flake8,linters 74 | ``` 75 | 76 | This command executes various linters and checkers to identify any potential issues or inconsistencies in your code. By 77 | following these guidelines, you can ensure a high-quality codebase. 78 | 79 | Thanks, 80 | 81 | Klemen Tusar -------------------------------------------------------------------------------- /tests/unit/test_build_create_table_sql_sqlglot_identifiers.py: -------------------------------------------------------------------------------- 1 | import re 2 | from unittest.mock import MagicMock, patch 3 | 4 | from mysql_to_sqlite3.transporter import MySQLtoSQLite 5 | 6 | 7 | def _make_base_instance(): 8 | with patch.object(MySQLtoSQLite, "__init__", return_value=None): 9 | inst = MySQLtoSQLite() # type: ignore[call-arg] 10 | inst._mysql_cur_dict = MagicMock() 11 | inst._mysql_database = "db" 12 | inst._sqlite_json1_extension_enabled = False 13 | inst._collation = "BINARY" 14 | inst._prefix_indices = False 15 | inst._without_tables = False 16 | inst._without_foreign_keys = True 17 | inst._logger = MagicMock() 18 | inst._sqlite_strict = False 19 | # Track index names for uniqueness 20 | inst._seen_sqlite_index_names = set() 21 | inst._sqlite_index_name_counters = {} 22 | return inst 23 | 24 | 25 | def test_show_columns_backticks_are_escaped_in_mysql_query() -> None: 26 | inst = _make_base_instance() 27 | 28 | # Capture executed SQL 29 | executed_sql = [] 30 | 31 | def capture_execute(sql: str, *_, **__): 32 | executed_sql.append(sql) 33 | 34 | inst._mysql_cur_dict.execute.side_effect = capture_execute 35 | 36 | # SHOW COLUMNS -> then STATISTICS query 37 | inst._mysql_cur_dict.fetchall.side_effect = [ 38 | [ 39 | { 40 | "Field": "id", 41 | "Type": "INT", 42 | "Null": "NO", 43 | "Default": None, 44 | "Key": "PRI", 45 | "Extra": "", 46 | } 47 | ], 48 | [], 49 | ] 50 | # TABLE collision check -> 0 51 | inst._mysql_cur_dict.fetchone.return_value = {"count": 0} 52 | 53 | sql = inst._build_create_table_sql("we`ird") 54 | assert sql.startswith('CREATE TABLE IF NOT EXISTS "we`ird" (') 55 | 56 | # First executed SQL should be SHOW COLUMNS with backticks escaped 57 | assert executed_sql 58 | assert executed_sql[0] == "SHOW COLUMNS FROM `we``ird`" 59 | 60 | 61 | def test_identifiers_with_double_quotes_are_safely_quoted_in_create_and_index() -> None: 62 | inst = _make_base_instance() 63 | inst._prefix_indices = True # ensure an index is emitted with a deterministic name prefix 64 | 65 | # SHOW COLUMNS first call, then STATISTICS rows 66 | inst._mysql_cur_dict.fetchall.side_effect = [ 67 | [ 68 | { 69 | "Field": 'na"me', 70 | "Type": "VARCHAR(10)", 71 | "Null": "YES", 72 | "Default": None, 73 | "Key": "", 74 | "Extra": "", 75 | }, 76 | ], 77 | [ 78 | { 79 | "name": "idx", 80 | "primary": 0, 81 | "unique": 0, 82 | "auto_increment": 0, 83 | "columns": 'na"me', 84 | "types": "VARCHAR(10)", 85 | } 86 | ], 87 | ] 88 | inst._mysql_cur_dict.fetchone.return_value = {"count": 0} 89 | 90 | sql = inst._build_create_table_sql('ta"ble') 91 | 92 | # Column should be quoted with doubled quotes inside 93 | assert '"na""me" VARCHAR(10)' in sql or '"na""me" TEXT' in sql 94 | 95 | # Index should quote table and column names with doubled quotes 96 | norm = re.sub(r"\s+", " ", sql) 97 | assert 'CREATE INDEX IF NOT EXISTS "ta""ble_idx" ON "ta""ble" ("na""me")' in norm 98 | -------------------------------------------------------------------------------- /tests/unit/test_indices_prefix_and_uniqueness.py: -------------------------------------------------------------------------------- 1 | from unittest.mock import MagicMock, patch 2 | 3 | from mysql_to_sqlite3.transporter import MySQLtoSQLite 4 | 5 | 6 | def _make_instance_with_mocks(): 7 | with patch.object(MySQLtoSQLite, "__init__", return_value=None): 8 | instance = MySQLtoSQLite() # type: ignore[call-arg] 9 | instance._mysql_cur_dict = MagicMock() 10 | instance._mysql_database = "db" 11 | instance._sqlite_json1_extension_enabled = False 12 | instance._collation = "BINARY" 13 | instance._prefix_indices = False 14 | instance._without_tables = False 15 | instance._without_foreign_keys = True 16 | instance._logger = MagicMock() 17 | instance._sqlite_strict = False 18 | # Track index names for uniqueness 19 | instance._seen_sqlite_index_names = set() 20 | instance._sqlite_index_name_counters = {} 21 | return instance 22 | 23 | 24 | def test_build_create_table_sql_prefix_indices_true_prefixes_index_names() -> None: 25 | inst = _make_instance_with_mocks() 26 | inst._prefix_indices = True 27 | 28 | # SHOW COLUMNS 29 | inst._mysql_cur_dict.fetchall.side_effect = [ 30 | [ 31 | {"Field": "id", "Type": "INT", "Null": "NO", "Default": None, "Key": "PRI", "Extra": ""}, 32 | {"Field": "name", "Type": "VARCHAR(10)", "Null": "YES", "Default": None, "Key": "", "Extra": ""}, 33 | ], 34 | # STATISTICS rows 35 | [ 36 | { 37 | "name": "idx_name", 38 | "primary": 0, 39 | "unique": 0, 40 | "auto_increment": 0, 41 | "columns": "name", 42 | "types": "VARCHAR(10)", 43 | } 44 | ], 45 | ] 46 | # TABLE collision check -> 0 47 | inst._mysql_cur_dict.fetchone.return_value = {"count": 0} 48 | 49 | sql = inst._build_create_table_sql("users") 50 | 51 | # With prefix_indices=True, the index name should be prefixed with table name 52 | assert 'CREATE INDEX IF NOT EXISTS "users_idx_name" ON "users" ("name");' in sql 53 | 54 | 55 | def test_build_create_table_sql_collision_renamed_and_uniqueness_suffix() -> None: 56 | inst = _make_instance_with_mocks() 57 | inst._prefix_indices = False 58 | 59 | # Pre-mark an index name as already used globally to force suffixing 60 | inst._seen_sqlite_index_names.add("dup") 61 | 62 | # SHOW COLUMNS 63 | inst._mysql_cur_dict.fetchall.side_effect = [ 64 | [ 65 | {"Field": "id", "Type": "INT", "Null": "NO", "Default": None, "Key": "", "Extra": ""}, 66 | ], 67 | # STATISTICS rows 68 | [ 69 | { 70 | "name": "dup", # collides globally 71 | "primary": 0, 72 | "unique": 1, 73 | "auto_increment": 0, 74 | "columns": "id", 75 | "types": "INT", 76 | } 77 | ], 78 | ] 79 | # TABLE collision check -> 1 so we also prefix with table name before uniqueness 80 | inst._mysql_cur_dict.fetchone.return_value = {"count": 1} 81 | 82 | sql = inst._build_create_table_sql("accounts") 83 | 84 | # Proposed becomes accounts_dup, and since dup already used, unique name stays accounts_dup (no clash) 85 | # or if accounts_dup was in the seen set, it would become accounts_dup_2. We only asserted the presence of accounts_ prefix. 86 | assert 'CREATE UNIQUE INDEX IF NOT EXISTS "accounts_dup" ON "accounts" ("id");' in sql 87 | -------------------------------------------------------------------------------- /tests/unit/test_types_sqlglot_augmented.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from mysql_to_sqlite3.transporter import MySQLtoSQLite 4 | 5 | 6 | class TestSqlglotAugmentedTypeTranslation: 7 | @pytest.mark.parametrize("mysql_type", ["double precision", "DOUBLE PRECISION", "DoUbLe PrEcIsIoN"]) 8 | def test_double_precision_maps_to_numeric_type(self, mysql_type: str) -> None: 9 | # Prior mapper would resolve this to TEXT; sqlglot fallback should improve it 10 | out = MySQLtoSQLite._translate_type_from_mysql_to_sqlite(mysql_type) 11 | assert out in {"DOUBLE", "REAL"} 12 | 13 | def test_fixed_maps_to_decimal(self) -> None: 14 | out = MySQLtoSQLite._translate_type_from_mysql_to_sqlite("fixed(10,2)") 15 | # Normalize to DECIMAL (without length) to match existing style 16 | assert out == "DECIMAL" 17 | 18 | def test_character_varying_keeps_length_as_varchar(self) -> None: 19 | out = MySQLtoSQLite._translate_type_from_mysql_to_sqlite("character varying(20)") 20 | assert out == "VARCHAR(20)" 21 | 22 | def test_char_varying_keeps_length_as_varchar(self) -> None: 23 | out = MySQLtoSQLite._translate_type_from_mysql_to_sqlite("char varying(12)") 24 | assert out == "VARCHAR(12)" 25 | 26 | def test_national_character_varying_maps_to_nvarchar(self) -> None: 27 | out = MySQLtoSQLite._translate_type_from_mysql_to_sqlite("national character varying(15)") 28 | assert out == "NVARCHAR(15)" 29 | 30 | def test_national_character_maps_to_nchar(self) -> None: 31 | out = MySQLtoSQLite._translate_type_from_mysql_to_sqlite("national character(5)") 32 | assert out == "NCHAR(5)" 33 | 34 | @pytest.mark.parametrize( 35 | "mysql_type,expected", 36 | [ 37 | ("int unsigned", "INTEGER"), 38 | ("mediumint unsigned", "MEDIUMINT"), 39 | ("smallint unsigned", "SMALLINT"), 40 | ("tinyint unsigned", "TINYINT"), 41 | ("bigint unsigned", "BIGINT"), 42 | ], 43 | ) 44 | def test_unsigned_variants_strip_unsigned(self, mysql_type: str, expected: str) -> None: 45 | out = MySQLtoSQLite._translate_type_from_mysql_to_sqlite(mysql_type) 46 | assert out == expected 47 | 48 | def test_timestamp_maps_to_datetime(self) -> None: 49 | out = MySQLtoSQLite._translate_type_from_mysql_to_sqlite("timestamp") 50 | assert out == "DATETIME" 51 | 52 | def test_varbinary_and_blobs_map_to_blob(self) -> None: 53 | assert MySQLtoSQLite._translate_type_from_mysql_to_sqlite("varbinary(16)") == "BLOB" 54 | assert MySQLtoSQLite._translate_type_from_mysql_to_sqlite("mediumblob") == "BLOB" 55 | 56 | def test_char_maps_to_character_with_length(self) -> None: 57 | out = MySQLtoSQLite._translate_type_from_mysql_to_sqlite("char(3)") 58 | assert out == "CHARACTER(3)" 59 | 60 | def test_json_mapping_respects_json1(self) -> None: 61 | assert ( 62 | MySQLtoSQLite._translate_type_from_mysql_to_sqlite("json", sqlite_json1_extension_enabled=False) == "TEXT" 63 | ) 64 | assert MySQLtoSQLite._translate_type_from_mysql_to_sqlite("json", sqlite_json1_extension_enabled=True) == "JSON" 65 | 66 | def test_fallback_to_text_on_unknown_type(self) -> None: 67 | out = MySQLtoSQLite._translate_type_from_mysql_to_sqlite("geography") 68 | assert out == "TEXT" 69 | 70 | def test_enum_remains_text(self) -> None: 71 | out = MySQLtoSQLite._translate_type_from_mysql_to_sqlite("enum('a','b')") 72 | assert out == "TEXT" 73 | -------------------------------------------------------------------------------- /src/mysql_to_sqlite3/debug_info.py: -------------------------------------------------------------------------------- 1 | """Module containing bug report helper(s). 2 | 3 | Adapted from https://github.com/psf/requests/blob/master/requests/help.py 4 | """ 5 | 6 | import platform 7 | import sqlite3 8 | import sys 9 | import typing as t 10 | from shutil import which 11 | from subprocess import check_output 12 | 13 | import click 14 | import mysql.connector 15 | import pytimeparse2 16 | import simplejson 17 | import slugify 18 | import sqlglot 19 | import tabulate 20 | import tqdm 21 | 22 | from . import __version__ as package_version 23 | 24 | 25 | def _implementation() -> str: 26 | """Return a dict with the Python implementation and version. 27 | 28 | Provide both the name and the version of the Python implementation 29 | currently running. For example, on CPython 2.7.5 it will return 30 | {'name': 'CPython', 'version': '2.7.5'}. 31 | 32 | This function works best on CPython and PyPy: in particular, it probably 33 | doesn't work for Jython or IronPython. Future investigation should be done 34 | to work out the correct shape of the code for those platforms. 35 | """ 36 | implementation: str = platform.python_implementation() 37 | 38 | if implementation == "CPython": 39 | implementation_version = platform.python_version() 40 | elif implementation == "PyPy": 41 | implementation_version = "%s.%s.%s" % ( 42 | sys.pypy_version_info.major, # type: ignore # noqa: ignore=E1101 pylint: disable=E1101 43 | sys.pypy_version_info.minor, # type: ignore # noqa: ignore=E1101 pylint: disable=E1101 44 | sys.pypy_version_info.micro, # type: ignore # noqa: ignore=E1101 pylint: disable=E1101 45 | ) 46 | rel = sys.pypy_version_info.releaselevel # type: ignore # noqa: ignore=E1101 pylint: disable=E1101 47 | if rel != "final": 48 | implementation_version = "".join([implementation_version, rel]) 49 | elif implementation == "Jython": 50 | implementation_version = platform.python_version() # Complete Guess 51 | elif implementation == "IronPython": 52 | implementation_version = platform.python_version() # Complete Guess 53 | else: 54 | implementation_version = "Unknown" 55 | 56 | return f"{implementation} {implementation_version}" 57 | 58 | 59 | def _mysql_version() -> str: 60 | if which("mysql") is not None: 61 | try: 62 | mysql_version: t.Union[str, bytes] = check_output(["mysql", "-V"]) 63 | try: 64 | return mysql_version.decode().strip() # type: ignore 65 | except (UnicodeDecodeError, AttributeError): 66 | return str(mysql_version) 67 | except Exception: # nosec pylint: disable=W0703 68 | pass 69 | return "MySQL client not found on the system" 70 | 71 | 72 | def info() -> t.List[t.List[str]]: 73 | """Generate information for a bug report.""" 74 | try: 75 | platform_info: str = f"{platform.system()} {platform.release()}" 76 | except IOError: 77 | platform_info = "Unknown" 78 | 79 | return [ 80 | ["mysql-to-sqlite3", package_version], 81 | ["", ""], 82 | ["Operating System", platform_info], 83 | ["Python", _implementation()], 84 | ["MySQL", _mysql_version()], 85 | ["SQLite", sqlite3.sqlite_version], 86 | ["", ""], 87 | ["click", str(click.__version__)], 88 | ["mysql-connector-python", mysql.connector.__version__], 89 | ["python-slugify", slugify.__version__], 90 | ["pytimeparse2", pytimeparse2.__version__], 91 | ["simplejson", simplejson.__version__], # type: ignore 92 | ["sqlglot", sqlglot.__version__], 93 | ["tabulate", tabulate.__version__], 94 | ["tqdm", tqdm.__version__], 95 | ] 96 | -------------------------------------------------------------------------------- /.github/workflows/publish.yml: -------------------------------------------------------------------------------- 1 | name: Publish package 2 | 3 | on: 4 | push: 5 | tags: 6 | - 'v[0-9]+.[0-9]+.[0-9]+*' 7 | defaults: 8 | run: 9 | shell: bash 10 | permissions: read-all 11 | env: 12 | NAME: mysql-to-sqlite3 13 | 14 | jobs: 15 | test: 16 | uses: ./.github/workflows/test.yml 17 | secrets: inherit 18 | publish: 19 | needs: test 20 | name: "Publish" 21 | runs-on: ubuntu-latest 22 | environment: 23 | name: pypi 24 | url: https://pypi.org/p/${{ env.NAME }} 25 | permissions: 26 | id-token: write 27 | contents: write 28 | concurrency: 29 | group: publish-${{ github.ref }} 30 | cancel-in-progress: false 31 | steps: 32 | - uses: actions/checkout@v6 33 | - name: Compare package version with ref/tag 34 | id: compare 35 | run: | 36 | set -e 37 | VERSION=$(awk -F'"' '/__version__/ {print $2}' src/mysql_to_sqlite3/__init__.py) 38 | TAG=${GITHUB_REF_NAME#v} 39 | if [[ "$VERSION" != "$TAG" ]]; then 40 | echo "Version in src/mysql_to_sqlite3/__init__.py ($VERSION) does not match tag ($TAG)" 41 | exit 1 42 | fi 43 | echo "VERSION=$VERSION" >> $GITHUB_ENV 44 | - name: Check CHANGELOG.md 45 | id: check_changelog 46 | run: | 47 | set -e 48 | if ! grep -q "# $VERSION" CHANGELOG.md; then 49 | echo "CHANGELOG.md does not contain a section for $VERSION" 50 | exit 1 51 | fi 52 | - name: Set up Python 53 | id: setup_python 54 | uses: actions/setup-python@v6 55 | with: 56 | python-version: "3.x" 57 | cache: "pip" 58 | - name: Install build dependencies 59 | id: install_build_dependencies 60 | run: | 61 | set -e 62 | python3 -m pip install -U pip 63 | python3 -m pip install -U build 64 | - name: Build a binary wheel and a source tarball 65 | id: build 66 | run: | 67 | set -e 68 | python3 -m build --sdist --wheel --outdir dist/ . 69 | - name: Verify package metadata (twine) 70 | id: twine_check 71 | run: | 72 | set -e 73 | python3 -m pip install -U twine 74 | twine check dist/* 75 | - name: Ensure py.typed is packaged 76 | id: assert_py_typed 77 | run: | 78 | set -e 79 | unzip -l dist/*.whl | grep -q 'mysql_to_sqlite3/py.typed' 80 | - name: Publish distribution package to PyPI 81 | id: publish 82 | if: startsWith(github.ref, 'refs/tags') 83 | uses: pypa/gh-action-pypi-publish@release/v1 84 | - name: Create tag-specific CHANGELOG 85 | id: create_changelog 86 | run: | 87 | set -e 88 | CHANGELOG_PATH=$RUNNER_TEMP/CHANGELOG.md 89 | awk '/^#[[:space:]].*/ { if (count == 1) exit; count++; print } count == 1 && !/^#[[:space:]].*/ { print }' CHANGELOG.md | sed -e :a -e '/^\n*$/{$d;N;ba' -e '}' > $CHANGELOG_PATH 90 | echo -en "\n[https://pypi.org/project/$NAME/$VERSION/](https://pypi.org/project/$NAME/$VERSION/)" >> $CHANGELOG_PATH 91 | echo "CHANGELOG_PATH=$CHANGELOG_PATH" >> $GITHUB_ENV 92 | - name: Github Release 93 | id: github_release 94 | uses: softprops/action-gh-release@v2 95 | with: 96 | name: ${{ env.VERSION }} 97 | tag_name: ${{ github.ref }} 98 | body_path: ${{ env.CHANGELOG_PATH }} 99 | files: | 100 | dist/*.whl 101 | dist/*.tar.gz 102 | - name: Cleanup 103 | if: ${{ always() }} 104 | run: | 105 | rm -rf dist 106 | rm -rf $CHANGELOG_PATH 107 | docker: 108 | needs: [ test, publish ] 109 | permissions: 110 | packages: write 111 | contents: read 112 | uses: ./.github/workflows/docker.yml 113 | secrets: inherit 114 | docs: 115 | uses: ./.github/workflows/docs.yml 116 | needs: [ test, publish ] 117 | permissions: 118 | contents: write 119 | secrets: inherit -------------------------------------------------------------------------------- /docs/index.rst: -------------------------------------------------------------------------------- 1 | MySQL to SQLite3 2 | ================ 3 | 4 | A simple Python tool to transfer data from MySQL to SQLite 3 5 | 6 | |PyPI| |PyPI - Downloads| |Homebrew Formula Downloads| |PyPI - Python Version| 7 | |MySQL Support| |MariaDB Support| |GitHub license| |Contributor Covenant| 8 | |PyPI - Format| |Code style: black| |Codacy Badge| |Test Status| |CodeQL Status| 9 | |Publish PyPI Package Status| |codecov| |GitHub Sponsors| |GitHub stars| 10 | 11 | Installation 12 | ------------ 13 | 14 | .. code:: bash 15 | 16 | pip install mysql-to-sqlite3 17 | 18 | Basic Usage 19 | ----------- 20 | 21 | .. code:: bash 22 | 23 | mysql2sqlite -f path/to/foo.sqlite -d foo_db -u foo_user -p 24 | 25 | .. toctree:: 26 | :maxdepth: 2 27 | :caption: Contents: 28 | 29 | README 30 | modules 31 | 32 | Indices and tables 33 | ================== 34 | 35 | * :ref:`genindex` 36 | * :ref:`modindex` 37 | * :ref:`search` 38 | 39 | .. |PyPI| image:: https://img.shields.io/pypi/v/mysql-to-sqlite3?logo=pypi 40 | :target: https://pypi.org/project/mysql-to-sqlite3/ 41 | .. |PyPI - Downloads| image:: https://img.shields.io/pypi/dm/mysql-to-sqlite3?logo=pypi&label=PyPI%20downloads 42 | :target: https://pypistats.org/packages/mysql-to-sqlite3 43 | .. |Homebrew Formula Downloads| image:: https://img.shields.io/homebrew/installs/dm/mysql-to-sqlite3?logo=homebrew&label=Homebrew%20downloads 44 | :target: https://formulae.brew.sh/formula/mysql-to-sqlite3 45 | .. |PyPI - Python Version| image:: https://img.shields.io/pypi/pyversions/mysql-to-sqlite3?logo=python 46 | :target: https://pypi.org/project/mysql-to-sqlite3/ 47 | .. |MySQL Support| image:: https://img.shields.io/static/v1?logo=mysql&label=MySQL&message=5.5+%7C+5.6+%7C+5.7+%7C+8.0&color=2b5d80 48 | :target: https://img.shields.io/static/v1?label=MySQL&message=5.6+%7C+5.7+%7C+8.0&color=2b5d80 49 | .. |MariaDB Support| image:: https://img.shields.io/static/v1?logo=mariadb&label=MariaDB&message=5.5+%7C+10.0+%7C+10.1+%7C+10.2+%7C+10.3+%7C+10.4+%7C+10.5+%7C+10.6%7C+10.11&color=C0765A 50 | :target: https://img.shields.io/static/v1?label=MariaDB&message=10.0+%7C+10.1+%7C+10.2+%7C+10.3+%7C+10.4+%7C+10.5&color=C0765A 51 | .. |GitHub license| image:: https://img.shields.io/github/license/techouse/mysql-to-sqlite3 52 | :target: https://github.com/techouse/mysql-to-sqlite3/blob/master/LICENSE 53 | .. |Contributor Covenant| image:: https://img.shields.io/badge/Contributor%20Covenant-2.1-4baaaa.svg?logo=contributorcovenant 54 | :target: CODE-OF-CONDUCT.md 55 | .. |PyPI - Format| image:: https://img.shields.io/pypi/format/mysql-to-sqlite3?logo=python 56 | :target: https://pypi.org/project/sqlite3-to-mysql/ 57 | .. |Code style: black| image:: https://img.shields.io/badge/code%20style-black-000000.svg?logo=python 58 | :target: https://github.com/ambv/black 59 | .. |Codacy Badge| image:: https://api.codacy.com/project/badge/Grade/64aae8e9599746d58d277852b35cc2bd 60 | :target: https://www.codacy.com/manual/techouse/mysql-to-sqlite3?utm_source=github.com&utm_medium=referral&utm_content=techouse/mysql-to-sqlite3&utm_campaign=Badge_Grade 61 | .. |Test Status| image:: https://github.com/techouse/mysql-to-sqlite3/actions/workflows/test.yml/badge.svg 62 | :target: https://github.com/techouse/mysql-to-sqlite3/actions/workflows/test.yml 63 | .. |CodeQL Status| image:: https://github.com/techouse/mysql-to-sqlite3/actions/workflows/github-code-scanning/codeql/badge.svg 64 | :target: https://github.com/techouse/mysql-to-sqlite3/actions/workflows/codeql-analysis.yml 65 | .. |Publish PyPI Package Status| image:: https://github.com/techouse/mysql-to-sqlite3/actions/workflows/publish.yml/badge.svg 66 | :target: https://github.com/techouse/mysql-to-sqlite3/actions/workflows/publish.yml 67 | .. |codecov| image:: https://codecov.io/gh/techouse/mysql-to-sqlite3/branch/master/graph/badge.svg 68 | :target: https://codecov.io/gh/techouse/mysql-to-sqlite3 69 | .. |GitHub Sponsors| image:: https://img.shields.io/github/sponsors/techouse?logo=github 70 | :target: https://github.com/sponsors/techouse 71 | .. |GitHub stars| image:: https://img.shields.io/github/stars/techouse/mysql-to-sqlite3.svg?style=social&label=Star&maxAge=2592000 72 | :target: https://github.com/techouse/mysql-to-sqlite3/stargazers -------------------------------------------------------------------------------- /docs/README.rst: -------------------------------------------------------------------------------- 1 | Usage 2 | ----- 3 | 4 | Options 5 | ^^^^^^^ 6 | 7 | The command line options for the ``mysql2sqlite`` tool are as follows: 8 | 9 | .. code-block:: bash 10 | 11 | mysql2sqlite [OPTIONS] 12 | 13 | Required Options 14 | """""""""""""""" 15 | 16 | - ``-f, --sqlite-file PATH``: SQLite3 database file. This option is required. 17 | - ``-d, --mysql-database TEXT``: MySQL database name. This option is required. 18 | - ``-u, --mysql-user TEXT``: MySQL user. This option is required. 19 | 20 | Password Options 21 | """""""""""""""" 22 | 23 | - ``-p, --prompt-mysql-password``: Prompt for MySQL password. 24 | - ``--mysql-password TEXT``: MySQL password. 25 | 26 | Table Options 27 | """"""""""""" 28 | 29 | - ``-t, --mysql-tables TUPLE``: Transfer only these specific tables (space separated table names). Implies --without-foreign-keys which inhibits the transfer of foreign keys. Can not be used together with --exclude-mysql-tables. 30 | - ``-e, --exclude-mysql-tables TUPLE``: Transfer all tables except these specific tables (space separated table names). Implies --without-foreign-keys which inhibits the transfer of foreign keys. Can not be used together with --mysql-tables. 31 | 32 | Transfer Options 33 | """""""""""""""" 34 | 35 | - ``-T, --mysql-views-as-tables``: Materialize MySQL VIEWs as SQLite tables (legacy behavior). 36 | - ``-L, --limit-rows INTEGER``: Transfer only a limited number of rows from each table. 37 | - ``-C, --collation [BINARY|NOCASE|RTRIM]``: Create datatypes of TEXT affinity using a specified collation sequence. The default is BINARY. 38 | - ``-K, --prefix-indices``: Prefix indices with their corresponding tables. This ensures that their names remain unique across the SQLite database. 39 | - ``-X, --without-foreign-keys``: Do not transfer foreign keys. 40 | - ``-Z, --without-tables``: Do not transfer tables, data only. 41 | - ``-W, --without-data``: Do not transfer table data, DDL only. 42 | - ``-M, --strict``: Create SQLite STRICT tables when supported. 43 | 44 | Connection Options 45 | """""""""""""""""" 46 | 47 | - ``-h, --mysql-host TEXT``: MySQL host. Defaults to localhost. 48 | - ``-P, --mysql-port INTEGER``: MySQL port. Defaults to 3306. 49 | - ``--mysql-charset TEXT``: MySQL database and table character set. The default is utf8mb4. 50 | - ``--mysql-collation TEXT``: MySQL database and table collation 51 | - ``-S, --skip-ssl``: Disable MySQL connection encryption. 52 | 53 | Other Options 54 | """"""""""""" 55 | 56 | - ``-c, --chunk INTEGER``: Chunk reading/writing SQL records. 57 | - ``-l, --log-file PATH``: Log file. 58 | - ``--json-as-text``: Transfer JSON columns as TEXT. 59 | - ``-V, --vacuum``: Use the VACUUM command to rebuild the SQLite database file, repacking it into a minimal amount of disk space. 60 | - ``--use-buffered-cursors``: Use MySQLCursorBuffered for reading the MySQL database. This can be useful in situations where multiple queries, with small result sets, need to be combined or computed with each other. 61 | - ``-q, --quiet``: Quiet. Display only errors. 62 | - ``--debug``: Debug mode. Will throw exceptions. 63 | - ``--version``: Show the version and exit. 64 | - ``--help``: Show this message and exit. 65 | 66 | Docker 67 | ^^^^^^ 68 | 69 | If you don’t want to install the tool on your system, you can use the 70 | Docker image instead. 71 | 72 | .. code:: bash 73 | 74 | docker run -it \ 75 | --workdir $(pwd) \ 76 | --volume $(pwd):$(pwd) \ 77 | --rm ghcr.io/techouse/mysql-to-sqlite3:latest \ 78 | --sqlite-file baz.db \ 79 | --mysql-user foo \ 80 | --mysql-password bar \ 81 | --mysql-database baz \ 82 | --mysql-host host.docker.internal 83 | 84 | This will mount your host current working directory (pwd) inside the 85 | Docker container as the current working directory. Any files Docker 86 | would write to the current working directory are written to the host 87 | directory where you did docker run. Note that you have to also use a 88 | `special 89 | hostname `__ 90 | ``host.docker.internal`` to access your host machine from inside the 91 | Docker container. 92 | 93 | Homebrew 94 | ^^^^^^^^ 95 | 96 | If you’re on macOS, you can install the tool using 97 | `Homebrew `__. 98 | 99 | .. code:: bash 100 | 101 | brew install mysql-to-sqlite3 102 | mysql2sqlite --help 103 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | requires = ["hatchling"] 3 | build-backend = "hatchling.build" 4 | 5 | [project] 6 | name = "mysql-to-sqlite3" 7 | description = "A simple Python tool to transfer data from MySQL to SQLite 3" 8 | readme = { file = "README.md", content-type = "text/markdown" } 9 | license = "MIT" 10 | license-files = ["LICENSE"] 11 | requires-python = ">=3.9" 12 | authors = [ 13 | { name = "Klemen Tusar", email = "techouse@gmail.com" }, 14 | ] 15 | keywords = [ 16 | "mysql", 17 | "sqlite3", 18 | "transfer", 19 | "data", 20 | "migrate", 21 | "migration", 22 | ] 23 | classifiers = [ 24 | "Development Status :: 5 - Production/Stable", 25 | "Environment :: Console", 26 | "Intended Audience :: End Users/Desktop", 27 | "Intended Audience :: Developers", 28 | "Intended Audience :: System Administrators", 29 | "License :: OSI Approved :: MIT License", 30 | "Operating System :: OS Independent", 31 | "Programming Language :: Python", 32 | "Programming Language :: Python :: 3", 33 | "Programming Language :: Python :: 3.9", 34 | "Programming Language :: Python :: 3.10", 35 | "Programming Language :: Python :: 3.11", 36 | "Programming Language :: Python :: 3.12", 37 | "Programming Language :: Python :: 3.13", 38 | "Programming Language :: Python :: 3.14", 39 | "Programming Language :: Python :: Implementation :: CPython", 40 | "Typing :: Typed", 41 | "Topic :: Database", 42 | ] 43 | dependencies = [ 44 | "click>=8.1.3", 45 | "mysql-connector-python>=9.0.0", 46 | "pytimeparse2", 47 | "python-dateutil>=2.9.0.post0", 48 | "python-slugify>=7.0.0", 49 | "simplejson>=3.19.0", 50 | "sqlglot>=27.27.0", 51 | "tqdm>=4.65.0", 52 | "tabulate", 53 | "typing-extensions; python_version < \"3.11\"", 54 | ] 55 | dynamic = ["version"] 56 | 57 | [project.urls] 58 | Homepage = "https://techouse.github.io/mysql-to-sqlite3/" 59 | Documentation = "https://techouse.github.io/mysql-to-sqlite3/" 60 | Repository = "https://github.com/techouse/mysql-to-sqlite3.git" 61 | Issues = "https://github.com/techouse/mysql-to-sqlite3" 62 | Changelog = "https://github.com/techouse/mysql-to-sqlite3/blob/master/CHANGELOG.md" 63 | Sponsor = "https://github.com/sponsors/techouse" 64 | PayPal = "https://paypal.me/ktusar" 65 | 66 | [project.optional-dependencies] 67 | dev = [ 68 | "docker>=6.1.3", 69 | "factory-boy", 70 | "Faker>=18.10.0", 71 | "mysqlclient>=2.1.1", 72 | "pytest>=7.3.1", 73 | "pytest-cov", 74 | "pytest-mock", 75 | "pytest-timeout", 76 | "types-python-dateutil", 77 | "types-python-slugify", 78 | "types-simplejson", 79 | "sqlalchemy>=2.0.0", 80 | "sqlalchemy-utils", 81 | "types-sqlalchemy-utils", 82 | "tox", 83 | "types-tqdm", 84 | "packaging", 85 | "types-tabulate", 86 | "requests", 87 | "types-requests", 88 | "mypy>=1.3.0", 89 | ] 90 | 91 | [tool.hatch.version] 92 | path = "src/mysql_to_sqlite3/__init__.py" 93 | 94 | [tool.hatch.build.targets.sdist] 95 | include = [ 96 | "src", 97 | "tests", 98 | "README.md", 99 | "CHANGELOG.md", 100 | "CODE-OF-CONDUCT.md", 101 | "LICENSE", 102 | "requirements_dev.txt", 103 | ] 104 | 105 | [tool.hatch.build.targets.wheel] 106 | packages = ["src/mysql_to_sqlite3"] 107 | include = ["src/mysql_to_sqlite3/py.typed"] 108 | 109 | [project.scripts] 110 | mysql2sqlite = "mysql_to_sqlite3.cli:cli" 111 | 112 | [tool.black] 113 | line-length = 120 114 | target-version = ["py39", "py310", "py311", "py312", "py313", "py314"] 115 | include = '\.pyi?$' 116 | exclude = ''' 117 | ( 118 | /( 119 | \.eggs 120 | | \.git 121 | | \.hg 122 | | \.mypy_cache 123 | | \.tox 124 | | \.venv 125 | | _build 126 | | buck-out 127 | | build 128 | | dist 129 | | docs 130 | )/ 131 | | foo.py 132 | ) 133 | ''' 134 | 135 | [tool.isort] 136 | line_length = 120 137 | profile = "black" 138 | lines_after_imports = 2 139 | known_first_party = "mysql_to_sqlite3" 140 | skip_gitignore = true 141 | 142 | [tool.pytest.ini_options] 143 | pythonpath = ["src"] 144 | testpaths = ["tests"] 145 | norecursedirs = [".*", "venv", "env", "*.egg", "dist", "build"] 146 | minversion = "7.3.1" 147 | addopts = "-rsxX --tb=short --strict-markers" 148 | timeout = 300 149 | markers = [ 150 | "init: Run the initialisation test functions", 151 | "transfer: Run the main transfer test functions", 152 | "exceptions: Run SQL exception test functions", 153 | "cli: Run the cli test functions", 154 | ] 155 | 156 | [tool.mypy] 157 | mypy_path = "src" 158 | python_version = "3.9" 159 | exclude = [ 160 | "tests", 161 | "docs", 162 | "build", 163 | "dist", 164 | "venv", 165 | "env", 166 | ] 167 | warn_return_any = true 168 | warn_unused_configs = true 169 | 170 | [[tool.mypy.overrides]] 171 | module = "pytimeparse2.*,factory.*,docker.*" 172 | ignore_missing_imports = true 173 | -------------------------------------------------------------------------------- /tests/factories.py: -------------------------------------------------------------------------------- 1 | import typing as t 2 | from os import environ 3 | 4 | import factory 5 | 6 | from . import faker_providers, models 7 | 8 | 9 | factory.Faker.add_provider(faker_providers.DateTimeProviders) 10 | 11 | 12 | class AuthorFactory(factory.Factory): 13 | class Meta: 14 | model: t.Type[models.Author] = models.Author 15 | 16 | name: factory.Faker = factory.Faker("name") 17 | 18 | 19 | class ImageFactory(factory.Factory): 20 | class Meta: 21 | model: t.Type[models.Image] = models.Image 22 | 23 | path: factory.Faker = factory.Faker("file_path", depth=3, extension="jpg") 24 | description: factory.Faker = factory.Faker("sentence", nb_words=12, variable_nb_words=True) 25 | 26 | 27 | class TagFactory(factory.Factory): 28 | class Meta: 29 | model: t.Type[models.Tag] = models.Tag 30 | 31 | name: factory.Faker = factory.Faker("sentence", nb_words=3, variable_nb_words=True) 32 | 33 | 34 | class MiscFactory(factory.Factory): 35 | class Meta: 36 | model: t.Type[models.Misc] = models.Misc 37 | 38 | big_integer_field: factory.Faker = factory.Faker("pyint", max_value=10**9) 39 | large_binary_field: factory.Faker = factory.Faker("binary", length=1024 * 10) 40 | boolean_field: factory.Faker = factory.Faker("boolean") 41 | char_field: factory.Faker = factory.Faker("text", max_nb_chars=255) 42 | date_field: factory.Faker = factory.Faker("date_this_decade") 43 | date_time_field: factory.Faker = factory.Faker("date_time_this_century_without_microseconds") 44 | decimal_field: factory.Faker = factory.Faker("pydecimal", left_digits=8, right_digits=2) 45 | float_field: factory.Faker = factory.Faker("pyfloat", left_digits=8, right_digits=4) 46 | integer_field: factory.Faker = factory.Faker("pyint", min_value=-(2**31), max_value=2**31 - 1) 47 | if environ.get("LEGACY_DB", "0") == "0": 48 | json_field: factory.Faker = factory.Faker("pydict") 49 | nchar_field: factory.Faker = factory.Faker("text", max_nb_chars=255) 50 | numeric_field: factory.Faker = factory.Faker("pyfloat", left_digits=8, right_digits=4) 51 | unicode_field: factory.Faker = factory.Faker("text", max_nb_chars=255) 52 | real_field: factory.Faker = factory.Faker("pyfloat", left_digits=8, right_digits=4) 53 | small_integer_field: factory.Faker = factory.Faker("pyint", min_value=-(2**15), max_value=2**15 - 1) 54 | string_field: factory.Faker = factory.Faker("text", max_nb_chars=255) 55 | text_field: factory.Faker = factory.Faker("text", max_nb_chars=1024) 56 | time_field: factory.Faker = factory.Faker("time_object_without_microseconds") 57 | varbinary_field: factory.Faker = factory.Faker("binary", length=255) 58 | varchar_field: factory.Faker = factory.Faker("text", max_nb_chars=255) 59 | timestamp_field: factory.Faker = factory.Faker("date_time_this_century_without_microseconds") 60 | 61 | 62 | class ArticleFactory(factory.Factory): 63 | class Meta: 64 | model: t.Type[models.Article] = models.Article 65 | 66 | hash: factory.Faker = factory.Faker("md5") 67 | title: factory.Faker = factory.Faker("sentence", nb_words=6) 68 | slug: factory.Faker = factory.Faker("slug") 69 | content: factory.Faker = factory.Faker("text", max_nb_chars=1024) 70 | status: factory.Faker = factory.Faker("pystr", max_chars=1) 71 | published: factory.Faker = factory.Faker("date_between", start_date="-1y", end_date="-1d") 72 | 73 | @factory.post_generation 74 | def authors(self, create, extracted, **kwargs): 75 | if not create: 76 | # Simple build, do nothing. 77 | return 78 | 79 | if extracted: 80 | # A list of authors were passed in, use them 81 | for author in extracted: 82 | self.authors.add(author) 83 | 84 | @factory.post_generation 85 | def tags(self, create, extracted, **kwargs): 86 | if not create: 87 | # Simple build, do nothing. 88 | return 89 | 90 | if extracted: 91 | # A list of authors were passed in, use them 92 | for tag in extracted: 93 | self.tags.add(tag) 94 | 95 | @factory.post_generation 96 | def images(self, create, extracted, **kwargs): 97 | if not create: 98 | # Simple build, do nothing. 99 | return 100 | 101 | if extracted: 102 | # A list of authors were passed in, use them 103 | for image in extracted: 104 | self.images.add(image) 105 | 106 | @factory.post_generation 107 | def misc(self, create, extracted, **kwargs): 108 | if not create: 109 | # Simple build, do nothing. 110 | return 111 | 112 | if extracted: 113 | # A list of authors were passed in, use them 114 | for misc in extracted: 115 | self.misc.add(misc) 116 | 117 | 118 | class CrazyNameFactory(factory.Factory): 119 | class Meta: 120 | model: t.Type[models.CrazyName] = models.CrazyName 121 | 122 | name: factory.Faker = factory.Faker("name") 123 | -------------------------------------------------------------------------------- /tests/unit/test_types_and_defaults_extra.py: -------------------------------------------------------------------------------- 1 | import sqlite3 2 | import typing as t 3 | 4 | import pytest 5 | 6 | from mysql_to_sqlite3.sqlite_utils import CollatingSequences 7 | from mysql_to_sqlite3.transporter import MySQLtoSQLite 8 | 9 | 10 | class TestTypesAndDefaultsExtra: 11 | def test_valid_column_type_and_length(self) -> None: 12 | # COLUMN_PATTERN should match the type name without length 13 | m = MySQLtoSQLite._valid_column_type("varchar(255)") 14 | assert m is not None 15 | assert m.group(0).lower() == "varchar" 16 | # No parenthesis -> no length suffix 17 | assert MySQLtoSQLite._column_type_length("int") == "" 18 | # With parenthesis -> returns the (N) 19 | assert MySQLtoSQLite._column_type_length("nvarchar(42)") == "(42)" 20 | 21 | def test_data_type_collation_sequence(self) -> None: 22 | # Collation applies to textual affinity types only 23 | assert ( 24 | MySQLtoSQLite._data_type_collation_sequence(collation=CollatingSequences.NOCASE, column_type="VARCHAR(10)") 25 | == f"COLLATE {CollatingSequences.NOCASE}" 26 | ) 27 | assert ( 28 | MySQLtoSQLite._data_type_collation_sequence(collation=CollatingSequences.NOCASE, column_type="INTEGER") 29 | == "" 30 | ) 31 | 32 | @pytest.mark.parametrize( 33 | "default,expected", 34 | [ 35 | ("curtime()", "DEFAULT CURRENT_TIME"), 36 | ("curdate()", "DEFAULT CURRENT_DATE"), 37 | ("now()", "DEFAULT CURRENT_TIMESTAMP"), 38 | ], 39 | ) 40 | def test_translate_default_common_keywords(self, default: str, expected: str) -> None: 41 | assert MySQLtoSQLite._translate_default_from_mysql_to_sqlite(default) == expected 42 | 43 | def test_translate_default_current_timestamp_precision_transpiled(self) -> None: 44 | # MySQL allows fractional seconds: CURRENT_TIMESTAMP(6). Ensure it's normalized to SQLite token. 45 | out = MySQLtoSQLite._translate_default_from_mysql_to_sqlite( 46 | "CURRENT_TIMESTAMP(6)", column_extra="DEFAULT_GENERATED" 47 | ) 48 | assert out == "DEFAULT CURRENT_TIMESTAMP" 49 | 50 | def test_translate_default_generated_expr_fallback_quotes(self) -> None: 51 | # Unknown expressions should fall back to quoted string default for safety 52 | out = MySQLtoSQLite._translate_default_from_mysql_to_sqlite("uuid()", column_extra="DEFAULT_GENERATED") 53 | assert out == "DEFAULT 'uuid()'" 54 | 55 | def test_translate_default_charset_introducer_str_hex_and_bin(self) -> None: 56 | # DEFAULT_GENERATED with charset introducer and hex (escaped as in MySQL) 57 | s = "_utf8mb4 X\\'41\\'" # hex for 'A' 58 | out = MySQLtoSQLite._translate_default_from_mysql_to_sqlite( 59 | s, column_type="BLOB", column_extra="DEFAULT_GENERATED" 60 | ) 61 | assert out == "DEFAULT x'41'" 62 | # DEFAULT_GENERATED with charset introducer and binary literal (escaped) 63 | s2 = "_utf8mb4 b\\'01000001\\'" # binary for 'A' 64 | out2 = MySQLtoSQLite._translate_default_from_mysql_to_sqlite( 65 | s2, column_type="BLOB", column_extra="DEFAULT_GENERATED" 66 | ) 67 | assert out2 == "DEFAULT 'A'" 68 | 69 | def test_translate_default_charset_introducer_bytes(self) -> None: 70 | # Escaped form in bytes 71 | s = b"_utf8mb4 x\\'41\\'" 72 | out = MySQLtoSQLite._translate_default_from_mysql_to_sqlite( 73 | s, column_type="BLOB", column_extra="DEFAULT_GENERATED" 74 | ) 75 | assert out == "DEFAULT x'41'" 76 | 77 | def test_translate_default_bool_non_boolean_type(self) -> None: 78 | # When column_type is not BOOLEAN, booleans become '1'/'0' 79 | assert MySQLtoSQLite._translate_default_from_mysql_to_sqlite(True, column_type="INTEGER") == "DEFAULT '1'" 80 | assert MySQLtoSQLite._translate_default_from_mysql_to_sqlite(False, column_type="INTEGER") == "DEFAULT '0'" 81 | 82 | def test_translate_default_bool_boolean_type(self, monkeypatch: pytest.MonkeyPatch) -> None: 83 | # Ensure SQLite version satisfies condition 84 | monkeypatch.setattr(sqlite3, "sqlite_version", "3.40.0") 85 | assert MySQLtoSQLite._translate_default_from_mysql_to_sqlite(True, column_type="BOOLEAN") == "DEFAULT(TRUE)" 86 | assert MySQLtoSQLite._translate_default_from_mysql_to_sqlite(False, column_type="BOOLEAN") == "DEFAULT(FALSE)" 87 | 88 | def test_translate_default_prequoted_string_literal(self) -> None: 89 | # MariaDB can report TEXT defaults already wrapped in single quotes; ensure they're normalized 90 | assert MySQLtoSQLite._translate_default_from_mysql_to_sqlite("'[]'") == "DEFAULT '[]'" 91 | assert MySQLtoSQLite._translate_default_from_mysql_to_sqlite("'It''s'") == "DEFAULT 'It''s'" 92 | assert MySQLtoSQLite._translate_default_from_mysql_to_sqlite("'a\\'b'") == "DEFAULT 'a''b'" 93 | assert MySQLtoSQLite._translate_default_from_mysql_to_sqlite("''") == "DEFAULT ''" 94 | assert MySQLtoSQLite._translate_default_from_mysql_to_sqlite("('value')") == "DEFAULT 'value'" 95 | assert MySQLtoSQLite._translate_default_from_mysql_to_sqlite("'tab\\there'") == "DEFAULT 'tab\there'" 96 | -------------------------------------------------------------------------------- /tests/unit/test_cli_error_paths.py: -------------------------------------------------------------------------------- 1 | from types import SimpleNamespace 2 | 3 | import pytest 4 | from click.testing import CliRunner 5 | 6 | from mysql_to_sqlite3.cli import cli as mysql2sqlite 7 | 8 | 9 | class _FakeConverter: 10 | def __init__(self, *args, **kwargs): 11 | pass 12 | 13 | def transfer(self): 14 | raise RuntimeError("should not run") 15 | 16 | 17 | def _fake_supported_charsets(charset=None): 18 | """Produce deterministic charset/collation pairs for tests.""" 19 | # When called without a charset, emulate the public API generator used by click.Choice. 20 | if charset is None: 21 | return iter( 22 | [ 23 | SimpleNamespace(id=0, charset="utf8mb4", collation="utf8mb4_general_ci"), 24 | SimpleNamespace(id=1, charset="latin1", collation="latin1_swedish_ci"), 25 | ] 26 | ) 27 | # When scoped to a particular charset, expose only the primary collation. 28 | return iter([SimpleNamespace(id=0, charset=charset, collation=f"{charset}_general_ci")]) 29 | 30 | 31 | class TestCliErrorPaths: 32 | def test_mysql_collation_must_match_charset(self, monkeypatch: pytest.MonkeyPatch) -> None: 33 | """Invalid charset/collation combinations should be rejected before transfer starts.""" 34 | monkeypatch.setattr("mysql_to_sqlite3.cli.mysql_supported_character_sets", _fake_supported_charsets) 35 | monkeypatch.setattr("mysql_to_sqlite3.cli.MySQLtoSQLite", _FakeConverter) 36 | 37 | runner = CliRunner() 38 | result = runner.invoke( 39 | mysql2sqlite, 40 | [ 41 | "-f", 42 | "out.sqlite3", 43 | "-d", 44 | "db", 45 | "-u", 46 | "user", 47 | "--mysql-charset", 48 | "utf8mb4", 49 | "--mysql-collation", 50 | "latin1_swedish_ci", 51 | ], 52 | ) 53 | assert result.exit_code == 1 54 | assert "Invalid value for '--collation'" in result.output 55 | 56 | def test_debug_reraises_keyboard_interrupt(self, monkeypatch: pytest.MonkeyPatch) -> None: 57 | """Debug mode should bubble up KeyboardInterrupt for easier debugging.""" 58 | 59 | class KeyboardInterruptConverter: 60 | def __init__(self, *args, **kwargs): 61 | pass 62 | 63 | def transfer(self): 64 | raise KeyboardInterrupt() 65 | 66 | monkeypatch.setattr("mysql_to_sqlite3.cli.MySQLtoSQLite", KeyboardInterruptConverter) 67 | 68 | kwargs = { 69 | "sqlite_file": "out.sqlite3", 70 | "mysql_user": "user", 71 | "prompt_mysql_password": False, 72 | "mysql_password": None, 73 | "mysql_database": "db", 74 | "mysql_tables": None, 75 | "exclude_mysql_tables": None, 76 | "mysql_views_as_tables": False, 77 | "limit_rows": 0, 78 | "collation": "BINARY", 79 | "prefix_indices": False, 80 | "without_foreign_keys": False, 81 | "without_tables": False, 82 | "without_data": False, 83 | "strict": False, 84 | "mysql_host": "localhost", 85 | "mysql_port": 3306, 86 | "mysql_charset": "utf8mb4", 87 | "mysql_collation": None, 88 | "skip_ssl": False, 89 | "chunk": 200000, 90 | "log_file": None, 91 | "json_as_text": False, 92 | "vacuum": False, 93 | "use_buffered_cursors": False, 94 | "quiet": False, 95 | "debug": True, 96 | } 97 | with pytest.raises(KeyboardInterrupt): 98 | mysql2sqlite.callback(**kwargs) 99 | 100 | def test_debug_reraises_generic_exception(self, monkeypatch: pytest.MonkeyPatch) -> None: 101 | """Debug mode should bubble up unexpected exceptions.""" 102 | 103 | class ExplodingConverter: 104 | def __init__(self, *args, **kwargs): 105 | pass 106 | 107 | def transfer(self): 108 | raise RuntimeError("boom") 109 | 110 | monkeypatch.setattr("mysql_to_sqlite3.cli.MySQLtoSQLite", ExplodingConverter) 111 | 112 | kwargs = { 113 | "sqlite_file": "out.sqlite3", 114 | "mysql_user": "user", 115 | "prompt_mysql_password": False, 116 | "mysql_password": None, 117 | "mysql_database": "db", 118 | "mysql_tables": None, 119 | "exclude_mysql_tables": None, 120 | "mysql_views_as_tables": False, 121 | "limit_rows": 0, 122 | "collation": "BINARY", 123 | "prefix_indices": False, 124 | "without_foreign_keys": False, 125 | "without_tables": False, 126 | "without_data": False, 127 | "strict": False, 128 | "mysql_host": "localhost", 129 | "mysql_port": 3306, 130 | "mysql_charset": "utf8mb4", 131 | "mysql_collation": None, 132 | "skip_ssl": False, 133 | "chunk": 200000, 134 | "log_file": None, 135 | "json_as_text": False, 136 | "vacuum": False, 137 | "use_buffered_cursors": False, 138 | "quiet": False, 139 | "debug": True, 140 | } 141 | with pytest.raises(RuntimeError): 142 | mysql2sqlite.callback(**kwargs) 143 | -------------------------------------------------------------------------------- /tests/unit/test_views_sqlglot.py: -------------------------------------------------------------------------------- 1 | import re 2 | from unittest.mock import patch 3 | 4 | import pytest 5 | 6 | from mysql_to_sqlite3.transporter import MySQLtoSQLite 7 | 8 | 9 | class TestViewsSqlglot: 10 | def test_mysql_viewdef_to_sqlite_strips_schema_and_transpiles(self) -> None: 11 | mysql_select = "SELECT `u`.`id`, `u`.`name` FROM `db`.`users` AS `u` WHERE `u`.`id` > 1" 12 | # Use an instance to ensure access to _mysql_database for stripping 13 | with patch.object(MySQLtoSQLite, "__init__", return_value=None): 14 | inst = MySQLtoSQLite() # type: ignore[call-arg] 15 | inst._mysql_database = "db" # type: ignore[attr-defined] 16 | sql = inst._mysql_viewdef_to_sqlite( 17 | view_select_sql=mysql_select, 18 | view_name="v_users", 19 | ) 20 | assert sql.startswith('CREATE VIEW IF NOT EXISTS "v_users" AS') 21 | # Ensure schema qualifier was removed 22 | assert '"db".' not in sql 23 | assert "`db`." not in sql 24 | # Ensure it targets sqlite dialect (identifiers quoted with ") 25 | assert 'FROM "users"' in sql 26 | # Ends with single semicolon 27 | assert re.search(r";\s*$", sql) is not None 28 | 29 | def test_mysql_viewdef_to_sqlite_parse_fallback(self, monkeypatch: pytest.MonkeyPatch) -> None: 30 | # Force parse_one to raise so we hit the fallback path 31 | from sqlglot.errors import ParseError 32 | 33 | def boom(*_, **__): 34 | raise ParseError("boom") 35 | 36 | monkeypatch.setattr("mysql_to_sqlite3.transporter.parse_one", boom) 37 | 38 | sql_in = "SELECT 1" 39 | with patch.object(MySQLtoSQLite, "__init__", return_value=None): 40 | inst = MySQLtoSQLite() # type: ignore[call-arg] 41 | inst._mysql_database = "db" # type: ignore[attr-defined] 42 | out = inst._mysql_viewdef_to_sqlite( 43 | view_select_sql=sql_in, 44 | view_name="v1", 45 | ) 46 | assert out.startswith('CREATE VIEW IF NOT EXISTS "v1" AS') 47 | assert "SELECT 1" in out 48 | assert out.strip().endswith(";") 49 | 50 | def test_mysql_viewdef_to_sqlite_parse_fallback_strips_schema(self, monkeypatch: pytest.MonkeyPatch) -> None: 51 | # Force parse_one to raise so we exercise the fallback path with schema qualifiers 52 | from sqlglot.errors import ParseError 53 | 54 | def boom(*_, **__): 55 | raise ParseError("boom") 56 | 57 | monkeypatch.setattr("mysql_to_sqlite3.transporter.parse_one", boom) 58 | 59 | mysql_select = "SELECT `u`.`id` FROM `db`.`users` AS `u` WHERE `u`.`id` > 1" 60 | with patch.object(MySQLtoSQLite, "__init__", return_value=None): 61 | inst = MySQLtoSQLite() # type: ignore[call-arg] 62 | inst._mysql_database = "db" # type: ignore[attr-defined] 63 | out = inst._mysql_viewdef_to_sqlite( 64 | view_select_sql=mysql_select, 65 | view_name="v_users", 66 | ) 67 | # Should not contain schema qualifier anymore 68 | assert "`db`." not in out and '"db".' not in out and " db." not in out 69 | # Should still reference the table name 70 | assert "FROM `users`" in out or 'FROM "users"' in out or "FROM users" in out 71 | assert out.strip().endswith(";") 72 | 73 | def test_mysql_viewdef_to_sqlite_strips_schema_from_qualified_columns_nested(self) -> None: 74 | # Based on the user-reported example with nested subquery and fully-qualified columns 75 | mysql_sql = ( 76 | "select `p`.`instrument_id` AS `instrument_id`,`p`.`price_date` AS `price_date`,`p`.`close` AS `close` " 77 | "from (`example`.`prices` `p` join (select `example`.`prices`.`instrument_id` AS `instrument_id`," 78 | "max(`example`.`prices`.`price_date`) AS `max_date` from `example`.`prices` group by " 79 | "`example`.`prices`.`instrument_id`) `t` on(((`t`.`instrument_id` = `p`.`instrument_id`) and " 80 | "(`t`.`max_date` = `p`.`price_date`))))" 81 | ) 82 | with patch.object(MySQLtoSQLite, "__init__", return_value=None): 83 | inst = MySQLtoSQLite() # type: ignore[call-arg] 84 | inst._mysql_database = "example" # type: ignore[attr-defined] 85 | out = inst._mysql_viewdef_to_sqlite(view_select_sql=mysql_sql, view_name="v_prices") 86 | # Ensure all schema qualifiers are removed, including on qualified columns inside subqueries 87 | assert '"example".' not in out and "`example`." not in out and " example." not in out 88 | # Still references the base table name 89 | assert 'FROM "prices"' in out or 'FROM ("prices"' in out or "FROM prices" in out 90 | assert out.strip().endswith(";") 91 | 92 | def test_mysql_viewdef_to_sqlite_strips_matching_schema_qualifiers(self) -> None: 93 | mysql_select = "SELECT `u`.`id` FROM `db`.`users` AS `u`" 94 | # Use instance for consistent attribute access 95 | with patch.object(MySQLtoSQLite, "__init__", return_value=None): 96 | inst = MySQLtoSQLite() # type: ignore[call-arg] 97 | inst._mysql_database = "db" # type: ignore[attr-defined] 98 | # Since keep_schema behavior is no longer parameterized, ensure that if schema matches current db, it is stripped 99 | sql = inst._mysql_viewdef_to_sqlite( 100 | view_select_sql=mysql_select, 101 | view_name="v_users", 102 | ) 103 | assert "`db`." not in sql and '"db".' not in sql 104 | assert sql.strip().endswith(";") 105 | -------------------------------------------------------------------------------- /tests/unit/test_click_utils.py: -------------------------------------------------------------------------------- 1 | import typing as t 2 | 3 | import click 4 | import pytest 5 | from click.testing import CliRunner 6 | from pytest_mock import MockFixture 7 | 8 | from mysql_to_sqlite3.click_utils import OptionEatAll, prompt_password, validate_positive_integer 9 | 10 | 11 | class TestOptionEatAll: 12 | def test_init_with_invalid_nargs(self) -> None: 13 | """Test OptionEatAll initialization with invalid nargs.""" 14 | with pytest.raises(ValueError) as excinfo: 15 | OptionEatAll("--test", nargs=1) 16 | assert "nargs, if set, must be -1 not 1" in str(excinfo.value) 17 | 18 | def test_init_with_valid_nargs(self) -> None: 19 | """Test OptionEatAll initialization with valid nargs and behavior.""" 20 | 21 | @click.command() 22 | @click.option("--test", cls=OptionEatAll, nargs=-1, help="Test option") 23 | def cli(test: t.Optional[t.Tuple[str, ...]] = None) -> None: 24 | # This just verifies that the option works when nargs=-1 25 | assert test is not None 26 | click.echo(f"Success: {len(test)} values") 27 | 28 | runner = CliRunner() 29 | result = runner.invoke(cli, ["--test", "value1", "value2", "value3"]) 30 | assert result.exit_code == 0 31 | assert "Success:" in result.output 32 | assert "values" in result.output 33 | 34 | def test_add_to_parser(self) -> None: 35 | """Test add_to_parser method.""" 36 | 37 | @click.command() 38 | @click.option("--test", cls=OptionEatAll, help="Test option") 39 | def cli(test: t.Optional[t.Tuple[str, ...]] = None) -> None: 40 | click.echo(f"Test: {test}") 41 | 42 | runner = CliRunner() 43 | result = runner.invoke(cli, ["--test", "value1", "value2", "value3"]) 44 | assert result.exit_code == 0 45 | assert "Test: ('value1', 'value2', 'value3')" in result.output 46 | 47 | def test_add_to_parser_with_other_options(self) -> None: 48 | """Test add_to_parser method with other options.""" 49 | 50 | @click.command() 51 | @click.option("--test", cls=OptionEatAll, help="Test option") 52 | @click.option("--other", help="Other option") 53 | def cli(test: t.Optional[t.Tuple[str, ...]] = None, other: t.Optional[str] = None) -> None: 54 | click.echo(f"Test: {test}, Other: {other}") 55 | 56 | runner = CliRunner() 57 | result = runner.invoke(cli, ["--test", "value1", "value2", "--other", "value3"]) 58 | assert result.exit_code == 0 59 | assert "Test: ('value1', 'value2'), Other: value3" in result.output 60 | 61 | def test_add_to_parser_without_save_other_options(self) -> None: 62 | """Test add_to_parser method without saving other options.""" 63 | 64 | @click.command() 65 | @click.option("--test", cls=OptionEatAll, save_other_options=False, help="Test option") 66 | @click.option("--other", help="Other option") 67 | def cli(test: t.Optional[t.Tuple[str, ...]] = None, other: t.Optional[str] = None) -> None: 68 | click.echo(f"Test: {test}, Other: {other}") 69 | 70 | runner = CliRunner() 71 | result = runner.invoke(cli, ["--test", "value1", "value2", "--other", "value3"]) 72 | assert result.exit_code == 0 73 | # All remaining args should be consumed by --test 74 | assert "Test: ('value1', 'value2', '--other', 'value3'), Other: None" in result.output 75 | 76 | 77 | class TestPromptPassword: 78 | def test_prompt_password_with_password(self) -> None: 79 | """Test prompt_password with password already provided.""" 80 | ctx = click.Context(click.Command("test")) 81 | ctx.params = {"mysql_password": "test_password"} 82 | 83 | result = prompt_password(ctx, None, True) 84 | assert result == "test_password" 85 | 86 | def test_prompt_password_without_password(self, mocker: MockFixture) -> None: 87 | """Test prompt_password without password provided.""" 88 | ctx = click.Context(click.Command("test")) 89 | ctx.params = {"mysql_password": None} 90 | 91 | mocker.patch("click.prompt", return_value="prompted_password") 92 | 93 | result = prompt_password(ctx, None, True) 94 | assert result == "prompted_password" 95 | 96 | def test_prompt_password_use_password_false(self) -> None: 97 | """Test prompt_password with use_password=False.""" 98 | ctx = click.Context(click.Command("test")) 99 | ctx.params = {"mysql_password": "test_password"} 100 | 101 | result = prompt_password(ctx, None, False) 102 | assert result is None 103 | 104 | 105 | class TestValidatePositiveInteger: 106 | def test_validate_positive_integer_valid(self) -> None: 107 | """Test validate_positive_integer with valid values.""" 108 | ctx = click.Context(click.Command("test")) 109 | 110 | assert validate_positive_integer(ctx, None, 0) == 0 111 | assert validate_positive_integer(ctx, None, 1) == 1 112 | assert validate_positive_integer(ctx, None, 100) == 100 113 | 114 | def test_validate_positive_integer_invalid(self) -> None: 115 | """Test validate_positive_integer with invalid values.""" 116 | ctx = click.Context(click.Command("test")) 117 | 118 | with pytest.raises(click.BadParameter) as excinfo: 119 | validate_positive_integer(ctx, None, -1) 120 | assert "Should be a positive integer or 0." in str(excinfo.value) 121 | 122 | with pytest.raises(click.BadParameter) as excinfo: 123 | validate_positive_integer(ctx, None, -100) 124 | assert "Should be a positive integer or 0." in str(excinfo.value) 125 | -------------------------------------------------------------------------------- /tests/unit/test_sqlite_utils.py: -------------------------------------------------------------------------------- 1 | from datetime import date, timedelta 2 | from decimal import Decimal 3 | 4 | import pytest 5 | 6 | from mysql_to_sqlite3.sqlite_utils import ( 7 | CollatingSequences, 8 | Integer_Types, 9 | adapt_decimal, 10 | adapt_timedelta, 11 | convert_date, 12 | convert_decimal, 13 | convert_timedelta, 14 | encode_data_for_sqlite, 15 | ) 16 | 17 | 18 | class TestSQLiteUtils: 19 | def test_adapt_decimal(self) -> None: 20 | """Test adapt_decimal function.""" 21 | assert adapt_decimal(Decimal("123.45")) == "123.45" 22 | assert adapt_decimal(Decimal("0")) == "0" 23 | assert adapt_decimal(Decimal("-123.45")) == "-123.45" 24 | assert adapt_decimal(Decimal("123456789.123456789")) == "123456789.123456789" 25 | 26 | def test_convert_decimal(self) -> None: 27 | """Test convert_decimal function.""" 28 | assert convert_decimal("123.45") == Decimal("123.45") 29 | assert convert_decimal("0") == Decimal("0") 30 | assert convert_decimal("-123.45") == Decimal("-123.45") 31 | assert convert_decimal("123456789.123456789") == Decimal("123456789.123456789") 32 | 33 | def test_adapt_timedelta(self) -> None: 34 | """Test adapt_timedelta function.""" 35 | assert adapt_timedelta(timedelta(hours=1, minutes=30, seconds=45)) == "01:30:45" 36 | assert adapt_timedelta(timedelta(hours=0, minutes=0, seconds=0)) == "00:00:00" 37 | assert adapt_timedelta(timedelta(hours=100, minutes=0, seconds=0)) == "100:00:00" 38 | assert adapt_timedelta(timedelta(hours=0, minutes=90, seconds=0)) == "01:30:00" 39 | assert adapt_timedelta(timedelta(hours=0, minutes=0, seconds=90)) == "00:01:30" 40 | 41 | def test_convert_timedelta(self) -> None: 42 | """Test convert_timedelta function.""" 43 | assert convert_timedelta("01:30:45") == timedelta(hours=1, minutes=30, seconds=45) 44 | assert convert_timedelta("00:00:00") == timedelta(hours=0, minutes=0, seconds=0) 45 | assert convert_timedelta("100:00:00") == timedelta(hours=100, minutes=0, seconds=0) 46 | assert convert_timedelta("01:30:00") == timedelta(hours=1, minutes=30, seconds=0) 47 | assert convert_timedelta("00:01:30") == timedelta(hours=0, minutes=1, seconds=30) 48 | 49 | def test_encode_data_for_sqlite_string(self) -> None: 50 | """Test encode_data_for_sqlite with string.""" 51 | assert encode_data_for_sqlite("test") == "test" 52 | 53 | def test_encode_data_for_sqlite_bytes_success(self) -> None: 54 | """Test encode_data_for_sqlite with bytes that can be decoded.""" 55 | assert encode_data_for_sqlite(b"test") == "test" 56 | 57 | def test_encode_data_for_sqlite_bytes_failure(self) -> None: 58 | """Test encode_data_for_sqlite with bytes that cannot be decoded.""" 59 | # Create invalid UTF-8 bytes 60 | invalid_bytes = b"\xff\xfe\xfd" 61 | result = encode_data_for_sqlite(invalid_bytes) 62 | # Should return a sqlite3.Binary object or something that behaves similarly 63 | # Check if it's either a Binary object or at least contains the original bytes 64 | if hasattr(result, "adapt"): 65 | assert result.adapt() == invalid_bytes 66 | else: 67 | # If it's a memoryview or other type, verify it contains our original bytes 68 | assert bytes(result) == invalid_bytes 69 | 70 | def test_encode_data_for_sqlite_non_bytes(self) -> None: 71 | """Test encode_data_for_sqlite with non-bytes object that has no decode method.""" 72 | result = encode_data_for_sqlite(123) 73 | # In our implementation, the function should either: 74 | # 1. Return a sqlite3.Binary object, or 75 | # 2. Return the original value if it can't be converted to Binary 76 | # Either way, the result should work with SQLite 77 | if hasattr(result, "adapt"): 78 | assert result.adapt() == 123 79 | else: 80 | assert result == 123 81 | 82 | def test_convert_date_valid_string(self) -> None: 83 | """Test convert_date with valid string.""" 84 | assert convert_date("2021-01-01") == date(2021, 1, 1) 85 | assert convert_date("2021/01/01") == date(2021, 1, 1) 86 | assert convert_date("Jan 1, 2021") == date(2021, 1, 1) 87 | 88 | def test_convert_date_valid_bytes(self) -> None: 89 | """Test convert_date with valid bytes.""" 90 | assert convert_date(b"2021-01-01") == date(2021, 1, 1) 91 | assert convert_date(b"2021/01/01") == date(2021, 1, 1) 92 | assert convert_date(b"Jan 1, 2021") == date(2021, 1, 1) 93 | 94 | def test_convert_date_invalid(self) -> None: 95 | """Test convert_date with invalid date string.""" 96 | with pytest.raises(ValueError) as excinfo: 97 | convert_date("not a date") 98 | assert "DATE field contains" in str(excinfo.value) 99 | 100 | def test_collating_sequences(self) -> None: 101 | """Test CollatingSequences class.""" 102 | assert CollatingSequences.BINARY == "BINARY" 103 | assert CollatingSequences.NOCASE == "NOCASE" 104 | assert CollatingSequences.RTRIM == "RTRIM" 105 | 106 | def test_integer_types(self) -> None: 107 | """Test Integer_Types set.""" 108 | assert "INTEGER" in Integer_Types 109 | assert "INT" in Integer_Types 110 | assert "BIGINT" in Integer_Types 111 | assert "SMALLINT" in Integer_Types 112 | assert "TINYINT" in Integer_Types 113 | assert "MEDIUMINT" in Integer_Types 114 | assert "NUMERIC" in Integer_Types 115 | # Check that unsigned variants are included 116 | assert "INTEGER UNSIGNED" in Integer_Types 117 | assert "INT UNSIGNED" in Integer_Types 118 | assert "BIGINT UNSIGNED" in Integer_Types 119 | assert "SMALLINT UNSIGNED" in Integer_Types 120 | assert "TINYINT UNSIGNED" in Integer_Types 121 | assert "MEDIUMINT UNSIGNED" in Integer_Types 122 | -------------------------------------------------------------------------------- /CODE-OF-CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Contributor Covenant Code of Conduct 2 | 3 | ## Our Pledge 4 | 5 | We as members, contributors, and leaders pledge to make participation in our 6 | community a harassment-free experience for everyone, regardless of age, body 7 | size, visible or invisible disability, ethnicity, sex characteristics, gender 8 | identity and expression, level of experience, education, socio-economic status, 9 | nationality, personal appearance, race, caste, color, religion, or sexual identity 10 | and orientation. 11 | 12 | We pledge to act and interact in ways that contribute to an open, welcoming, 13 | diverse, inclusive, and healthy community. 14 | 15 | ## Our Standards 16 | 17 | Examples of behavior that contributes to a positive environment for our 18 | community include: 19 | 20 | * Demonstrating empathy and kindness toward other people 21 | * Being respectful of differing opinions, viewpoints, and experiences 22 | * Giving and gracefully accepting constructive feedback 23 | * Accepting responsibility and apologizing to those affected by our mistakes, 24 | and learning from the experience 25 | * Focusing on what is best not just for us as individuals, but for the 26 | overall community 27 | 28 | Examples of unacceptable behavior include: 29 | 30 | * The use of sexualized language or imagery, and sexual attention or 31 | advances of any kind 32 | * Trolling, insulting or derogatory comments, and personal or political attacks 33 | * Public or private harassment 34 | * Publishing others' private information, such as a physical or email 35 | address, without their explicit permission 36 | * Other conduct which could reasonably be considered inappropriate in a 37 | professional setting 38 | 39 | ## Enforcement Responsibilities 40 | 41 | Community leaders are responsible for clarifying and enforcing our standards of 42 | acceptable behavior and will take appropriate and fair corrective action in 43 | response to any behavior that they deem inappropriate, threatening, offensive, 44 | or harmful. 45 | 46 | Community leaders have the right and responsibility to remove, edit, or reject 47 | comments, commits, code, wiki edits, issues, and other contributions that are 48 | not aligned to this Code of Conduct, and will communicate reasons for moderation 49 | decisions when appropriate. 50 | 51 | ## Scope 52 | 53 | This Code of Conduct applies within all community spaces, and also applies when 54 | an individual is officially representing the community in public spaces. 55 | Examples of representing our community include using an official e-mail address, 56 | posting via an official social media account, or acting as an appointed 57 | representative at an online or offline event. 58 | 59 | ## Enforcement 60 | 61 | Instances of abusive, harassing, or otherwise unacceptable behavior may be 62 | reported to the community leaders responsible for enforcement at 63 | [techouse@gmail.com](mailto:techouse@gmail.com). 64 | All complaints will be reviewed and investigated promptly and fairly. 65 | 66 | All community leaders are obligated to respect the privacy and security of the 67 | reporter of any incident. 68 | 69 | ## Enforcement Guidelines 70 | 71 | Community leaders will follow these Community Impact Guidelines in determining 72 | the consequences for any action they deem in violation of this Code of Conduct: 73 | 74 | ### 1. Correction 75 | 76 | **Community Impact**: Use of inappropriate language or other behavior deemed 77 | unprofessional or unwelcome in the community. 78 | 79 | **Consequence**: A private, written warning from community leaders, providing 80 | clarity around the nature of the violation and an explanation of why the 81 | behavior was inappropriate. A public apology may be requested. 82 | 83 | ### 2. Warning 84 | 85 | **Community Impact**: A violation through a single incident or series 86 | of actions. 87 | 88 | **Consequence**: A warning with consequences for continued behavior. No 89 | interaction with the people involved, including unsolicited interaction with 90 | those enforcing the Code of Conduct, for a specified period of time. This 91 | includes avoiding interactions in community spaces as well as external channels 92 | like social media. Violating these terms may lead to a temporary or 93 | permanent ban. 94 | 95 | ### 3. Temporary Ban 96 | 97 | **Community Impact**: A serious violation of community standards, including 98 | sustained inappropriate behavior. 99 | 100 | **Consequence**: A temporary ban from any sort of interaction or public 101 | communication with the community for a specified period of time. No public or 102 | private interaction with the people involved, including unsolicited interaction 103 | with those enforcing the Code of Conduct, is allowed during this period. 104 | Violating these terms may lead to a permanent ban. 105 | 106 | ### 4. Permanent Ban 107 | 108 | **Community Impact**: Demonstrating a pattern of violation of community 109 | standards, including sustained inappropriate behavior, harassment of an 110 | individual, or aggression toward or disparagement of classes of individuals. 111 | 112 | **Consequence**: A permanent ban from any sort of public interaction within 113 | the community. 114 | 115 | ## Attribution 116 | 117 | This Code of Conduct is adapted from the [Contributor Covenant][homepage], 118 | version 2.1, available at 119 | [https://www.contributor-covenant.org/version/2/1/code_of_conduct.html][v2.1]. 120 | 121 | Community Impact Guidelines were inspired by 122 | [Mozilla's code of conduct enforcement ladder][Mozilla CoC]. 123 | 124 | For answers to common questions about this code of conduct, see the FAQ at 125 | [https://www.contributor-covenant.org/faq][FAQ]. Translations are available 126 | at [https://www.contributor-covenant.org/translations][translations]. 127 | 128 | [homepage]: https://www.contributor-covenant.org 129 | [v2.1]: https://www.contributor-covenant.org/version/2/1/code_of_conduct.html 130 | [Mozilla CoC]: https://github.com/mozilla/diversity 131 | [FAQ]: https://www.contributor-covenant.org/faq 132 | [translations]: https://www.contributor-covenant.org/translations 133 | -------------------------------------------------------------------------------- /.github/copilot-instructions.md: -------------------------------------------------------------------------------- 1 | # AI Assistant Project Instructions 2 | 3 | Concise, project-specific guidance for code-generation agents working in this repository. 4 | 5 | ## 1. Purpose & High-Level Architecture 6 | 7 | This package provides a robust one-shot transfer of a MySQL/MariaDB database schema + data into a single SQLite file via 8 | a CLI (`mysql2sqlite`). Core orchestration lives in `transporter.py` (`MySQLtoSQLite` class) and is invoked by the Click 9 | CLI defined in `cli.py`. The transfer pipeline: 10 | 11 | 1. Parse/validate CLI options (mutually exclusive flags, charset/collation validation). 12 | 2. Introspect MySQL schema via `information_schema` (columns, indices, foreign keys). 13 | 3. Generate SQLite DDL (type & default translation, index naming, optional foreign keys, JSON1 detection). 14 | 4. Stream or chunk table data rows from MySQL → SQLite with optional chunk size and progress bars (`tqdm`). 15 | 5. Post-process (optional `VACUUM`). 16 | 17 | ## 2. Key Modules & Responsibilities 18 | 19 | - `src/mysql_to_sqlite3/cli.py`: Click command. Central place to add new user-facing options (update docs + README + 20 | `docs/index.rst` if changed). 21 | - `src/mysql_to_sqlite3/transporter.py`: Core transfer logic, schema introspection, type/default translation, batching, 22 | logging, reconnection handling. 23 | - `sqlite_utils.py` / `mysql_utils.py`: Helpers for charset/collation sets, adapting/encoding values and SQLite 24 | extension capability detection. 25 | - `types.py`: Typed parameter & attribute Protocols / TypedDict-like structures for constructor kwargs (mypy relies on 26 | this; keep hints exhaustive). 27 | - `debug_info.py`: Version table printed with `--version` (tabulated output). 28 | 29 | ## 3. Patterns & Conventions 30 | 31 | - Strict option conflict handling: e.g. `--mysql-tables` vs `--exclude-mysql-tables`; `--without-tables` + 32 | `--without-data` is invalid. Mirror same validation in new features. 33 | - AUTO_INCREMENT single primary key columns converted to `INTEGER PRIMARY KEY AUTOINCREMENT` only when underlying 34 | translated type is an integer (see `Integer_Types`). Log a warning otherwise. 35 | - Index naming collision avoidance: when an index name equals a table name or `--prefix-indices` set, prefix with 36 | `_` to ensure uniqueness. 37 | - Default value translation centralised in `_translate_default_from_mysql_to_sqlite`; extend here for new MySQL 38 | constructs instead of sprinkling conditionals elsewhere. 39 | - JSON handling: if `JSON` column and SQLite JSON1 compiled (`PRAGMA compile_options`), map to `JSON`; else fallback to 40 | `TEXT` unless `--json-as-text` provided. Preserve logic when adding new types. 41 | - Foreign keys generation skipped if any table subset restriction applied (ensures referential integrity isn’t partially 42 | generated). 43 | - Logging: use `_setup_logger` — do not instantiate new loggers ad hoc. 44 | 45 | ## 4. Testing Approach 46 | 47 | - Use `pytest` framework; tests in `tests/` mirror `src/mysql_to_sqlite3/` structure. 48 | - Unit tests in `tests/unit/` focus on isolated helpers (e.g. type/default translation, JSON1 detection, constructor 49 | validation). When adding logic, create a targeted test file `test_.py`. 50 | - Functional / transfer tests in `tests/func/` (may require live MySQL). Respect existing pytest markers: add 51 | `@pytest.mark.transfer` for heavy DB flows, `@pytest.mark.cli` for CLI option parsing. 52 | - Keep mypy passing (configured for Python 3.9 baseline). Avoid untyped dynamic attributes; update `types.py` if 53 | constructor kwargs change. 54 | 55 | ## 5. Developer Workflows 56 | 57 | - Dev install: `pip install -e .` then `pip install -r requirements_dev.txt`. 58 | - Run full test + coverage: `pytest -v --cov=src/mysql_to_sqlite3` (tox orchestrates matrix via `tox`). 59 | - Lint suite: `tox -e linters` (runs black, isort, flake8, pylint, bandit, mypy). Ensure new files adhere to 120-char 60 | lines (black config) and import ordering (isort profile=black). 61 | - Build distribution: `python -m build` or `hatch build`. 62 | 63 | ## 6. Performance & Reliability Considerations 64 | 65 | - Large tables: prefer `--chunk` to bound memory; logic uses `fetchmany(self._chunk_size)` with `executemany` inserts 66 | for efficiency. 67 | - Reconnection: On `CR_SERVER_LOST` errors during schema creation or data transfer, a single reconnect attempt is made; 68 | preserve pattern if extending. 69 | - Foreign keys disabled (`PRAGMA foreign_keys=OFF`) during bulk load, then re-enabled in `finally`; ensure any 70 | early-return path still re-enables. 71 | - Use `INSERT OR IGNORE` to gracefully handle potential duplicates. 72 | 73 | ## 7. Adding New CLI Flags 74 | 75 | 1. Add `@click.option` in `cli.py` (keep mutually exclusive logic consistent). 76 | 2. Thread parameter through `MySQLtoSQLite` constructor (update typing + tests). 77 | 3. Update docs: `README.md` + `docs/index.rst` + optionally changelog. 78 | 4. Add at least one unit test exercising the new behavior / validation. 79 | 80 | ## 8. Error Handling Philosophy 81 | 82 | - Fail fast on invalid configuration (raise `click.ClickException` or `UsageError` in CLI, `ValueError` in constructor). 83 | - Swallow and print only when `--debug` not set; with `--debug` re-raise for stack inspection. 84 | 85 | ## 9. Security & Credentials 86 | 87 | - Never log raw passwords. Current design only accepts password via argument or prompt; continue that pattern. 88 | - MySQL SSL can be disabled via `--skip-ssl`; default is encrypted where possible—don’t silently change. 89 | 90 | ## 10. Common Extension Points 91 | 92 | - Type mapping additions: edit `_translate_type_from_mysql_to_sqlite` and corresponding tests. 93 | - Default expression support: `_translate_default_from_mysql_to_sqlite` + tests. 94 | - Progress/UI changes: centralize around `tqdm` usage; respect `--quiet` flag. 95 | 96 | ## 11. Examples 97 | 98 | - Chunked transfer: `mysql2sqlite -f out.db -d db -u user -p -c 50000` (efficient large table copy). 99 | - Subset tables (no FKs): `mysql2sqlite -f out.db -d db -u user -t users orders`. 100 | 101 | ## 12. PR Expectations 102 | 103 | - Include before/after CLI snippet when adding flags. 104 | - Keep coverage steady; add or adapt tests for new branches. 105 | - Update `SECURITY.md` only if security-relevant surface changes (e.g., new credential flag). 106 | 107 | --- 108 | Questions or unclear areas? Ask which section needs refinement or provide the diff you’re planning; guidance here should 109 | remain minimal but precise. 110 | -------------------------------------------------------------------------------- /tests/unit/test_debug_info.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import typing as t 3 | from unittest.mock import MagicMock, patch 4 | 5 | import pytest 6 | from pytest_mock import MockFixture 7 | 8 | from mysql_to_sqlite3.debug_info import _implementation, _mysql_version, info 9 | 10 | 11 | class TestDebugInfo: 12 | def test_implementation_cpython(self, mocker: MockFixture) -> None: 13 | """Test _implementation function with CPython.""" 14 | mocker.patch("platform.python_implementation", return_value="CPython") 15 | mocker.patch("platform.python_version", return_value="3.8.10") 16 | 17 | result = _implementation() 18 | assert result == "CPython 3.8.10" 19 | 20 | def test_implementation_pypy(self, mocker: MockFixture) -> None: 21 | """Test _implementation function with PyPy.""" 22 | mocker.patch("platform.python_implementation", return_value="PyPy") 23 | 24 | # Create a mock for pypy_version_info 25 | mock_version_info = MagicMock() 26 | mock_version_info.major = 3 27 | mock_version_info.minor = 7 28 | mock_version_info.micro = 4 29 | mock_version_info.releaselevel = "final" 30 | 31 | # Need to use patch instead of mocker.patch for sys module attributes 32 | with patch.object(sys, "pypy_version_info", mock_version_info, create=True): 33 | result = _implementation() 34 | assert result == "PyPy 3.7.4" 35 | 36 | def test_implementation_pypy_non_final(self, mocker: MockFixture) -> None: 37 | """Test _implementation function with PyPy non-final release.""" 38 | mocker.patch("platform.python_implementation", return_value="PyPy") 39 | 40 | # Create a mock for pypy_version_info 41 | mock_version_info = MagicMock() 42 | mock_version_info.major = 3 43 | mock_version_info.minor = 7 44 | mock_version_info.micro = 4 45 | mock_version_info.releaselevel = "beta" 46 | 47 | # Need to use patch instead of mocker.patch for sys module attributes 48 | with patch.object(sys, "pypy_version_info", mock_version_info, create=True): 49 | result = _implementation() 50 | assert result == "PyPy 3.7.4beta" 51 | 52 | def test_implementation_jython(self, mocker: MockFixture) -> None: 53 | """Test _implementation function with Jython.""" 54 | mocker.patch("platform.python_implementation", return_value="Jython") 55 | mocker.patch("platform.python_version", return_value="2.7.2") 56 | 57 | result = _implementation() 58 | assert result == "Jython 2.7.2" 59 | 60 | def test_implementation_ironpython(self, mocker: MockFixture) -> None: 61 | """Test _implementation function with IronPython.""" 62 | mocker.patch("platform.python_implementation", return_value="IronPython") 63 | mocker.patch("platform.python_version", return_value="2.7.9") 64 | 65 | result = _implementation() 66 | assert result == "IronPython 2.7.9" 67 | 68 | def test_implementation_unknown(self, mocker: MockFixture) -> None: 69 | """Test _implementation function with unknown implementation.""" 70 | mocker.patch("platform.python_implementation", return_value="UnknownPython") 71 | 72 | result = _implementation() 73 | assert result == "UnknownPython Unknown" 74 | 75 | def test_mysql_version_success(self, mocker: MockFixture) -> None: 76 | """Test _mysql_version function when mysql client is available.""" 77 | mocker.patch("mysql_to_sqlite3.debug_info.which", return_value="/usr/bin/mysql") 78 | mocker.patch( 79 | "mysql_to_sqlite3.debug_info.check_output", 80 | return_value=b"mysql Ver 8.0.26 for Linux on x86_64", 81 | ) 82 | 83 | result = _mysql_version() 84 | assert result == "mysql Ver 8.0.26 for Linux on x86_64" 85 | 86 | def test_mysql_version_bytes_decode_error(self, mocker: MockFixture) -> None: 87 | """Test _mysql_version function when bytes decoding fails.""" 88 | mocker.patch("mysql_to_sqlite3.debug_info.which", return_value="/usr/bin/mysql") 89 | mock_output = MagicMock() 90 | mock_output.decode.side_effect = UnicodeDecodeError("utf-8", b"", 0, 1, "invalid") 91 | mocker.patch( 92 | "mysql_to_sqlite3.debug_info.check_output", 93 | return_value=mock_output, 94 | ) 95 | 96 | result = _mysql_version() 97 | assert isinstance(result, str) 98 | 99 | def test_mysql_version_exception(self, mocker: MockFixture) -> None: 100 | """Test _mysql_version function when an exception occurs.""" 101 | mocker.patch("mysql_to_sqlite3.debug_info.which", return_value="/usr/bin/mysql") 102 | mocker.patch( 103 | "mysql_to_sqlite3.debug_info.check_output", 104 | side_effect=Exception("Command failed"), 105 | ) 106 | 107 | result = _mysql_version() 108 | assert result == "MySQL client not found on the system" 109 | 110 | def test_mysql_version_not_found(self, mocker: MockFixture) -> None: 111 | """Test _mysql_version function when mysql client is not found.""" 112 | mocker.patch("mysql_to_sqlite3.debug_info.which", return_value=None) 113 | 114 | result = _mysql_version() 115 | assert result == "MySQL client not found on the system" 116 | 117 | def test_info_success(self, mocker: MockFixture) -> None: 118 | """Test info function.""" 119 | mocker.patch("platform.system", return_value="Linux") 120 | mocker.patch("platform.release", return_value="5.4.0-80-generic") 121 | mocker.patch("mysql_to_sqlite3.debug_info._implementation", return_value="CPython 3.8.10") 122 | mocker.patch("mysql_to_sqlite3.debug_info._mysql_version", return_value="mysql Ver 8.0.26 for Linux on x86_64") 123 | 124 | result = info() 125 | assert isinstance(result, list) 126 | assert len(result) > 0 127 | assert result[2] == ["Operating System", "Linux 5.4.0-80-generic"] 128 | assert result[3] == ["Python", "CPython 3.8.10"] 129 | assert result[4] == ["MySQL", "mysql Ver 8.0.26 for Linux on x86_64"] 130 | 131 | def test_info_platform_error(self, mocker: MockFixture) -> None: 132 | """Test info function when platform.system raises IOError.""" 133 | mocker.patch("platform.system", side_effect=IOError("Platform error")) 134 | 135 | result = info() 136 | assert isinstance(result, list) 137 | assert len(result) > 0 138 | assert result[2] == ["Operating System", "Unknown"] 139 | -------------------------------------------------------------------------------- /tests/models.py: -------------------------------------------------------------------------------- 1 | import typing as t 2 | from datetime import date, datetime, time 3 | from decimal import Decimal 4 | from os import environ 5 | 6 | from sqlalchemy import ( 7 | CHAR, 8 | DECIMAL, 9 | JSON, 10 | NCHAR, 11 | REAL, 12 | TIMESTAMP, 13 | VARBINARY, 14 | VARCHAR, 15 | BigInteger, 16 | Column, 17 | ForeignKey, 18 | Integer, 19 | LargeBinary, 20 | Numeric, 21 | SmallInteger, 22 | String, 23 | Table, 24 | Text, 25 | Time, 26 | Unicode, 27 | ) 28 | from sqlalchemy.dialects.mysql import BIGINT, INTEGER, MEDIUMINT, SMALLINT, TINYINT 29 | from sqlalchemy.orm import DeclarativeBase, Mapped, backref, mapped_column, relationship 30 | from sqlalchemy.sql.functions import current_timestamp 31 | 32 | 33 | class Base(DeclarativeBase): 34 | pass 35 | 36 | 37 | class Author(Base): 38 | __tablename__ = "authors" 39 | id: Mapped[int] = mapped_column(primary_key=True) 40 | name: Mapped[str] = mapped_column(String(128), nullable=False, index=True) 41 | dupe: Mapped[bool] = mapped_column(index=True, default=False) 42 | 43 | def __repr__(self): 44 | return f"" 45 | 46 | 47 | article_authors = Table( 48 | "article_authors", 49 | Base.metadata, 50 | Column("article_id", Integer, ForeignKey("articles.id"), primary_key=True), 51 | Column("author_id", Integer, ForeignKey("authors.id"), primary_key=True), 52 | ) 53 | 54 | 55 | class Image(Base): 56 | __tablename__ = "images" 57 | id: Mapped[int] = mapped_column(primary_key=True) 58 | path: Mapped[str] = mapped_column(String(255), index=True) 59 | description: Mapped[str] = mapped_column(String(255), nullable=True) 60 | dupe: Mapped[bool] = mapped_column(index=True, default=False) 61 | 62 | def __repr__(self): 63 | return f"" 64 | 65 | 66 | article_images = Table( 67 | "article_images", 68 | Base.metadata, 69 | Column("article_id", Integer, ForeignKey("articles.id"), primary_key=True), 70 | Column("image_id", Integer, ForeignKey("images.id"), primary_key=True), 71 | ) 72 | 73 | 74 | class Tag(Base): 75 | __tablename__ = "tags" 76 | id: Mapped[int] = mapped_column(primary_key=True) 77 | name: Mapped[str] = mapped_column(String(128), nullable=False, index=True) 78 | dupe: Mapped[bool] = mapped_column(index=True, default=False) 79 | 80 | def __repr__(self): 81 | return f"" 82 | 83 | 84 | article_tags = Table( 85 | "article_tags", 86 | Base.metadata, 87 | Column("article_id", Integer, ForeignKey("articles.id"), primary_key=True), 88 | Column("tag_id", Integer, ForeignKey("tags.id"), primary_key=True), 89 | ) 90 | 91 | 92 | class Misc(Base): 93 | """This model contains all possible MySQL types""" 94 | 95 | __tablename__ = "misc" 96 | id: Mapped[int] = mapped_column(primary_key=True) 97 | big_integer_field: Mapped[int] = mapped_column(BigInteger, default=0) 98 | big_integer_unsigned_field: Mapped[int] = mapped_column(BIGINT(unsigned=True), default=0) 99 | if environ.get("LEGACY_DB", "0") == "0": 100 | large_binary_field: Mapped[bytes] = mapped_column(LargeBinary, nullable=True, default=b"Lorem ipsum dolor") 101 | else: 102 | large_binary_field = mapped_column(LargeBinary, nullable=True) 103 | boolean_field: Mapped[bool] = mapped_column(default=False) 104 | char_field: Mapped[str] = mapped_column(CHAR(255), nullable=True) 105 | date_field: Mapped[date] = mapped_column(nullable=True) 106 | date_time_field: Mapped[datetime] = mapped_column(nullable=True) 107 | decimal_field: Mapped[Decimal] = mapped_column(DECIMAL(10, 2), nullable=True) 108 | float_field: Mapped[Decimal] = mapped_column(DECIMAL(12, 4), default=0) 109 | integer_field: Mapped[int] = mapped_column(default=0) 110 | integer_unsigned_field: Mapped[int] = mapped_column(INTEGER(unsigned=True), default=0) 111 | tinyint_field: Mapped[int] = mapped_column(TINYINT, default=0) 112 | tinyint_unsigned_field: Mapped[int] = mapped_column(TINYINT(unsigned=True), default=0) 113 | mediumint_field: Mapped[int] = mapped_column(MEDIUMINT, default=0) 114 | mediumint_unsigned_field: Mapped[int] = mapped_column(MEDIUMINT(unsigned=True), default=0) 115 | if environ.get("LEGACY_DB", "0") == "0": 116 | json_field: Mapped[t.Mapping[str, t.Any]] = mapped_column(JSON, nullable=True) 117 | nchar_field: Mapped[str] = mapped_column(NCHAR(255), nullable=True) 118 | numeric_field: Mapped[float] = mapped_column(Numeric(12, 4), default=0) 119 | unicode_field: Mapped[str] = mapped_column(Unicode(255), nullable=True) 120 | real_field: Mapped[float] = mapped_column(REAL(12), default=0) 121 | small_integer_field: Mapped[int] = mapped_column(SmallInteger, default=0) 122 | small_integer_unsigned_field: Mapped[int] = mapped_column(SMALLINT(unsigned=True), default=0) 123 | string_field: Mapped[str] = mapped_column(String(255), nullable=True) 124 | text_field: Mapped[str] = mapped_column(Text, nullable=True) 125 | time_field: Mapped[time] = mapped_column(Time, nullable=True) 126 | varbinary_field: Mapped[bytes] = mapped_column(VARBINARY(255), nullable=True) 127 | varchar_field: Mapped[str] = mapped_column(VARCHAR(255), nullable=True) 128 | timestamp_field: Mapped[datetime] = mapped_column(TIMESTAMP, default=current_timestamp()) 129 | dupe: Mapped[bool] = mapped_column(index=True, default=False) 130 | 131 | 132 | article_misc = Table( 133 | "article_misc", 134 | Base.metadata, 135 | Column("article_id", Integer, ForeignKey("articles.id"), primary_key=True), 136 | Column("misc_id", Integer, ForeignKey("misc.id"), primary_key=True), 137 | ) 138 | 139 | 140 | class Article(Base): 141 | __tablename__ = "articles" 142 | id: Mapped[int] = mapped_column(primary_key=True) 143 | hash: Mapped[str] = mapped_column(String(32), unique=True) 144 | slug: Mapped[str] = mapped_column(String(255), index=True) 145 | title: Mapped[str] = mapped_column(String(255), index=True) 146 | content: Mapped[str] = mapped_column(Text, nullable=True) 147 | status: Mapped[str] = mapped_column(CHAR(1), index=True) 148 | published: Mapped[datetime] = mapped_column(nullable=True) 149 | dupe: Mapped[bool] = mapped_column(index=True, default=False) 150 | # relationships 151 | authors: Mapped[t.List[Author]] = relationship( 152 | "Author", 153 | secondary=article_authors, 154 | backref=backref("authors", lazy="dynamic"), 155 | lazy="dynamic", 156 | ) 157 | tags: Mapped[t.List[Tag]] = relationship( 158 | "Tag", 159 | secondary=article_tags, 160 | backref=backref("tags", lazy="dynamic"), 161 | lazy="dynamic", 162 | ) 163 | images: Mapped[t.List[Image]] = relationship( 164 | "Image", 165 | secondary=article_images, 166 | backref=backref("images", lazy="dynamic"), 167 | lazy="dynamic", 168 | ) 169 | misc: Mapped[t.List[Misc]] = relationship( 170 | "Misc", 171 | secondary=article_misc, 172 | backref=backref("misc", lazy="dynamic"), 173 | lazy="dynamic", 174 | ) 175 | 176 | def __repr__(self): 177 | return f"" 178 | 179 | 180 | class CrazyName(Base): 181 | __tablename__ = "crazy_name." 182 | id: Mapped[int] = mapped_column(primary_key=True) 183 | name: Mapped[str] = mapped_column(String(128), nullable=False, index=True) 184 | dupe: Mapped[bool] = mapped_column(index=True, default=False) 185 | 186 | def __repr__(self): 187 | return f"" 188 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [![PyPI](https://img.shields.io/pypi/v/mysql-to-sqlite3?logo=pypi)](https://pypi.org/project/mysql-to-sqlite3/) 2 | [![PyPI - Downloads](https://img.shields.io/pypi/dm/mysql-to-sqlite3?logo=pypi&label=PyPI%20downloads)](https://pypistats.org/packages/mysql-to-sqlite3) 3 | [![Homebrew Formula Downloads](https://img.shields.io/homebrew/installs/dm/mysql-to-sqlite3?logo=homebrew&label=Homebrew%20downloads)](https://formulae.brew.sh/formula/mysql-to-sqlite3) 4 | [![PyPI - Python Version](https://img.shields.io/pypi/pyversions/mysql-to-sqlite3?logo=python)](https://pypi.org/project/mysql-to-sqlite3/) 5 | [![MySQL Support](https://img.shields.io/static/v1?logo=mysql&label=MySQL&message=5.5+|+5.6+|+5.7+|+8.0+|+8.4&color=2b5d80)](https://img.shields.io/static/v1?label=MySQL&message=5.5+|+5.6+|+5.7+|+8.0+|+8.4&color=2b5d80) 6 | [![MariaDB Support](https://img.shields.io/static/v1?logo=mariadb&label=MariaDB&message=5.5+|+10.0+|+10.6+|+10.11+|+11.4+|+11.6+|+11.8&color=C0765A)](https://img.shields.io/static/v1?label=MariaDB&message=5.5+|+10.0+|+10.6+|+10.11+|+11.4+|+11.6+|+11.8&color=C0765A) 7 | [![GitHub license](https://img.shields.io/github/license/techouse/mysql-to-sqlite3)](https://github.com/techouse/mysql-to-sqlite3/blob/master/LICENSE) 8 | [![Contributor Covenant](https://img.shields.io/badge/Contributor%20Covenant-2.1-4baaaa.svg?logo=contributorcovenant)](CODE-OF-CONDUCT.md) 9 | [![PyPI - Format](https://img.shields.io/pypi/format/mysql-to-sqlite3?logo=python)](https://pypi.org/project/sqlite3-to-mysql/) 10 | [![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg?logo=python)](https://github.com/ambv/black) 11 | [![Codacy Badge](https://api.codacy.com/project/badge/Grade/64aae8e9599746d58d277852b35cc2bd)](https://www.codacy.com/manual/techouse/mysql-to-sqlite3?utm_source=github.com&utm_medium=referral&utm_content=techouse/mysql-to-sqlite3&utm_campaign=Badge_Grade) 12 | [![Test Status](https://github.com/techouse/mysql-to-sqlite3/actions/workflows/test.yml/badge.svg)](https://github.com/techouse/mysql-to-sqlite3/actions/workflows/test.yml) 13 | [![CodeQL Status](https://github.com/techouse/mysql-to-sqlite3/actions/workflows/github-code-scanning/codeql/badge.svg)](https://github.com/techouse/mysql-to-sqlite3/actions/workflows/github-code-scanning/codeql) 14 | [![Publish PyPI Package Status](https://github.com/techouse/mysql-to-sqlite3/actions/workflows/publish.yml/badge.svg)](https://github.com/techouse/mysql-to-sqlite3/actions/workflows/publish.yml) 15 | [![codecov](https://codecov.io/gh/techouse/mysql-to-sqlite3/branch/master/graph/badge.svg)](https://codecov.io/gh/techouse/mysql-to-sqlite3) 16 | [![GitHub Sponsors](https://img.shields.io/github/sponsors/techouse?logo=github)](https://github.com/sponsors/techouse) 17 | [![GitHub stars](https://img.shields.io/github/stars/techouse/mysql-to-sqlite3.svg?style=social&label=Star&maxAge=2592000)](https://github.com/techouse/mysql-to-sqlite3/stargazers) 18 | 19 | # MySQL to SQLite3 20 | 21 | #### A simple Python tool to transfer data from MySQL to SQLite 3. 22 | 23 | ### How to run 24 | 25 | ```bash 26 | pip install mysql-to-sqlite3 27 | mysql2sqlite --help 28 | ``` 29 | 30 | ### Usage 31 | 32 | ``` 33 | Usage: mysql2sqlite [OPTIONS] 34 | 35 | Options: 36 | -f, --sqlite-file PATH SQLite3 database file [required] 37 | -d, --mysql-database TEXT MySQL database name [required] 38 | -u, --mysql-user TEXT MySQL user [required] 39 | -p, --prompt-mysql-password Prompt for MySQL password 40 | --mysql-password TEXT MySQL password 41 | -t, --mysql-tables TUPLE Transfer only these specific tables (space 42 | separated table names). Implies --without- 43 | foreign-keys which inhibits the transfer of 44 | foreign keys. Can not be used together with 45 | --exclude-mysql-tables. 46 | -e, --exclude-mysql-tables TUPLE 47 | Transfer all tables except these specific 48 | tables (space separated table names). 49 | Implies --without-foreign-keys which 50 | inhibits the transfer of foreign keys. Can 51 | not be used together with --mysql-tables. 52 | -T, --mysql-views-as-tables Materialize MySQL VIEWs as SQLite tables 53 | (legacy behavior). 54 | -L, --limit-rows INTEGER Transfer only a limited number of rows from 55 | each table. 56 | -C, --collation [BINARY|NOCASE|RTRIM] 57 | Create datatypes of TEXT affinity using a 58 | specified collation sequence. [default: 59 | BINARY] 60 | -K, --prefix-indices Prefix indices with their corresponding 61 | tables. This ensures that their names remain 62 | unique across the SQLite database. 63 | -X, --without-foreign-keys Do not transfer foreign keys. 64 | -Z, --without-tables Do not transfer tables, data only. 65 | -W, --without-data Do not transfer table data, DDL only. 66 | -M, --strict Create SQLite STRICT tables when supported. 67 | -h, --mysql-host TEXT MySQL host. Defaults to localhost. 68 | -P, --mysql-port INTEGER MySQL port. Defaults to 3306. 69 | --mysql-charset TEXT MySQL database and table character set 70 | [default: utf8mb4] 71 | --mysql-collation TEXT MySQL database and table collation 72 | -S, --skip-ssl Disable MySQL connection encryption. 73 | -c, --chunk INTEGER Chunk reading/writing SQL records 74 | -l, --log-file PATH Log file 75 | --json-as-text Transfer JSON columns as TEXT. 76 | -V, --vacuum Use the VACUUM command to rebuild the SQLite 77 | database file, repacking it into a minimal 78 | amount of disk space 79 | --use-buffered-cursors Use MySQLCursorBuffered for reading the 80 | MySQL database. This can be useful in 81 | situations where multiple queries, with 82 | small result sets, need to be combined or 83 | computed with each other. 84 | -q, --quiet Quiet. Display only errors. 85 | --debug Debug mode. Will throw exceptions. 86 | --version Show the version and exit. 87 | --help Show this message and exit. 88 | ``` 89 | 90 | #### Docker 91 | 92 | If you don't want to install the tool on your system, you can use the Docker image instead. 93 | 94 | ```bash 95 | docker run -it \ 96 | --workdir $(pwd) \ 97 | --volume $(pwd):$(pwd) \ 98 | --rm ghcr.io/techouse/mysql-to-sqlite3:latest \ 99 | --sqlite-file baz.db \ 100 | --mysql-user foo \ 101 | --mysql-password bar \ 102 | --mysql-database baz \ 103 | --mysql-host host.docker.internal 104 | ``` 105 | 106 | This will mount your host current working directory (pwd) inside the Docker container as the current working directory. 107 | Any files Docker would write to the current working directory are written to the host directory where you did docker 108 | run. Note that you have to also use a 109 | [special hostname](https://docs.docker.com/desktop/networking/#use-cases-and-workarounds-for-all-platforms) 110 | `host.docker.internal` 111 | to access your host machine from inside the Docker container. 112 | 113 | #### Homebrew 114 | 115 | If you're on macOS, you can install the tool using [Homebrew](https://brew.sh/). 116 | 117 | ```bash 118 | brew install mysql-to-sqlite3 119 | mysql2sqlite --help 120 | ``` 121 | -------------------------------------------------------------------------------- /tests/unit/test_mysql_utils.py: -------------------------------------------------------------------------------- 1 | """Unit tests for the mysql_utils module.""" 2 | 3 | import typing as t 4 | from unittest import mock 5 | 6 | import pytest 7 | from mysql.connector import CharacterSet 8 | 9 | from mysql_to_sqlite3.mysql_utils import ( 10 | CHARSET_INTRODUCERS, 11 | CharSet, 12 | mysql_supported_character_sets, 13 | ) 14 | 15 | 16 | class TestMySQLUtils: 17 | """Unit tests for the mysql_utils module.""" 18 | 19 | def test_charset_introducers(self) -> None: 20 | """Test that CHARSET_INTRODUCERS contains the expected values.""" 21 | assert isinstance(CHARSET_INTRODUCERS, tuple) 22 | assert len(CHARSET_INTRODUCERS) > 0 23 | assert all(isinstance(intro, str) for intro in CHARSET_INTRODUCERS) 24 | assert all(intro.startswith("_") for intro in CHARSET_INTRODUCERS) 25 | 26 | def test_charset_named_tuple(self) -> None: 27 | """Test the CharSet named tuple.""" 28 | charset = CharSet(id=1, charset="utf8", collation="utf8_general_ci") 29 | assert charset.id == 1 30 | assert charset.charset == "utf8" 31 | assert charset.collation == "utf8_general_ci" 32 | 33 | def test_mysql_supported_character_sets_with_charset(self) -> None: 34 | """Test mysql_supported_character_sets with a specific charset.""" 35 | test_charset = "utf8mb4" 36 | results = list(mysql_supported_character_sets(test_charset)) 37 | assert len(results) > 0 38 | for result in results: 39 | assert result.charset == test_charset 40 | assert isinstance(result.id, int) 41 | assert isinstance(result.collation, str) 42 | 43 | @mock.patch("mysql_to_sqlite3.mysql_utils.MYSQL_CHARACTER_SETS", [(None, None), ("utf8", "utf8_general_ci", True)]) 44 | def test_mysql_supported_character_sets_with_charset_keyerror(self) -> None: 45 | """Test handling KeyError in mysql_supported_character_sets with charset.""" 46 | # Override the MYSQL_CHARACTER_SETS behavior to raise KeyError 47 | with mock.patch("mysql_to_sqlite3.mysql_utils.MYSQL_CHARACTER_SETS") as mock_charset_sets: 48 | mock_charset_sets.__getitem__.side_effect = KeyError("Test KeyError") 49 | 50 | # This should not raise any exceptions 51 | results = list(mysql_supported_character_sets("utf8")) 52 | assert len(results) == 0 53 | 54 | @mock.patch("mysql_to_sqlite3.mysql_utils.CharacterSet") 55 | def test_mysql_supported_character_sets_no_charset(self, mock_charset_class) -> None: 56 | """Test mysql_supported_character_sets with no charset.""" 57 | # Create a custom mock class for CharacterSet that has get_supported method 58 | mock_instance = mock.MagicMock() 59 | mock_instance.get_supported.return_value = ["utf8"] 60 | mock_charset_class.return_value = mock_instance 61 | 62 | with mock.patch( 63 | "mysql_to_sqlite3.mysql_utils.MYSQL_CHARACTER_SETS", 64 | [ 65 | None, # This will be skipped due to None check 66 | ("utf8", "utf8_general_ci", True), # This will be processed 67 | ], 68 | ): 69 | results = list(mysql_supported_character_sets()) 70 | 71 | mock_instance.get_supported.assert_called_once() 72 | 73 | assert len(results) > 0 74 | charset_results = [r.charset for r in results] 75 | assert "utf8" in charset_results 76 | 77 | @mock.patch("mysql_to_sqlite3.mysql_utils.CharacterSet") 78 | def test_mysql_supported_character_sets_no_charset_keyerror(self, mock_charset_class) -> None: 79 | """Test handling KeyError in mysql_supported_character_sets without charset.""" 80 | # Setup mock to return specific values 81 | mock_instance = mock.MagicMock() 82 | mock_instance.get_supported.return_value = ["utf8"] 83 | mock_charset_class.return_value = mock_instance 84 | 85 | # Create a mock object to return either valid info or raise KeyError 86 | mock_char_sets = mock.MagicMock() 87 | mock_char_sets.__len__.return_value = 2 88 | 89 | # Make the first item None and the second one raise KeyError when accessed 90 | def getitem_side_effect(idx): 91 | if idx == 0: 92 | return (None, None, None) 93 | else: 94 | raise KeyError("Test KeyError") 95 | 96 | mock_char_sets.__getitem__.side_effect = getitem_side_effect 97 | 98 | # Patch with the mock object 99 | with mock.patch("mysql_to_sqlite3.mysql_utils.MYSQL_CHARACTER_SETS", mock_char_sets): 100 | # Should continue without raising exceptions despite KeyError 101 | results = list(mysql_supported_character_sets()) 102 | assert len(results) == 0 103 | 104 | # Verify the get_supported method was called 105 | mock_instance.get_supported.assert_called_once() 106 | 107 | def test_mysql_supported_character_sets_complete_coverage(self) -> None: 108 | """Test mysql_supported_character_sets to target specific edge cases for full coverage.""" 109 | # Test with a charset that doesn't match any entries 110 | with mock.patch( 111 | "mysql_to_sqlite3.mysql_utils.MYSQL_CHARACTER_SETS", 112 | [("utf8", "utf8_general_ci", True), None, ("latin1", "latin1_swedish_ci", True)], # Test None handling 113 | ): 114 | # Should return empty when charset doesn't match any entries 115 | results = list(mysql_supported_character_sets("non_existent_charset")) 116 | assert len(results) == 0 117 | 118 | # Should process all valid charsets when no specific charset is requested 119 | with mock.patch("mysql_to_sqlite3.mysql_utils.CharacterSet") as mock_charset_class: 120 | mock_instance = mock.MagicMock() 121 | mock_instance.get_supported.return_value = ["utf8", "latin1", "invalid_charset"] 122 | mock_charset_class.return_value = mock_instance 123 | 124 | # Test when no charset is specified - should process all entries 125 | results = list(mysql_supported_character_sets()) 126 | # Should have entries for utf8 and latin1 127 | assert len([r for r in results if r.charset in ["utf8", "latin1"]]) > 0 128 | 129 | def test_mysql_supported_character_sets_with_specific_keyerror(self) -> None: 130 | """Test mysql_supported_character_sets with specific KeyError scenarios.""" 131 | # Test the specific KeyError scenario in the first branch (with charset specified) 132 | with mock.patch( 133 | "mysql_to_sqlite3.mysql_utils.MYSQL_CHARACTER_SETS", 134 | [ 135 | None, 136 | mock.MagicMock(side_effect=KeyError("Key error in info[0]")), # Trigger KeyError on info[0] 137 | ("latin1", "latin1_swedish_ci", True), 138 | ], 139 | ): 140 | # This should not raise exceptions despite the KeyError 141 | results = list(mysql_supported_character_sets("utf8")) 142 | assert len(results) == 0 143 | 144 | @mock.patch("mysql_to_sqlite3.mysql_utils.CharacterSet") 145 | def test_mysql_supported_character_sets_with_specific_info_keyerror(self, mock_charset_class) -> None: 146 | """Test mysql_supported_character_sets with KeyError on info[1] access.""" 147 | mock_instance = mock.MagicMock() 148 | mock_instance.get_supported.return_value = ["utf8"] 149 | mock_charset_class.return_value = mock_instance 150 | 151 | # Create a special mock that will raise KeyError on accessing index 1 152 | class InfoMock: 153 | def __getitem__(self, key): 154 | if key == 0: 155 | return "utf8" 156 | elif key == 1: 157 | raise KeyError("Test KeyError on info[1]") 158 | return None 159 | 160 | # Set up MYSQL_CHARACTER_SETS with our special mock 161 | with mock.patch( 162 | "mysql_to_sqlite3.mysql_utils.MYSQL_CHARACTER_SETS", 163 | [ 164 | None, 165 | InfoMock(), # Will raise KeyError on info[1] 166 | ], 167 | ): 168 | # This should not raise exceptions despite the KeyError 169 | results = list(mysql_supported_character_sets()) 170 | assert len(results) == 0 171 | 172 | # Now test with a specific charset to cover both branches 173 | results = list(mysql_supported_character_sets("utf8")) 174 | assert len(results) == 0 175 | -------------------------------------------------------------------------------- /src/mysql_to_sqlite3/cli.py: -------------------------------------------------------------------------------- 1 | """The command line interface of MySQLtoSQLite.""" 2 | 3 | import os 4 | import sys 5 | import typing as t 6 | from datetime import datetime 7 | 8 | import click 9 | from mysql.connector import CharacterSet 10 | from tabulate import tabulate 11 | 12 | from . import MySQLtoSQLite 13 | from . import __version__ as package_version 14 | from .click_utils import OptionEatAll, prompt_password, validate_positive_integer 15 | from .debug_info import info 16 | from .mysql_utils import mysql_supported_character_sets 17 | from .sqlite_utils import CollatingSequences 18 | 19 | 20 | _copyright_header: str = f"mysql2sqlite version {package_version} Copyright (c) 2019-{datetime.now().year} Klemen Tusar" 21 | 22 | 23 | @click.command( 24 | name="mysql2sqlite", 25 | help=_copyright_header, 26 | no_args_is_help=True, 27 | epilog="For more information, visit https://github.com/techouse/mysql-to-sqlite3", 28 | ) 29 | @click.option( 30 | "-f", 31 | "--sqlite-file", 32 | type=click.Path(), 33 | default=None, 34 | help="SQLite3 database file", 35 | required=True, 36 | ) 37 | @click.option("-d", "--mysql-database", default=None, help="MySQL database name", required=True) 38 | @click.option("-u", "--mysql-user", default=None, help="MySQL user", required=True) 39 | @click.option( 40 | "-p", 41 | "--prompt-mysql-password", 42 | is_flag=True, 43 | default=False, 44 | callback=prompt_password, 45 | help="Prompt for MySQL password", 46 | ) 47 | @click.option("--mysql-password", default=None, help="MySQL password") 48 | @click.option( 49 | "-t", 50 | "--mysql-tables", 51 | type=tuple, 52 | cls=OptionEatAll, 53 | help="Transfer only these specific tables (space separated table names). " 54 | "Implies --without-foreign-keys which inhibits the transfer of foreign keys. " 55 | "Can not be used together with --exclude-mysql-tables.", 56 | ) 57 | @click.option( 58 | "-e", 59 | "--exclude-mysql-tables", 60 | type=tuple, 61 | cls=OptionEatAll, 62 | help="Transfer all tables except these specific tables (space separated table names). " 63 | "Implies --without-foreign-keys which inhibits the transfer of foreign keys. " 64 | "Can not be used together with --mysql-tables.", 65 | ) 66 | @click.option( 67 | "-T", 68 | "--mysql-views-as-tables", 69 | is_flag=True, 70 | help="Materialize MySQL VIEWs as SQLite tables (legacy behavior).", 71 | ) 72 | @click.option( 73 | "-L", 74 | "--limit-rows", 75 | type=int, 76 | callback=validate_positive_integer, 77 | default=0, 78 | help="Transfer only a limited number of rows from each table.", 79 | ) 80 | @click.option( 81 | "-C", 82 | "--collation", 83 | type=click.Choice( 84 | [ 85 | CollatingSequences.BINARY, 86 | CollatingSequences.NOCASE, 87 | CollatingSequences.RTRIM, 88 | ], 89 | case_sensitive=False, 90 | ), 91 | default=CollatingSequences.BINARY, 92 | show_default=True, 93 | help="Create datatypes of TEXT affinity using a specified collation sequence.", 94 | ) 95 | @click.option( 96 | "-K", 97 | "--prefix-indices", 98 | is_flag=True, 99 | help="Prefix indices with their corresponding tables. " 100 | "This ensures that their names remain unique across the SQLite database.", 101 | ) 102 | @click.option("-X", "--without-foreign-keys", is_flag=True, help="Do not transfer foreign keys.") 103 | @click.option( 104 | "-Z", 105 | "--without-tables", 106 | is_flag=True, 107 | help="Do not transfer tables, data only.", 108 | ) 109 | @click.option( 110 | "-W", 111 | "--without-data", 112 | is_flag=True, 113 | help="Do not transfer table data, DDL only.", 114 | ) 115 | @click.option( 116 | "-M", 117 | "--strict", 118 | is_flag=True, 119 | help="Create SQLite STRICT tables when supported.", 120 | ) 121 | @click.option("-h", "--mysql-host", default="localhost", help="MySQL host. Defaults to localhost.") 122 | @click.option("-P", "--mysql-port", type=int, default=3306, help="MySQL port. Defaults to 3306.") 123 | @click.option( 124 | "--mysql-charset", 125 | metavar="TEXT", 126 | type=click.Choice(list(CharacterSet().get_supported()), case_sensitive=False), 127 | default="utf8mb4", 128 | show_default=True, 129 | help="MySQL database and table character set", 130 | ) 131 | @click.option( 132 | "--mysql-collation", 133 | metavar="TEXT", 134 | type=click.Choice( 135 | [charset.collation for charset in mysql_supported_character_sets()], 136 | case_sensitive=False, 137 | ), 138 | default=None, 139 | help="MySQL database and table collation", 140 | ) 141 | @click.option("-S", "--skip-ssl", is_flag=True, help="Disable MySQL connection encryption.") 142 | @click.option( 143 | "-c", 144 | "--chunk", 145 | type=int, 146 | default=200000, # this default is here for performance reasons 147 | help="Chunk reading/writing SQL records", 148 | ) 149 | @click.option("-l", "--log-file", type=click.Path(), help="Log file") 150 | @click.option("--json-as-text", is_flag=True, help="Transfer JSON columns as TEXT.") 151 | @click.option( 152 | "-V", 153 | "--vacuum", 154 | is_flag=True, 155 | help="Use the VACUUM command to rebuild the SQLite database file, " 156 | "repacking it into a minimal amount of disk space", 157 | ) 158 | @click.option( 159 | "--use-buffered-cursors", 160 | is_flag=True, 161 | help="Use MySQLCursorBuffered for reading the MySQL database. This " 162 | "can be useful in situations where multiple queries, with small " 163 | "result sets, need to be combined or computed with each other.", 164 | ) 165 | @click.option("-q", "--quiet", is_flag=True, help="Quiet. Display only errors.") 166 | @click.option("--debug", is_flag=True, help="Debug mode. Will throw exceptions.") 167 | @click.version_option(message=tabulate(info(), headers=["software", "version"], tablefmt="github")) 168 | def cli( 169 | sqlite_file: t.Union[str, "os.PathLike[t.Any]"], 170 | mysql_user: str, 171 | prompt_mysql_password: bool, 172 | mysql_password: str, 173 | mysql_database: str, 174 | mysql_tables: t.Optional[t.Sequence[str]], 175 | exclude_mysql_tables: t.Optional[t.Sequence[str]], 176 | mysql_views_as_tables: bool, 177 | limit_rows: int, 178 | collation: t.Optional[str], 179 | prefix_indices: bool, 180 | without_foreign_keys: bool, 181 | without_tables: bool, 182 | without_data: bool, 183 | strict: bool, 184 | mysql_host: str, 185 | mysql_port: int, 186 | mysql_charset: str, 187 | mysql_collation: str, 188 | skip_ssl: bool, 189 | chunk: int, 190 | log_file: t.Union[str, "os.PathLike[t.Any]"], 191 | json_as_text: bool, 192 | vacuum: bool, 193 | use_buffered_cursors: bool, 194 | quiet: bool, 195 | debug: bool, 196 | ) -> None: 197 | """Transfer MySQL to SQLite using the provided CLI options.""" 198 | click.echo(_copyright_header) 199 | try: 200 | if mysql_collation: 201 | charset_collations: t.Tuple[str, ...] = tuple( 202 | cs.collation for cs in mysql_supported_character_sets(mysql_charset.lower()) 203 | ) 204 | if mysql_collation not in set(charset_collations): 205 | raise click.ClickException( 206 | f"Error: Invalid value for '--collation' of charset '{mysql_charset}': '{mysql_collation}' " 207 | f"""is not one of {"'" + "', '".join(charset_collations) + "'"}.""" 208 | ) 209 | 210 | # check if both mysql_skip_create_table and mysql_skip_transfer_data are True 211 | if without_tables and without_data: 212 | raise click.ClickException( 213 | "Error: Both -Z/--without-tables and -W/--without-data are set. There is nothing to do. Exiting..." 214 | ) 215 | 216 | if mysql_tables is not None and exclude_mysql_tables is not None: 217 | raise click.UsageError("Illegal usage: --mysql-tables and --exclude-mysql-tables are mutually exclusive!") 218 | 219 | converter = MySQLtoSQLite( 220 | sqlite_file=sqlite_file, 221 | mysql_user=mysql_user, 222 | mysql_password=mysql_password or prompt_mysql_password, 223 | mysql_database=mysql_database, 224 | mysql_tables=mysql_tables, 225 | exclude_mysql_tables=exclude_mysql_tables, 226 | views_as_views=not mysql_views_as_tables, 227 | limit_rows=limit_rows, 228 | collation=collation, 229 | prefix_indices=prefix_indices, 230 | without_foreign_keys=without_foreign_keys or bool(mysql_tables) or bool(exclude_mysql_tables), 231 | without_tables=without_tables, 232 | without_data=without_data, 233 | sqlite_strict=strict, 234 | mysql_host=mysql_host, 235 | mysql_port=mysql_port, 236 | mysql_charset=mysql_charset, 237 | mysql_collation=mysql_collation, 238 | mysql_ssl_disabled=skip_ssl, 239 | chunk=chunk, 240 | json_as_text=json_as_text, 241 | vacuum=vacuum, 242 | buffered=use_buffered_cursors, 243 | log_file=log_file, 244 | quiet=quiet, 245 | ) 246 | converter.transfer() 247 | except KeyboardInterrupt: 248 | if debug: 249 | raise 250 | click.echo("\nProcess interrupted. Exiting...") 251 | sys.exit(1) 252 | except Exception as err: # pylint: disable=W0703 253 | if debug: 254 | raise 255 | click.echo(err) 256 | sys.exit(1) 257 | -------------------------------------------------------------------------------- /.bandit.yml: -------------------------------------------------------------------------------- 1 | ### This config may optionally select a subset of tests to run or skip by 2 | ### filling out the 'tests' and 'skips' lists given below. If no tests are 3 | ### specified for inclusion then it is assumed all tests are desired. The skips 4 | ### set will remove specific tests from the include set. This can be controlled 5 | ### using the -t/-s CLI options. Note that the same test ID should not appear 6 | ### in both 'tests' and 'skips', this would be nonsensical and is detected by 7 | ### Bandit at runtime. 8 | 9 | # Available tests: 10 | # B101 : assert_used 11 | # B102 : exec_used 12 | # B103 : set_bad_file_permissions 13 | # B104 : hardcoded_bind_all_interfaces 14 | # B105 : hardcoded_password_string 15 | # B106 : hardcoded_password_funcarg 16 | # B107 : hardcoded_password_default 17 | # B108 : hardcoded_tmp_directory 18 | # B110 : try_except_pass 19 | # B112 : try_except_continue 20 | # B201 : flask_debug_true 21 | # B301 : pickle 22 | # B302 : marshal 23 | # B303 : md5 24 | # B304 : ciphers 25 | # B305 : cipher_modes 26 | # B306 : mktemp_q 27 | # B307 : eval 28 | # B308 : mark_safe 29 | # B309 : httpsconnection 30 | # B310 : urllib_urlopen 31 | # B311 : random 32 | # B312 : telnetlib 33 | # B313 : xml_bad_cElementTree 34 | # B314 : xml_bad_ElementTree 35 | # B315 : xml_bad_expatreader 36 | # B316 : xml_bad_expatbuilder 37 | # B317 : xml_bad_sax 38 | # B318 : xml_bad_minidom 39 | # B319 : xml_bad_pulldom 40 | # B320 : xml_bad_etree 41 | # B321 : ftplib 42 | # B322 : input 43 | # B323 : unverified_context 44 | # B324 : hashlib_new_insecure_functions 45 | # B325 : tempnam 46 | # B401 : import_telnetlib 47 | # B402 : import_ftplib 48 | # B403 : import_pickle 49 | # B404 : import_subprocess 50 | # B405 : import_xml_etree 51 | # B406 : import_xml_sax 52 | # B407 : import_xml_expat 53 | # B408 : import_xml_minidom 54 | # B409 : import_xml_pulldom 55 | # B410 : import_lxml 56 | # B411 : import_xmlrpclib 57 | # B412 : import_httpoxy 58 | # B413 : import_pycrypto 59 | # B501 : request_with_no_cert_validation 60 | # B502 : ssl_with_bad_version 61 | # B503 : ssl_with_bad_defaults 62 | # B504 : ssl_with_no_version 63 | # B505 : weak_cryptographic_key 64 | # B506 : yaml_load 65 | # B507 : ssh_no_host_key_verification 66 | # B601 : paramiko_calls 67 | # B602 : subprocess_popen_with_shell_equals_true 68 | # B603 : subprocess_without_shell_equals_true 69 | # B604 : any_other_function_with_shell_equals_true 70 | # B605 : start_process_with_a_shell 71 | # B606 : start_process_with_no_shell 72 | # B607 : start_process_with_partial_path 73 | # B608 : hardcoded_sql_expressions 74 | # B609 : linux_commands_wildcard_injection 75 | # B610 : django_extra_used 76 | # B611 : django_rawsql_used 77 | # B701 : jinja2_autoescape_false 78 | # B702 : use_of_mako_templates 79 | # B703 : django_mark_safe 80 | 81 | # (optional) list included test IDs here, eg '[B101, B406]': 82 | tests: 83 | 84 | # (optional) list skipped test IDs here, eg '[B101, B406]': 85 | skips: 86 | - B404 87 | - B603 88 | - B607 89 | - B608 90 | 91 | ### (optional) plugin settings - some test plugins require configuration data 92 | ### that may be given here, per-plugin. All bandit test plugins have a built in 93 | ### set of sensible defaults and these will be used if no configuration is 94 | ### provided. It is not necessary to provide settings for every (or any) plugin 95 | ### if the defaults are acceptable. 96 | 97 | any_other_function_with_shell_equals_true: 98 | no_shell: 99 | - os.execl 100 | - os.execle 101 | - os.execlp 102 | - os.execlpe 103 | - os.execv 104 | - os.execve 105 | - os.execvp 106 | - os.execvpe 107 | - os.spawnl 108 | - os.spawnle 109 | - os.spawnlp 110 | - os.spawnlpe 111 | - os.spawnv 112 | - os.spawnve 113 | - os.spawnvp 114 | - os.spawnvpe 115 | - os.startfile 116 | shell: 117 | - os.system 118 | - os.popen 119 | - os.popen2 120 | - os.popen3 121 | - os.popen4 122 | - popen2.popen2 123 | - popen2.popen3 124 | - popen2.popen4 125 | - popen2.Popen3 126 | - popen2.Popen4 127 | - commands.getoutput 128 | - commands.getstatusoutput 129 | subprocess: 130 | - subprocess.Popen 131 | - subprocess.call 132 | - subprocess.check_call 133 | - subprocess.check_output 134 | - subprocess.run 135 | hardcoded_tmp_directory: 136 | tmp_dirs: 137 | - /tmp 138 | - /var/tmp 139 | - /dev/shm 140 | linux_commands_wildcard_injection: 141 | no_shell: 142 | - os.execl 143 | - os.execle 144 | - os.execlp 145 | - os.execlpe 146 | - os.execv 147 | - os.execve 148 | - os.execvp 149 | - os.execvpe 150 | - os.spawnl 151 | - os.spawnle 152 | - os.spawnlp 153 | - os.spawnlpe 154 | - os.spawnv 155 | - os.spawnve 156 | - os.spawnvp 157 | - os.spawnvpe 158 | - os.startfile 159 | shell: 160 | - os.system 161 | - os.popen 162 | - os.popen2 163 | - os.popen3 164 | - os.popen4 165 | - popen2.popen2 166 | - popen2.popen3 167 | - popen2.popen4 168 | - popen2.Popen3 169 | - popen2.Popen4 170 | - commands.getoutput 171 | - commands.getstatusoutput 172 | subprocess: 173 | - subprocess.Popen 174 | - subprocess.call 175 | - subprocess.check_call 176 | - subprocess.check_output 177 | - subprocess.run 178 | ssl_with_bad_defaults: 179 | bad_protocol_versions: 180 | - PROTOCOL_SSLv2 181 | - SSLv2_METHOD 182 | - SSLv23_METHOD 183 | - PROTOCOL_SSLv3 184 | - PROTOCOL_TLSv1 185 | - SSLv3_METHOD 186 | - TLSv1_METHOD 187 | ssl_with_bad_version: 188 | bad_protocol_versions: 189 | - PROTOCOL_SSLv2 190 | - SSLv2_METHOD 191 | - SSLv23_METHOD 192 | - PROTOCOL_SSLv3 193 | - PROTOCOL_TLSv1 194 | - SSLv3_METHOD 195 | - TLSv1_METHOD 196 | start_process_with_a_shell: 197 | no_shell: 198 | - os.execl 199 | - os.execle 200 | - os.execlp 201 | - os.execlpe 202 | - os.execv 203 | - os.execve 204 | - os.execvp 205 | - os.execvpe 206 | - os.spawnl 207 | - os.spawnle 208 | - os.spawnlp 209 | - os.spawnlpe 210 | - os.spawnv 211 | - os.spawnve 212 | - os.spawnvp 213 | - os.spawnvpe 214 | - os.startfile 215 | shell: 216 | - os.system 217 | - os.popen 218 | - os.popen2 219 | - os.popen3 220 | - os.popen4 221 | - popen2.popen2 222 | - popen2.popen3 223 | - popen2.popen4 224 | - popen2.Popen3 225 | - popen2.Popen4 226 | - commands.getoutput 227 | - commands.getstatusoutput 228 | subprocess: 229 | - subprocess.Popen 230 | - subprocess.call 231 | - subprocess.check_call 232 | - subprocess.check_output 233 | - subprocess.run 234 | start_process_with_no_shell: 235 | no_shell: 236 | - os.execl 237 | - os.execle 238 | - os.execlp 239 | - os.execlpe 240 | - os.execv 241 | - os.execve 242 | - os.execvp 243 | - os.execvpe 244 | - os.spawnl 245 | - os.spawnle 246 | - os.spawnlp 247 | - os.spawnlpe 248 | - os.spawnv 249 | - os.spawnve 250 | - os.spawnvp 251 | - os.spawnvpe 252 | - os.startfile 253 | shell: 254 | - os.system 255 | - os.popen 256 | - os.popen2 257 | - os.popen3 258 | - os.popen4 259 | - popen2.popen2 260 | - popen2.popen3 261 | - popen2.popen4 262 | - popen2.Popen3 263 | - popen2.Popen4 264 | - commands.getoutput 265 | - commands.getstatusoutput 266 | subprocess: 267 | - subprocess.Popen 268 | - subprocess.call 269 | - subprocess.check_call 270 | - subprocess.check_output 271 | - subprocess.run 272 | start_process_with_partial_path: 273 | no_shell: 274 | - os.execl 275 | - os.execle 276 | - os.execlp 277 | - os.execlpe 278 | - os.execv 279 | - os.execve 280 | - os.execvp 281 | - os.execvpe 282 | - os.spawnl 283 | - os.spawnle 284 | - os.spawnlp 285 | - os.spawnlpe 286 | - os.spawnv 287 | - os.spawnve 288 | - os.spawnvp 289 | - os.spawnvpe 290 | - os.startfile 291 | shell: 292 | - os.system 293 | - os.popen 294 | - os.popen2 295 | - os.popen3 296 | - os.popen4 297 | - popen2.popen2 298 | - popen2.popen3 299 | - popen2.popen4 300 | - popen2.Popen3 301 | - popen2.Popen4 302 | - commands.getoutput 303 | - commands.getstatusoutput 304 | subprocess: 305 | - subprocess.Popen 306 | - subprocess.call 307 | - subprocess.check_call 308 | - subprocess.check_output 309 | - subprocess.run 310 | subprocess_popen_with_shell_equals_true: 311 | no_shell: 312 | - os.execl 313 | - os.execle 314 | - os.execlp 315 | - os.execlpe 316 | - os.execv 317 | - os.execve 318 | - os.execvp 319 | - os.execvpe 320 | - os.spawnl 321 | - os.spawnle 322 | - os.spawnlp 323 | - os.spawnlpe 324 | - os.spawnv 325 | - os.spawnve 326 | - os.spawnvp 327 | - os.spawnvpe 328 | - os.startfile 329 | shell: 330 | - os.system 331 | - os.popen 332 | - os.popen2 333 | - os.popen3 334 | - os.popen4 335 | - popen2.popen2 336 | - popen2.popen3 337 | - popen2.popen4 338 | - popen2.Popen3 339 | - popen2.Popen4 340 | - commands.getoutput 341 | - commands.getstatusoutput 342 | subprocess: 343 | - subprocess.Popen 344 | - subprocess.call 345 | - subprocess.check_call 346 | - subprocess.check_output 347 | - subprocess.run 348 | subprocess_without_shell_equals_true: 349 | no_shell: 350 | - os.execl 351 | - os.execle 352 | - os.execlp 353 | - os.execlpe 354 | - os.execv 355 | - os.execve 356 | - os.execvp 357 | - os.execvpe 358 | - os.spawnl 359 | - os.spawnle 360 | - os.spawnlp 361 | - os.spawnlpe 362 | - os.spawnv 363 | - os.spawnve 364 | - os.spawnvp 365 | - os.spawnvpe 366 | - os.startfile 367 | shell: 368 | - os.system 369 | - os.popen 370 | - os.popen2 371 | - os.popen3 372 | - os.popen4 373 | - popen2.popen2 374 | - popen2.popen3 375 | - popen2.popen4 376 | - popen2.Popen3 377 | - popen2.Popen4 378 | - commands.getoutput 379 | - commands.getstatusoutput 380 | subprocess: 381 | - subprocess.Popen 382 | - subprocess.call 383 | - subprocess.check_call 384 | - subprocess.check_output 385 | - subprocess.run 386 | try_except_continue: 387 | check_typed_exception: false 388 | try_except_pass: 389 | check_typed_exception: false 390 | weak_cryptographic_key: 391 | weak_key_size_dsa_high: 1024 392 | weak_key_size_dsa_medium: 2048 393 | weak_key_size_ec_high: 160 394 | weak_key_size_ec_medium: 224 395 | weak_key_size_rsa_high: 1024 396 | weak_key_size_rsa_medium: 2048 397 | 398 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # 2.5.5 2 | 3 | * [FIX] removed the Hatch `sources = ["src"]` rewrite so `sdists` now keep the `src/mysql_to_sqlite3` tree, letting 4 | wheels built from the sdist include the actual package 5 | 6 | # 2.5.4 7 | 8 | * [FIX] enhance handling of MySQL string literals and default values 9 | 10 | # 2.5.3 11 | 12 | * [CHORE] add Python 3.14 support 13 | 14 | # 2.5.2 15 | 16 | * [FEAT] transfer MySQL views as native SQLite views 17 | * [CHORE] improve robustness of data transfer by leveraging sqlglot for SQL translation 18 | 19 | # 2.5.1 20 | 21 | * [FIX] prevent non-unique SQLite indices from being quietly omitted 22 | 23 | # 2.5.0 24 | 25 | * [FEAT] add support for creating SQLite STRICT tables via `-M/--strict` CLI switch 26 | 27 | # 2.4.5 28 | 29 | * [FIX] fix importing `typing_extensions` on Python >= 3.11 30 | 31 | # 2.4.4 32 | 33 | * [FIX] fix pyproject.toml build sources and specify package inclusion for sdist and wheel 34 | 35 | # 2.4.3 36 | 37 | * [CHORE] update pyproject.toml to improve metadata, dependency management, and wheel build config 38 | 39 | # 2.4.2 40 | 41 | * [FIX] handle `curtime()`, `curdate()`, `current_timestamp()`, and `now()` MariaDB-specific translations in default 42 | values 43 | * [CHORE] add MariaDB 11.8 LTS tests 44 | * [CHORE] drop redundant MariaDB 10.1, 10.2, 10.3, 10.4 and 10.5 tests 45 | 46 | # 2.4.1 47 | 48 | * [FIX] fix passwordless login 49 | 50 | # 2.4.0 51 | 52 | * [CHORE] drop support for Python 3.8 53 | 54 | # 2.3.0 55 | 56 | * [FEAT] add MySQL 8.4 and MariaDB 11.4 support 57 | 58 | # 2.2.2 59 | 60 | * [FIX] use `dateutil.parse` to parse SQLite dates 61 | 62 | # 2.2.1 63 | 64 | * [FIX] fix transferring composite primary keys when AUTOINCREMENT present 65 | 66 | # 2.2.0 67 | 68 | * [FEAT] add --without-tables option 69 | 70 | # 2.1.12 71 | 72 | * [CHORE] update MySQL Connector/Python to 8.4.0 73 | * [CHORE] add Sphinx documentation 74 | 75 | # 2.1.11 76 | 77 | * [CHORE] migrate package from flat layout to src layout 78 | 79 | # 2.1.10 80 | 81 | * [FEAT] add support for AUTOINCREMENT 82 | 83 | # 2.1.9 84 | 85 | * [FIX] pin MySQL Connector/Python to 8.3.0 86 | 87 | # 2.1.8 88 | 89 | * [FIX] ensure index names do not collide with table names 90 | 91 | # 2.1.7 92 | 93 | * [FIX] use more precise foreign key constraints 94 | 95 | # 2.1.6 96 | 97 | * [FEAT] build both linux/amd64 and linux/arm64 Docker images 98 | 99 | # 2.1.5 100 | 101 | * [CHORE] fix Docker package publishing from Github Workflow 102 | 103 | # 2.1.4 104 | 105 | * [FIX] fix invalid column_type error message 106 | 107 | # 2.1.3 108 | 109 | * [CHORE] maintenance release to publish first containerized release 110 | 111 | # 2.1.2 112 | 113 | * [FIX] throw more comprehensive error messages when translating column types 114 | 115 | # 2.1.1 116 | 117 | * [CHORE] add support for Python 3.12 118 | * [CHORE] bump minimum version of MySQL Connector/Python to 8.2.0 119 | 120 | # 2.1.0 121 | 122 | * [CHORE] drop support for Python 3.7 123 | 124 | # 2.0.3 125 | 126 | * [FIX] import MySQLConnectionAbstract instead of concrete implementations 127 | 128 | # 2.0.2 129 | 130 | * [FIX] properly import CMySQLConnection 131 | 132 | # 2.0.1 133 | 134 | * [FEAT] add support for MySQL character set introducers in DEFAULT clause 135 | 136 | # 2.0.0 137 | 138 | * [CHORE] drop support for Python 2.7, 3.5 and 3.6 139 | * [CHORE] migrate pytest.ini configuration into pyproject.toml 140 | * [CHORE] migrate from setuptools to hatch / hatchling 141 | * [CHORE] update dependencies 142 | * [CHORE] add types 143 | * [CHORE] add types to tests 144 | * [CHORE] update dependencies 145 | * [CHORE] use f-strings where appropriate 146 | 147 | # 1.4.18 148 | 149 | * [CHORE] update dependencies 150 | * [CHORE] use [black](https://github.com/psf/black) and [isort](https://github.com/PyCQA/isort) in tox linters 151 | 152 | # 1.4.17 153 | 154 | * [CHORE] migrate from setup.py to pyproject.toml 155 | * [CHORE] update the publishing workflow 156 | 157 | # 1.4.16 158 | 159 | * [CHORE] add MariaDB 10.11 CI tests 160 | * [CHORE] add Python 3.11 support 161 | 162 | # 1.4.15 163 | 164 | * [FIX] fix BLOB default value 165 | * [CHORE] remove CI tests for Python 3.5, 3.6, add tests for Python 3.11 166 | 167 | # 1.4.14 168 | 169 | * [FIX] pin mysql-connector-python to <8.0.30 170 | * [CHORE] update CI actions/checkout to v3 171 | * [CHORE] update CI actions/setup-python to v4 172 | * [CHORE] update CI actions/cache to v3 173 | * [CHORE] update CI github/codeql-action/init to v2 174 | * [CHORE] update CI github/codeql-action/analyze to v2 175 | 176 | # 1.4.13 177 | 178 | * [FEAT] add option to exclude specific MySQL tables 179 | * [CHORE] update CI codecov/codecov-action to v2 180 | 181 | # 1.4.12 182 | 183 | * [FIX] fix SQLite convert_date converter 184 | * [CHORE] update tests 185 | 186 | # 1.4.11 187 | 188 | * [FIX] pin python-slugify to <6.0.0 189 | 190 | # 1.4.10 191 | 192 | * [FEAT] add feature to transfer tables without any data (DDL only) 193 | 194 | # 1.4.9 195 | 196 | * [CHORE] add Python 3.10 support 197 | * [CHORE] add Python 3.10 tests 198 | 199 | # 1.4.8 200 | 201 | * [FEAT] transfer JSON columns as JSON 202 | 203 | # 1.4.7 204 | 205 | * [CHORE] add experimental tests for Python 3.10-dev 206 | * [CHORE] add tests for MariaDB 10.6 207 | 208 | # 1.4.6 209 | 210 | * [FIX] pin Click to <8.0 211 | 212 | # 1.4.5 213 | 214 | * [FEAT] add -K, --prefix-indices CLI option to prefix indices with table names. This used to be the default behavior 215 | until now. To keep the old behavior simply use this CLI option. 216 | 217 | # 1.4.4 218 | 219 | * [FEAT] add --limit-rows CLI option 220 | * [FEAT] add --collation CLI option to specify SQLite collation sequence 221 | 222 | # 1.4.3 223 | 224 | * [FIX] pin python-tabulate to <0.8.6 for Python 3.4 or less 225 | * [FIX] pin python-slugify to <5.0.0 for Python 3.5 or less 226 | * [FIX] pin Click to 7.x for Python 3.5 or less 227 | 228 | # 1.4.2 229 | 230 | * [FIX] fix default column value not getting converted 231 | 232 | # 1.4.1 233 | 234 | * [FIX] get table list error when Click package is 8.0+ 235 | 236 | # 1.4.0 237 | 238 | * [FEAT] add password prompt. This changes the default behavior of -p 239 | * [FEAT] add option to disable MySQL connection encryption 240 | * [FEAT] add non-chunked progress bar 241 | * [FIX] pin mysql-connector-python to <8.0.24 for Python 3.5 or lower 242 | * [FIX] require sqlalchemy <1.4.0 to make compatible with sqlalchemy-utils 243 | 244 | # 1.3.8 245 | 246 | * [FIX] some MySQL integer column definitions result in TEXT fields in sqlite3 247 | * [FIX] fix CI tests 248 | 249 | # 1.3.7 250 | 251 | * [CHORE] transition from Travis CI to GitHub Actions 252 | 253 | # 1.3.6 254 | 255 | * [FIX] Fix Python 3.9 tests 256 | 257 | # 1.3.5 258 | 259 | * [FIX] add IF NOT EXISTS to the CREATE INDEX SQL command 260 | * [CHORE] add Python 3.9 CI tests 261 | 262 | # 1.3.4 263 | 264 | * [FEAT] add --quiet option 265 | 266 | # 1.3.3 267 | 268 | * [FIX] test for mysql client more gracefully 269 | 270 | # 1.3.2 271 | 272 | * [FEAT] simpler access to the debug version info using the --version switch 273 | * [FEAT] add debug_info module to be used in bug reports 274 | * [CHORE] remove PyPy and PyPy3 CI tests 275 | * [CHORE] add tabulate to development dependencies 276 | * [CHORE] use pytest fixture fom Faker 4.1.0 in Python 3 tests 277 | * [CHORE] omit debug_info.py in coverage reports 278 | 279 | # 1.3.1 280 | 281 | * [FIX] fix information_schema issue introduced with MySQL 8.0.21 282 | * [FIX] fix MySQL 8 bug where column types would sometimes be returned as bytes instead of strings 283 | * [FIX] sqlalchemy-utils dropped Python 2.7 support in v0.36.7 284 | * [CHORE] use MySQL Client instead of PyMySQL in tests 285 | * [CHORE] add MySQL version output to CI tests 286 | * [CHORE] add Python 3.9 to the CI tests 287 | * [CHORE] add MariaDB 10.5 to the CI tests 288 | * [CHORE] remove Python 2.7 from allowed CI test failures 289 | * [CHORE] use Ubuntu Bionic instead of Ubuntu Xenial in CI tests 290 | * [CHORE] use Ubuntu Xenial only for MariaDB 10.4 CI tests 291 | * [CHORE] test legacy databases in CI tests 292 | 293 | # 1.3.0 294 | 295 | * [FEAT] add option to transfer only specific tables using -t 296 | * [CHORE] add tests for transferring only certain tables 297 | 298 | # 1.2.11 299 | 300 | * [FIX] duplicate foreign keys 301 | 302 | # 1.2.10 303 | 304 | * [FIX] properly escape SQLite index names 305 | * [FIX] fix SQLite global index name scoping 306 | * [CHORE] test the successful transfer of an unorthodox table name 307 | * [CHORE] test the successful transfer of indices with same names 308 | 309 | # 1.2.9 310 | 311 | * [FIX] differentiate better between MySQL and SQLite errors 312 | * [CHORE] add Python 3.8 and 3.8-dev test build 313 | 314 | # 1.2.8 315 | 316 | * [CHORE] add support for Python 3.8 317 | * [CHORE] update mysql-connector-python to a minimum version of 8.0.18 to support Python 3.8 318 | * [CHORE] update development dependencies 319 | * [CHORE] add [bandit](https://github.com/PyCQA/bandit) tests 320 | 321 | # 1.2.7 322 | 323 | * [FEAT] transfer unique indices 324 | * [FIX] improve index transport 325 | * [CHORE] test transfer of indices 326 | 327 | # 1.2.6 328 | 329 | * [CHORE] include tests in the PyPI package 330 | 331 | # 1.2.5 332 | 333 | * [FEAT] transfer foreign keys 334 | * [CHORE] removed duplicate import in test database models 335 | 336 | # 1.2.4 337 | 338 | * [CHORE] reformat MySQLtoSQLite constructor 339 | * [CHORE] reformat translator function 340 | * [CHORE] add more tests 341 | 342 | # 1.2.3 343 | 344 | * [CHORE] add more tests 345 | 346 | # 1.2.2 347 | 348 | * [CHORE] refactor package 349 | * [CHORE] fix CI tests 350 | * [CHORE] add linter rules 351 | 352 | # 1.2.1 353 | 354 | * [FEAT] add Python 2.7 support 355 | 356 | # 1.2.0 357 | 358 | * [CHORE] add CI tests 359 | * [CHORE] achieve 100% test coverage 360 | 361 | # 1.1.2 362 | 363 | * [FIX] fix error of transferring tables without primary keys 364 | * [FIX] fix error of transferring empty tables 365 | 366 | # 1.1.1 367 | 368 | * [FEAT] add option to use MySQLCursorBuffered cursors 369 | * [FEAT] add MySQL port 370 | * [FEAT] update --help hints 371 | * [FIX] fix slugify import 372 | * [FIX] cursor error 373 | 374 | # 1.1.0 375 | 376 | * [FEAT] add VACUUM option 377 | 378 | # 1.0.0 379 | 380 | Initial commit 381 | -------------------------------------------------------------------------------- /tests/conftest.py: -------------------------------------------------------------------------------- 1 | import json 2 | import os 3 | import socket 4 | import typing as t 5 | from codecs import open 6 | from contextlib import contextmanager 7 | from os.path import abspath, dirname, isfile, join 8 | from pathlib import Path 9 | from random import choice 10 | from string import ascii_lowercase, ascii_uppercase, digits 11 | from time import sleep 12 | 13 | import docker 14 | import mysql.connector 15 | import pytest 16 | from _pytest._py.path import LocalPath 17 | from _pytest.config import Config 18 | from _pytest.config.argparsing import Parser 19 | from _pytest.legacypath import TempdirFactory 20 | from click.testing import CliRunner 21 | from docker import DockerClient 22 | from docker.errors import NotFound 23 | from docker.models.containers import Container 24 | from faker import Faker 25 | from mysql.connector import MySQLConnection, errorcode 26 | from mysql.connector.connection_cext import CMySQLConnection 27 | from mysql.connector.pooling import PooledMySQLConnection 28 | from requests import HTTPError 29 | from sqlalchemy.exc import IntegrityError 30 | from sqlalchemy.orm import Session 31 | from sqlalchemy_utils import database_exists, drop_database 32 | 33 | from . import database, factories, models 34 | 35 | 36 | def pytest_addoption(parser: "Parser"): 37 | parser.addoption( 38 | "--mysql-user", 39 | dest="mysql_user", 40 | default="tester", 41 | help="MySQL user. Defaults to 'tester'.", 42 | ) 43 | 44 | parser.addoption( 45 | "--mysql-password", 46 | dest="mysql_password", 47 | default="testpass", 48 | help="MySQL password. Defaults to 'testpass'.", 49 | ) 50 | 51 | parser.addoption( 52 | "--mysql-database", 53 | dest="mysql_database", 54 | default="test_db", 55 | help="MySQL database name. Defaults to 'test_db'.", 56 | ) 57 | 58 | parser.addoption( 59 | "--mysql-host", 60 | dest="mysql_host", 61 | default="0.0.0.0", 62 | help="Test against a MySQL server running on this host. Defaults to '0.0.0.0'.", 63 | ) 64 | 65 | parser.addoption( 66 | "--mysql-port", 67 | dest="mysql_port", 68 | type=int, 69 | default=None, 70 | help="The TCP port of the MySQL server.", 71 | ) 72 | 73 | parser.addoption( 74 | "--no-docker", 75 | dest="use_docker", 76 | default=True, 77 | action="store_false", 78 | help="Do not use a Docker MySQL image to run the tests. " 79 | "If you decide to use this switch you will have to use a physical MySQL server.", 80 | ) 81 | 82 | parser.addoption( 83 | "--docker-mysql-image", 84 | dest="docker_mysql_image", 85 | default="mysql:latest", 86 | help="Run the tests against a specific MySQL Docker image. Defaults to mysql:latest. " 87 | "Check all supported versions here https://hub.docker.com/_/mysql", 88 | ) 89 | 90 | 91 | @pytest.fixture(scope="session", autouse=True) 92 | def cleanup_hanged_docker_containers() -> None: 93 | try: 94 | client: DockerClient = docker.from_env() 95 | for container in client.containers.list(): 96 | if container.name == "pytest_mysql_to_sqlite3": 97 | container.kill() 98 | break 99 | except Exception: 100 | pass 101 | 102 | 103 | def pytest_keyboard_interrupt() -> None: 104 | try: 105 | client: DockerClient = docker.from_env() 106 | for container in client.containers.list(): 107 | if container.name == "pytest_mysql_to_sqlite3": 108 | container.kill() 109 | break 110 | except Exception: 111 | pass 112 | 113 | 114 | class Helpers: 115 | @staticmethod 116 | @contextmanager 117 | def not_raises(exception: t.Type[Exception]) -> t.Generator: 118 | try: 119 | yield 120 | except exception: 121 | raise pytest.fail(f"DID RAISE {exception}") 122 | 123 | @staticmethod 124 | @contextmanager 125 | def session_scope(db: database.Database) -> t.Generator: 126 | """Provide a transactional scope around a series of operations.""" 127 | session: Session = db.Session() 128 | try: 129 | yield session 130 | session.commit() 131 | except Exception: 132 | session.rollback() 133 | raise 134 | finally: 135 | session.close() 136 | 137 | 138 | @pytest.fixture 139 | def helpers() -> t.Type[Helpers]: 140 | return Helpers 141 | 142 | 143 | @pytest.fixture() 144 | def sqlite_database(tmpdir: LocalPath) -> t.Union[str, Path, "os.PathLike[t.Any]"]: 145 | db_name: str = "".join(choice(ascii_uppercase + ascii_lowercase + digits) for _ in range(32)) 146 | return Path(tmpdir.join(Path(f"{db_name}.sqlite3"))) 147 | 148 | 149 | def is_port_in_use(port: int, host: str = "0.0.0.0") -> bool: 150 | with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: 151 | return s.connect_ex((host, port)) == 0 152 | 153 | 154 | class MySQLCredentials(t.NamedTuple): 155 | """MySQL credentials.""" 156 | 157 | user: str 158 | password: str 159 | host: str 160 | port: int 161 | database: str 162 | 163 | 164 | @pytest.fixture(scope="session") 165 | def mysql_credentials(pytestconfig: Config) -> MySQLCredentials: 166 | db_credentials_file: str = abspath(join(dirname(__file__), "db_credentials.json")) 167 | if isfile(db_credentials_file): 168 | with open(db_credentials_file, "r", "utf-8") as fh: 169 | db_credentials: t.Dict[str, t.Any] = json.load(fh) 170 | return MySQLCredentials( 171 | user=db_credentials["mysql_user"], 172 | password=db_credentials["mysql_password"], 173 | database=db_credentials["mysql_database"], 174 | host=db_credentials["mysql_host"], 175 | port=db_credentials["mysql_port"], 176 | ) 177 | 178 | port: int = pytestconfig.getoption("mysql_port") or 3306 179 | if pytestconfig.getoption("use_docker"): 180 | while is_port_in_use(port, pytestconfig.getoption("mysql_host")): 181 | if port >= 2**16 - 1: 182 | pytest.fail(f"No ports appear to be available on the host {pytestconfig.getoption('mysql_host')}") 183 | port += 1 184 | 185 | return MySQLCredentials( 186 | user=pytestconfig.getoption("mysql_user") or "tester", 187 | password=pytestconfig.getoption("mysql_password") or "testpass", 188 | database=pytestconfig.getoption("mysql_database") or "test_db", 189 | host=pytestconfig.getoption("mysql_host") or "0.0.0.0", 190 | port=port, 191 | ) 192 | 193 | 194 | @pytest.fixture(scope="session") 195 | def mysql_instance(mysql_credentials: MySQLCredentials, pytestconfig: Config) -> t.Iterator[MySQLConnection]: 196 | container: t.Optional[Container] = None 197 | mysql_connection: t.Optional[t.Union[PooledMySQLConnection, MySQLConnection, CMySQLConnection]] = None 198 | mysql_available: bool = False 199 | mysql_connection_retries: int = 15 # failsafe 200 | 201 | db_credentials_file = abspath(join(dirname(__file__), "db_credentials.json")) 202 | if isfile(db_credentials_file): 203 | use_docker = False 204 | else: 205 | use_docker = pytestconfig.getoption("use_docker") 206 | 207 | if use_docker: 208 | """Connecting to a MySQL server within a Docker container is quite tricky :P 209 | Read more on the issue here https://hub.docker.com/_/mysql#no-connections-until-mysql-init-completes 210 | """ 211 | try: 212 | client = docker.from_env() 213 | except Exception as err: 214 | pytest.fail(str(err)) 215 | 216 | docker_mysql_image = pytestconfig.getoption("docker_mysql_image") or "mysql:latest" 217 | 218 | if not any(docker_mysql_image in image.tags for image in client.images.list()): 219 | print(f"Attempting to download Docker image {docker_mysql_image}'") 220 | try: 221 | client.images.pull(docker_mysql_image) 222 | except (HTTPError, NotFound) as err: 223 | pytest.fail(str(err)) 224 | 225 | container = client.containers.run( 226 | image=docker_mysql_image, 227 | name="pytest_mysql_to_sqlite3", 228 | ports={"3306/tcp": (mysql_credentials.host, f"{mysql_credentials.port}/tcp")}, 229 | environment={ 230 | "MYSQL_RANDOM_ROOT_PASSWORD": "yes", 231 | "MYSQL_USER": mysql_credentials.user, 232 | "MYSQL_PASSWORD": mysql_credentials.password, 233 | "MYSQL_DATABASE": mysql_credentials.database, 234 | }, 235 | command=[ 236 | "--character-set-server=utf8mb4", 237 | "--collation-server=utf8mb4_unicode_ci", 238 | ], 239 | detach=True, 240 | auto_remove=True, 241 | ) 242 | 243 | while not mysql_available and mysql_connection_retries > 0: 244 | try: 245 | mysql_connection = mysql.connector.connect( 246 | user=mysql_credentials.user, 247 | password=mysql_credentials.password, 248 | host=mysql_credentials.host, 249 | port=mysql_credentials.port, 250 | charset="utf8mb4", 251 | collation="utf8mb4_unicode_ci", 252 | ) 253 | except mysql.connector.Error as err: 254 | if err.errno == errorcode.CR_SERVER_LOST: 255 | # sleep for two seconds and retry the connection 256 | sleep(2) 257 | else: 258 | raise 259 | finally: 260 | mysql_connection_retries -= 1 261 | if mysql_connection and mysql_connection.is_connected(): 262 | mysql_available = True 263 | mysql_connection.close() 264 | else: 265 | if not mysql_available and mysql_connection_retries <= 0: 266 | raise ConnectionAbortedError("Maximum MySQL connection retries exhausted! Are you sure MySQL is running?") 267 | 268 | yield # type: ignore[misc] 269 | 270 | if use_docker and container is not None: 271 | container.kill() 272 | 273 | 274 | @pytest.fixture(scope="session") 275 | def mysql_database( 276 | tmpdir_factory: TempdirFactory, 277 | mysql_instance: MySQLConnection, 278 | mysql_credentials: MySQLCredentials, 279 | _session_faker: Faker, 280 | ) -> t.Iterator[database.Database]: 281 | temp_image_dir: LocalPath = tmpdir_factory.mktemp("images") 282 | 283 | db: database.Database = database.Database( 284 | f"mysql+mysqldb://{mysql_credentials.user}:{mysql_credentials.password}@{mysql_credentials.host}:{mysql_credentials.port}/{mysql_credentials.database}" 285 | ) 286 | 287 | with Helpers.session_scope(db) as session: 288 | for _ in range(_session_faker.pyint(min_value=12, max_value=24)): 289 | article: models.Article = factories.ArticleFactory() 290 | article.authors.append(factories.AuthorFactory()) 291 | article.tags.append(factories.TagFactory()) 292 | article.misc.append(factories.MiscFactory()) 293 | for _ in range(_session_faker.pyint(min_value=1, max_value=4)): 294 | article.images.append( 295 | factories.ImageFactory( 296 | path=join( 297 | str(temp_image_dir), 298 | _session_faker.year(), 299 | _session_faker.month(), 300 | _session_faker.day_of_month(), 301 | _session_faker.file_name(extension="jpg"), 302 | ) 303 | ) 304 | ) 305 | session.add(article) 306 | 307 | for _ in range(_session_faker.pyint(min_value=12, max_value=24)): 308 | session.add(factories.CrazyNameFactory()) 309 | try: 310 | session.commit() 311 | except IntegrityError: 312 | session.rollback() 313 | 314 | yield db 315 | 316 | if database_exists(db.engine.url): 317 | drop_database(db.engine.url) 318 | 319 | 320 | @pytest.fixture() 321 | def cli_runner() -> t.Iterator[CliRunner]: 322 | yield CliRunner() 323 | -------------------------------------------------------------------------------- /.github/workflows/test.yml: -------------------------------------------------------------------------------- 1 | name: Test 2 | 3 | on: 4 | push: 5 | branches: 6 | - master 7 | pull_request: 8 | branches: 9 | - master 10 | workflow_call: 11 | defaults: 12 | run: 13 | shell: bash 14 | permissions: read-all 15 | concurrency: 16 | group: test-${{ github.ref }} 17 | cancel-in-progress: true 18 | 19 | jobs: 20 | analyze: 21 | name: "Analyze" 22 | runs-on: ubuntu-latest 23 | steps: 24 | - uses: actions/checkout@v6 25 | - name: Set up Python 26 | uses: actions/setup-python@v6 27 | with: 28 | python-version: "3.x" 29 | - name: Install dependencies 30 | run: | 31 | python3 -m pip install --upgrade pip 32 | pip install -r requirements_dev.txt 33 | - name: Run static analysis 34 | run: tox -e linters 35 | test: 36 | name: "Test" 37 | needs: analyze 38 | runs-on: ubuntu-latest 39 | strategy: 40 | fail-fast: false 41 | max-parallel: 8 42 | matrix: 43 | include: 44 | - toxenv: "python3.9" 45 | db: "mariadb:5.5" 46 | legacy_db: 1 47 | experimental: true 48 | py: "3.9" 49 | 50 | - toxenv: "python3.10" 51 | db: "mariadb:5.5" 52 | legacy_db: 1 53 | experimental: true 54 | py: "3.10" 55 | 56 | - toxenv: "python3.11" 57 | db: "mariadb:5.5" 58 | legacy_db: 1 59 | experimental: true 60 | py: "3.11" 61 | 62 | - toxenv: "python3.12" 63 | db: "mariadb:5.5" 64 | legacy_db: 1 65 | experimental: true 66 | py: "3.12" 67 | 68 | - toxenv: "python3.13" 69 | db: "mariadb:5.5" 70 | legacy_db: 1 71 | experimental: true 72 | py: "3.13" 73 | 74 | - toxenv: "python3.14" 75 | db: "mariadb:5.5" 76 | legacy_db: 1 77 | experimental: true 78 | py: "3.14" 79 | 80 | - toxenv: "python3.9" 81 | db: "mariadb:10.0" 82 | legacy_db: 1 83 | experimental: true 84 | py: "3.9" 85 | 86 | - toxenv: "python3.10" 87 | db: "mariadb:10.0" 88 | legacy_db: 1 89 | experimental: true 90 | py: "3.10" 91 | 92 | - toxenv: "python3.11" 93 | db: "mariadb:10.0" 94 | legacy_db: 1 95 | experimental: true 96 | py: "3.11" 97 | 98 | - toxenv: "python3.12" 99 | db: "mariadb:10.0" 100 | legacy_db: 1 101 | experimental: true 102 | py: "3.12" 103 | 104 | - toxenv: "python3.13" 105 | db: "mariadb:10.0" 106 | legacy_db: 1 107 | experimental: true 108 | py: "3.13" 109 | 110 | - toxenv: "python3.14" 111 | db: "mariadb:10.0" 112 | legacy_db: 1 113 | experimental: true 114 | py: "3.14" 115 | 116 | - toxenv: "python3.9" 117 | db: "mariadb:10.6" 118 | legacy_db: 0 119 | experimental: false 120 | py: "3.9" 121 | 122 | - toxenv: "python3.10" 123 | db: "mariadb:10.6" 124 | legacy_db: 0 125 | experimental: false 126 | py: "3.10" 127 | 128 | - toxenv: "python3.11" 129 | db: "mariadb:10.6" 130 | legacy_db: 0 131 | experimental: false 132 | py: "3.11" 133 | 134 | - toxenv: "python3.12" 135 | db: "mariadb:10.6" 136 | legacy_db: 0 137 | experimental: false 138 | py: "3.12" 139 | 140 | - toxenv: "python3.13" 141 | db: "mariadb:10.6" 142 | legacy_db: 0 143 | experimental: false 144 | py: "3.13" 145 | 146 | - toxenv: "python3.14" 147 | db: "mariadb:10.6" 148 | legacy_db: 0 149 | experimental: false 150 | py: "3.14" 151 | 152 | - toxenv: "python3.9" 153 | db: "mariadb:10.11" 154 | legacy_db: 0 155 | experimental: false 156 | py: "3.9" 157 | 158 | - toxenv: "python3.10" 159 | db: "mariadb:10.11" 160 | legacy_db: 0 161 | experimental: false 162 | py: "3.10" 163 | 164 | - toxenv: "python3.11" 165 | db: "mariadb:10.11" 166 | legacy_db: 0 167 | experimental: false 168 | py: "3.11" 169 | 170 | - toxenv: "python3.12" 171 | db: "mariadb:10.11" 172 | legacy_db: 0 173 | experimental: false 174 | py: "3.12" 175 | 176 | - toxenv: "python3.13" 177 | db: "mariadb:10.11" 178 | legacy_db: 0 179 | experimental: false 180 | py: "3.13" 181 | 182 | - toxenv: "python3.14" 183 | db: "mariadb:10.11" 184 | legacy_db: 0 185 | experimental: false 186 | py: "3.14" 187 | 188 | - toxenv: "python3.9" 189 | db: "mariadb:11.4" 190 | legacy_db: 0 191 | experimental: false 192 | py: "3.9" 193 | 194 | - toxenv: "python3.10" 195 | db: "mariadb:11.4" 196 | legacy_db: 0 197 | experimental: false 198 | py: "3.10" 199 | 200 | - toxenv: "python3.11" 201 | db: "mariadb:11.4" 202 | legacy_db: 0 203 | experimental: false 204 | py: "3.11" 205 | 206 | - toxenv: "python3.12" 207 | db: "mariadb:11.4" 208 | legacy_db: 0 209 | experimental: false 210 | py: "3.12" 211 | 212 | - toxenv: "python3.13" 213 | db: "mariadb:11.4" 214 | legacy_db: 0 215 | experimental: false 216 | py: "3.13" 217 | 218 | - toxenv: "python3.14" 219 | db: "mariadb:11.4" 220 | legacy_db: 0 221 | experimental: false 222 | py: "3.14" 223 | 224 | - toxenv: "python3.9" 225 | db: "mariadb:11.8" 226 | legacy_db: 0 227 | experimental: false 228 | py: "3.9" 229 | 230 | - toxenv: "python3.10" 231 | db: "mariadb:11.8" 232 | legacy_db: 0 233 | experimental: false 234 | py: "3.10" 235 | 236 | - toxenv: "python3.11" 237 | db: "mariadb:11.8" 238 | legacy_db: 0 239 | experimental: false 240 | py: "3.11" 241 | 242 | - toxenv: "python3.12" 243 | db: "mariadb:11.8" 244 | legacy_db: 0 245 | experimental: false 246 | py: "3.12" 247 | 248 | - toxenv: "python3.13" 249 | db: "mariadb:11.8" 250 | legacy_db: 0 251 | experimental: false 252 | py: "3.13" 253 | 254 | - toxenv: "python3.14" 255 | db: "mariadb:11.8" 256 | legacy_db: 0 257 | experimental: false 258 | py: "3.14" 259 | 260 | - toxenv: "python3.9" 261 | db: "mysql:5.5" 262 | legacy_db: 1 263 | experimental: true 264 | py: "3.9" 265 | 266 | - toxenv: "python3.10" 267 | db: "mysql:5.5" 268 | legacy_db: 1 269 | experimental: true 270 | py: "3.10" 271 | 272 | - toxenv: "python3.11" 273 | db: "mysql:5.5" 274 | legacy_db: 1 275 | experimental: true 276 | py: "3.11" 277 | 278 | - toxenv: "python3.12" 279 | db: "mysql:5.5" 280 | legacy_db: 1 281 | experimental: true 282 | py: "3.12" 283 | 284 | - toxenv: "python3.13" 285 | db: "mysql:5.5" 286 | legacy_db: 1 287 | experimental: true 288 | py: "3.13" 289 | 290 | - toxenv: "python3.14" 291 | db: "mysql:5.5" 292 | legacy_db: 1 293 | experimental: true 294 | py: "3.14" 295 | 296 | - toxenv: "python3.9" 297 | db: "mysql:5.6" 298 | legacy_db: 1 299 | experimental: true 300 | py: "3.9" 301 | 302 | - toxenv: "python3.10" 303 | db: "mysql:5.6" 304 | legacy_db: 1 305 | experimental: true 306 | py: "3.10" 307 | 308 | - toxenv: "python3.11" 309 | db: "mysql:5.6" 310 | legacy_db: 1 311 | experimental: true 312 | py: "3.11" 313 | 314 | - toxenv: "python3.12" 315 | db: "mysql:5.6" 316 | legacy_db: 1 317 | experimental: true 318 | py: "3.12" 319 | 320 | - toxenv: "python3.13" 321 | db: "mysql:5.6" 322 | legacy_db: 1 323 | experimental: true 324 | py: "3.13" 325 | 326 | - toxenv: "python3.14" 327 | db: "mysql:5.6" 328 | legacy_db: 1 329 | experimental: true 330 | py: "3.14" 331 | 332 | - toxenv: "python3.9" 333 | db: "mysql:5.7" 334 | legacy_db: 0 335 | experimental: true 336 | py: "3.9" 337 | 338 | - toxenv: "python3.10" 339 | db: "mysql:5.7" 340 | legacy_db: 0 341 | experimental: true 342 | py: "3.10" 343 | 344 | - toxenv: "python3.11" 345 | db: "mysql:5.7" 346 | legacy_db: 0 347 | experimental: true 348 | py: "3.11" 349 | 350 | - toxenv: "python3.12" 351 | db: "mysql:5.7" 352 | legacy_db: 0 353 | experimental: true 354 | py: "3.12" 355 | 356 | - toxenv: "python3.13" 357 | db: "mysql:5.7" 358 | legacy_db: 0 359 | experimental: true 360 | py: "3.13" 361 | 362 | - toxenv: "python3.14" 363 | db: "mysql:5.7" 364 | legacy_db: 0 365 | experimental: true 366 | py: "3.14" 367 | 368 | - toxenv: "python3.9" 369 | db: "mysql:8.0" 370 | legacy_db: 0 371 | experimental: false 372 | py: "3.9" 373 | 374 | - toxenv: "python3.10" 375 | db: "mysql:8.0" 376 | legacy_db: 0 377 | experimental: false 378 | py: "3.10" 379 | 380 | - toxenv: "python3.11" 381 | db: "mysql:8.0" 382 | legacy_db: 0 383 | experimental: false 384 | py: "3.11" 385 | 386 | - toxenv: "python3.12" 387 | db: "mysql:8.0" 388 | legacy_db: 0 389 | experimental: false 390 | py: "3.12" 391 | 392 | - toxenv: "python3.13" 393 | db: "mysql:8.0" 394 | legacy_db: 0 395 | experimental: false 396 | py: "3.13" 397 | 398 | - toxenv: "python3.14" 399 | db: "mysql:8.0" 400 | legacy_db: 0 401 | experimental: false 402 | py: "3.14" 403 | 404 | - toxenv: "python3.9" 405 | db: "mysql:8.4" 406 | legacy_db: 0 407 | experimental: true 408 | py: "3.9" 409 | 410 | - toxenv: "python3.10" 411 | db: "mysql:8.4" 412 | legacy_db: 0 413 | experimental: true 414 | py: "3.10" 415 | 416 | - toxenv: "python3.11" 417 | db: "mysql:8.4" 418 | legacy_db: 0 419 | experimental: true 420 | py: "3.11" 421 | 422 | - toxenv: "python3.12" 423 | db: "mysql:8.4" 424 | legacy_db: 0 425 | experimental: true 426 | py: "3.12" 427 | 428 | - toxenv: "python3.13" 429 | db: "mysql:8.4" 430 | legacy_db: 0 431 | experimental: true 432 | py: "3.13" 433 | 434 | - toxenv: "python3.14" 435 | db: "mysql:8.4" 436 | legacy_db: 0 437 | experimental: true 438 | py: "3.14" 439 | continue-on-error: ${{ matrix.experimental }} 440 | services: 441 | mysql: 442 | image: ${{ matrix.db }} 443 | ports: 444 | - 3306:3306 445 | env: 446 | MYSQL_ALLOW_EMPTY_PASSWORD: yes 447 | MARIADB_ALLOW_EMPTY_ROOT_PASSWORD: yes 448 | options: >- 449 | --name=mysqld 450 | --health-start-period=60s 451 | --health-cmd="command -v healthcheck.sh >/dev/null 2>&1 && healthcheck.sh --connect --innodb_initialized || mysqladmin ping -h 127.0.0.1 --silent" 452 | --health-interval=10s 453 | --health-timeout=5s 454 | --health-retries=30 455 | steps: 456 | - uses: actions/checkout@v6 457 | - name: Set up Python ${{ matrix.py }} 458 | uses: actions/setup-python@v6 459 | with: 460 | python-version: ${{ matrix.py }} 461 | cache: "pip" 462 | cache-dependency-path: requirements_dev.txt 463 | - name: Install dependencies 464 | run: | 465 | set -e 466 | python -m pip install --upgrade pip 467 | python -m pip install -U tox-gh-actions 468 | pip install -r requirements_dev.txt 469 | - name: Install MySQL client 470 | run: | 471 | set -e 472 | sudo apt-get update 473 | sudo apt-get install -y mysql-client 474 | - name: Set up MySQL 475 | env: 476 | DB: ${{ matrix.db }} 477 | MYSQL_USER: tester 478 | MYSQL_PASSWORD: testpass 479 | MYSQL_DATABASE: test_db 480 | MYSQL_HOST: 0.0.0.0 481 | MYSQL_PORT: 3306 482 | run: | 483 | set -e 484 | 485 | case "$DB" in 486 | 'mysql:8.0'|'mysql:8.4') 487 | mysql -h127.0.0.1 -uroot -e "SET GLOBAL local_infile=on" 488 | docker cp mysqld:/var/lib/mysql/public_key.pem "${HOME}" 489 | docker cp mysqld:/var/lib/mysql/ca.pem "${HOME}" 490 | docker cp mysqld:/var/lib/mysql/server-cert.pem "${HOME}" 491 | docker cp mysqld:/var/lib/mysql/client-key.pem "${HOME}" 492 | docker cp mysqld:/var/lib/mysql/client-cert.pem "${HOME}" 493 | ;; 494 | esac 495 | 496 | USER_CREATION_COMMANDS='' 497 | WITH_PLUGIN='' 498 | 499 | if [ "$DB" == 'mysql:8.0' ]; then 500 | WITH_PLUGIN='with mysql_native_password' 501 | USER_CREATION_COMMANDS=' 502 | CREATE USER 503 | user_sha256 IDENTIFIED WITH "sha256_password" BY "pass_sha256", 504 | nopass_sha256 IDENTIFIED WITH "sha256_password", 505 | user_caching_sha2 IDENTIFIED WITH "caching_sha2_password" BY "pass_caching_sha2", 506 | nopass_caching_sha2 IDENTIFIED WITH "caching_sha2_password" 507 | PASSWORD EXPIRE NEVER; 508 | GRANT RELOAD ON *.* TO user_caching_sha2;' 509 | elif [ "$DB" == 'mysql:8.4' ]; then 510 | WITH_PLUGIN='with caching_sha2_password' 511 | USER_CREATION_COMMANDS=' 512 | CREATE USER 513 | user_caching_sha2 IDENTIFIED WITH "caching_sha2_password" BY "pass_caching_sha2", 514 | nopass_caching_sha2 IDENTIFIED WITH "caching_sha2_password" 515 | PASSWORD EXPIRE NEVER; 516 | GRANT RELOAD ON *.* TO user_caching_sha2;' 517 | fi 518 | 519 | if [ ! -z "$USER_CREATION_COMMANDS" ]; then 520 | mysql -uroot -h127.0.0.1 -e "$USER_CREATION_COMMANDS" 521 | fi 522 | 523 | mysql -h127.0.0.1 -uroot -e "create database $MYSQL_DATABASE DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci" 524 | mysql -h127.0.0.1 -uroot -e "create user $MYSQL_USER identified $WITH_PLUGIN by '${MYSQL_PASSWORD}'; grant all on ${MYSQL_DATABASE}.* to ${MYSQL_USER};" 525 | mysql -h127.0.0.1 -uroot -e "create user ${MYSQL_USER}@localhost identified $WITH_PLUGIN by '${MYSQL_PASSWORD}'; grant all on ${MYSQL_DATABASE}.* to ${MYSQL_USER}@localhost;" 526 | - name: Create db_credentials.json 527 | env: 528 | MYSQL_USER: tester 529 | MYSQL_PASSWORD: testpass 530 | MYSQL_DATABASE: test_db 531 | MYSQL_HOST: 0.0.0.0 532 | MYSQL_PORT: 3306 533 | run: | 534 | set -e 535 | jq -n \ 536 | --arg mysql_user "$MYSQL_USER" \ 537 | --arg mysql_password "$MYSQL_PASSWORD" \ 538 | --arg mysql_database "$MYSQL_DATABASE" \ 539 | --arg mysql_host "$MYSQL_HOST" \ 540 | --arg mysql_port $MYSQL_PORT \ 541 | '$ARGS.named' > tests/db_credentials.json 542 | - name: Test with tox 543 | env: 544 | LEGACY_DB: ${{ matrix.legacy_db }} 545 | TOXENV: ${{ matrix.toxenv }} 546 | run: tox 547 | - name: Upload coverage to Codecov 548 | env: 549 | OS: ubuntu-latest 550 | PYTHON: ${{ matrix.py }} 551 | uses: codecov/codecov-action@v5 552 | continue-on-error: true 553 | with: 554 | token: ${{ secrets.CODECOV_TOKEN }} 555 | slug: techouse/mysql-to-sqlite3 556 | files: ./coverage.xml 557 | env_vars: OS,PYTHON 558 | verbose: true 559 | - name: Cleanup 560 | if: ${{ always() }} 561 | run: | 562 | rm -rf tests/db_credentials.json 563 | --------------------------------------------------------------------------------