├── tests ├── __init__.py ├── func │ ├── __init__.py │ └── sqlite3_to_mysql_test.py ├── unit │ ├── __init__.py │ ├── sqlite_utils_test.py │ ├── debug_info_test.py │ ├── types_test.py │ ├── click_utils_test.py │ └── mysql_utils_test.py ├── faker_providers.py ├── database.py ├── factories.py ├── models.py └── conftest.py ├── src └── sqlite3_to_mysql │ ├── py.typed │ ├── __init__.py │ ├── click_utils.py │ ├── sqlite_utils.py │ ├── types.py │ ├── debug_info.py │ ├── mysql_utils.py │ └── cli.py ├── docs ├── requirements.txt ├── modules.rst ├── Makefile ├── make.bat ├── conf.py ├── sqlite3_to_mysql.rst ├── index.rst └── README.rst ├── .github ├── FUNDING.yml ├── dependabot.yml ├── ISSUE_TEMPLATE │ ├── feature_request.md │ └── bug_report.md ├── workflows │ ├── docs.yml │ ├── docker.yml │ ├── publish.yml │ └── test.yml ├── pull_request_template.md └── copilot-instructions.md ├── .flake8 ├── Dockerfile ├── requirements_dev.txt ├── LICENSE ├── SECURITY.md ├── .gitignore ├── tox.ini ├── CONTRIBUTING.md ├── pyproject.toml ├── CODE-OF-CONDUCT.md ├── README.md ├── .bandit.yml └── CHANGELOG.md /tests/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/func/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/unit/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /src/sqlite3_to_mysql/py.typed: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /docs/requirements.txt: -------------------------------------------------------------------------------- 1 | Sphinx==8.2.3 -------------------------------------------------------------------------------- /.github/FUNDING.yml: -------------------------------------------------------------------------------- 1 | github: techouse 2 | custom: [ "https://paypal.me/ktusar" ] 3 | -------------------------------------------------------------------------------- /.flake8: -------------------------------------------------------------------------------- 1 | [flake8] 2 | ignore = I100,I201,I202,D203,D401,W503,E203,F401,F403,C901,E501 3 | exclude = tests 4 | max-line-length = 88 5 | -------------------------------------------------------------------------------- /docs/modules.rst: -------------------------------------------------------------------------------- 1 | sqlite3_to_mysql 2 | ================ 3 | 4 | .. toctree:: 5 | :maxdepth: 4 6 | 7 | sqlite3_to_mysql 8 | -------------------------------------------------------------------------------- /src/sqlite3_to_mysql/__init__.py: -------------------------------------------------------------------------------- 1 | """Utility to transfer data from SQLite 3 to MySQL.""" 2 | 3 | __version__ = "2.5.5" 4 | 5 | from .transporter import SQLite3toMySQL 6 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.14-alpine 2 | 3 | LABEL maintainer="https://github.com/techouse" 4 | 5 | RUN pip install --no-cache-dir --upgrade pip && \ 6 | pip install --no-cache-dir sqlite3-to-mysql 7 | 8 | ENTRYPOINT ["sqlite3mysql"] -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | - package-ecosystem: "pip" 4 | directory: "/" 5 | schedule: 6 | interval: "weekly" 7 | - package-ecosystem: "github-actions" 8 | directory: "/" 9 | schedule: 10 | interval: "weekly" -------------------------------------------------------------------------------- /requirements_dev.txt: -------------------------------------------------------------------------------- 1 | Click>=8.1.3 2 | docker>=6.1.3 3 | factory-boy 4 | Faker>=18.10.0 5 | mysql-connector-python>=9.0.0 6 | PyMySQL>=1.0.3 7 | pytest>=7.3.1 8 | pytest-cov 9 | pytest-mock 10 | pytest-timeout 11 | pytimeparse2 12 | python-dateutil>=2.9.0.post0 13 | types-python-dateutil 14 | simplejson>=3.19.1 15 | types-simplejson 16 | sqlglot>=27.27.0 17 | sqlalchemy>=2.0.0 18 | sqlalchemy-utils 19 | types-sqlalchemy-utils 20 | tox 21 | tqdm>=4.65.0 22 | types-tqdm 23 | packaging 24 | tabulate 25 | types-tabulate 26 | Unidecode>=1.3.6 27 | typing-extensions; python_version < "3.11" 28 | types-Pygments 29 | types-colorama 30 | types-mock 31 | types-setuptools -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Minimal makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line, and also 5 | # from the environment for the first two. 6 | SPHINXOPTS ?= 7 | SPHINXBUILD ?= sphinx-build 8 | SOURCEDIR = . 9 | BUILDDIR = _build 10 | 11 | # Put it first so that "make" without argument is like "make help". 12 | help: 13 | @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 14 | 15 | .PHONY: help Makefile 16 | 17 | # Catch-all target: route all unknown targets to Sphinx using the new 18 | # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). 19 | %: Makefile 20 | @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 21 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature_request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Feature request 3 | about: Suggest an idea for this project 4 | title: '' 5 | labels: enhancement 6 | assignees: techouse 7 | 8 | --- 9 | 10 | **Is your feature request related to a problem? Please describe.** 11 | A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] 12 | 13 | **Describe the solution you'd like** 14 | A clear and concise description of what you want to happen. 15 | 16 | **Describe alternatives you've considered** 17 | A clear and concise description of any alternative solutions or features you've considered. 18 | 19 | **Additional context** 20 | Add any other context or screenshots about the feature request here. 21 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/bug_report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug report 3 | about: Create a report to help us improve 4 | title: '' 5 | labels: bug 6 | assignees: techouse 7 | 8 | --- 9 | 10 | **Describe the bug** 11 | A clear and concise description of what the bug is. 12 | 13 | **Expected behaviour** 14 | What you expected. 15 | 16 | **Actual result** 17 | What happened instead. 18 | 19 | **System Information** 20 | 21 | ```bash 22 | $ sqlite3mysql --version 23 | ``` 24 | 25 | ``` 26 | 27 | ``` 28 | 29 | This command is only available on v1.3.6 and greater. Otherwise, please provide some basic information about your system (Python version, operating system, etc.). 30 | 31 | **Additional context** 32 | Add any other context about the problem here. 33 | 34 | In case of errors please run the same command with `--debug`. This option is only available on v1.4.12 or greater. 35 | -------------------------------------------------------------------------------- /tests/faker_providers.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | from typing import Optional 3 | 4 | from faker.providers import BaseProvider, date_time 5 | from faker.typing import DateParseType 6 | 7 | 8 | class DateTimeProviders(BaseProvider): 9 | def time_object_without_microseconds(self, end_datetime: Optional[DateParseType] = None) -> datetime.time: 10 | return date_time.Provider(self.generator).time_object(end_datetime).replace(microsecond=0) 11 | 12 | def date_time_this_century_without_microseconds( 13 | self, 14 | before_now: bool = True, 15 | after_now: bool = False, 16 | tzinfo: Optional[datetime.tzinfo] = None, 17 | ) -> datetime.datetime: 18 | return ( 19 | date_time.Provider(self.generator) 20 | .date_time_this_century(before_now, after_now, tzinfo) 21 | .replace(microsecond=0) 22 | ) 23 | -------------------------------------------------------------------------------- /docs/make.bat: -------------------------------------------------------------------------------- 1 | @ECHO OFF 2 | 3 | pushd %~dp0 4 | 5 | REM Command file for Sphinx documentation 6 | 7 | if "%SPHINXBUILD%" == "" ( 8 | set SPHINXBUILD=sphinx-build 9 | ) 10 | set SOURCEDIR=. 11 | set BUILDDIR=_build 12 | 13 | %SPHINXBUILD% >NUL 2>NUL 14 | if errorlevel 9009 ( 15 | echo. 16 | echo.The 'sphinx-build' command was not found. Make sure you have Sphinx 17 | echo.installed, then set the SPHINXBUILD environment variable to point 18 | echo.to the full path of the 'sphinx-build' executable. Alternatively you 19 | echo.may add the Sphinx directory to PATH. 20 | echo. 21 | echo.If you don't have Sphinx installed, grab it from 22 | echo.https://www.sphinx-doc.org/ 23 | exit /b 1 24 | ) 25 | 26 | if "%1" == "" goto help 27 | 28 | %SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% 29 | goto end 30 | 31 | :help 32 | %SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% 33 | 34 | :end 35 | popd 36 | -------------------------------------------------------------------------------- /.github/workflows/docs.yml: -------------------------------------------------------------------------------- 1 | name: Docs 2 | 3 | on: 4 | workflow_dispatch: 5 | workflow_call: 6 | 7 | permissions: 8 | contents: write 9 | 10 | concurrency: 11 | group: docs-${{ github.ref }} 12 | cancel-in-progress: false 13 | 14 | jobs: 15 | docs: 16 | name: "Docs" 17 | runs-on: ubuntu-latest 18 | steps: 19 | - uses: actions/checkout@v6 20 | - uses: actions/setup-python@v6 21 | with: 22 | python-version: "3.x" 23 | cache: "pip" 24 | - name: Install package 25 | run: pip install -e . 26 | - name: Install Sphinx dependencies 27 | working-directory: docs 28 | run: pip install -r requirements.txt 29 | - name: Sphinx build 30 | working-directory: docs 31 | run: sphinx-build . _build 32 | - name: Deploy to GitHub Pages 33 | uses: peaceiris/actions-gh-pages@v4 34 | with: 35 | publish_branch: gh-pages 36 | github_token: ${{ secrets.GITHUB_TOKEN }} 37 | publish_dir: docs/_build/ 38 | force_orphan: true -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2024 Klemen Tusar 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /.github/pull_request_template.md: -------------------------------------------------------------------------------- 1 | # Pull Request Template 2 | 3 | ## Description 4 | 5 | Please include a summary of the change and which issue is fixed. Please also include relevant motivation and context. 6 | List any dependencies that are required for this change. 7 | 8 | Fixes # (issue) 9 | 10 | ## Type of change 11 | 12 | Please delete options that are not relevant. 13 | 14 | - [ ] Bug fix (non-breaking change which fixes an issue) 15 | - [ ] New feature (non-breaking change which adds functionality) 16 | - [ ] Breaking change (fix or feature that would cause existing functionality to not work as expected) 17 | - [ ] This change requires a documentation update 18 | 19 | ## How Has This Been Tested? 20 | 21 | Please describe the tests that you ran to verify your changes. Provide instructions so we can reproduce. Please also 22 | list any relevant details for your test configuration 23 | 24 | - [ ] Test A 25 | - [ ] Test B 26 | 27 | ## Checklist: 28 | 29 | - [ ] My code follows the style guidelines of this project 30 | - [ ] I have performed a self-review of my own code 31 | - [ ] I have commented my code, particularly in hard-to-understand areas 32 | - [ ] I have made corresponding changes to the documentation 33 | - [ ] My changes generate no new warnings 34 | - [ ] I have added tests that prove my fix is effective or that my feature works 35 | - [ ] New and existing unit tests pass locally with my changes 36 | - [ ] Any dependent changes have been merged and published in downstream modules -------------------------------------------------------------------------------- /tests/database.py: -------------------------------------------------------------------------------- 1 | import typing as t 2 | from datetime import datetime, timedelta 3 | from decimal import Decimal 4 | 5 | import simplejson as json 6 | from sqlalchemy import create_engine 7 | from sqlalchemy.engine import Engine 8 | from sqlalchemy.orm import sessionmaker 9 | from sqlalchemy_utils import database_exists 10 | 11 | from .models import Base 12 | 13 | 14 | class Database: 15 | engine: Engine 16 | Session: sessionmaker 17 | 18 | def __init__(self, database_uri): 19 | self.Session = sessionmaker() 20 | self.engine = create_engine(database_uri, json_serializer=self.dumps, json_deserializer=json.loads) 21 | if not database_exists(database_uri): 22 | self._create_db_tables() 23 | self.Session.configure(bind=self.engine) 24 | 25 | def _create_db_tables(self) -> None: 26 | Base.metadata.create_all(self.engine) 27 | 28 | @classmethod 29 | def dumps(cls, data: t.Any) -> str: 30 | return json.dumps(data, default=cls.json_serializer) 31 | 32 | @staticmethod 33 | def json_serializer(data: t.Any) -> t.Optional[str]: 34 | if isinstance(data, datetime): 35 | return data.isoformat() 36 | if isinstance(data, Decimal): 37 | return str(data) 38 | if isinstance(data, timedelta): 39 | hours, remainder = divmod(data.total_seconds(), 3600) 40 | minutes, seconds = divmod(remainder, 60) 41 | return "{:02}:{:02}:{:02}".format(int(hours), int(minutes), int(seconds)) 42 | return None 43 | -------------------------------------------------------------------------------- /docs/conf.py: -------------------------------------------------------------------------------- 1 | # Configuration file for the Sphinx documentation builder. 2 | # 3 | # For the full list of built-in configuration values, see the documentation: 4 | # https://www.sphinx-doc.org/en/master/usage/configuration.html 5 | import os 6 | import re 7 | import sys 8 | from pathlib import Path 9 | 10 | 11 | sys.path.insert(0, os.path.abspath("../src")) 12 | 13 | _ROOT = Path(__file__).resolve().parents[1] 14 | _ver_file = _ROOT / "src" / "sqlite3_to_mysql" / "__init__.py" 15 | _m = re.search(r'^__version__\s*=\s*[\'"]([^\'"]+)[\'"]', _ver_file.read_text(encoding="utf-8"), re.M) 16 | __version__ = _m.group(1) if _m else "0+unknown" 17 | 18 | 19 | # -- Project information ----------------------------------------------------- 20 | # https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information 21 | 22 | project = "sqlite3-to-mysql" 23 | copyright = "%Y, Klemen Tusar" 24 | author = "Klemen Tusar" 25 | release = __version__ 26 | 27 | # -- General configuration --------------------------------------------------- 28 | # https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration 29 | 30 | extensions = ["sphinx.ext.autodoc", "sphinx.ext.viewcode", "sphinx.ext.napoleon"] 31 | 32 | napoleon_google_docstring = True 33 | napoleon_include_init_with_doc = True 34 | 35 | templates_path = ["_templates"] 36 | exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"] 37 | 38 | 39 | # -- Options for HTML output ------------------------------------------------- 40 | # https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output 41 | 42 | html_theme = "alabaster" 43 | html_static_path = ["_static"] 44 | -------------------------------------------------------------------------------- /docs/sqlite3_to_mysql.rst: -------------------------------------------------------------------------------- 1 | sqlite3\_to\_mysql package 2 | ========================== 3 | 4 | Submodules 5 | ---------- 6 | 7 | sqlite3\_to\_mysql.cli module 8 | ----------------------------- 9 | 10 | .. automodule:: sqlite3_to_mysql.cli 11 | :members: 12 | :undoc-members: 13 | :show-inheritance: 14 | 15 | sqlite3\_to\_mysql.click\_utils module 16 | -------------------------------------- 17 | 18 | .. automodule:: sqlite3_to_mysql.click_utils 19 | :members: 20 | :undoc-members: 21 | :show-inheritance: 22 | 23 | sqlite3\_to\_mysql.debug\_info module 24 | ------------------------------------- 25 | 26 | .. automodule:: sqlite3_to_mysql.debug_info 27 | :members: 28 | :undoc-members: 29 | :show-inheritance: 30 | 31 | sqlite3\_to\_mysql.mysql\_utils module 32 | -------------------------------------- 33 | 34 | .. automodule:: sqlite3_to_mysql.mysql_utils 35 | :members: 36 | :undoc-members: 37 | :show-inheritance: 38 | 39 | sqlite3\_to\_mysql.sqlite\_utils module 40 | --------------------------------------- 41 | 42 | .. automodule:: sqlite3_to_mysql.sqlite_utils 43 | :members: 44 | :undoc-members: 45 | :show-inheritance: 46 | 47 | sqlite3\_to\_mysql.transporter module 48 | ------------------------------------- 49 | 50 | .. automodule:: sqlite3_to_mysql.transporter 51 | :members: 52 | :undoc-members: 53 | :show-inheritance: 54 | 55 | sqlite3\_to\_mysql.types module 56 | ------------------------------- 57 | 58 | .. automodule:: sqlite3_to_mysql.types 59 | :members: 60 | :undoc-members: 61 | :show-inheritance: 62 | 63 | Module contents 64 | --------------- 65 | 66 | .. automodule:: sqlite3_to_mysql 67 | :members: 68 | :undoc-members: 69 | :show-inheritance: 70 | -------------------------------------------------------------------------------- /SECURITY.md: -------------------------------------------------------------------------------- 1 | # Security Policy 2 | 3 | ## Supported Versions 4 | 5 | | Version | Supported | 6 | |---------|--------------------| 7 | | 2.1.x | :white_check_mark: | 8 | | 2.0.x | :x: | 9 | | 1.x.x | :x: | 10 | 11 | ## Reporting a Vulnerability 12 | 13 | We take the security of our software seriously. If you believe you have found a security vulnerability, please report it 14 | to us as described below. 15 | 16 | **DO NOT CREATE A GITHUB ISSUE** reporting the vulnerability. 17 | 18 | Instead, send an email to [techouse@gmail.com](mailto:techouse@gmail.com). 19 | 20 | In the report, please include the following: 21 | 22 | - Your name and affiliation (if any). 23 | - A description of the technical details of the vulnerabilities. It is very important to let us know how we can 24 | reproduce your findings. 25 | - An explanation who can exploit this vulnerability, and what they gain when doing so -- write an attack scenario. This 26 | will help us evaluate your submission quickly, especially if it is a complex or creative vulnerability. 27 | - Whether this vulnerability is public or known to third parties. If it is, please provide details. 28 | 29 | If you don’t get an acknowledgment from us or have heard nothing from us in a week, please contact us again. 30 | 31 | We will send a response indicating the next steps in handling your report. We will keep you informed about the progress 32 | towards a fix and full announcement. 33 | 34 | We will not disclose your identity to the public without your permission. We strive to credit researchers in our 35 | advisories when we release a fix, but only after getting your permission. 36 | 37 | We appreciate your efforts to responsibly disclose your findings, and will make every effort to acknowledge your 38 | contributions. -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | *.egg-info/ 24 | .installed.cfg 25 | *.egg 26 | MANIFEST 27 | 28 | # PyInstaller 29 | # Usually these files are written by a python script from a template 30 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 31 | *.manifest 32 | *.spec 33 | 34 | # Installer logs 35 | pip-log.txt 36 | pip-delete-this-directory.txt 37 | 38 | # Unit test / coverage reports 39 | htmlcov/ 40 | .tox/ 41 | .coverage 42 | .coverage.* 43 | .cache 44 | nosetests.xml 45 | coverage.xml 46 | *.cover 47 | .hypothesis/ 48 | .pytest_cache/ 49 | 50 | # Translations 51 | *.mo 52 | *.pot 53 | 54 | # Django stuff: 55 | *.log 56 | local_settings.py 57 | db.sqlite3 58 | 59 | # Flask stuff: 60 | instance/ 61 | .webassets-cache 62 | 63 | # Scrapy stuff: 64 | .scrapy 65 | 66 | # Sphinx documentation 67 | docs/_build/ 68 | 69 | # PyBuilder 70 | target/ 71 | 72 | # Jupyter Notebook 73 | .ipynb_checkpoints 74 | 75 | # pyenv 76 | .python-version 77 | 78 | # celery beat schedule file 79 | celerybeat-schedule 80 | 81 | # SageMath parsed files 82 | *.sage.py 83 | 84 | # Environments 85 | .env 86 | .venv 87 | env/ 88 | venv/ 89 | ENV/ 90 | env.bak/ 91 | venv.bak/ 92 | 93 | # Spyder project settings 94 | .spyderproject 95 | .spyproject 96 | 97 | # Rope project settings 98 | .ropeproject 99 | 100 | # mkdocs documentation 101 | /site 102 | 103 | # mypy 104 | .mypy_cache/ 105 | 106 | # PyCharm specific files 107 | .idea 108 | 109 | # macOS specific 110 | .DS_Store 111 | 112 | # Potential leftovers 113 | tests/db_credentials.json 114 | test.db -------------------------------------------------------------------------------- /.github/workflows/docker.yml: -------------------------------------------------------------------------------- 1 | name: Publish Docker image 2 | 3 | on: 4 | workflow_call: 5 | defaults: 6 | run: 7 | shell: bash 8 | 9 | concurrency: 10 | group: docker-${{ github.ref }} 11 | cancel-in-progress: false 12 | 13 | jobs: 14 | push_to_registry: 15 | name: Push Docker image to Docker Hub 16 | runs-on: ubuntu-latest 17 | permissions: 18 | packages: write 19 | contents: read 20 | environment: 21 | name: docker 22 | url: https://hub.docker.com/r/${{ vars.DOCKERHUB_REPOSITORY }} 23 | steps: 24 | - name: Check out the repo 25 | uses: actions/checkout@v6 26 | - name: Set up QEMU 27 | uses: docker/setup-qemu-action@v3 28 | - name: Set up Docker Buildx 29 | uses: docker/setup-buildx-action@v3 30 | - name: Docker meta 31 | id: meta 32 | uses: docker/metadata-action@v5 33 | with: 34 | images: | 35 | ${{ vars.DOCKERHUB_REPOSITORY }} 36 | ghcr.io/${{ github.repository }} 37 | tags: | 38 | type=ref,event=branch 39 | type=ref,event=pr 40 | type=semver,pattern={{version}} 41 | type=semver,pattern={{major}}.{{minor}}.{{patch}} 42 | - name: Log in to Docker Hub 43 | uses: docker/login-action@v3 44 | with: 45 | username: ${{ secrets.DOCKERHUB_USERNAME }} 46 | password: ${{ secrets.DOCKERHUB_TOKEN }} 47 | - name: Log in to the Container registry 48 | uses: docker/login-action@v3 49 | with: 50 | registry: ghcr.io 51 | username: ${{ github.actor }} 52 | password: ${{ secrets.GITHUB_TOKEN }} 53 | - name: Build and push 54 | uses: docker/build-push-action@v6 55 | with: 56 | context: . 57 | platforms: linux/amd64,linux/arm64 58 | push: true 59 | tags: ${{ steps.meta.outputs.tags }} 60 | labels: ${{ steps.meta.outputs.labels }} 61 | -------------------------------------------------------------------------------- /src/sqlite3_to_mysql/click_utils.py: -------------------------------------------------------------------------------- 1 | """Click utilities.""" 2 | 3 | import typing as t 4 | 5 | import click 6 | 7 | 8 | class OptionEatAll(click.Option): 9 | """Taken from https://stackoverflow.com/questions/48391777/nargs-equivalent-for-options-in-click#answer-48394004.""" # noqa: ignore=E501 pylint: disable=C0301 10 | 11 | def __init__(self, *args, **kwargs): 12 | """Override.""" 13 | self.save_other_options = kwargs.pop("save_other_options", True) 14 | nargs = kwargs.pop("nargs", -1) 15 | if nargs != -1: 16 | raise ValueError(f"nargs, if set, must be -1 not {nargs}") 17 | super(OptionEatAll, self).__init__(*args, **kwargs) 18 | self._previous_parser_process = None 19 | self._eat_all_parser = None 20 | 21 | def add_to_parser(self, parser, ctx) -> None: 22 | """Override.""" 23 | 24 | def parser_process(value, state) -> None: 25 | # method to hook to the parser.process 26 | done: bool = False 27 | value = [value] 28 | if self.save_other_options: 29 | # grab everything up to the next option 30 | while state.rargs and not done: 31 | for prefix in self._eat_all_parser.prefixes: 32 | if state.rargs[0].startswith(prefix): 33 | done = True 34 | if not done: 35 | value.append(state.rargs.pop(0)) 36 | else: 37 | # grab everything remaining 38 | value += state.rargs 39 | state.rargs[:] = [] 40 | value = tuple(value) 41 | 42 | # call the actual process 43 | self._previous_parser_process(value, state) 44 | 45 | retval = super(OptionEatAll, self).add_to_parser(parser, ctx) # pylint: disable=E1111 46 | for name in self.opts: 47 | # pylint: disable=W0212 48 | our_parser = parser._long_opt.get(name) or parser._short_opt.get(name) 49 | if our_parser: 50 | self._eat_all_parser = our_parser 51 | self._previous_parser_process = our_parser.process 52 | our_parser.process = parser_process 53 | break 54 | return retval 55 | 56 | 57 | def prompt_password(ctx: click.core.Context, param: t.Any, use_password: bool): # pylint: disable=W0613 58 | """Prompt for password.""" 59 | if use_password: 60 | mysql_password = ctx.params.get("mysql_password") 61 | if not mysql_password: 62 | mysql_password = click.prompt("MySQL password", hide_input=True) 63 | 64 | return mysql_password 65 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | isolated_build = true 3 | envlist = 4 | python3.9, 5 | python3.10, 6 | python3.11, 7 | python3.12, 8 | python3.13, 9 | python3.14, 10 | black, 11 | flake8, 12 | linters, 13 | skip_missing_interpreters = true 14 | 15 | [gh-actions] 16 | python = 17 | 3.9: python3.9 18 | 3.10: python3.10 19 | 3.11: python3.11 20 | 3.12: python3.12 21 | 3.13: python3.13 22 | 3.14: python3.14 23 | 24 | [testenv] 25 | passenv = 26 | LANG 27 | LEGACY_DB 28 | deps = 29 | -rrequirements_dev.txt 30 | commands = 31 | pytest -v --cov=src/sqlite3_to_mysql --cov-report=xml 32 | 33 | [testenv:black] 34 | basepython = python3 35 | skip_install = true 36 | deps = 37 | black 38 | commands = 39 | black src/sqlite3_to_mysql tests/ 40 | 41 | [testenv:isort] 42 | basepython = python3 43 | skip_install = true 44 | deps = 45 | isort 46 | commands = 47 | isort --check-only --diff . 48 | 49 | [testenv:flake8] 50 | basepython = python3 51 | skip_install = true 52 | deps = 53 | flake8 54 | flake8-colors 55 | flake8-docstrings 56 | flake8-import-order 57 | flake8-typing-imports 58 | pep8-naming 59 | commands = flake8 src/sqlite3_to_mysql 60 | 61 | [testenv:pylint] 62 | basepython = python3 63 | skip_install = true 64 | deps = 65 | pylint 66 | -rrequirements_dev.txt 67 | commands = 68 | pylint --rcfile=tox.ini src/sqlite3_to_mysql 69 | 70 | [testenv:bandit] 71 | basepython = python3 72 | skip_install = true 73 | deps = 74 | bandit 75 | commands = 76 | bandit -r src/sqlite3_to_mysql -c .bandit.yml 77 | 78 | [testenv:mypy] 79 | basepython = python3 80 | skip_install = true 81 | deps = 82 | mypy>=1.3.0 83 | -rrequirements_dev.txt 84 | commands = 85 | mypy src/sqlite3_to_mysql 86 | 87 | [testenv:linters] 88 | basepython = python3 89 | skip_install = true 90 | deps = 91 | {[testenv:black]deps} 92 | {[testenv:isort]deps} 93 | {[testenv:flake8]deps} 94 | {[testenv:pylint]deps} 95 | {[testenv:bandit]deps} 96 | {[testenv:mypy]deps} 97 | commands = 98 | {[testenv:black]commands} 99 | {[testenv:isort]commands} 100 | {[testenv:flake8]commands} 101 | {[testenv:pylint]commands} 102 | {[testenv:bandit]commands} 103 | {[testenv:mypy]commands} 104 | 105 | [flake8] 106 | ignore = I100,I201,I202,D203,D401,W503,E203,F401,F403,C901,E501 107 | exclude = 108 | *__init__.py 109 | .tox 110 | max-complexity = 10 111 | max-line-length = 88 112 | import-order-style = pycharm 113 | application-import-names = flake8 114 | 115 | [pylint] 116 | disable = C0209,C0301,C0302,C0411,R,W0107,W0622,C0103 117 | -------------------------------------------------------------------------------- /src/sqlite3_to_mysql/sqlite_utils.py: -------------------------------------------------------------------------------- 1 | """SQLite adapters and converters for unsupported data types.""" 2 | 3 | import typing as t 4 | from datetime import date, timedelta 5 | from decimal import Decimal 6 | 7 | from dateutil.parser import ParserError 8 | from dateutil.parser import parse as dateutil_parse 9 | from packaging import version 10 | from packaging.version import Version 11 | from pytimeparse2 import parse 12 | from unidecode import unidecode 13 | 14 | 15 | def adapt_decimal(value) -> str: 16 | """Convert decimal.Decimal to string.""" 17 | return str(value) 18 | 19 | 20 | def convert_decimal(value) -> Decimal: 21 | """Convert string to decimalDecimal.""" 22 | return Decimal(str(value.decode())) 23 | 24 | 25 | def adapt_timedelta(value) -> str: 26 | """Convert datetime.timedelta to %H:%M:%S string.""" 27 | hours, remainder = divmod(value.total_seconds(), 3600) 28 | minutes, seconds = divmod(remainder, 60) 29 | return "{:02}:{:02}:{:02}".format(int(hours), int(minutes), int(seconds)) 30 | 31 | 32 | def convert_timedelta(value) -> timedelta: 33 | """Convert %H:%M:%S string to datetime.timedelta.""" 34 | return timedelta(seconds=parse(value.decode())) 35 | 36 | 37 | def unicase_compare(string_1: str, string_2: str) -> int: 38 | """Taken from https://github.com/patarapolw/ankisync2/issues/3#issuecomment-768687431.""" 39 | _string_1: str = unidecode(string_1).lower() 40 | _string_2: str = unidecode(string_2).lower() 41 | return 1 if _string_1 > _string_2 else -1 if _string_1 < _string_2 else 0 42 | 43 | 44 | def convert_date(value: t.Union[str, bytes]) -> date: 45 | """Handle SQLite date conversion.""" 46 | try: 47 | return dateutil_parse(value.decode() if isinstance(value, bytes) else value).date() 48 | except ParserError as err: 49 | raise ValueError(f"DATE field contains {err}") # pylint: disable=W0707 50 | 51 | 52 | def check_sqlite_table_xinfo_support(version_string: str) -> bool: 53 | """Check for SQLite table_xinfo support.""" 54 | sqlite_version: Version = version.parse(version_string) 55 | return sqlite_version.major > 3 or (sqlite_version.major == 3 and sqlite_version.minor >= 26) 56 | 57 | 58 | def check_sqlite_jsonb_support(version_string: str) -> bool: 59 | """Check for SQLite JSONB support.""" 60 | sqlite_version: Version = version.parse(version_string) 61 | return sqlite_version.major > 3 or (sqlite_version.major == 3 and sqlite_version.minor >= 45) 62 | 63 | 64 | def sqlite_jsonb_column_expression(quoted_column_name: str) -> str: 65 | """Return a SELECT expression that converts JSONB blobs to textual JSON while preserving NULLs.""" 66 | return 'CASE WHEN "{name}" IS NULL THEN NULL ELSE json("{name}") END AS "{name}"'.format(name=quoted_column_name) 67 | -------------------------------------------------------------------------------- /src/sqlite3_to_mysql/types.py: -------------------------------------------------------------------------------- 1 | """Types for sqlite3-to-mysql.""" 2 | 3 | import os 4 | import typing as t 5 | from logging import Logger 6 | from sqlite3 import Connection, Cursor 7 | 8 | from mysql.connector import MySQLConnection 9 | from mysql.connector.cursor import MySQLCursor 10 | 11 | 12 | try: 13 | # Python 3.11+ 14 | from typing import TypedDict # type: ignore[attr-defined] 15 | except ImportError: 16 | # Python < 3.11 17 | from typing_extensions import TypedDict # type: ignore 18 | 19 | 20 | class SQLite3toMySQLParams(TypedDict): 21 | """SQLite3toMySQL parameters.""" 22 | 23 | sqlite_file: t.Union[str, "os.PathLike[t.Any]"] 24 | sqlite_tables: t.Optional[t.Sequence[str]] 25 | exclude_sqlite_tables: t.Optional[t.Sequence[str]] 26 | sqlite_views_as_tables: t.Optional[bool] 27 | without_foreign_keys: t.Optional[bool] 28 | mysql_user: t.Optional[str] 29 | mysql_password: t.Optional[t.Union[str, bool]] 30 | mysql_host: t.Optional[str] 31 | mysql_port: t.Optional[int] 32 | mysql_socket: t.Optional[t.Union[str, "os.PathLike[t.Any]"]] 33 | mysql_ssl_disabled: t.Optional[bool] 34 | chunk: t.Optional[int] 35 | quiet: t.Optional[bool] 36 | log_file: t.Optional[t.Union[str, "os.PathLike[t.Any]"]] 37 | mysql_database: t.Optional[str] 38 | mysql_integer_type: t.Optional[str] 39 | mysql_create_tables: t.Optional[bool] 40 | mysql_truncate_tables: t.Optional[bool] 41 | mysql_transfer_data: t.Optional[bool] 42 | mysql_charset: t.Optional[str] 43 | mysql_collation: t.Optional[str] 44 | ignore_duplicate_keys: t.Optional[bool] 45 | use_fulltext: t.Optional[bool] 46 | with_rowid: t.Optional[bool] 47 | mysql_insert_method: t.Optional[str] 48 | mysql_string_type: t.Optional[str] 49 | mysql_text_type: t.Optional[str] 50 | 51 | 52 | class SQLite3toMySQLAttributes: 53 | """SQLite3toMySQL attributes.""" 54 | 55 | _sqlite_file: t.Union[str, "os.PathLike[t.Any]"] 56 | _sqlite_tables: t.Sequence[str] 57 | _exclude_sqlite_tables: t.Sequence[str] 58 | _sqlite_views_as_tables: bool 59 | _without_foreign_keys: bool 60 | _mysql_user: str 61 | _mysql_password: t.Optional[str] 62 | _mysql_host: t.Optional[str] 63 | _mysql_port: t.Optional[int] 64 | _mysql_socket: t.Optional[t.Union[str, "os.PathLike[t.Any]"]] 65 | _mysql_ssl_disabled: bool 66 | _chunk_size: t.Optional[int] 67 | _quiet: bool 68 | _logger: Logger 69 | _log_file: t.Union[str, "os.PathLike[t.Any]"] 70 | _mysql_database: str 71 | _mysql_insert_method: str 72 | _mysql_create_tables: bool 73 | _mysql_truncate_tables: bool 74 | _mysql_transfer_data: bool 75 | _mysql_integer_type: str 76 | _mysql_string_type: str 77 | _mysql_text_type: str 78 | _mysql_charset: str 79 | _mysql_collation: str 80 | _ignore_duplicate_keys: bool 81 | _use_fulltext: bool 82 | _with_rowid: bool 83 | _sqlite: Connection 84 | _sqlite_cur: Cursor 85 | _sqlite_version: str 86 | _sqlite_table_xinfo_support: bool 87 | _mysql: MySQLConnection 88 | _mysql_cur: MySQLCursor 89 | _mysql_version: str 90 | _is_mariadb: bool 91 | _mysql_json_support: bool 92 | _mysql_fulltext_support: bool 93 | _allow_expr_defaults: bool 94 | _allow_current_ts_dt: bool 95 | _allow_fsp: bool 96 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing 2 | 3 | I greatly appreciate your interest in reading this message, as this project requires volunteer developers to assist 4 | in developing and maintaining it. 5 | 6 | Before making any changes to this repository, please first discuss the proposed modifications with the repository owners 7 | through an issue, email, or any other appropriate communication channel. 8 | 9 | Please be aware that a [code of conduct](CODE-OF-CONDUCT.md) is in place, and should be adhered to during all 10 | interactions related to the project. 11 | 12 | ## Python version support 13 | 14 | Ensuring backward compatibility is an imperative requirement. 15 | 16 | Currently, the tool supports Python versions 3.9, 3.10, 3.11, 3.12, 3.13, and 3.14. 17 | 18 | ## MySQL version support 19 | 20 | This tool is intended to fully support MySQL versions 5.5, 5.6, 5.7, and 8.0, including major forks like MariaDB. 21 | We should prioritize and be dedicated to maintaining compatibility with these versions for a smooth user experience. 22 | 23 | ## Testing 24 | 25 | As this project/tool involves the critical process of transferring data between different database types, it is of 26 | utmost importance to ensure thorough testing. Please remember to write tests for any new code you create, utilizing the 27 | [pytest](https://docs.pytest.org/en/latest/) framework for all test cases. 28 | 29 | ### Running the test suite 30 | 31 | In order to run the test suite run these commands using a Docker MySQL image. 32 | 33 | **Requires a running Docker instance!** 34 | 35 | ```bash 36 | git clone https://github.com/techouse/sqlite3-to-mysql 37 | cd sqlite3-to-mysql 38 | python3 -m venv env 39 | source env/bin/activate 40 | pip install -e . 41 | pip install -r requirements_dev.txt 42 | tox 43 | ``` 44 | 45 | ## Submitting changes 46 | 47 | To contribute to this project, please submit a 48 | new [pull request](https://github.com/techouse/sqlite3-to-mysql/pull/new/master) and provide a clear list of your 49 | modifications. For guidance on creating pull requests, you can refer 50 | to [this resource](http://help.github.com/pull-requests/). 51 | 52 | When sending a pull request, we highly appreciate the inclusion of [pytest](https://docs.pytest.org/en/latest/) tests, 53 | as we strive to enhance our test coverage. Following our coding conventions is essential, and it would be ideal if you 54 | ensure that each commit focuses on a single feature. 55 | 56 | For commits, please write clear log messages. While concise one-line messages are suitable for small changes, more 57 | substantial modifications should follow a format similar to the example below: 58 | 59 | ```bash 60 | git commit -m "A brief summary of the commit 61 | > 62 | > A paragraph describing what changed and its impact." 63 | ``` 64 | 65 | ## Coding standards 66 | 67 | It is essential to prioritize code readability and conciseness. To achieve this, we recommend 68 | using [Black](https://github.com/psf/black) for code formatting. 69 | 70 | Once your work is deemed complete, it is advisable to run the following command: 71 | 72 | ```bash 73 | tox -e flake8,linters 74 | ``` 75 | 76 | This command executes various linters and checkers to identify any potential issues or inconsistencies in your code. By 77 | following these guidelines, you can ensure a high-quality codebase. 78 | 79 | Thanks, 80 | 81 | Klemen Tusar 82 | -------------------------------------------------------------------------------- /src/sqlite3_to_mysql/debug_info.py: -------------------------------------------------------------------------------- 1 | """Module containing bug report helper(s). 2 | 3 | Adapted from https://github.com/psf/requests/blob/master/requests/help.py 4 | """ 5 | 6 | import platform 7 | import sqlite3 8 | import sys 9 | import typing as t 10 | from shutil import which 11 | from subprocess import check_output 12 | 13 | import click 14 | import mysql.connector 15 | import pytimeparse2 16 | import simplejson 17 | import sqlglot 18 | import tabulate 19 | import tqdm 20 | 21 | from . import __version__ as package_version 22 | 23 | 24 | def _implementation() -> str: 25 | """Return a dict with the Python implementation and version. 26 | 27 | Provide both the name and the version of the Python implementation 28 | currently running. For example, on CPython 2.7.5 it will return 29 | {'name': 'CPython', 'version': '2.7.5'}. 30 | 31 | This function works best on CPython and PyPy: in particular, it probably 32 | doesn't work for Jython or IronPython. Future investigation should be done 33 | to work out the correct shape of the code for those platforms. 34 | """ 35 | implementation: str = platform.python_implementation() 36 | 37 | if implementation == "CPython": 38 | implementation_version = platform.python_version() 39 | elif implementation == "PyPy": 40 | implementation_version = "%s.%s.%s" % ( 41 | sys.pypy_version_info.major, # type: ignore # noqa: ignore=E1101 pylint: disable=E1101 42 | sys.pypy_version_info.minor, # type: ignore # noqa: ignore=E1101 pylint: disable=E1101 43 | sys.pypy_version_info.micro, # type: ignore # noqa: ignore=E1101 pylint: disable=E1101 44 | ) 45 | rel = sys.pypy_version_info.releaselevel # type: ignore # noqa: ignore=E1101 pylint: disable=E1101 46 | if rel != "final": 47 | implementation_version = "".join([implementation_version, rel]) 48 | elif implementation == "Jython": 49 | implementation_version = platform.python_version() # Complete Guess 50 | elif implementation == "IronPython": 51 | implementation_version = platform.python_version() # Complete Guess 52 | else: 53 | implementation_version = "Unknown" 54 | 55 | return f"{implementation} {implementation_version}" 56 | 57 | 58 | def _mysql_version() -> str: 59 | if which("mysql") is not None: 60 | try: 61 | mysql_version: t.Union[str, bytes] = check_output(["mysql", "-V"]) 62 | try: 63 | return mysql_version.decode().strip() # type: ignore 64 | except (UnicodeDecodeError, AttributeError): 65 | return str(mysql_version) 66 | except Exception: # nosec pylint: disable=W0703 67 | pass 68 | return "MySQL client not found on the system" 69 | 70 | 71 | def info() -> t.List[t.List[str]]: 72 | """Generate information for a bug report.""" 73 | try: 74 | platform_info = f"{platform.system()} {platform.release()}" 75 | except IOError: 76 | platform_info = "Unknown" 77 | 78 | return [ 79 | ["sqlite3-to-mysql", package_version], 80 | ["", ""], 81 | ["Operating System", platform_info], 82 | ["Python", _implementation()], 83 | ["MySQL", _mysql_version()], 84 | ["SQLite", sqlite3.sqlite_version], 85 | ["", ""], 86 | ["click", str(click.__version__)], 87 | ["mysql-connector-python", mysql.connector.__version__], 88 | ["pytimeparse2", pytimeparse2.__version__], 89 | ["simplejson", simplejson.__version__], # type: ignore 90 | ["sqlglot", sqlglot.__version__], 91 | ["tabulate", tabulate.__version__], 92 | ["tqdm", tqdm.__version__], 93 | ] 94 | -------------------------------------------------------------------------------- /tests/unit/sqlite_utils_test.py: -------------------------------------------------------------------------------- 1 | from datetime import date, timedelta 2 | from decimal import Decimal 3 | from unittest.mock import patch 4 | 5 | import pytest 6 | from dateutil.parser import ParserError 7 | from packaging.version import Version 8 | 9 | from sqlite3_to_mysql.sqlite_utils import ( 10 | adapt_decimal, 11 | adapt_timedelta, 12 | check_sqlite_table_xinfo_support, 13 | convert_date, 14 | convert_decimal, 15 | convert_timedelta, 16 | unicase_compare, 17 | ) 18 | 19 | 20 | class TestSQLiteUtils: 21 | def test_adapt_decimal(self) -> None: 22 | """Test adapt_decimal function.""" 23 | assert adapt_decimal(Decimal("123.45")) == "123.45" 24 | assert adapt_decimal(Decimal("0")) == "0" 25 | assert adapt_decimal(Decimal("-123.45")) == "-123.45" 26 | 27 | def test_convert_decimal(self) -> None: 28 | """Test convert_decimal function.""" 29 | assert convert_decimal(b"123.45") == Decimal("123.45") 30 | assert convert_decimal(b"0") == Decimal("0") 31 | assert convert_decimal(b"-123.45") == Decimal("-123.45") 32 | 33 | def test_adapt_timedelta(self) -> None: 34 | """Test adapt_timedelta function.""" 35 | assert adapt_timedelta(timedelta(hours=1, minutes=30, seconds=45)) == "01:30:45" 36 | assert adapt_timedelta(timedelta(hours=0, minutes=0, seconds=0)) == "00:00:00" 37 | assert adapt_timedelta(timedelta(hours=100, minutes=30, seconds=45)) == "100:30:45" 38 | 39 | def test_convert_timedelta(self) -> None: 40 | """Test convert_timedelta function.""" 41 | assert convert_timedelta(b"01:30:45") == timedelta(hours=1, minutes=30, seconds=45) 42 | assert convert_timedelta(b"00:00:00") == timedelta(hours=0, minutes=0, seconds=0) 43 | assert convert_timedelta(b"100:30:45") == timedelta(hours=100, minutes=30, seconds=45) 44 | 45 | def test_unicase_compare(self) -> None: 46 | """Test unicase_compare function.""" 47 | # Test with lowercase strings 48 | assert unicase_compare("abc", "def") == -1 49 | assert unicase_compare("def", "abc") == 1 50 | assert unicase_compare("abc", "abc") == 0 51 | 52 | # Test with mixed case strings 53 | assert unicase_compare("Abc", "def") == -1 54 | assert unicase_compare("DEF", "abc") == 1 55 | assert unicase_compare("ABC", "abc") == 0 56 | 57 | # Test with accented characters 58 | assert unicase_compare("café", "cafe") == 0 59 | assert unicase_compare("café", "cafz") == -1 60 | assert unicase_compare("cafz", "café") == 1 61 | 62 | def test_convert_date(self) -> None: 63 | """Test convert_date function.""" 64 | # Test with string 65 | assert convert_date("2020-01-01") == date(2020, 1, 1) 66 | assert convert_date("2020/01/01") == date(2020, 1, 1) 67 | assert convert_date("Jan 1, 2020") == date(2020, 1, 1) 68 | 69 | # Test with bytes 70 | assert convert_date(b"2020-01-01") == date(2020, 1, 1) 71 | assert convert_date(b"2020/01/01") == date(2020, 1, 1) 72 | assert convert_date(b"Jan 1, 2020") == date(2020, 1, 1) 73 | 74 | def test_convert_date_error(self) -> None: 75 | """Test convert_date function with invalid date.""" 76 | with pytest.raises(ValueError): 77 | convert_date("not a date") 78 | 79 | with pytest.raises(ValueError): 80 | convert_date(b"not a date") 81 | 82 | @pytest.mark.parametrize( 83 | "version_string,expected", 84 | [ 85 | ("3.25.0", False), 86 | ("3.26.0", True), 87 | ("3.27.0", True), 88 | ("4.0.0", True), 89 | ], 90 | ) 91 | def test_check_sqlite_table_xinfo_support(self, version_string: str, expected: bool) -> None: 92 | """Test check_sqlite_table_xinfo_support function.""" 93 | assert check_sqlite_table_xinfo_support(version_string) == expected 94 | -------------------------------------------------------------------------------- /.github/workflows/publish.yml: -------------------------------------------------------------------------------- 1 | name: Publish package 2 | 3 | on: 4 | push: 5 | tags: 6 | - 'v[0-9]+.[0-9]+.[0-9]+*' 7 | defaults: 8 | run: 9 | shell: bash 10 | permissions: read-all 11 | 12 | env: 13 | NAME: sqlite3-to-mysql 14 | 15 | jobs: 16 | test: 17 | uses: ./.github/workflows/test.yml 18 | secrets: inherit 19 | publish: 20 | needs: test 21 | name: "Publish" 22 | runs-on: ubuntu-latest 23 | environment: 24 | name: pypi 25 | url: https://pypi.org/p/${{ env.NAME }} 26 | permissions: 27 | id-token: write 28 | contents: write 29 | concurrency: 30 | group: publish-${{ github.ref }} 31 | cancel-in-progress: false 32 | steps: 33 | - uses: actions/checkout@v6 34 | - name: Compare package version with ref/tag 35 | id: compare 36 | run: | 37 | set -e 38 | VERSION=$(awk -F'"' '/__version__/ {print $2}' src/sqlite3_to_mysql/__init__.py) 39 | TAG=${GITHUB_REF_NAME#v} 40 | if [[ "$VERSION" != "$TAG" ]]; then 41 | echo "Version in src/sqlite3_to_mysql/__init__.py ($VERSION) does not match tag ($TAG)" 42 | exit 1 43 | fi 44 | echo "VERSION=$VERSION" >> $GITHUB_ENV 45 | - name: Check CHANGELOG.md 46 | id: check_changelog 47 | run: | 48 | set -e 49 | if ! grep -q "# $VERSION" CHANGELOG.md; then 50 | echo "CHANGELOG.md does not contain a section for $VERSION" 51 | exit 1 52 | fi 53 | - name: Set up Python 54 | id: setup_python 55 | uses: actions/setup-python@v6 56 | with: 57 | python-version: "3.x" 58 | cache: "pip" 59 | - name: Install build dependencies 60 | id: install_build_dependencies 61 | run: | 62 | set -e 63 | python3 -m pip install -U pip 64 | python3 -m pip install -U build 65 | - name: Build a binary wheel and a source tarball 66 | id: build 67 | run: | 68 | set -e 69 | python3 -m build --sdist --wheel --outdir dist/ . 70 | - name: Verify package metadata (twine) 71 | id: twine_check 72 | run: | 73 | set -e 74 | python3 -m pip install -U twine 75 | twine check dist/* 76 | - name: Ensure py.typed is packaged 77 | id: assert_py_typed 78 | run: | 79 | set -e 80 | unzip -l dist/*.whl | grep -q 'sqlite3_to_mysql/py.typed' 81 | - name: Publish distribution package to PyPI 82 | id: publish 83 | if: startsWith(github.ref, 'refs/tags') 84 | uses: pypa/gh-action-pypi-publish@release/v1 85 | - name: Create tag-specific CHANGELOG 86 | id: create_changelog 87 | run: | 88 | set -e 89 | CHANGELOG_PATH=$RUNNER_TEMP/CHANGELOG.md 90 | awk '/^#[[:space:]].*/ { if (count == 1) exit; count++; print } count == 1 && !/^#[[:space:]].*/ { print }' CHANGELOG.md | sed -e :a -e '/^\n*$/{$d;N;ba' -e '}' > $CHANGELOG_PATH 91 | echo -en "\n[https://pypi.org/project/$NAME/$VERSION/](https://pypi.org/project/$NAME/$VERSION/)" >> $CHANGELOG_PATH 92 | echo "CHANGELOG_PATH=$CHANGELOG_PATH" >> $GITHUB_ENV 93 | - name: Github Release 94 | id: github_release 95 | uses: softprops/action-gh-release@v2 96 | with: 97 | name: ${{ env.VERSION }} 98 | tag_name: ${{ github.ref }} 99 | body_path: ${{ env.CHANGELOG_PATH }} 100 | files: | 101 | dist/*.whl 102 | dist/*.tar.gz 103 | - name: Cleanup 104 | if: ${{ always() }} 105 | run: | 106 | rm -rf dist 107 | rm -rf $CHANGELOG_PATH 108 | docker: 109 | needs: [ test, publish ] 110 | permissions: 111 | packages: write 112 | contents: read 113 | uses: ./.github/workflows/docker.yml 114 | secrets: inherit 115 | docs: 116 | uses: ./.github/workflows/docs.yml 117 | needs: [ test, publish ] 118 | permissions: 119 | contents: write 120 | secrets: inherit -------------------------------------------------------------------------------- /docs/index.rst: -------------------------------------------------------------------------------- 1 | SQLite3 to MySQL 2 | ================ 3 | 4 | A simple Python tool to transfer data from SQLite 3 to MySQL 5 | 6 | |PyPI| |PyPI - Downloads| |Homebrew Formula Downloads| |PyPI - Python Version| 7 | |MySQL Support| |MariaDB Support| |GitHub license| |Contributor Covenant| 8 | |PyPI - Format| |Code style: black| |Codacy Badge| |Test Status| |CodeQL Status| 9 | |Publish PyPI Package Status| |codecov| |GitHub Sponsors| |GitHub stars| 10 | 11 | Installation 12 | ------------ 13 | 14 | .. code:: bash 15 | 16 | pip install sqlite3-to-mysql 17 | 18 | Basic Usage 19 | ----------- 20 | 21 | .. code:: bash 22 | 23 | sqlite3mysql -f path/to/foo.sqlite -d foo_db -u foo_user -p 24 | 25 | .. toctree:: 26 | :maxdepth: 2 27 | :caption: Contents: 28 | 29 | README 30 | modules 31 | 32 | Indices and tables 33 | ================== 34 | 35 | * :ref:`genindex` 36 | * :ref:`modindex` 37 | * :ref:`search` 38 | 39 | .. |PyPI| image:: https://img.shields.io/pypi/v/sqlite3-to-mysql?logo=pypi 40 | :target: https://pypi.org/project/sqlite3-to-mysql/ 41 | .. |PyPI - Downloads| image:: https://img.shields.io/pypi/dm/sqlite3-to-mysql?logo=pypi&label=PyPI%20downloads 42 | :target: https://pypistats.org/packages/sqlite3-to-mysql 43 | .. |Homebrew Formula Downloads| image:: https://img.shields.io/homebrew/installs/dm/sqlite3-to-mysql?logo=homebrew&label=Homebrew%20downloads 44 | :target: https://formulae.brew.sh/formula/sqlite3-to-mysql 45 | .. |PyPI - Python Version| image:: https://img.shields.io/pypi/pyversions/sqlite3-to-mysql?logo=python 46 | :target: https://pypi.org/project/sqlite3-to-mysql/ 47 | .. |MySQL Support| image:: https://img.shields.io/static/v1?logo=mysql&label=MySQL&message=5.5+%7C+5.6+%7C+5.7+%7C+8.0&color=2b5d80 48 | :target: https://img.shields.io/static/v1?label=MySQL&message=5.6+%7C+5.7+%7C+8.0&color=2b5d80 49 | .. |MariaDB Support| image:: https://img.shields.io/static/v1?logo=mariadb&label=MariaDB&message=5.5+%7C+10.0+%7C+10.1+%7C+10.2+%7C+10.3+%7C+10.4+%7C+10.5+%7C+10.6%7C+10.11&color=C0765A 50 | :target: https://img.shields.io/static/v1?label=MariaDB&message=10.0+%7C+10.1+%7C+10.2+%7C+10.3+%7C+10.4+%7C+10.5&color=C0765A 51 | .. |GitHub license| image:: https://img.shields.io/github/license/techouse/sqlite3-to-mysql 52 | :target: https://github.com/techouse/sqlite3-to-mysql/blob/master/LICENSE 53 | .. |Contributor Covenant| image:: https://img.shields.io/badge/Contributor%20Covenant-2.1-4baaaa.svg?logo=contributorcovenant 54 | :target: CODE-OF-CONDUCT.md 55 | .. |PyPI - Format| image:: https://img.shields.io/pypi/format/sqlite3-to-mysql?logo=python 56 | :target: (https://pypi.org/project/sqlite3-to-mysql/) 57 | .. |Code style: black| image:: https://img.shields.io/badge/code%20style-black-000000.svg?logo=python 58 | :target: https://github.com/ambv/black 59 | .. |Codacy Badge| image:: https://api.codacy.com/project/badge/Grade/d33b59d35b924711aae9418741a923ae 60 | :target: https://www.codacy.com/manual/techouse/sqlite3-to-mysql?utm_source=github.com&utm_medium=referral&utm_content=techouse/sqlite3-to-mysql&utm_campaign=Badge_Grade 61 | .. |Test Status| image:: https://github.com/techouse/sqlite3-to-mysql/actions/workflows/test.yml/badge.svg 62 | :target: https://github.com/techouse/sqlite3-to-mysql/actions/workflows/test.yml 63 | .. |CodeQL Status| image:: https://github.com/techouse/sqlite3-to-mysql/actions/workflows/github-code-scanning/codeql/badge.svg 64 | :target: https://github.com/techouse/sqlite3-to-mysql/actions/workflows/codeql-analysis.yml 65 | .. |Publish PyPI Package Status| image:: https://github.com/techouse/sqlite3-to-mysql/actions/workflows/publish.yml/badge.svg 66 | :target: https://github.com/techouse/sqlite3-to-mysql/actions/workflows/publish.yml 67 | .. |codecov| image:: https://codecov.io/gh/techouse/sqlite3-to-mysql/branch/master/graph/badge.svg 68 | :target: https://codecov.io/gh/techouse/sqlite3-to-mysql 69 | .. |GitHub Sponsors| image:: https://img.shields.io/github/sponsors/techouse?logo=github 70 | :target: https://github.com/sponsors/techouse 71 | .. |GitHub stars| image:: https://img.shields.io/github/stars/techouse/sqlite3-to-mysql.svg?style=social&label=Star&maxAge=2592000 72 | :target: https://github.com/techouse/sqlite3-to-mysql/stargazers -------------------------------------------------------------------------------- /docs/README.rst: -------------------------------------------------------------------------------- 1 | Usage 2 | ----- 3 | 4 | Options 5 | ^^^^^^^ 6 | 7 | The command line options for the ``sqlite3mysql`` tool are as follows: 8 | 9 | .. code-block:: bash 10 | 11 | sqlite3mysql [OPTIONS] 12 | 13 | Required Options 14 | """""""""""""""" 15 | 16 | - ``-f, --sqlite-file PATH``: SQLite3 database file [required] 17 | - ``-d, --mysql-database TEXT``: MySQL database name [required] 18 | - ``-u, --mysql-user TEXT``: MySQL user [required] 19 | 20 | Password Options 21 | """""""""""""""" 22 | 23 | - ``-p, --prompt-mysql-password``: Prompt for MySQL password 24 | - ``--mysql-password TEXT``: MySQL password 25 | 26 | Connection Options 27 | """""""""""""""""" 28 | 29 | - ``-h, --mysql-host TEXT``: MySQL host. Defaults to localhost. 30 | - ``-P, --mysql-port INTEGER``: MySQL port. Defaults to 3306. 31 | - ``-S, --skip-ssl``: Disable MySQL connection encryption. 32 | - ``--mysql-socket TEXT``: Path to MySQL unix socket file. 33 | 34 | Transfer Options 35 | """""""""""""""" 36 | 37 | - ``-t, --sqlite-tables TUPLE``: Transfer only these specific tables (space separated table names). Implies ``--without-foreign-keys`` which inhibits the transfer of foreign keys. Can not be used together with ``--exclude-sqlite-tables``. 38 | - ``-e, --exclude-sqlite-tables TUPLE``: Exclude these specific tables (space separated table names). Implies ``--without-foreign-keys`` which inhibits the transfer of foreign keys. Can not be used together with ``--sqlite-tables``. 39 | - ``-A, --sqlite-views-as-tables``: Materialize SQLite views as tables in MySQL instead of creating matching MySQL views (legacy behavior). 40 | - ``-E, --mysql-truncate-tables``: Truncates existing tables before inserting data. 41 | - ``-K, --mysql-skip-create-tables``: Skip creating tables in MySQL. 42 | - ``-i, --mysql-insert-method [UPDATE|IGNORE|DEFAULT]``: MySQL insert method. DEFAULT will throw errors when encountering duplicate records; UPDATE will update existing rows; IGNORE will ignore insert errors. Defaults to IGNORE. 43 | - ``-J, --mysql-skip-transfer-data``: Skip transferring data to MySQL. 44 | - ``--mysql-integer-type TEXT``: MySQL default integer field type. Defaults to INT(11). 45 | - ``--mysql-string-type TEXT``: MySQL default string field type. Defaults to VARCHAR(255). 46 | - ``--mysql-text-type [LONGTEXT|MEDIUMTEXT|TEXT|TINYTEXT]``: MySQL default text field type. Defaults to TEXT. 47 | - ``--mysql-charset TEXT``: MySQL database and table character set. Defaults to utf8mb4. 48 | ` ``--mysql-collation TEXT``: MySQL database and table collation 49 | - ``-T, --use-fulltext``: Use FULLTEXT indexes on TEXT columns. Will throw an error if your MySQL version does not support InnoDB FULLTEXT indexes! 50 | - ``-X, --without-foreign-keys``: Do not transfer foreign keys. 51 | - ``-W, --ignore-duplicate-keys``: Ignore duplicate keys. The default behavior is to create new ones with a numerical suffix, e.g. 'existing_key' -> 'existing_key_1' 52 | - ``--with-rowid``: Transfer rowid columns. 53 | - ``-c, --chunk INTEGER``: Chunk reading/writing SQL records 54 | 55 | Other Options 56 | """"""""""""" 57 | 58 | - ``-l, --log-file PATH``: Log file 59 | - ``-q, --quiet``: Quiet. Display only errors. 60 | - ``--debug``: Debug mode. Will throw exceptions. 61 | - ``--version``: Show the version and exit. 62 | - ``--help``: Show help message and exit. 63 | 64 | Docker 65 | ^^^^^^ 66 | 67 | If you don’t want to install the tool on your system, you can use the 68 | Docker image instead. 69 | 70 | .. code:: bash 71 | 72 | docker run -it \ 73 | --workdir $(pwd) \ 74 | --volume $(pwd):$(pwd) \ 75 | --rm ghcr.io/techouse/sqlite3-to-mysql:latest \ 76 | --sqlite-file baz.db \ 77 | --mysql-user foo \ 78 | --mysql-password bar \ 79 | --mysql-database baz \ 80 | --mysql-host host.docker.internal 81 | 82 | This will mount your host current working directory (pwd) inside the 83 | Docker container as the current working directory. Any files Docker 84 | would write to the current working directory are written to the host 85 | directory where you did docker run. Note that you have to also use a 86 | `special 87 | hostname `__ 88 | ``host.docker.internal`` to access your host machine from inside the 89 | Docker container. 90 | 91 | Homebrew 92 | ^^^^^^^^ 93 | 94 | If you’re on macOS, you can install the tool using 95 | `Homebrew `__. 96 | 97 | .. code:: bash 98 | 99 | brew install sqlite3-to-mysql 100 | sqlite3mysql --help -------------------------------------------------------------------------------- /.github/copilot-instructions.md: -------------------------------------------------------------------------------- 1 | # AI Coding Agent Instructions for `sqlite3-to-mysql` 2 | 3 | > Purpose: Help agents quickly contribute features or fixes to this SQLite → MySQL transfer tool while respecting project workflows and constraints. 4 | 5 | ## Architecture Snapshot 6 | - Core package lives in `src/sqlite3_to_mysql/`. 7 | - `cli.py` defines the `sqlite3mysql` Click CLI and validates option interplay (mutual exclusions, implied flags). 8 | - `transporter.py` implements the end-to-end transfer in class `SQLite3toMySQL.transfer()`: create DB (if missing), create tables, optionally truncate, bulk insert (chunked or streamed), then indices and foreign keys. 9 | - `mysql_utils.py` & `sqlite_utils.py` encapsulate dialect/version feature checks, type adaptation and identifier safety (`safe_identifier_length`). Keep new MySQL capability gates here. 10 | - `types.py` provides typed param/attribute structures consumed via `Unpack` in `SQLite3toMySQL.__init__`. 11 | - Data flow: CLI → construct `SQLite3toMySQL` → introspect SQLite schema via PRAGMA → create MySQL schema → transfer rows (streamed / chunked) → add indices → add foreign keys. 12 | - Generated/ephemeral outputs: logs (optional), progress bars (`tqdm`), created MySQL objects. 13 | 14 | ## Key Behaviors & Patterns 15 | - Table creation logic retries without DEFAULTs if MySQL rejects expression defaults (`_create_table` with `skip_default=True`). Preserve this two-pass approach when modifying default handling. 16 | - MySQL feature detection stored as booleans (`_mysql_fulltext_support`, `_allow_expr_defaults`, etc.) set early in `__init__`. New conditional behaviors should follow this pattern. 17 | - Column type translation centralised in `_translate_type_from_sqlite_to_mysql`; prefer extending mappings there rather than scattering conversions. 18 | - DEFAULT value normalization handled by `_translate_default_for_mysql`; supports CURRENT_* and boolean coercion. Extend cautiously—return empty string to suppress invalid defaults. 19 | - Identifier shortening: always wrap dynamic table/index/column names with `safe_identifier_length(...)` before emitting SQL. 20 | - Index creation may recurse on duplicate names, appending numeric suffix. Respect `_ignore_duplicate_keys` to skip retries. 21 | - Chunked data transfer controlled by `--chunk`; when present uses `fetchmany` loop, else full `fetchall` with `tqdm` progress. 22 | - Foreign keys only added when neither table include/exclude filters nor `--without-foreign-keys` effective. 23 | 24 | ## CLI Option Conventions 25 | - Mutually exclusive: `--sqlite-tables` vs `--exclude-sqlite-tables`; setting either implies `--without-foreign-keys`. 26 | - Disallow simultaneous `-K` (skip create) and `-J` (skip transfer) — early exit. 27 | - Insert method: `IGNORE` (default), `UPDATE` (uses ON DUPLICATE KEY UPDATE with optional VALUES alias), `DEFAULT` (no modifiers). 28 | 29 | ## Development Workflow 30 | - Local env: `python3 -m venv env && source env/bin/activate && pip install -e . && pip install -r requirements_dev.txt`. 31 | - Run tests with coverage: `pytest -v --cov=src/sqlite3_to_mysql` (unit: `tests/unit`, functional/CLI: `tests/func`). Functional tests need running MySQL (e.g. `docker run --rm -d -e MYSQL_ROOT_PASSWORD=test -p 3306:3306 mysql:8`). Credentials from `tests/db_credentials.json`. 32 | - Full quality gate: `tox -e linters` (runs black, isort, flake8, pylint, bandit, mypy). Use `tox -e python3.12` for test subset. 33 | - Formatting: Black (120 cols) + isort profile=black; Flake8 enforces 88-col soft cap—avoid long chained expressions in one line. 34 | 35 | ## Adding Features Safely 36 | - Add new MySQL/SQLite capability checks in `mysql_utils.py` / `sqlite_utils.py`, expose booleans during `__init__` for downstream logic. 37 | - For new CLI flags: define in `cli.py` above `cli()` with clear help text; maintain mutual exclusion patterns and keep error messages consistent with existing ones. 38 | - Ensure new behavior gets unit tests (isolate pure functions) plus at least one functional test hitting the CLI. 39 | - Avoid editing files in `build/`, `docs/_build`, `htmlcov/`—treat them as generated. 40 | 41 | ## Performance Considerations 42 | - Prefer prepared cursors (`cursor(prepared=True)`) like existing code; batch inserts via `executemany`. 43 | - Large transfers: encourage using `--chunk` to reduce memory footprint; any new bulk path must preserve commit granularity. 44 | 45 | ## Logging & Debugging 46 | - Use `_logger` from class; don't print directly. Respect `quiet` flag for progress/INFO output; errors still emitted. 47 | - Debug mode (`--debug`) surfaces exceptions instead of swallowing. 48 | 49 | ## Contribution Style 50 | - Single-concern commits with gitmoji prefix (e.g., `:sparkles:` for features). Update `CHANGELOG.md` for user-facing changes. 51 | - Preserve type hints; new public interfaces should be annotated and, if user-facing, reflected in docs (`docs/`). 52 | 53 | --- 54 | Feedback welcome: Clarify any missing workflow, edge case, or pattern you need—request updates and this doc will iterate. 55 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | requires = ["hatchling"] 3 | build-backend = "hatchling.build" 4 | 5 | [project] 6 | name = "sqlite3-to-mysql" 7 | description = "A simple Python tool to transfer data from SQLite 3 to MySQL" 8 | readme = { file = "README.md", content-type = "text/markdown" } 9 | license = "MIT" 10 | license-files = ["LICENSE"] 11 | requires-python = ">=3.9" 12 | authors = [ 13 | { name = "Klemen Tusar", email = "techouse@gmail.com" }, 14 | ] 15 | keywords = [ 16 | "sqlite3", 17 | "mysql", 18 | "transfer", 19 | "data", 20 | "migrate", 21 | "migration", 22 | ] 23 | classifiers = [ 24 | "Development Status :: 5 - Production/Stable", 25 | "Environment :: Console", 26 | "Intended Audience :: End Users/Desktop", 27 | "Intended Audience :: Developers", 28 | "Intended Audience :: System Administrators", 29 | "License :: OSI Approved :: MIT License", 30 | "Operating System :: OS Independent", 31 | "Programming Language :: Python", 32 | "Programming Language :: Python :: 3", 33 | "Programming Language :: Python :: 3.9", 34 | "Programming Language :: Python :: 3.10", 35 | "Programming Language :: Python :: 3.11", 36 | "Programming Language :: Python :: 3.12", 37 | "Programming Language :: Python :: 3.13", 38 | "Programming Language :: Python :: 3.14", 39 | "Programming Language :: Python :: Implementation :: CPython", 40 | "Typing :: Typed", 41 | "Topic :: Database", 42 | ] 43 | dependencies = [ 44 | "Click>=8.1.3", 45 | "mysql-connector-python>=9.0.0", 46 | "pytimeparse2", 47 | "python-dateutil>=2.9.0.post0", 48 | "simplejson>=3.19.1", 49 | "sqlglot>=27.27.0", 50 | "tqdm>=4.65.0", 51 | "tabulate", 52 | "Unidecode>=1.3.6", 53 | "typing-extensions; python_version < \"3.11\"", 54 | "packaging", 55 | ] 56 | dynamic = ["version"] 57 | 58 | [project.urls] 59 | Homepage = "https://techouse.github.io/sqlite3-to-mysql/" 60 | Documentation = "https://techouse.github.io/sqlite3-to-mysql/" 61 | Repository = "https://github.com/techouse/sqlite3-to-mysql.git" 62 | Issues = "https://github.com/techouse/sqlite3-to-mysql/issues" 63 | Changelog = "https://github.com/techouse/sqlite3-to-mysql/blob/master/CHANGELOG.md" 64 | Sponsor = "https://github.com/sponsors/techouse" 65 | PayPal = "https://paypal.me/ktusar" 66 | 67 | [project.optional-dependencies] 68 | dev = [ 69 | "docker>=6.1.3", 70 | "factory-boy", 71 | "Faker>=18.10.0", 72 | "PyMySQL>=1.0.3", 73 | "pytest>=7.3.1", 74 | "pytest-cov", 75 | "pytest-mock", 76 | "pytest-timeout", 77 | "types-python-dateutil", 78 | "types-simplejson", 79 | "sqlalchemy>=2.0.0", 80 | "sqlalchemy-utils", 81 | "types-sqlalchemy-utils", 82 | "tox", 83 | "types-tqdm", 84 | "packaging", 85 | "types-tabulate", 86 | "types-Pygments", 87 | "types-colorama", 88 | "types-mock", 89 | "types-setuptools" 90 | ] 91 | 92 | [tool.hatch.version] 93 | path = "src/sqlite3_to_mysql/__init__.py" 94 | 95 | [tool.hatch.build.targets.sdist] 96 | include = [ 97 | "src", 98 | "tests", 99 | "README.md", 100 | "CHANGELOG.md", 101 | "CODE-OF-CONDUCT.md", 102 | "LICENSE", 103 | "requirements_dev.txt", 104 | ] 105 | 106 | [tool.hatch.build.targets.wheel] 107 | packages = ["src/sqlite3_to_mysql"] 108 | include = ["src/sqlite3_to_mysql/py.typed"] 109 | 110 | [project.scripts] 111 | sqlite3mysql = "sqlite3_to_mysql.cli:cli" 112 | 113 | [tool.black] 114 | line-length = 120 115 | target-version = ["py39", "py310", "py311", "py312", "py313", "py314"] 116 | include = '\.pyi?$' 117 | exclude = ''' 118 | ( 119 | /( 120 | \.eggs 121 | | \.git 122 | | \.hg 123 | | \.mypy_cache 124 | | \.tox 125 | | \.venv 126 | | _build 127 | | buck-out 128 | | build 129 | | dist 130 | | docs 131 | )/ 132 | | foo.py 133 | ) 134 | ''' 135 | 136 | [tool.isort] 137 | line_length = 120 138 | profile = "black" 139 | lines_after_imports = 2 140 | known_first_party = "sqlite3_to_mysql" 141 | skip_gitignore = true 142 | 143 | [tool.pytest.ini_options] 144 | pythonpath = ["src"] 145 | testpaths = ["tests"] 146 | norecursedirs = [".*", "venv", "env", "*.egg", "dist", "build"] 147 | minversion = "7.3.1" 148 | addopts = "-rsxX --tb=short --strict-markers" 149 | timeout = 300 150 | markers = [ 151 | "init: Run the initialisation test functions", 152 | "transfer: Run the main transfer test functions", 153 | "cli: Run the cli test functions", 154 | ] 155 | 156 | [tool.mypy] 157 | mypy_path = "src" 158 | python_version = "3.9" 159 | exclude = [ 160 | "tests", 161 | "docs", 162 | "build", 163 | "dist", 164 | "venv", 165 | "env", 166 | ] 167 | warn_return_any = true 168 | warn_unused_configs = true 169 | 170 | [[tool.mypy.overrides]] 171 | module = "pytimeparse2.*,factory.*,docker.*" 172 | ignore_missing_imports = true 173 | -------------------------------------------------------------------------------- /tests/factories.py: -------------------------------------------------------------------------------- 1 | import typing as t 2 | from os import environ 3 | 4 | import factory 5 | 6 | from . import faker_providers, models 7 | 8 | 9 | factory.Faker.add_provider(faker_providers.DateTimeProviders) 10 | 11 | 12 | class AuthorFactory(factory.Factory): 13 | class Meta: 14 | model: t.Type[models.Author] = models.Author 15 | 16 | name: factory.Faker = factory.Faker("name") 17 | 18 | 19 | class ImageFactory(factory.Factory): 20 | class Meta: 21 | model: t.Type[models.Image] = models.Image 22 | 23 | path: factory.Faker = factory.Faker("file_path", depth=3, extension="jpg") 24 | description: factory.Faker = factory.Faker("sentence", nb_words=12, variable_nb_words=True) 25 | 26 | 27 | class TagFactory(factory.Factory): 28 | class Meta: 29 | model: t.Type[models.Tag] = models.Tag 30 | 31 | name: factory.Faker = factory.Faker("sentence", nb_words=3, variable_nb_words=True) 32 | 33 | 34 | class MiscFactory(factory.Factory): 35 | class Meta: 36 | model: t.Type[models.Misc] = models.Misc 37 | 38 | big_integer_field: factory.Faker = factory.Faker("pyint", max_value=10**9) 39 | blob_field: factory.Faker = factory.Faker("binary", length=1024 * 10) 40 | boolean_field: factory.Faker = factory.Faker("boolean") 41 | char_field: factory.Faker = factory.Faker("text", max_nb_chars=255) 42 | date_field: factory.Faker = factory.Faker("date_this_decade") 43 | date_time_field: factory.Faker = factory.Faker("date_time_this_century_without_microseconds") 44 | decimal_field: factory.Faker = factory.Faker("pydecimal", left_digits=8, right_digits=2) 45 | float_field: factory.Faker = factory.Faker("pyfloat", left_digits=8, right_digits=4) 46 | integer_field: factory.Faker = factory.Faker("pyint", min_value=-(2**31), max_value=2**31 - 1) 47 | if environ.get("LEGACY_DB", "0") == "0": 48 | json_field = factory.Faker( 49 | "pydict", 50 | nb_elements=10, 51 | variable_nb_elements=True, 52 | value_types=["str", "int", "float", "boolean", "date_time"], 53 | ) 54 | numeric_field: factory.Faker = factory.Faker("pyfloat", left_digits=8, right_digits=4) 55 | real_field: factory.Faker = factory.Faker("pyfloat", left_digits=8, right_digits=4) 56 | small_integer_field: factory.Faker = factory.Faker("pyint", min_value=-(2**15), max_value=2**15 - 1) 57 | string_field: factory.Faker = factory.Faker("text", max_nb_chars=255) 58 | text_field: factory.Faker = factory.Faker("text", max_nb_chars=1024) 59 | time_field: factory.Faker = factory.Faker("time_object_without_microseconds") 60 | varchar_field: factory.Faker = factory.Faker("text", max_nb_chars=255) 61 | timestamp_field: factory.Faker = factory.Faker("date_time_this_century_without_microseconds") 62 | my_type_field: factory.Faker = factory.Faker("text", max_nb_chars=255) 63 | 64 | 65 | class ArticleFactory(factory.Factory): 66 | class Meta: 67 | model: t.Type[models.Article] = models.Article 68 | 69 | hash: factory.Faker = factory.Faker("md5") 70 | title: factory.Faker = factory.Faker("sentence", nb_words=6) 71 | slug: factory.Faker = factory.Faker("slug") 72 | content: factory.Faker = factory.Faker("text", max_nb_chars=1024) 73 | status: factory.Faker = factory.Faker("pystr", max_chars=1) 74 | published: factory.Faker = factory.Faker("date_between", start_date="-1y", end_date="-1d") 75 | 76 | @factory.post_generation 77 | def authors(self, create, extracted, **kwargs): 78 | if not create: 79 | # Simple build, do nothing. 80 | return 81 | 82 | if extracted: 83 | # A list of authors were passed in, use them 84 | for author in extracted: 85 | self.authors.add(author) 86 | 87 | @factory.post_generation 88 | def tags(self, create, extracted, **kwargs): 89 | if not create: 90 | # Simple build, do nothing. 91 | return 92 | 93 | if extracted: 94 | # A list of authors were passed in, use them 95 | for tag in extracted: 96 | self.tags.add(tag) 97 | 98 | @factory.post_generation 99 | def images(self, create, extracted, **kwargs): 100 | if not create: 101 | # Simple build, do nothing. 102 | return 103 | 104 | if extracted: 105 | # A list of authors were passed in, use them 106 | for image in extracted: 107 | self.images.add(image) 108 | 109 | @factory.post_generation 110 | def misc(self, create, extracted, **kwargs): 111 | if not create: 112 | # Simple build, do nothing. 113 | return 114 | 115 | if extracted: 116 | # A list of authors were passed in, use them 117 | for misc in extracted: 118 | self.misc.add(misc) 119 | 120 | 121 | class MediaFactory(factory.Factory): 122 | class Meta: 123 | model: t.Type[models.Media] = models.Media 124 | 125 | id: factory.Faker = factory.Faker("sha256", raw_output=False) 126 | title: factory.Faker = factory.Faker("sentence", nb_words=6) 127 | description: factory.Faker = factory.Faker("sentence", nb_words=12, variable_nb_words=True) 128 | -------------------------------------------------------------------------------- /src/sqlite3_to_mysql/mysql_utils.py: -------------------------------------------------------------------------------- 1 | """MySQL helpers.""" 2 | 3 | import re 4 | import typing as t 5 | 6 | from mysql.connector import CharacterSet 7 | from mysql.connector.charsets import MYSQL_CHARACTER_SETS 8 | from packaging import version 9 | 10 | 11 | # Shamelessly copied from SQLAlchemy's dialects/mysql/__init__.py 12 | MYSQL_COLUMN_TYPES: t.Tuple[str, ...] = ( 13 | "BIGINT", 14 | "BINARY", 15 | "BIT", 16 | "BLOB", 17 | "BOOLEAN", 18 | "CHAR", 19 | "DATE", 20 | "DATETIME", 21 | "DECIMAL", 22 | "DOUBLE", 23 | "ENUM", 24 | "FLOAT", 25 | "INTEGER", 26 | "JSON", 27 | "LONGBLOB", 28 | "LONGTEXT", 29 | "MEDIUMBLOB", 30 | "MEDIUMINT", 31 | "MEDIUMTEXT", 32 | "NCHAR", 33 | "NVARCHAR", 34 | "NUMERIC", 35 | "SET", 36 | "SMALLINT", 37 | "REAL", 38 | "TEXT", 39 | "TIME", 40 | "TIMESTAMP", 41 | "TINYBLOB", 42 | "TINYINT", 43 | "TINYTEXT", 44 | "VARBINARY", 45 | "VARCHAR", 46 | "YEAR", 47 | ) 48 | 49 | MYSQL_TEXT_COLUMN_TYPES: t.Tuple[str, ...] = ( 50 | "LONGTEXT", 51 | "MEDIUMTEXT", 52 | "TEXT", 53 | "TINYTEXT", 54 | ) 55 | 56 | MYSQL_TEXT_COLUMN_TYPES_WITH_JSON: t.Tuple[str, ...] = ("JSON",) + MYSQL_TEXT_COLUMN_TYPES 57 | 58 | MYSQL_BLOB_COLUMN_TYPES: t.Tuple[str, ...] = ( 59 | "LONGBLOB", 60 | "MEDIUMBLOB", 61 | "BLOB", 62 | "TINYBLOB", 63 | ) 64 | 65 | MYSQL_COLUMN_TYPES_WITHOUT_DEFAULT: t.Tuple[str, ...] = ( 66 | ("GEOMETRY",) + MYSQL_TEXT_COLUMN_TYPES_WITH_JSON + MYSQL_BLOB_COLUMN_TYPES 67 | ) 68 | 69 | 70 | MYSQL_INSERT_METHOD: t.Tuple[str, ...] = ( 71 | "DEFAULT", 72 | "IGNORE", 73 | "UPDATE", 74 | ) 75 | 76 | 77 | class CharSet(t.NamedTuple): 78 | """MySQL character set as a named tuple.""" 79 | 80 | id: int 81 | charset: str 82 | collation: str 83 | 84 | 85 | def mysql_supported_character_sets(charset: t.Optional[str] = None) -> t.Iterator[CharSet]: 86 | """Get supported MySQL character sets.""" 87 | index: int 88 | info: t.Optional[t.Tuple[str, str, bool]] 89 | if charset is not None: 90 | for index, info in enumerate(MYSQL_CHARACTER_SETS): 91 | if info is not None: 92 | try: 93 | if info[0] == charset: 94 | yield CharSet(index, charset, info[1]) 95 | except KeyError: 96 | continue 97 | else: 98 | for charset in CharacterSet().get_supported(): 99 | for index, info in enumerate(MYSQL_CHARACTER_SETS): 100 | if info is not None: 101 | try: 102 | yield CharSet(index, charset, info[1]) 103 | except KeyError: 104 | continue 105 | 106 | 107 | def get_mysql_version(version_string: str) -> version.Version: 108 | """Get MySQL version.""" 109 | return version.parse(re.sub("-.*$", "", version_string)) 110 | 111 | 112 | def check_mysql_json_support(version_string: str) -> bool: 113 | """Check for MySQL JSON support.""" 114 | mysql_version: version.Version = get_mysql_version(version_string) 115 | if "-mariadb" in version_string.lower(): 116 | return mysql_version >= version.parse("10.2.7") 117 | return mysql_version >= version.parse("5.7.8") 118 | 119 | 120 | def check_mysql_values_alias_support(version_string: str) -> bool: 121 | """Check for VALUES alias support. 122 | 123 | Returns: 124 | bool: True if VALUES alias is supported (MySQL 8.0.19+), False for MariaDB 125 | or older MySQL versions. 126 | """ 127 | mysql_version: version.Version = get_mysql_version(version_string) 128 | if "-mariadb" in version_string.lower(): 129 | return False 130 | # Only MySQL 8.0.19 and later support VALUES alias 131 | return mysql_version >= version.parse("8.0.19") 132 | 133 | 134 | def check_mysql_fulltext_support(version_string: str) -> bool: 135 | """Check for FULLTEXT indexing support.""" 136 | mysql_version: version.Version = get_mysql_version(version_string) 137 | if "-mariadb" in version_string.lower(): 138 | return mysql_version >= version.parse("10.0.5") 139 | return mysql_version >= version.parse("5.6.0") 140 | 141 | 142 | def check_mysql_expression_defaults_support(version_string: str) -> bool: 143 | """Check for expression defaults support.""" 144 | mysql_version: version.Version = get_mysql_version(version_string) 145 | if "-mariadb" in version_string.lower(): 146 | return mysql_version >= version.parse("10.2.0") 147 | return mysql_version >= version.parse("8.0.13") 148 | 149 | 150 | def check_mysql_current_timestamp_datetime_support(version_string: str) -> bool: 151 | """Check for CURRENT_TIMESTAMP support for DATETIME fields.""" 152 | mysql_version: version.Version = get_mysql_version(version_string) 153 | if "-mariadb" in version_string.lower(): 154 | return mysql_version >= version.parse("10.0.1") 155 | return mysql_version >= version.parse("5.6.5") 156 | 157 | 158 | def check_mysql_fractional_seconds_support(version_string: str) -> bool: 159 | """Check for fractional seconds support.""" 160 | mysql_version: version.Version = get_mysql_version(version_string) 161 | if "-mariadb" in version_string.lower(): 162 | return mysql_version >= version.parse("10.1.2") 163 | return mysql_version >= version.parse("5.6.4") 164 | 165 | 166 | def safe_identifier_length(identifier_name: str, max_length: int = 64) -> str: 167 | """https://dev.mysql.com/doc/refman/8.0/en/identifier-length.html.""" 168 | return str(identifier_name)[:max_length] 169 | -------------------------------------------------------------------------------- /CODE-OF-CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Contributor Covenant Code of Conduct 2 | 3 | ## Our Pledge 4 | 5 | We as members, contributors, and leaders pledge to make participation in our 6 | community a harassment-free experience for everyone, regardless of age, body 7 | size, visible or invisible disability, ethnicity, sex characteristics, gender 8 | identity and expression, level of experience, education, socio-economic status, 9 | nationality, personal appearance, race, caste, color, religion, or sexual identity 10 | and orientation. 11 | 12 | We pledge to act and interact in ways that contribute to an open, welcoming, 13 | diverse, inclusive, and healthy community. 14 | 15 | ## Our Standards 16 | 17 | Examples of behavior that contributes to a positive environment for our 18 | community include: 19 | 20 | * Demonstrating empathy and kindness toward other people 21 | * Being respectful of differing opinions, viewpoints, and experiences 22 | * Giving and gracefully accepting constructive feedback 23 | * Accepting responsibility and apologizing to those affected by our mistakes, 24 | and learning from the experience 25 | * Focusing on what is best not just for us as individuals, but for the 26 | overall community 27 | 28 | Examples of unacceptable behavior include: 29 | 30 | * The use of sexualized language or imagery, and sexual attention or 31 | advances of any kind 32 | * Trolling, insulting or derogatory comments, and personal or political attacks 33 | * Public or private harassment 34 | * Publishing others' private information, such as a physical or email 35 | address, without their explicit permission 36 | * Other conduct which could reasonably be considered inappropriate in a 37 | professional setting 38 | 39 | ## Enforcement Responsibilities 40 | 41 | Community leaders are responsible for clarifying and enforcing our standards of 42 | acceptable behavior and will take appropriate and fair corrective action in 43 | response to any behavior that they deem inappropriate, threatening, offensive, 44 | or harmful. 45 | 46 | Community leaders have the right and responsibility to remove, edit, or reject 47 | comments, commits, code, wiki edits, issues, and other contributions that are 48 | not aligned to this Code of Conduct, and will communicate reasons for moderation 49 | decisions when appropriate. 50 | 51 | ## Scope 52 | 53 | This Code of Conduct applies within all community spaces, and also applies when 54 | an individual is officially representing the community in public spaces. 55 | Examples of representing our community include using an official e-mail address, 56 | posting via an official social media account, or acting as an appointed 57 | representative at an online or offline event. 58 | 59 | ## Enforcement 60 | 61 | Instances of abusive, harassing, or otherwise unacceptable behavior may be 62 | reported to the community leaders responsible for enforcement at 63 | [techouse@gmail.com](mailto:techouse@gmail.com). 64 | All complaints will be reviewed and investigated promptly and fairly. 65 | 66 | All community leaders are obligated to respect the privacy and security of the 67 | reporter of any incident. 68 | 69 | ## Enforcement Guidelines 70 | 71 | Community leaders will follow these Community Impact Guidelines in determining 72 | the consequences for any action they deem in violation of this Code of Conduct: 73 | 74 | ### 1. Correction 75 | 76 | **Community Impact**: Use of inappropriate language or other behavior deemed 77 | unprofessional or unwelcome in the community. 78 | 79 | **Consequence**: A private, written warning from community leaders, providing 80 | clarity around the nature of the violation and an explanation of why the 81 | behavior was inappropriate. A public apology may be requested. 82 | 83 | ### 2. Warning 84 | 85 | **Community Impact**: A violation through a single incident or series 86 | of actions. 87 | 88 | **Consequence**: A warning with consequences for continued behavior. No 89 | interaction with the people involved, including unsolicited interaction with 90 | those enforcing the Code of Conduct, for a specified period of time. This 91 | includes avoiding interactions in community spaces as well as external channels 92 | like social media. Violating these terms may lead to a temporary or 93 | permanent ban. 94 | 95 | ### 3. Temporary Ban 96 | 97 | **Community Impact**: A serious violation of community standards, including 98 | sustained inappropriate behavior. 99 | 100 | **Consequence**: A temporary ban from any sort of interaction or public 101 | communication with the community for a specified period of time. No public or 102 | private interaction with the people involved, including unsolicited interaction 103 | with those enforcing the Code of Conduct, is allowed during this period. 104 | Violating these terms may lead to a permanent ban. 105 | 106 | ### 4. Permanent Ban 107 | 108 | **Community Impact**: Demonstrating a pattern of violation of community 109 | standards, including sustained inappropriate behavior, harassment of an 110 | individual, or aggression toward or disparagement of classes of individuals. 111 | 112 | **Consequence**: A permanent ban from any sort of public interaction within 113 | the community. 114 | 115 | ## Attribution 116 | 117 | This Code of Conduct is adapted from the [Contributor Covenant][homepage], 118 | version 2.1, available at 119 | [https://www.contributor-covenant.org/version/2/1/code_of_conduct.html][v2.1]. 120 | 121 | Community Impact Guidelines were inspired by 122 | [Mozilla's code of conduct enforcement ladder][Mozilla CoC]. 123 | 124 | For answers to common questions about this code of conduct, see the FAQ at 125 | [https://www.contributor-covenant.org/faq][FAQ]. Translations are available 126 | at [https://www.contributor-covenant.org/translations][translations]. 127 | 128 | [homepage]: https://www.contributor-covenant.org 129 | [v2.1]: https://www.contributor-covenant.org/version/2/1/code_of_conduct.html 130 | [Mozilla CoC]: https://github.com/mozilla/diversity 131 | [FAQ]: https://www.contributor-covenant.org/faq 132 | [translations]: https://www.contributor-covenant.org/translations 133 | -------------------------------------------------------------------------------- /tests/unit/debug_info_test.py: -------------------------------------------------------------------------------- 1 | import platform 2 | import sqlite3 3 | import sys 4 | from unittest.mock import MagicMock, patch 5 | 6 | import pytest 7 | 8 | from sqlite3_to_mysql.debug_info import _implementation, _mysql_version, info 9 | 10 | 11 | class TestDebugInfo: 12 | def test_implementation_cpython(self) -> None: 13 | """Test _implementation function with CPython.""" 14 | with patch("platform.python_implementation", return_value="CPython"): 15 | with patch("platform.python_version", return_value="3.8.0"): 16 | assert _implementation() == "CPython 3.8.0" 17 | 18 | def test_implementation_pypy(self) -> None: 19 | """Test _implementation function with PyPy.""" 20 | with patch("platform.python_implementation", return_value="PyPy"): 21 | # Mock sys.pypy_version_info 22 | mock_version_info = MagicMock() 23 | mock_version_info.major = 7 24 | mock_version_info.minor = 3 25 | mock_version_info.micro = 1 26 | mock_version_info.releaselevel = "final" 27 | 28 | with patch.object(sys, "pypy_version_info", mock_version_info, create=True): 29 | assert _implementation() == "PyPy 7.3.1" 30 | 31 | def test_implementation_pypy_non_final(self) -> None: 32 | """Test _implementation function with PyPy non-final release.""" 33 | with patch("platform.python_implementation", return_value="PyPy"): 34 | # Mock sys.pypy_version_info 35 | mock_version_info = MagicMock() 36 | mock_version_info.major = 7 37 | mock_version_info.minor = 3 38 | mock_version_info.micro = 1 39 | mock_version_info.releaselevel = "beta" 40 | 41 | with patch.object(sys, "pypy_version_info", mock_version_info, create=True): 42 | assert _implementation() == "PyPy 7.3.1beta" 43 | 44 | def test_implementation_jython(self) -> None: 45 | """Test _implementation function with Jython.""" 46 | with patch("platform.python_implementation", return_value="Jython"): 47 | with patch("platform.python_version", return_value="2.7.2"): 48 | assert _implementation() == "Jython 2.7.2" 49 | 50 | def test_implementation_ironpython(self) -> None: 51 | """Test _implementation function with IronPython.""" 52 | with patch("platform.python_implementation", return_value="IronPython"): 53 | with patch("platform.python_version", return_value="2.7.9"): 54 | assert _implementation() == "IronPython 2.7.9" 55 | 56 | def test_implementation_unknown(self) -> None: 57 | """Test _implementation function with unknown implementation.""" 58 | with patch("platform.python_implementation", return_value="Unknown"): 59 | assert _implementation() == "Unknown Unknown" 60 | 61 | def test_mysql_version_found(self) -> None: 62 | """Test _mysql_version function when MySQL is found.""" 63 | with patch("sqlite3_to_mysql.debug_info.which", return_value="/usr/bin/mysql"): 64 | with patch("sqlite3_to_mysql.debug_info.check_output", return_value=b"mysql Ver 8.0.23"): 65 | assert _mysql_version() == "mysql Ver 8.0.23" 66 | 67 | def test_mysql_version_decode_error(self) -> None: 68 | """Test _mysql_version function with decode error.""" 69 | with patch("sqlite3_to_mysql.debug_info.which", return_value="/usr/bin/mysql"): 70 | with patch("sqlite3_to_mysql.debug_info.check_output", return_value=b"\xff\xfe"): 71 | # This should trigger a UnicodeDecodeError 72 | result = _mysql_version() 73 | assert "b'" in result # Should contain the string representation of bytes 74 | 75 | def test_mysql_version_exception(self) -> None: 76 | """Test _mysql_version function with exception.""" 77 | with patch("sqlite3_to_mysql.debug_info.which", return_value="/usr/bin/mysql"): 78 | with patch("sqlite3_to_mysql.debug_info.check_output", side_effect=Exception("Command failed")): 79 | assert _mysql_version() == "MySQL client not found on the system" 80 | 81 | def test_mysql_version_not_found(self) -> None: 82 | """Test _mysql_version function when MySQL is not found.""" 83 | with patch("sqlite3_to_mysql.debug_info.which", return_value=None): 84 | assert _mysql_version() == "MySQL client not found on the system" 85 | 86 | def test_info(self) -> None: 87 | """Test info function.""" 88 | with patch("platform.system", return_value="Linux"): 89 | with patch("platform.release", return_value="5.4.0"): 90 | with patch("sqlite3_to_mysql.debug_info._implementation", return_value="CPython 3.8.0"): 91 | with patch("sqlite3_to_mysql.debug_info._mysql_version", return_value="mysql Ver 8.0.23"): 92 | with patch.object(sqlite3, "sqlite_version", "3.32.3"): 93 | result = info() 94 | assert result[0][0] == "sqlite3-to-mysql" 95 | assert result[2][0] == "Operating System" 96 | assert result[2][1] == "Linux 5.4.0" 97 | assert result[3][0] == "Python" 98 | assert result[3][1] == "CPython 3.8.0" 99 | assert result[4][0] == "MySQL" 100 | assert result[4][1] == "mysql Ver 8.0.23" 101 | assert result[5][0] == "SQLite" 102 | assert result[5][1] == "3.32.3" 103 | 104 | def test_info_platform_error(self) -> None: 105 | """Test info function with platform error.""" 106 | with patch("platform.system", side_effect=IOError("Platform error")): 107 | with patch("sqlite3_to_mysql.debug_info._implementation", return_value="CPython 3.8.0"): 108 | with patch("sqlite3_to_mysql.debug_info._mysql_version", return_value="mysql Ver 8.0.23"): 109 | with patch.object(sqlite3, "sqlite_version", "3.32.3"): 110 | result = info() 111 | assert result[0][0] == "sqlite3-to-mysql" 112 | assert result[2][0] == "Operating System" 113 | assert result[2][1] == "Unknown" 114 | assert result[3][0] == "Python" 115 | assert result[3][1] == "CPython 3.8.0" 116 | assert result[4][0] == "MySQL" 117 | assert result[4][1] == "mysql Ver 8.0.23" 118 | assert result[5][0] == "SQLite" 119 | assert result[5][1] == "3.32.3" 120 | -------------------------------------------------------------------------------- /tests/models.py: -------------------------------------------------------------------------------- 1 | import typing as t 2 | from datetime import date, datetime, time 3 | from decimal import Decimal 4 | from os import environ 5 | 6 | import sqlalchemy.types as types 7 | from sqlalchemy import ( 8 | BLOB, 9 | CHAR, 10 | DECIMAL, 11 | JSON, 12 | REAL, 13 | TIMESTAMP, 14 | VARCHAR, 15 | BigInteger, 16 | Column, 17 | Dialect, 18 | ForeignKey, 19 | Integer, 20 | SmallInteger, 21 | String, 22 | Table, 23 | Text, 24 | ) 25 | from sqlalchemy.orm import DeclarativeBase, Mapped, backref, mapped_column, relationship 26 | from sqlalchemy.sql.functions import current_timestamp 27 | 28 | 29 | class MyCustomType(types.TypeDecorator): 30 | impl: t.Type[String] = types.String 31 | 32 | def load_dialect_impl(self, dialect: Dialect) -> t.Any: 33 | return dialect.type_descriptor(types.VARCHAR(self.length)) 34 | 35 | def process_bind_param(self, value: t.Any, dialect: Dialect) -> str: 36 | return str(value) 37 | 38 | def process_result_value(self, value: t.Any, dialect: Dialect) -> str: 39 | return str(value) 40 | 41 | 42 | class Base(DeclarativeBase): 43 | pass 44 | 45 | 46 | class Author(Base): 47 | __tablename__ = "authors" 48 | id: Mapped[int] = mapped_column(primary_key=True) 49 | name: Mapped[str] = mapped_column(String(128), nullable=False, index=True) 50 | 51 | def __repr__(self): 52 | return f"" 53 | 54 | 55 | article_authors: Table = Table( 56 | "article_authors", 57 | Base.metadata, 58 | Column("article_id", Integer, ForeignKey("articles.id"), primary_key=True), 59 | Column("author_id", Integer, ForeignKey("authors.id"), primary_key=True), 60 | ) 61 | 62 | 63 | class Image(Base): 64 | __tablename__ = "images" 65 | id: Mapped[int] = mapped_column(primary_key=True) 66 | path: Mapped[str] = mapped_column(String(255), index=True) 67 | description: Mapped[str] = mapped_column(String(255), nullable=True) 68 | 69 | def __repr__(self): 70 | return f"" 71 | 72 | 73 | article_images: Table = Table( 74 | "article_images", 75 | Base.metadata, 76 | Column("article_id", Integer, ForeignKey("articles.id"), primary_key=True), 77 | Column("image_id", Integer, ForeignKey("images.id"), primary_key=True), 78 | ) 79 | 80 | 81 | class Tag(Base): 82 | __tablename__ = "tags" 83 | id: Mapped[int] = mapped_column(primary_key=True) 84 | name: Mapped[str] = mapped_column(String(128), nullable=False, index=True) 85 | 86 | def __repr__(self): 87 | return f"" 88 | 89 | 90 | article_tags = Table( 91 | "article_tags", 92 | Base.metadata, 93 | Column("article_id", Integer, ForeignKey("articles.id"), primary_key=True), 94 | Column("tag_id", Integer, ForeignKey("tags.id"), primary_key=True), 95 | ) 96 | 97 | 98 | class Misc(Base): 99 | """This model contains all possible MySQL types""" 100 | 101 | __tablename__ = "misc" 102 | id: Mapped[int] = mapped_column(primary_key=True) 103 | big_integer_field: Mapped[int] = mapped_column(BigInteger, default=0) 104 | blob_field: Mapped[bytes] = mapped_column(BLOB, nullable=True, index=True) 105 | boolean_field: Mapped[bool] = mapped_column(default=False) 106 | char_field: Mapped[str] = mapped_column(CHAR(255), nullable=True) 107 | date_field: Mapped[date] = mapped_column(nullable=True) 108 | date_time_field: Mapped[datetime] = mapped_column(nullable=True) 109 | decimal_field: Mapped[Decimal] = mapped_column(DECIMAL(10, 2), nullable=True) 110 | float_field: Mapped[Decimal] = mapped_column(DECIMAL(12, 4), default=0) 111 | integer_field: Mapped[int] = mapped_column(default=0) 112 | if environ.get("LEGACY_DB", "0") == "0": 113 | json_field: Mapped[t.Mapping[str, t.Any]] = mapped_column(JSON, nullable=True) 114 | numeric_field: Mapped[Decimal] = mapped_column(DECIMAL(12, 4), default=0) 115 | real_field: Mapped[float] = mapped_column(REAL(12), default=0) 116 | small_integer_field: Mapped[int] = mapped_column(SmallInteger, default=0) 117 | string_field: Mapped[str] = mapped_column(String(255), nullable=True) 118 | text_field: Mapped[str] = mapped_column(Text, nullable=True) 119 | time_field: Mapped[time] = mapped_column(nullable=True) 120 | varchar_field: Mapped[str] = mapped_column(VARCHAR(255), nullable=True) 121 | timestamp_field: Mapped[datetime] = mapped_column(TIMESTAMP, default=current_timestamp()) 122 | my_type_field: Mapped[t.Any] = mapped_column(MyCustomType(255), nullable=True) 123 | 124 | 125 | article_misc: Table = Table( 126 | "article_misc", 127 | Base.metadata, 128 | Column("article_id", Integer, ForeignKey("articles.id"), primary_key=True), 129 | Column("misc_id", Integer, ForeignKey("misc.id"), primary_key=True), 130 | ) 131 | 132 | 133 | class Media(Base): 134 | __tablename__ = "media" 135 | id: Mapped[str] = mapped_column(CHAR(64), primary_key=True) 136 | title: Mapped[str] = mapped_column(String(255), index=True) 137 | description: Mapped[str] = mapped_column(String(255), nullable=True) 138 | 139 | def __repr__(self): 140 | return f"" 141 | 142 | 143 | article_media = Table( 144 | "article_media", 145 | Base.metadata, 146 | Column("article_id", Integer, ForeignKey("articles.id"), primary_key=True), 147 | Column("media_id", CHAR(64), ForeignKey("media.id"), primary_key=True), 148 | ) 149 | 150 | 151 | class Article(Base): 152 | __tablename__ = "articles" 153 | id: Mapped[int] = mapped_column(primary_key=True) 154 | hash: Mapped[str] = mapped_column(String(32), unique=True) 155 | slug: Mapped[str] = mapped_column(String(255), index=True) 156 | title: Mapped[str] = mapped_column(String(255), index=True) 157 | content: Mapped[str] = mapped_column(Text, nullable=True, index=True) 158 | status: Mapped[str] = mapped_column(CHAR(1), index=True) 159 | published: Mapped[datetime] = mapped_column(nullable=True) 160 | # relationships 161 | authors: Mapped[t.List[Author]] = relationship( 162 | "Author", 163 | secondary=article_authors, 164 | backref=backref("authors", lazy="dynamic"), 165 | lazy="dynamic", 166 | ) 167 | tags: Mapped[t.List[Tag]] = relationship( 168 | "Tag", 169 | secondary=article_tags, 170 | backref=backref("tags", lazy="dynamic"), 171 | lazy="dynamic", 172 | ) 173 | images: Mapped[t.List[Image]] = relationship( 174 | "Image", 175 | secondary=article_images, 176 | backref=backref("images", lazy="dynamic"), 177 | lazy="dynamic", 178 | ) 179 | media: Mapped[t.List[Media]] = relationship( 180 | "Media", 181 | secondary=article_media, 182 | backref=backref("media", lazy="dynamic"), 183 | lazy="dynamic", 184 | ) 185 | misc: Mapped[t.List[Misc]] = relationship( 186 | "Misc", 187 | secondary=article_misc, 188 | backref=backref("misc", lazy="dynamic"), 189 | lazy="dynamic", 190 | ) 191 | 192 | def __repr__(self): 193 | return f"" 194 | -------------------------------------------------------------------------------- /tests/unit/types_test.py: -------------------------------------------------------------------------------- 1 | import os 2 | import typing as t 3 | from logging import Logger 4 | from sqlite3 import Connection, Cursor 5 | from unittest.mock import MagicMock 6 | 7 | import pytest 8 | from mysql.connector import MySQLConnection 9 | from mysql.connector.cursor import MySQLCursor 10 | 11 | from sqlite3_to_mysql.types import SQLite3toMySQLAttributes, SQLite3toMySQLParams 12 | 13 | 14 | class TestTypes: 15 | def test_sqlite3_to_mysql_params_typing(self) -> None: 16 | """Test SQLite3toMySQLParams typing.""" 17 | # Create a valid params dict 18 | params: SQLite3toMySQLParams = { 19 | "sqlite_file": "test.db", 20 | "sqlite_tables": ["table1", "table2"], 21 | "exclude_sqlite_tables": ["skip_this"], 22 | "sqlite_views_as_tables": False, 23 | "without_foreign_keys": False, 24 | "mysql_user": "user", 25 | "mysql_password": "password", 26 | "mysql_host": "localhost", 27 | "mysql_port": 3306, 28 | "mysql_socket": "/var/run/mysqld/mysqld.sock", 29 | "mysql_ssl_disabled": True, 30 | "chunk": 1000, 31 | "quiet": False, 32 | "log_file": "log.txt", 33 | "mysql_database": "test_db", 34 | "mysql_integer_type": "INT", 35 | "mysql_create_tables": True, 36 | "mysql_truncate_tables": False, 37 | "mysql_transfer_data": True, 38 | "mysql_charset": "utf8mb4", 39 | "mysql_collation": "utf8mb4_unicode_ci", 40 | "ignore_duplicate_keys": False, 41 | "use_fulltext": True, 42 | "with_rowid": False, 43 | "mysql_insert_method": "INSERT", 44 | "mysql_string_type": "VARCHAR", 45 | "mysql_text_type": "TEXT", 46 | } 47 | 48 | # Test that all fields are accessible 49 | assert params["sqlite_file"] == "test.db" 50 | assert params["sqlite_tables"] == ["table1", "table2"] 51 | assert params["exclude_sqlite_tables"] == ["skip_this"] 52 | assert params["sqlite_views_as_tables"] is False 53 | assert params["without_foreign_keys"] is False 54 | assert params["mysql_user"] == "user" 55 | assert params["mysql_password"] == "password" 56 | assert params["mysql_host"] == "localhost" 57 | assert params["mysql_port"] == 3306 58 | assert params["mysql_socket"] == "/var/run/mysqld/mysqld.sock" 59 | assert params["mysql_ssl_disabled"] is True 60 | assert params["chunk"] == 1000 61 | assert params["quiet"] is False 62 | assert params["log_file"] == "log.txt" 63 | assert params["mysql_database"] == "test_db" 64 | assert params["mysql_integer_type"] == "INT" 65 | assert params["mysql_create_tables"] is True 66 | assert params["mysql_truncate_tables"] is False 67 | assert params["mysql_transfer_data"] is True 68 | assert params["mysql_charset"] == "utf8mb4" 69 | assert params["mysql_collation"] == "utf8mb4_unicode_ci" 70 | assert params["ignore_duplicate_keys"] is False 71 | assert params["use_fulltext"] is True 72 | assert params["with_rowid"] is False 73 | assert params["mysql_insert_method"] == "INSERT" 74 | assert params["mysql_string_type"] == "VARCHAR" 75 | assert params["mysql_text_type"] == "TEXT" 76 | 77 | # Test with optional fields omitted 78 | minimal_params: SQLite3toMySQLParams = {"sqlite_file": "test.db"} 79 | assert minimal_params["sqlite_file"] == "test.db" 80 | 81 | # Test with PathLike object 82 | path_like_params: SQLite3toMySQLParams = {"sqlite_file": os.path.join("path", "to", "test.db")} 83 | assert path_like_params["sqlite_file"] == os.path.join("path", "to", "test.db") 84 | 85 | def test_sqlite3_to_mysql_attributes_typing(self) -> None: 86 | """Test SQLite3toMySQLAttributes typing.""" 87 | 88 | # Create a mock class that inherits from SQLite3toMySQLAttributes 89 | class MockSQLite3toMySQL(SQLite3toMySQLAttributes): 90 | def __init__(self) -> None: 91 | # Initialize all required attributes 92 | self._sqlite_file = "test.db" 93 | self._sqlite_tables = ["table1", "table2"] 94 | self._exclude_sqlite_tables = [] 95 | self._sqlite_views_as_tables = False 96 | self._without_foreign_keys = False 97 | self._mysql_user = "user" 98 | self._mysql_password = "password" 99 | self._mysql_host = "localhost" 100 | self._mysql_port = 3306 101 | self._mysql_socket = "/var/run/mysqld/mysqld.sock" 102 | self._mysql_ssl_disabled = True 103 | self._chunk_size = 1000 104 | self._quiet = False 105 | self._logger = MagicMock(spec=Logger) 106 | self._log_file = "log.txt" 107 | self._mysql_database = "test_db" 108 | self._mysql_insert_method = "INSERT" 109 | self._mysql_create_tables = True 110 | self._mysql_truncate_tables = False 111 | self._mysql_transfer_data = True 112 | self._mysql_integer_type = "INT" 113 | self._mysql_string_type = "VARCHAR" 114 | self._mysql_text_type = "TEXT" 115 | self._mysql_charset = "utf8mb4" 116 | self._mysql_collation = "utf8mb4_unicode_ci" 117 | self._ignore_duplicate_keys = False 118 | self._use_fulltext = True 119 | self._with_rowid = False 120 | self._sqlite = MagicMock(spec=Connection) 121 | self._sqlite_cur = MagicMock(spec=Cursor) 122 | self._sqlite_version = "3.32.3" 123 | self._sqlite_table_xinfo_support = True 124 | self._mysql = MagicMock(spec=MySQLConnection) 125 | self._mysql_cur = MagicMock(spec=MySQLCursor) 126 | self._mysql_version = "8.0.23" 127 | self._mysql_json_support = True 128 | self._mysql_fulltext_support = True 129 | 130 | # Create an instance of the mock class 131 | instance = MockSQLite3toMySQL() 132 | 133 | # Test that all attributes are accessible 134 | assert instance._sqlite_file == "test.db" 135 | assert instance._sqlite_tables == ["table1", "table2"] 136 | assert instance._exclude_sqlite_tables == [] 137 | assert instance._sqlite_views_as_tables is False 138 | assert instance._without_foreign_keys is False 139 | assert instance._mysql_user == "user" 140 | assert instance._mysql_password == "password" 141 | assert instance._mysql_host == "localhost" 142 | assert instance._mysql_port == 3306 143 | assert instance._mysql_socket == "/var/run/mysqld/mysqld.sock" 144 | assert instance._mysql_ssl_disabled is True 145 | assert instance._chunk_size == 1000 146 | assert instance._quiet is False 147 | assert instance._log_file == "log.txt" 148 | assert instance._mysql_database == "test_db" 149 | assert instance._mysql_insert_method == "INSERT" 150 | assert instance._mysql_create_tables is True 151 | assert instance._mysql_truncate_tables is False 152 | assert instance._mysql_transfer_data is True 153 | assert instance._mysql_integer_type == "INT" 154 | assert instance._mysql_string_type == "VARCHAR" 155 | assert instance._mysql_text_type == "TEXT" 156 | assert instance._mysql_charset == "utf8mb4" 157 | assert instance._mysql_collation == "utf8mb4_unicode_ci" 158 | assert instance._ignore_duplicate_keys is False 159 | assert instance._use_fulltext is True 160 | assert instance._with_rowid is False 161 | assert instance._sqlite_version == "3.32.3" 162 | assert instance._sqlite_table_xinfo_support is True 163 | assert instance._mysql_version == "8.0.23" 164 | assert instance._mysql_json_support is True 165 | assert instance._mysql_fulltext_support is True 166 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [![PyPI](https://img.shields.io/pypi/v/sqlite3-to-mysql?logo=pypi)](https://pypi.org/project/sqlite3-to-mysql/) 2 | [![PyPI - Downloads](https://img.shields.io/pypi/dm/sqlite3-to-mysql?logo=pypi&label=PyPI%20downloads)](https://pypistats.org/packages/sqlite3-to-mysql) 3 | [![Homebrew Formula Downloads](https://img.shields.io/homebrew/installs/dm/sqlite3-to-mysql?logo=homebrew&label=Homebrew%20downloads)](https://formulae.brew.sh/formula/sqlite3-to-mysql) 4 | [![PyPI - Python Version](https://img.shields.io/pypi/pyversions/sqlite3-to-mysql?logo=python)](https://pypi.org/project/sqlite3-to-mysql/) 5 | [![MySQL Support](https://img.shields.io/static/v1?logo=mysql&label=MySQL&message=5.5+|+5.6+|+5.7+|+8.0+|+8.4&color=2b5d80)](https://img.shields.io/static/v1?label=MySQL&message=5.5+|+5.6+|+5.7+|+8.0+|+8.4&color=2b5d80) 6 | [![MariaDB Support](https://img.shields.io/static/v1?logo=mariadb&label=MariaDB&message=5.5+|+10.0+|+10.6+|+10.11+|+11.4+|+11.6+|+11.8&color=C0765A)](https://img.shields.io/static/v1?label=MariaDB&message=5.5+|+10.0+|+10.6+|+10.11+|+11.4+|+11.6+|+11.8&color=C0765A) 7 | [![GitHub license](https://img.shields.io/github/license/techouse/sqlite3-to-mysql)](https://github.com/techouse/sqlite3-to-mysql/blob/master/LICENSE) 8 | [![Contributor Covenant](https://img.shields.io/badge/Contributor%20Covenant-2.1-4baaaa.svg?logo=contributorcovenant)](CODE-OF-CONDUCT.md) 9 | [![PyPI - Format](https://img.shields.io/pypi/format/sqlite3-to-mysql?logo=python)]((https://pypi.org/project/sqlite3-to-mysql/)) 10 | [![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg?logo=python)](https://github.com/ambv/black) 11 | [![Codacy Badge](https://api.codacy.com/project/badge/Grade/d33b59d35b924711aae9418741a923ae)](https://www.codacy.com/manual/techouse/sqlite3-to-mysql?utm_source=github.com&utm_medium=referral&utm_content=techouse/sqlite3-to-mysql&utm_campaign=Badge_Grade) 12 | [![Test Status](https://github.com/techouse/sqlite3-to-mysql/actions/workflows/test.yml/badge.svg)](https://github.com/techouse/sqlite3-to-mysql/actions/workflows/test.yml) 13 | [![CodeQL Status](https://github.com/techouse/sqlite3-to-mysql/actions/workflows/github-code-scanning/codeql/badge.svg)](https://github.com/techouse/sqlite3-to-mysql/actions/workflows/github-code-scanning/codeql) 14 | [![Publish PyPI Package Status](https://github.com/techouse/sqlite3-to-mysql/actions/workflows/publish.yml/badge.svg)](https://github.com/techouse/sqlite3-to-mysql/actions/workflows/publish.yml) 15 | [![codecov](https://codecov.io/gh/techouse/sqlite3-to-mysql/branch/master/graph/badge.svg)](https://codecov.io/gh/techouse/sqlite3-to-mysql) 16 | [![GitHub Sponsors](https://img.shields.io/github/sponsors/techouse?logo=github)](https://github.com/sponsors/techouse) 17 | [![GitHub stars](https://img.shields.io/github/stars/techouse/sqlite3-to-mysql.svg?style=social&label=Star&maxAge=2592000)](https://github.com/techouse/sqlite3-to-mysql/stargazers) 18 | 19 | # SQLite3 to MySQL 20 | 21 | #### A simple Python tool to transfer data from SQLite 3 to MySQL. 22 | 23 | ### How to run 24 | 25 | ```bash 26 | pip install sqlite3-to-mysql 27 | sqlite3mysql --help 28 | ``` 29 | 30 | ### Usage 31 | 32 | ``` 33 | Usage: sqlite3mysql [OPTIONS] 34 | 35 | Options: 36 | -f, --sqlite-file PATH SQLite3 database file [required] 37 | -t, --sqlite-tables TUPLE Transfer only these specific tables (space 38 | separated table names). Implies --without- 39 | foreign-keys which inhibits the transfer of 40 | foreign keys. Can not be used together with 41 | --exclude-sqlite-tables. 42 | -e, --exclude-sqlite-tables TUPLE 43 | Transfer all tables except these specific 44 | tables (space separated table names). 45 | Implies --without-foreign-keys which 46 | inhibits the transfer of foreign keys. Can 47 | not be used together with --sqlite-tables. 48 | -A, --sqlite-views-as-tables Materialize SQLite views as tables in MySQL 49 | instead of creating matching MySQL views 50 | (legacy behavior). 51 | -X, --without-foreign-keys Do not transfer foreign keys. 52 | -W, --ignore-duplicate-keys Ignore duplicate keys. The default behavior 53 | is to create new ones with a numerical 54 | suffix, e.g. 'exising_key' -> 55 | 'existing_key_1' 56 | -d, --mysql-database TEXT MySQL database name [required] 57 | -u, --mysql-user TEXT MySQL user [required] 58 | -p, --prompt-mysql-password Prompt for MySQL password 59 | --mysql-password TEXT MySQL password 60 | -h, --mysql-host TEXT MySQL host. Defaults to localhost. 61 | -P, --mysql-port INTEGER MySQL port. Defaults to 3306. 62 | -k, --mysql-socket PATH Path to MySQL unix socket file. 63 | -S, --skip-ssl Disable MySQL connection encryption. 64 | -i, --mysql-insert-method [DEFAULT|IGNORE|UPDATE] 65 | MySQL insert method. DEFAULT will throw 66 | errors when encountering duplicate records; 67 | UPDATE will update existing rows; IGNORE 68 | will ignore insert errors. Defaults to 69 | IGNORE. 70 | -E, --mysql-truncate-tables Truncates existing tables before inserting 71 | data. 72 | --mysql-integer-type TEXT MySQL default integer field type. Defaults 73 | to INT(11). 74 | --mysql-string-type TEXT MySQL default string field type. Defaults to 75 | VARCHAR(255). 76 | --mysql-text-type [LONGTEXT|MEDIUMTEXT|TEXT|TINYTEXT] 77 | MySQL default text field type. Defaults to 78 | TEXT. 79 | --mysql-charset TEXT MySQL database and table character set 80 | [default: utf8mb4] 81 | --mysql-collation TEXT MySQL database and table collation 82 | -T, --use-fulltext Use FULLTEXT indexes on TEXT columns. Will 83 | throw an error if your MySQL version does 84 | not support InnoDB FULLTEXT indexes! 85 | --with-rowid Transfer rowid columns. 86 | -c, --chunk INTEGER Chunk reading/writing SQL records 87 | -K, --mysql-skip-create-tables Skip creating tables in MySQL. 88 | -J, --mysql-skip-transfer-data Skip transferring data to MySQL. 89 | -l, --log-file PATH Log file 90 | -q, --quiet Quiet. Display only errors. 91 | --debug Debug mode. Will throw exceptions. 92 | --version Show the version and exit. 93 | --help Show this message and exit. 94 | ``` 95 | 96 | #### Docker 97 | 98 | If you don't want to install the tool on your system, you can use the Docker image instead. 99 | 100 | ```bash 101 | docker run -it \ 102 | --workdir $(pwd) \ 103 | --volume $(pwd):$(pwd) \ 104 | --rm ghcr.io/techouse/sqlite3-to-mysql:latest \ 105 | --sqlite-file baz.db \ 106 | --mysql-user foo \ 107 | --mysql-password bar \ 108 | --mysql-database baz \ 109 | --mysql-host host.docker.internal 110 | ``` 111 | 112 | This will mount your host current working directory (pwd) inside the Docker container as the current working directory. 113 | Any files Docker would write to the current working directory are written to the host directory where you did docker 114 | run. Note that you have to also use a 115 | [special hostname](https://docs.docker.com/desktop/networking/#use-cases-and-workarounds-for-all-platforms) 116 | `host.docker.internal` 117 | to access your host machine from inside the Docker container. 118 | 119 | #### Homebrew 120 | 121 | If you're on macOS, you can install the tool using [Homebrew](https://brew.sh/). 122 | 123 | ```bash 124 | brew install sqlite3-to-mysql 125 | sqlite3mysql --help 126 | ``` 127 | -------------------------------------------------------------------------------- /tests/unit/click_utils_test.py: -------------------------------------------------------------------------------- 1 | import typing as t 2 | from unittest.mock import MagicMock, patch 3 | 4 | import click 5 | import pytest 6 | from click.parser import Option, OptionParser 7 | 8 | from sqlite3_to_mysql.click_utils import OptionEatAll, prompt_password 9 | 10 | 11 | class TestClickUtils: 12 | def test_option_eat_all_init(self) -> None: 13 | """Test OptionEatAll initialization.""" 14 | option = OptionEatAll(["--test"], save_other_options=True) 15 | assert option.save_other_options is True 16 | 17 | option = OptionEatAll(["--test"], save_other_options=False) 18 | assert option.save_other_options is False 19 | 20 | # Test with invalid nargs 21 | with pytest.raises(ValueError): 22 | OptionEatAll(["--test"], nargs=1) 23 | 24 | def test_option_eat_all_parser_process(self) -> None: 25 | """Test OptionEatAll parser_process function.""" 26 | # This is a simplified test that just verifies the parser_process function works 27 | # Create a mock state object 28 | state = MagicMock() 29 | state.rargs = ["value1", "value2", "--next-option"] 30 | 31 | # Create a mock parser process function 32 | process_mock = MagicMock() 33 | 34 | # Create a parser_process function similar to the one in OptionEatAll 35 | def parser_process(value, state_obj): 36 | done = False 37 | value = [value] 38 | # Grab everything up to the next option 39 | while state_obj.rargs and not done: 40 | if state_obj.rargs[0].startswith("--"): 41 | done = True 42 | if not done: 43 | value.append(state_obj.rargs.pop(0)) 44 | value = tuple(value) 45 | process_mock(value, state_obj) 46 | 47 | # Call the function 48 | parser_process("initial", state) 49 | 50 | # Check that the process_mock was called with the expected values 51 | process_mock.assert_called_once() 52 | args, kwargs = process_mock.call_args 53 | assert args[0] == ("initial", "value1", "value2") 54 | assert args[1] == state 55 | assert state.rargs == ["--next-option"] 56 | 57 | def test_prompt_password_with_password(self) -> None: 58 | """Test prompt_password function with password provided.""" 59 | ctx = MagicMock() 60 | ctx.params = {"mysql_password": "test_password"} 61 | 62 | result = prompt_password(ctx, None, True) 63 | assert result == "test_password" 64 | 65 | def test_prompt_password_without_password(self) -> None: 66 | """Test prompt_password function without password provided.""" 67 | ctx = MagicMock() 68 | ctx.params = {"mysql_password": None} 69 | 70 | with patch("click.prompt", return_value="prompted_password"): 71 | result = prompt_password(ctx, None, True) 72 | assert result == "prompted_password" 73 | 74 | def test_prompt_password_not_used(self) -> None: 75 | """Test prompt_password function when not used.""" 76 | ctx = MagicMock() 77 | ctx.params = {"mysql_password": "test_password"} 78 | 79 | result = prompt_password(ctx, None, False) 80 | assert result is None 81 | 82 | def test_prompt_password_not_used_no_password(self) -> None: 83 | """Test prompt_password function when not used and no password is provided.""" 84 | ctx = MagicMock() 85 | ctx.params = {"mysql_password": None} 86 | 87 | result = prompt_password(ctx, None, False) 88 | assert result is None 89 | 90 | def test_option_eat_all_add_to_parser(self) -> None: 91 | """Test OptionEatAll add_to_parser method with a real parser.""" 92 | # Create a real parser 93 | parser = OptionParser() 94 | ctx = MagicMock() 95 | 96 | # Create an OptionEatAll instance 97 | option = OptionEatAll(["--test"], save_other_options=True) 98 | 99 | # Add it to the parser 100 | option.add_to_parser(parser, ctx) 101 | 102 | # Verify that the parser has our option 103 | assert "--test" in parser._long_opt 104 | 105 | # Verify that the process method has been replaced 106 | assert parser._long_opt["--test"].process != option._previous_parser_process 107 | 108 | def test_option_eat_all_save_other_options_false(self) -> None: 109 | """Test OptionEatAll parser_process function with save_other_options=False.""" 110 | # Create a mock state object 111 | state = MagicMock() 112 | state.rargs = ["value1", "value2", "--next-option"] 113 | 114 | # Create a mock process function 115 | process_mock = MagicMock() 116 | 117 | # Create a simplified parser_process function that directly tests the behavior 118 | # we're interested in (save_other_options=False) 119 | def parser_process(value, state_obj): 120 | done = False 121 | value = [value] 122 | # This is the branch we want to test (save_other_options=False) 123 | # grab everything remaining 124 | value += state_obj.rargs 125 | state_obj.rargs[:] = [] 126 | value = tuple(value) 127 | process_mock(value, state_obj) 128 | 129 | # Call the function 130 | parser_process("initial", state) 131 | 132 | # Check that the process_mock was called with the expected values 133 | process_mock.assert_called_once() 134 | args, kwargs = process_mock.call_args 135 | # With save_other_options=False, all remaining args should be consumed 136 | assert args[0] == ("initial", "value1", "value2", "--next-option") 137 | assert args[1] == state 138 | # The state.rargs should be empty 139 | assert state.rargs == [] 140 | 141 | def test_option_eat_all_actual_implementation(self) -> None: 142 | """Test the actual implementation of OptionEatAll parser_process method.""" 143 | # Create a real OptionEatAll instance 144 | option = OptionEatAll(["--test"], save_other_options=True) 145 | 146 | # Create a mock parser with prefixes 147 | parser = MagicMock() 148 | parser.prefixes = ["--", "-"] 149 | 150 | # Create a mock option that will be returned by parser._long_opt.get() 151 | mock_option = MagicMock() 152 | mock_option.prefixes = ["--", "-"] # This is needed for the parser_process method 153 | parser._long_opt = {"--test": mock_option} 154 | parser._short_opt = {} 155 | 156 | # Create a state object with rargs 157 | state = MagicMock() 158 | 159 | # Test case 1: save_other_options=True, with non-option arguments 160 | state.rargs = ["value1", "value2", "--next-option"] 161 | 162 | # Call the parser_process method 163 | option.add_to_parser(parser, MagicMock()) # This sets up the parser_process method 164 | 165 | # Now mock_option.process should have been replaced with our parser_process 166 | # Call it directly to simulate what would happen in the real parser 167 | mock_option.process("initial", state) 168 | 169 | # Check that the previous_parser_process was called with the expected values 170 | option._previous_parser_process.assert_called_once() 171 | args, kwargs = option._previous_parser_process.call_args 172 | assert args[0] == ("initial", "value1", "value2") 173 | assert args[1] == state 174 | assert state.rargs == ["--next-option"] 175 | 176 | # Reset mocks 177 | option._previous_parser_process.reset_mock() 178 | 179 | # Test case 2: save_other_options=False 180 | option.save_other_options = False 181 | state.rargs = ["value1", "value2", "--next-option"] 182 | 183 | # Call the parser_process method 184 | mock_option.process("initial", state) 185 | 186 | # Check that the previous_parser_process was called with the expected values 187 | option._previous_parser_process.assert_called_once() 188 | args, kwargs = option._previous_parser_process.call_args 189 | assert args[0] == ("initial", "value1", "value2", "--next-option") 190 | assert args[1] == state 191 | assert state.rargs == [] 192 | 193 | def test_option_eat_all_add_to_parser_with_short_opt(self) -> None: 194 | """Test OptionEatAll add_to_parser method with short option.""" 195 | # Create a real parser 196 | parser = OptionParser() 197 | ctx = MagicMock() 198 | 199 | # Set up the parser with a short option 200 | mock_option = MagicMock() 201 | parser._short_opt = {"-t": mock_option} 202 | parser._long_opt = {} 203 | 204 | # Create an OptionEatAll instance with a short option 205 | option = OptionEatAll(["-t"], save_other_options=True) 206 | 207 | # Add it to the parser 208 | option.add_to_parser(parser, ctx) 209 | 210 | # Verify that the parser has our option 211 | assert "-t" in parser._short_opt 212 | 213 | # Verify that the process method has been replaced 214 | assert hasattr(option, "_previous_parser_process") 215 | assert option._eat_all_parser is not None 216 | -------------------------------------------------------------------------------- /.bandit.yml: -------------------------------------------------------------------------------- 1 | ### This config may optionally select a subset of tests to run or skip by 2 | ### filling out the 'tests' and 'skips' lists given below. If no tests are 3 | ### specified for inclusion then it is assumed all tests are desired. The skips 4 | ### set will remove specific tests from the include set. This can be controlled 5 | ### using the -t/-s CLI options. Note that the same test ID should not appear 6 | ### in both 'tests' and 'skips', this would be nonsensical and is detected by 7 | ### Bandit at runtime. 8 | 9 | # Available tests: 10 | # B101 : assert_used 11 | # B102 : exec_used 12 | # B103 : set_bad_file_permissions 13 | # B104 : hardcoded_bind_all_interfaces 14 | # B105 : hardcoded_password_string 15 | # B106 : hardcoded_password_funcarg 16 | # B107 : hardcoded_password_default 17 | # B108 : hardcoded_tmp_directory 18 | # B110 : try_except_pass 19 | # B112 : try_except_continue 20 | # B201 : flask_debug_true 21 | # B301 : pickle 22 | # B302 : marshal 23 | # B303 : md5 24 | # B304 : ciphers 25 | # B305 : cipher_modes 26 | # B306 : mktemp_q 27 | # B307 : eval 28 | # B308 : mark_safe 29 | # B309 : httpsconnection 30 | # B310 : urllib_urlopen 31 | # B311 : random 32 | # B312 : telnetlib 33 | # B313 : xml_bad_cElementTree 34 | # B314 : xml_bad_ElementTree 35 | # B315 : xml_bad_expatreader 36 | # B316 : xml_bad_expatbuilder 37 | # B317 : xml_bad_sax 38 | # B318 : xml_bad_minidom 39 | # B319 : xml_bad_pulldom 40 | # B320 : xml_bad_etree 41 | # B321 : ftplib 42 | # B322 : input 43 | # B323 : unverified_context 44 | # B324 : hashlib_new_insecure_functions 45 | # B325 : tempnam 46 | # B401 : import_telnetlib 47 | # B402 : import_ftplib 48 | # B403 : import_pickle 49 | # B404 : import_subprocess 50 | # B405 : import_xml_etree 51 | # B406 : import_xml_sax 52 | # B407 : import_xml_expat 53 | # B408 : import_xml_minidom 54 | # B409 : import_xml_pulldom 55 | # B410 : import_lxml 56 | # B411 : import_xmlrpclib 57 | # B412 : import_httpoxy 58 | # B413 : import_pycrypto 59 | # B501 : request_with_no_cert_validation 60 | # B502 : ssl_with_bad_version 61 | # B503 : ssl_with_bad_defaults 62 | # B504 : ssl_with_no_version 63 | # B505 : weak_cryptographic_key 64 | # B506 : yaml_load 65 | # B507 : ssh_no_host_key_verification 66 | # B601 : paramiko_calls 67 | # B602 : subprocess_popen_with_shell_equals_true 68 | # B603 : subprocess_without_shell_equals_true 69 | # B604 : any_other_function_with_shell_equals_true 70 | # B605 : start_process_with_a_shell 71 | # B606 : start_process_with_no_shell 72 | # B607 : start_process_with_partial_path 73 | # B608 : hardcoded_sql_expressions 74 | # B609 : linux_commands_wildcard_injection 75 | # B610 : django_extra_used 76 | # B611 : django_rawsql_used 77 | # B701 : jinja2_autoescape_false 78 | # B702 : use_of_mako_templates 79 | # B703 : django_mark_safe 80 | 81 | # (optional) list included test IDs here, eg '[B101, B406]': 82 | tests: 83 | 84 | # (optional) list skipped test IDs here, eg '[B101, B406]': 85 | skips: 86 | - B404 87 | - B603 88 | - B607 89 | - B608 90 | 91 | ### (optional) plugin settings - some test plugins require configuration data 92 | ### that may be given here, per-plugin. All bandit test plugins have a built in 93 | ### set of sensible defaults and these will be used if no configuration is 94 | ### provided. It is not necessary to provide settings for every (or any) plugin 95 | ### if the defaults are acceptable. 96 | 97 | any_other_function_with_shell_equals_true: 98 | no_shell: 99 | - os.execl 100 | - os.execle 101 | - os.execlp 102 | - os.execlpe 103 | - os.execv 104 | - os.execve 105 | - os.execvp 106 | - os.execvpe 107 | - os.spawnl 108 | - os.spawnle 109 | - os.spawnlp 110 | - os.spawnlpe 111 | - os.spawnv 112 | - os.spawnve 113 | - os.spawnvp 114 | - os.spawnvpe 115 | - os.startfile 116 | shell: 117 | - os.system 118 | - os.popen 119 | - os.popen2 120 | - os.popen3 121 | - os.popen4 122 | - popen2.popen2 123 | - popen2.popen3 124 | - popen2.popen4 125 | - popen2.Popen3 126 | - popen2.Popen4 127 | - commands.getoutput 128 | - commands.getstatusoutput 129 | subprocess: 130 | - subprocess.Popen 131 | - subprocess.call 132 | - subprocess.check_call 133 | - subprocess.check_output 134 | - subprocess.run 135 | hardcoded_tmp_directory: 136 | tmp_dirs: 137 | - /tmp 138 | - /var/tmp 139 | - /dev/shm 140 | linux_commands_wildcard_injection: 141 | no_shell: 142 | - os.execl 143 | - os.execle 144 | - os.execlp 145 | - os.execlpe 146 | - os.execv 147 | - os.execve 148 | - os.execvp 149 | - os.execvpe 150 | - os.spawnl 151 | - os.spawnle 152 | - os.spawnlp 153 | - os.spawnlpe 154 | - os.spawnv 155 | - os.spawnve 156 | - os.spawnvp 157 | - os.spawnvpe 158 | - os.startfile 159 | shell: 160 | - os.system 161 | - os.popen 162 | - os.popen2 163 | - os.popen3 164 | - os.popen4 165 | - popen2.popen2 166 | - popen2.popen3 167 | - popen2.popen4 168 | - popen2.Popen3 169 | - popen2.Popen4 170 | - commands.getoutput 171 | - commands.getstatusoutput 172 | subprocess: 173 | - subprocess.Popen 174 | - subprocess.call 175 | - subprocess.check_call 176 | - subprocess.check_output 177 | - subprocess.run 178 | ssl_with_bad_defaults: 179 | bad_protocol_versions: 180 | - PROTOCOL_SSLv2 181 | - SSLv2_METHOD 182 | - SSLv23_METHOD 183 | - PROTOCOL_SSLv3 184 | - PROTOCOL_TLSv1 185 | - SSLv3_METHOD 186 | - TLSv1_METHOD 187 | ssl_with_bad_version: 188 | bad_protocol_versions: 189 | - PROTOCOL_SSLv2 190 | - SSLv2_METHOD 191 | - SSLv23_METHOD 192 | - PROTOCOL_SSLv3 193 | - PROTOCOL_TLSv1 194 | - SSLv3_METHOD 195 | - TLSv1_METHOD 196 | start_process_with_a_shell: 197 | no_shell: 198 | - os.execl 199 | - os.execle 200 | - os.execlp 201 | - os.execlpe 202 | - os.execv 203 | - os.execve 204 | - os.execvp 205 | - os.execvpe 206 | - os.spawnl 207 | - os.spawnle 208 | - os.spawnlp 209 | - os.spawnlpe 210 | - os.spawnv 211 | - os.spawnve 212 | - os.spawnvp 213 | - os.spawnvpe 214 | - os.startfile 215 | shell: 216 | - os.system 217 | - os.popen 218 | - os.popen2 219 | - os.popen3 220 | - os.popen4 221 | - popen2.popen2 222 | - popen2.popen3 223 | - popen2.popen4 224 | - popen2.Popen3 225 | - popen2.Popen4 226 | - commands.getoutput 227 | - commands.getstatusoutput 228 | subprocess: 229 | - subprocess.Popen 230 | - subprocess.call 231 | - subprocess.check_call 232 | - subprocess.check_output 233 | - subprocess.run 234 | start_process_with_no_shell: 235 | no_shell: 236 | - os.execl 237 | - os.execle 238 | - os.execlp 239 | - os.execlpe 240 | - os.execv 241 | - os.execve 242 | - os.execvp 243 | - os.execvpe 244 | - os.spawnl 245 | - os.spawnle 246 | - os.spawnlp 247 | - os.spawnlpe 248 | - os.spawnv 249 | - os.spawnve 250 | - os.spawnvp 251 | - os.spawnvpe 252 | - os.startfile 253 | shell: 254 | - os.system 255 | - os.popen 256 | - os.popen2 257 | - os.popen3 258 | - os.popen4 259 | - popen2.popen2 260 | - popen2.popen3 261 | - popen2.popen4 262 | - popen2.Popen3 263 | - popen2.Popen4 264 | - commands.getoutput 265 | - commands.getstatusoutput 266 | subprocess: 267 | - subprocess.Popen 268 | - subprocess.call 269 | - subprocess.check_call 270 | - subprocess.check_output 271 | - subprocess.run 272 | start_process_with_partial_path: 273 | no_shell: 274 | - os.execl 275 | - os.execle 276 | - os.execlp 277 | - os.execlpe 278 | - os.execv 279 | - os.execve 280 | - os.execvp 281 | - os.execvpe 282 | - os.spawnl 283 | - os.spawnle 284 | - os.spawnlp 285 | - os.spawnlpe 286 | - os.spawnv 287 | - os.spawnve 288 | - os.spawnvp 289 | - os.spawnvpe 290 | - os.startfile 291 | shell: 292 | - os.system 293 | - os.popen 294 | - os.popen2 295 | - os.popen3 296 | - os.popen4 297 | - popen2.popen2 298 | - popen2.popen3 299 | - popen2.popen4 300 | - popen2.Popen3 301 | - popen2.Popen4 302 | - commands.getoutput 303 | - commands.getstatusoutput 304 | subprocess: 305 | - subprocess.Popen 306 | - subprocess.call 307 | - subprocess.check_call 308 | - subprocess.check_output 309 | - subprocess.run 310 | subprocess_popen_with_shell_equals_true: 311 | no_shell: 312 | - os.execl 313 | - os.execle 314 | - os.execlp 315 | - os.execlpe 316 | - os.execv 317 | - os.execve 318 | - os.execvp 319 | - os.execvpe 320 | - os.spawnl 321 | - os.spawnle 322 | - os.spawnlp 323 | - os.spawnlpe 324 | - os.spawnv 325 | - os.spawnve 326 | - os.spawnvp 327 | - os.spawnvpe 328 | - os.startfile 329 | shell: 330 | - os.system 331 | - os.popen 332 | - os.popen2 333 | - os.popen3 334 | - os.popen4 335 | - popen2.popen2 336 | - popen2.popen3 337 | - popen2.popen4 338 | - popen2.Popen3 339 | - popen2.Popen4 340 | - commands.getoutput 341 | - commands.getstatusoutput 342 | subprocess: 343 | - subprocess.Popen 344 | - subprocess.call 345 | - subprocess.check_call 346 | - subprocess.check_output 347 | - subprocess.run 348 | subprocess_without_shell_equals_true: 349 | no_shell: 350 | - os.execl 351 | - os.execle 352 | - os.execlp 353 | - os.execlpe 354 | - os.execv 355 | - os.execve 356 | - os.execvp 357 | - os.execvpe 358 | - os.spawnl 359 | - os.spawnle 360 | - os.spawnlp 361 | - os.spawnlpe 362 | - os.spawnv 363 | - os.spawnve 364 | - os.spawnvp 365 | - os.spawnvpe 366 | - os.startfile 367 | shell: 368 | - os.system 369 | - os.popen 370 | - os.popen2 371 | - os.popen3 372 | - os.popen4 373 | - popen2.popen2 374 | - popen2.popen3 375 | - popen2.popen4 376 | - popen2.Popen3 377 | - popen2.Popen4 378 | - commands.getoutput 379 | - commands.getstatusoutput 380 | subprocess: 381 | - subprocess.Popen 382 | - subprocess.call 383 | - subprocess.check_call 384 | - subprocess.check_output 385 | - subprocess.run 386 | try_except_continue: 387 | check_typed_exception: false 388 | try_except_pass: 389 | check_typed_exception: false 390 | weak_cryptographic_key: 391 | weak_key_size_dsa_high: 1024 392 | weak_key_size_dsa_medium: 2048 393 | weak_key_size_ec_high: 160 394 | weak_key_size_ec_medium: 224 395 | weak_key_size_rsa_high: 1024 396 | weak_key_size_rsa_medium: 2048 397 | 398 | -------------------------------------------------------------------------------- /src/sqlite3_to_mysql/cli.py: -------------------------------------------------------------------------------- 1 | """The command line interface of SQLite3toMySQL.""" 2 | 3 | import os 4 | import sys 5 | import typing as t 6 | from datetime import datetime 7 | 8 | import click 9 | from mysql.connector import CharacterSet 10 | from tabulate import tabulate 11 | 12 | from . import SQLite3toMySQL 13 | from . import __version__ as package_version 14 | from .click_utils import OptionEatAll, prompt_password 15 | from .debug_info import info 16 | from .mysql_utils import MYSQL_INSERT_METHOD, MYSQL_TEXT_COLUMN_TYPES, mysql_supported_character_sets 17 | 18 | 19 | _copyright_header: str = f"sqlite3mysql version {package_version} Copyright (c) 2018-{datetime.now().year} Klemen Tusar" 20 | 21 | 22 | @click.command( 23 | name="sqlite3mysql", 24 | help=_copyright_header, 25 | no_args_is_help=True, 26 | epilog="For more information, visit https://github.com/techouse/sqlite3-to-mysql", 27 | ) 28 | @click.option( 29 | "-f", 30 | "--sqlite-file", 31 | type=click.Path(exists=True), 32 | default=None, 33 | help="SQLite3 database file", 34 | required=True, 35 | ) 36 | @click.option( 37 | "-t", 38 | "--sqlite-tables", 39 | type=tuple, 40 | cls=OptionEatAll, 41 | help="Transfer only these specific tables (space separated table names). " 42 | "Implies --without-foreign-keys which inhibits the transfer of foreign keys. " 43 | "Can not be used together with --exclude-sqlite-tables.", 44 | ) 45 | @click.option( 46 | "-e", 47 | "--exclude-sqlite-tables", 48 | type=tuple, 49 | cls=OptionEatAll, 50 | help="Transfer all tables except these specific tables (space separated table names). " 51 | "Implies --without-foreign-keys which inhibits the transfer of foreign keys. " 52 | "Can not be used together with --sqlite-tables.", 53 | ) 54 | @click.option( 55 | "-A", 56 | "--sqlite-views-as-tables", 57 | is_flag=True, 58 | help="Materialize SQLite views as tables in MySQL instead of creating matching MySQL views (legacy behavior).", 59 | ) 60 | @click.option("-X", "--without-foreign-keys", is_flag=True, help="Do not transfer foreign keys.") 61 | @click.option( 62 | "-W", 63 | "--ignore-duplicate-keys", 64 | is_flag=True, 65 | help="Ignore duplicate keys. The default behavior is to create new ones with a numerical suffix, e.g. " 66 | "'exising_key' -> 'existing_key_1'", 67 | ) 68 | @click.option("-d", "--mysql-database", default=None, help="MySQL database name", required=True) 69 | @click.option("-u", "--mysql-user", default=None, help="MySQL user", required=True) 70 | @click.option( 71 | "-p", 72 | "--prompt-mysql-password", 73 | is_flag=True, 74 | default=False, 75 | callback=prompt_password, 76 | help="Prompt for MySQL password", 77 | ) 78 | @click.option("--mysql-password", default=None, help="MySQL password") 79 | @click.option("-h", "--mysql-host", default="localhost", help="MySQL host. Defaults to localhost.") 80 | @click.option("-P", "--mysql-port", type=int, default=3306, help="MySQL port. Defaults to 3306.") 81 | @click.option( 82 | "-k", 83 | "--mysql-socket", 84 | type=click.Path(exists=True), 85 | default=None, 86 | help="Path to MySQL unix socket file.", 87 | ) 88 | @click.option("-S", "--skip-ssl", is_flag=True, help="Disable MySQL connection encryption.") 89 | @click.option( 90 | "-i", 91 | "--mysql-insert-method", 92 | type=click.Choice(MYSQL_INSERT_METHOD, case_sensitive=False), 93 | default="IGNORE", 94 | help="MySQL insert method. DEFAULT will throw errors when encountering duplicate records; " 95 | "UPDATE will update existing rows; IGNORE will ignore insert errors. Defaults to IGNORE.", 96 | ) 97 | @click.option( 98 | "-E", 99 | "--mysql-truncate-tables", 100 | is_flag=True, 101 | help="Truncates existing tables before inserting data.", 102 | ) 103 | @click.option( 104 | "--mysql-integer-type", 105 | default="INT(11)", 106 | help="MySQL default integer field type. Defaults to INT(11).", 107 | ) 108 | @click.option( 109 | "--mysql-string-type", 110 | default="VARCHAR(255)", 111 | help="MySQL default string field type. Defaults to VARCHAR(255).", 112 | ) 113 | @click.option( 114 | "--mysql-text-type", 115 | type=click.Choice(MYSQL_TEXT_COLUMN_TYPES, case_sensitive=False), 116 | default="TEXT", 117 | help="MySQL default text field type. Defaults to TEXT.", 118 | ) 119 | @click.option( 120 | "--mysql-charset", 121 | metavar="TEXT", 122 | type=click.Choice(list(CharacterSet().get_supported()), case_sensitive=False), 123 | default="utf8mb4", 124 | show_default=True, 125 | help="MySQL database and table character set", 126 | ) 127 | @click.option( 128 | "--mysql-collation", 129 | metavar="TEXT", 130 | type=click.Choice( 131 | [charset.collation for charset in mysql_supported_character_sets()], 132 | case_sensitive=False, 133 | ), 134 | default=None, 135 | help="MySQL database and table collation", 136 | ) 137 | @click.option( 138 | "-T", 139 | "--use-fulltext", 140 | is_flag=True, 141 | help="Use FULLTEXT indexes on TEXT columns. " 142 | "Will throw an error if your MySQL version does not support InnoDB FULLTEXT indexes!", 143 | ) 144 | @click.option("--with-rowid", is_flag=True, help="Transfer rowid columns.") 145 | @click.option("-c", "--chunk", type=int, default=None, help="Chunk reading/writing SQL records") 146 | @click.option("-K", "--mysql-skip-create-tables", is_flag=True, help="Skip creating tables in MySQL.") 147 | @click.option("-J", "--mysql-skip-transfer-data", is_flag=True, help="Skip transferring data to MySQL.") 148 | @click.option("-l", "--log-file", type=click.Path(), help="Log file") 149 | @click.option("-q", "--quiet", is_flag=True, help="Quiet. Display only errors.") 150 | @click.option("--debug", is_flag=True, help="Debug mode. Will throw exceptions.") 151 | @click.version_option(message=tabulate(info(), headers=["software", "version"], tablefmt="github")) 152 | def cli( 153 | sqlite_file: t.Union[str, "os.PathLike[t.Any]"], 154 | sqlite_tables: t.Optional[t.Sequence[str]], 155 | exclude_sqlite_tables: t.Optional[t.Sequence[str]], 156 | sqlite_views_as_tables: bool, 157 | without_foreign_keys: bool, 158 | ignore_duplicate_keys: bool, 159 | mysql_user: str, 160 | prompt_mysql_password: bool, 161 | mysql_password: str, 162 | mysql_database: str, 163 | mysql_host: str, 164 | mysql_port: int, 165 | mysql_socket: t.Optional[str], 166 | skip_ssl: bool, 167 | mysql_insert_method: str, 168 | mysql_truncate_tables: bool, 169 | mysql_integer_type: str, 170 | mysql_string_type: str, 171 | mysql_text_type: str, 172 | mysql_charset: str, 173 | mysql_collation: str, 174 | use_fulltext: bool, 175 | with_rowid: bool, 176 | chunk: int, 177 | mysql_skip_create_tables: bool, 178 | mysql_skip_transfer_data: bool, 179 | log_file: t.Union[str, "os.PathLike[t.Any]"], 180 | quiet: bool, 181 | debug: bool, 182 | ) -> None: 183 | """Transfer SQLite to MySQL using the provided CLI options.""" 184 | click.echo(_copyright_header) 185 | try: 186 | if mysql_collation: 187 | charset_collations: t.Tuple[str, ...] = tuple( 188 | cs.collation for cs in mysql_supported_character_sets(mysql_charset.lower()) 189 | ) 190 | if mysql_collation not in set(charset_collations): 191 | raise click.ClickException( 192 | f"Error: Invalid value for '--collation' of charset '{mysql_charset}': '{mysql_collation}' " 193 | f"""is not one of {"'" + "', '".join(charset_collations) + "'"}.""" 194 | ) 195 | 196 | # check if both mysql_skip_create_table and mysql_skip_transfer_data are True 197 | if mysql_skip_create_tables and mysql_skip_transfer_data: 198 | raise click.ClickException( 199 | "Error: Both -K/--mysql-skip-create-tables and -J/--mysql-skip-transfer-data are set. " 200 | "There is nothing to do. Exiting..." 201 | ) 202 | 203 | if sqlite_tables and exclude_sqlite_tables: 204 | raise click.ClickException( 205 | "Error: Both -t/--sqlite-tables and -e/--exclude-sqlite-tables options are set. " 206 | "Please use only one of them." 207 | ) 208 | 209 | SQLite3toMySQL( 210 | sqlite_file=sqlite_file, 211 | sqlite_tables=sqlite_tables or tuple(), 212 | exclude_sqlite_tables=exclude_sqlite_tables or tuple(), 213 | sqlite_views_as_tables=sqlite_views_as_tables, 214 | without_foreign_keys=without_foreign_keys or bool(sqlite_tables) or bool(exclude_sqlite_tables), 215 | mysql_user=mysql_user, 216 | mysql_password=mysql_password or prompt_mysql_password, 217 | mysql_database=mysql_database, 218 | mysql_host=mysql_host, 219 | mysql_port=None if mysql_socket else mysql_port, 220 | mysql_socket=mysql_socket, 221 | mysql_ssl_disabled=skip_ssl, 222 | mysql_insert_method=mysql_insert_method, 223 | mysql_truncate_tables=mysql_truncate_tables, 224 | mysql_integer_type=mysql_integer_type, 225 | mysql_string_type=mysql_string_type, 226 | mysql_text_type=mysql_text_type, 227 | mysql_charset=mysql_charset.lower() if mysql_charset else "utf8mb4", 228 | mysql_collation=mysql_collation.lower() if mysql_collation else None, 229 | ignore_duplicate_keys=ignore_duplicate_keys, 230 | use_fulltext=use_fulltext, 231 | with_rowid=with_rowid, 232 | chunk=chunk, 233 | mysql_create_tables=not mysql_skip_create_tables, 234 | mysql_transfer_data=not mysql_skip_transfer_data, 235 | log_file=log_file, 236 | quiet=quiet, 237 | ).transfer() 238 | except KeyboardInterrupt: 239 | if debug: 240 | raise 241 | click.echo("\nProcess interrupted. Exiting...") 242 | sys.exit(1) 243 | except Exception as err: # pylint: disable=W0703 244 | if debug: 245 | raise 246 | click.echo(err) 247 | sys.exit(1) 248 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # 2.5.5 2 | 3 | * [FEAT] add support for SQLite `JSONB` column type 4 | * [FIX] enhance the translation of SQLite timestamp expressions that use `current_timestamp` within date/time functions 5 | 6 | # 2.5.4 7 | 8 | * [FEAT] add support for SQLite shorthand foreign key references that implicitly reference primary keys 9 | 10 | # 2.5.3 11 | 12 | * [FIX] transfer `TEXT` default 13 | * [FIX] removed the Hatch `sources = ["src"]` rewrite so `sdists` now keep the `src/sqlite3_to_mysql` tree, letting 14 | wheels built from the sdist include the actual package 15 | 16 | # 2.5.2 17 | 18 | * [FEAT] transfer SQLite views as MySQL native views 19 | * [CHORE] improve type and default value translation logic 20 | * [CHORE] add Python 3.14 support 21 | 22 | # 2.5.1 23 | 24 | * [FEAT] enhance table creation logic to handle default values and improve error handling 25 | 26 | # 2.5.0 27 | 28 | * [FEAT] translate SQLite defaults to MySQL 29 | * [FEAT] exclude transferring SQLite tables via `-e`/`--exclude-sqlite-tables` 30 | 31 | # 2.4.6 32 | 33 | * [FIX] fix importing `typing_extensions` on Python >= 3.11 34 | 35 | # 2.4.5 36 | 37 | * [FIX] fix MySQL connection parameters to handle optional socket, host and port 38 | 39 | # 2.4.4 40 | 41 | * [FIX] fix pyproject.toml build sources and specify package inclusion for sdist and wheel 42 | 43 | # 2.4.3 44 | 45 | * [CHORE] update pyproject.toml for improved metadata, dependency management, and dev requirements 46 | 47 | # 2.4.2 48 | 49 | * [FIX] fix conflict between MySQL port and socket options in CLI 50 | 51 | # 2.4.1 52 | 53 | * [FEAT] add MySQL Unix socket connection support via the `--mysql-socket` option 54 | 55 | # 2.4.0 56 | 57 | * [CHORE] drop support for Python 3.8 58 | 59 | # 2.3.2 60 | 61 | * [FIX] fix --mysql-insert-method 62 | * [FIX] modify the existing `check_mysql_json_support` and `check_mysql_fulltext_support` to improve detection of 63 | MariaDB versions 64 | * [FIX] fix connecting with empty MySQL password 65 | 66 | # 2.3.1 67 | 68 | * [FIX] fix conversion of SQLite `NUMERIC` data type with precision and scale to MySQL `DECIMAL` with precision and 69 | scale 70 | 71 | # 2.3.0 72 | 73 | * [FEAT] add MySQL 8.4 and MariaDB 11.4 support 74 | 75 | # 2.2.1 76 | 77 | * [FIX] use `dateutil.parse` to parse SQLite dates 78 | 79 | # 2.2.0 80 | 81 | * [FEAT] add `--mysql-skip-create-tables` and `--mysql-skip-transfer-data` options 82 | * [FIX] fix default parameter parsing 83 | * [CHORE] update Sphinx documentation 84 | 85 | # 2.1.10 86 | 87 | * [CHORE] add Sphinx documentation 88 | 89 | # 2.1.9 90 | 91 | * [FEAT] add conversion of SQLite custom `BOOL` data type to MySQL `TINYINT(1)` 92 | 93 | # 2.1.8 94 | 95 | * [CHORE] migrate package from flat layout to src layout 96 | 97 | # 2.1.7 98 | 99 | * [FEAT] add copyright header 100 | 101 | # 2.1.6 102 | 103 | * [FEAT] build both linux/amd64 and linux/arm64 Docker images 104 | 105 | # 2.1.5 106 | 107 | * [FEAT] add support for UNSIGNED numeric data type conversion 108 | * [FIX] fix invalid column_type error message 109 | 110 | # 2.1.4 111 | 112 | * [CHORE] maintenance release to publish first containerized release 113 | 114 | # 2.1.3 115 | 116 | * [FIX] add packaging as a dependency 117 | 118 | # 2.1.2 119 | 120 | * [FIX] throw more comprehensive error messages when translating column types 121 | 122 | # 2.1.1 123 | 124 | * [CHORE] add support for Python 3.12 125 | * [CHORE] bump minimum version of MySQL Connector/Python to 8.2.0 126 | 127 | # 2.1.0 128 | 129 | * [CHORE] drop support for Python 3.7 130 | 131 | # 2.0.3 132 | 133 | * [FIX] prevent AUTO_INCREMENT-ing fields from having a DEFAULT value 134 | 135 | # 2.0.2 136 | 137 | * [FIX] properly import CMySQLConnection 138 | 139 | # 2.0.1 140 | 141 | * [FIX] fix types 142 | 143 | # 2.0.0 144 | 145 | * [CHORE] drop support for Python 2.7, 3.5 and 3.6 146 | * [CHORE] migrate pytest.ini configuration into pyproject.toml 147 | * [CHORE] migrate from setuptools to hatch / hatchling 148 | * [CHORE] add types 149 | * [CHORE] add types to tests 150 | * [CHORE] update dependencies 151 | * [CHORE] use f-strings where appropriate 152 | 153 | # 1.4.20 154 | 155 | * [CHORE] update dependencies 156 | * [CHORE] use [black](https://github.com/psf/black) and [isort](https://github.com/PyCQA/isort) in tox linters 157 | 158 | # 1.4.19 159 | 160 | * [FEAT] handle generated columns 161 | 162 | # 1.4.18 163 | 164 | * [CHORE] migrate from setup.py to pyproject.toml 165 | 166 | # 1.4.17 167 | 168 | * [CHORE] add publishing workflow 169 | * [CHORE] add Python 3.11 support 170 | * [CHORE] Remove CI tests for Python 3.5, 3.6, add CI tests for Python 3.11 171 | * [CHORE] add MariaDB 10.11 CI tests 172 | 173 | # 1.4.16 174 | 175 | * [FIX] pin mysql-connector-python to <8.0.30 176 | * [CHORE] update CI actions/checkout to v3 177 | * [CHORE] update CI actions/setup-python to v4 178 | * [CHORE] update CI actions/cache to v3 179 | * [CHORE] update CI github/codeql-action/init to v2 180 | * [CHORE] update CI github/codeql-action/analyze to v2 181 | * [CHORE] update CI codecov/codecov-action to v2 182 | 183 | # 1.4.15 184 | 185 | * [FEAT] add option to truncate existing tables before inserting data 186 | 187 | # 1.4.14 188 | 189 | * [FIX] fix safe_identifier_length 190 | 191 | # 1.4.13 192 | 193 | * [FEAT] add option to update duplicate records 194 | * [FEAT] add option to skip duplicate index creation if key name already exists 195 | * [CHORE] mark test_quiet with xfail 196 | * [CHORE] fix CLI test 197 | * [CHORE] remove Fix MySQL GA Github Action step 198 | 199 | # 1.4.12 200 | 201 | * [FEAT] add --debug switch 202 | * [FIX] import backports-datetime-fromisoformat only for Python 3.4, 3.5 and 3.6 203 | * [FIX] handle SQLite date conversion 204 | 205 | # 1.4.11 206 | 207 | * [FIX] fix regression introduced in v1.4.9 208 | 209 | # 1.4.10 210 | 211 | * [FEAT] add ability to change default text type using --mysql-text-type 212 | * [FIX] fix BOOLEAN conversion to TINYINT(1) 213 | 214 | # 1.4.9 215 | 216 | * [FEAT] add support for DEFAULT statements 217 | 218 | # 1.4.8 219 | 220 | * [CHORE] fix tests 221 | 222 | # 1.4.7 223 | 224 | * [CHORE] add support for Python 3.10 225 | * [CHORE] add Python 3.10 tests 226 | 227 | # 1.4.6 228 | 229 | * [FEAT] add CLI options for custom charset and collation 230 | * [FEAT] add unicase custom collation 231 | * [FIX] limit MySQL identifier to 64 characters 232 | * [FIX] handle multiple column FULLTEXT index transfer error 233 | * [FIX] fix multiple column index length #28 234 | * [CHORE] move some MySQL helper methods out of the main transporter 235 | * [CHORE] refactor package 236 | * [CHORE] add experimental tests for Python 3.10-dev 237 | 238 | # 1.4.5 239 | 240 | * [FIX] revert change introduced in v1.4.4 241 | * [FIX] fix Click 8.0 OptionEatAll wrong type 242 | * [CHORE] add tests for MariaDB 10.6 243 | 244 | # 1.4.4 245 | 246 | * [FIX] pin Click to <8.0 247 | 248 | # 1.4.3 249 | 250 | * [FIX] pin python-tabulate to <0.8.6 for Python 3.4 or less 251 | * [FIX] pin Click to <8.0 only for Python 3.5 or less 252 | 253 | # 1.4.2 254 | 255 | * [FIX] fix auto_increment 256 | * [CHORE] add DECIMAL test 257 | * [FIX] pin Click to <8.0 258 | 259 | # 1.4.1 260 | 261 | * [FIX] pin mysql-connector-python to <8.0.24 for Python 3.5 or lower 262 | 263 | # 1.4.0 264 | 265 | * [FEAT] add password prompt. This changes the default behavior of -p 266 | * [FEAT] add option to disable MySQL connection encryption 267 | * [FEAT] add progress bar 268 | * [FEAT] implement feature to transport custom data types as strings 269 | * [FIX] require sqlalchemy <1.4.0 to make compatible with sqlalchemy-utils 270 | * [CHORE] fix CI tests 271 | 272 | # 1.3.12 273 | 274 | * [FIX] handle duplicate indices 275 | * [CHORE] transition from Travis CI to GitHub Actions 276 | 277 | # 1.3.11 278 | 279 | * [CHORE] add Python 3.9 tests 280 | 281 | # 1.3.10 282 | 283 | * [FEAT] add --use-fulltext option 284 | * [FIX] use FULLTEXT index only if all columns are TEXT 285 | 286 | # 1.3.9 287 | 288 | * [FEAT] add --quiet option 289 | 290 | # 1.3.8 291 | 292 | * [FIX] test for mysql client more gracefully 293 | 294 | # 1.3.7 295 | 296 | * [FEAT] transfer composite primary keys 297 | 298 | # 1.3.6 299 | 300 | * [FEAT] simpler access to the debug version info using the --version switch 301 | * [FEAT] add debug_info module to be used in bug reports 302 | * [CHORE] use pytest fixture fom Faker 4.1.0 in Python 3 tests 303 | * [CHORE] omit debug_info.py in coverage reports 304 | 305 | # 1.3.5 306 | 307 | * [FEAT] set default collation of newly created databases and tables to utf8mb4_general_ci 308 | * [FEAT] optional transfer of implicit column rowid using --with-rowid 309 | * [CHORE] test non-numeric primary keys 310 | * [CHORE] add rowid transfer tests 311 | * [CHORE] fix tests 312 | 313 | # 1.3.4 314 | 315 | * [FIX] fix information_schema issue introduced with MySQL 8.0.21 316 | * [FIX] sqlalchemy-utils dropped Python 2.7 support in v0.36.7 317 | * [CHORE] add MySQL version output to CI tests 318 | * [CHORE] add Python 3.9 to the CI tests 319 | * [CHORE] add MariaDB 10.5 to the CI tests 320 | * [CHORE] remove Python 2.7 from allowed CI test failures 321 | * [CHORE] use Ubuntu Bionic instead of Ubuntu Xenial in CI tests 322 | 323 | # 1.3.3 324 | 325 | * [FEAT] add support for SQLite STRING and translate it as MySQL TEXT 326 | 327 | # 1.3.2 328 | 329 | * [FIX] force not null on primary-key columns 330 | 331 | # 1.3.1 332 | 333 | * [CHORE] test legacy databases in CI tests 334 | * [CHORE] fix MySQL 8 CI tests 335 | 336 | # 1.3.0 337 | 338 | * [FEAT] add option to transfer only specific tables using -t 339 | * [CHORE] add tests for transferring only certain tables 340 | 341 | # 1.2.17 342 | 343 | * [FIX] properly escape foreign keys names 344 | 345 | # 1.2.16 346 | 347 | * [FIX] differentiate better between MySQL and SQLite errors 348 | * [CHORE] add Python 3.8 and 3.8-dev test build 349 | 350 | # 1.2.15 351 | 352 | * [CHORE] update Readme on PyPI 353 | 354 | # 1.2.14 355 | 356 | * [FIX] add INT64 as an alias for NUMERIC 357 | * [CHORE] add support for Python 3.8 358 | * [CHORE] add INT64 tests 359 | 360 | # 1.2.13 361 | 362 | * [CHORE] add [bandit](https://github.com/PyCQA/bandit) tests 363 | * [CHORE] add more tests to increase test coverage 364 | * [CHORE] fix tests 365 | 366 | # 1.2.12 367 | 368 | * [FEAT] transfer indices 369 | * [CHORE] add additional index transfer tests 370 | 371 | # 1.2.11 372 | 373 | * [FIX] remove redundant SQL cleanup 374 | * [CHORE] clean up a test 375 | 376 | # 1.2.10 377 | 378 | * [CHORE] update development requirements 379 | 380 | # 1.2.9 381 | 382 | * [FIX] change the way foreign keys are added. 383 | * [FIX] change default MySQL character set to utf8mb4 384 | * [CHORE] add more verbosity 385 | 386 | # 1.2.8 387 | 388 | * [FIX] disable FOREIGN_KEY_CHECKS before inserting the foreign keys and enable FOREIGN_KEY_CHECKS back once finished 389 | 390 | # 1.2.7 391 | 392 | * [FEAT] transfer foreign keys 393 | * [FIX] in Python 2 MySQL binary protocol can not handle 'buffer' objects so we have to convert them to strings 394 | * [CHORE] test transfer of foreign keys 395 | * [CHORE] only test databases that support JSON 396 | * [CHORE] fix tests 397 | 398 | # 1.2.6 399 | 400 | * [CHORE] refactor package 401 | 402 | # 1.2.5 403 | 404 | * [CHORE] update Readme 405 | 406 | # 1.2.4 407 | 408 | * [CHORE] fix CI tests 409 | * [CHORE] add linter rules 410 | 411 | # 1.2.3 412 | 413 | * [CHORE] refactor package 414 | * [CHORE] test the CLI interface 415 | * [CHORE] fix tests 416 | 417 | # 1.2.2 418 | 419 | * [FEAT] add Python 2.7 support 420 | * [CHORE] refactor package 421 | * [CHORE] fix tests 422 | * [CHORE] add option to test against a real SQLite file 423 | 424 | # 1.2.1 425 | 426 | * [FIX] catch exceptions 427 | * [FIX] default mysql_string_type from VARCHAR(300) to VARCHAR(255) 428 | * [CHORE] fix CI tests 429 | * [CHORE] option to run tests against a physical MySQL server instance as well as a Docker one 430 | * [CHORE] run tests against any Docker image with a MySQL/MariaDB database 431 | * [CHORE] clean up hanged Docker images with the name "pytest_sqlite3_to_mysql" 432 | * [CHORE] 100% code coverage 433 | 434 | # 1.2.0 435 | 436 | * [CHORE] add more tests 437 | 438 | # 1.1.2 439 | 440 | * [FIX] fix creation of tables with non-numeric primary keys 441 | 442 | # 1.1.1 443 | 444 | * [FIX] fix error of transferring empty tables 445 | 446 | # 1.1.0 447 | 448 | * [CHORE] update to work with MySQL Connector/Python v8.0.11+ 449 | 450 | # 1.0.3 451 | 452 | * [FIX] don't autoincrement if primary key is TEXT/VARCHAR 453 | 454 | # 1.0.2 455 | 456 | * [CHORE] refactor package 457 | 458 | # 1.0.1 459 | 460 | * [CHORE] change license from GPL to MIT 461 | 462 | # 1.0.0 463 | 464 | Initial commit -------------------------------------------------------------------------------- /tests/conftest.py: -------------------------------------------------------------------------------- 1 | import json 2 | import socket 3 | import typing as t 4 | from codecs import open 5 | from contextlib import contextmanager 6 | from os.path import abspath, dirname, isfile, join, realpath 7 | from pathlib import Path 8 | from time import sleep 9 | 10 | import docker 11 | import mysql.connector 12 | import pytest 13 | from _pytest._py.path import LocalPath 14 | from _pytest.config import Config 15 | from _pytest.config.argparsing import Parser 16 | from _pytest.legacypath import TempdirFactory 17 | from click.testing import CliRunner 18 | from docker import DockerClient 19 | from docker.errors import NotFound 20 | from docker.models.containers import Container 21 | from faker import Faker 22 | from mysql.connector import MySQLConnection, errorcode 23 | from mysql.connector.connection_cext import CMySQLConnection 24 | from mysql.connector.pooling import PooledMySQLConnection 25 | from requests import HTTPError 26 | from sqlalchemy import create_engine 27 | from sqlalchemy.engine import Engine 28 | from sqlalchemy.exc import IntegrityError 29 | from sqlalchemy.orm import Session 30 | from sqlalchemy_utils import database_exists, drop_database 31 | 32 | from . import database, factories 33 | 34 | 35 | def pytest_addoption(parser: "Parser") -> None: 36 | parser.addoption( 37 | "--sqlite-file", 38 | dest="sqlite_file", 39 | default=None, 40 | help="SQLite database file. Defaults to none and generates one internally.", 41 | ) 42 | 43 | parser.addoption( 44 | "--mysql-user", 45 | dest="mysql_user", 46 | default="tester", 47 | help="MySQL user. Defaults to 'tester'.", 48 | ) 49 | 50 | parser.addoption( 51 | "--mysql-password", 52 | dest="mysql_password", 53 | default="testpass", 54 | help="MySQL password. Defaults to 'testpass'.", 55 | ) 56 | 57 | parser.addoption( 58 | "--mysql-database", 59 | dest="mysql_database", 60 | default="test_db", 61 | help="MySQL database name. Defaults to 'test_db'.", 62 | ) 63 | 64 | parser.addoption( 65 | "--mysql-host", 66 | dest="mysql_host", 67 | default="0.0.0.0", 68 | help="Test against a MySQL server running on this host. Defaults to '0.0.0.0'.", 69 | ) 70 | 71 | parser.addoption( 72 | "--mysql-port", 73 | dest="mysql_port", 74 | type=int, 75 | default=None, 76 | help="The TCP port of the MySQL server.", 77 | ) 78 | 79 | parser.addoption( 80 | "--no-docker", 81 | dest="use_docker", 82 | default=True, 83 | action="store_false", 84 | help="Do not use a Docker MySQL image to run the tests. " 85 | "If you decide to use this switch you will have to use a physical MySQL server.", 86 | ) 87 | 88 | parser.addoption( 89 | "--docker-mysql-image", 90 | dest="docker_mysql_image", 91 | default="mysql:latest", 92 | help="Run the tests against a specific MySQL Docker image. Defaults to mysql:latest. " 93 | "Check all supported versions here https://hub.docker.com/_/mysql", 94 | ) 95 | 96 | 97 | @pytest.fixture(scope="session", autouse=True) 98 | def cleanup_hanged_docker_containers() -> None: 99 | try: 100 | client: DockerClient = docker.from_env() 101 | for container in client.containers.list(): 102 | if container.name == "pytest_sqlite3_to_mysql": 103 | container.kill() 104 | break 105 | except Exception: 106 | pass 107 | 108 | 109 | def pytest_keyboard_interrupt() -> None: 110 | try: 111 | client: DockerClient = docker.from_env() 112 | for container in client.containers.list(): 113 | if container.name == "pytest_sqlite3_to_mysql": 114 | container.kill() 115 | break 116 | except Exception: 117 | pass 118 | 119 | 120 | class Helpers: 121 | @staticmethod 122 | @contextmanager 123 | def not_raises(exception: t.Type[Exception]) -> t.Generator: 124 | try: 125 | yield 126 | except exception: 127 | raise pytest.fail(f"DID RAISE {exception}") 128 | 129 | @staticmethod 130 | @contextmanager 131 | def session_scope(db: database.Database) -> t.Generator: 132 | """Provide a transactional scope around a series of operations.""" 133 | session: Session = db.Session() 134 | try: 135 | yield session 136 | session.commit() 137 | except Exception: 138 | session.rollback() 139 | raise 140 | finally: 141 | session.close() 142 | 143 | 144 | @pytest.fixture 145 | def helpers() -> t.Type[Helpers]: 146 | return Helpers 147 | 148 | 149 | @pytest.fixture(scope="session") 150 | def sqlite_database(pytestconfig: Config, _session_faker: Faker, tmpdir_factory: TempdirFactory) -> str: 151 | db_file: LocalPath = pytestconfig.getoption("sqlite_file") 152 | if db_file: 153 | if not isfile(realpath(db_file)): 154 | pytest.fail(f"{db_file} does not exist") 155 | return str(realpath(db_file)) 156 | 157 | temp_data_dir: LocalPath = tmpdir_factory.mktemp("data") 158 | temp_image_dir: LocalPath = tmpdir_factory.mktemp("images") 159 | db_file = temp_data_dir.join(Path("db.sqlite3")) 160 | db: database.Database = database.Database(f"sqlite:///{db_file}") 161 | 162 | with Helpers.session_scope(db) as session: 163 | for _ in range(_session_faker.pyint(min_value=12, max_value=24)): 164 | article = factories.ArticleFactory() 165 | article.authors.append(factories.AuthorFactory()) 166 | article.tags.append(factories.TagFactory()) 167 | article.misc.append(factories.MiscFactory()) 168 | article.media.append(factories.MediaFactory()) 169 | for _ in range(_session_faker.pyint(min_value=1, max_value=4)): 170 | article.images.append( 171 | factories.ImageFactory( 172 | path=join( 173 | str(temp_image_dir), 174 | _session_faker.year(), 175 | _session_faker.month(), 176 | _session_faker.day_of_month(), 177 | _session_faker.file_name(extension="jpg"), 178 | ) 179 | ) 180 | ) 181 | session.add(article) 182 | try: 183 | session.commit() 184 | except IntegrityError: 185 | session.rollback() 186 | 187 | return str(db_file) 188 | 189 | 190 | def is_port_in_use(port: int, host: str = "0.0.0.0") -> bool: 191 | with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: 192 | return s.connect_ex((host, port)) == 0 193 | 194 | 195 | class MySQLCredentials(t.NamedTuple): 196 | """MySQL credentials.""" 197 | 198 | user: str 199 | password: str 200 | host: str 201 | port: int 202 | database: str 203 | 204 | 205 | @pytest.fixture(scope="session") 206 | def mysql_credentials(pytestconfig: Config) -> MySQLCredentials: 207 | db_credentials_file: str = abspath(join(dirname(__file__), "db_credentials.json")) 208 | if isfile(db_credentials_file): 209 | with open(db_credentials_file, "r", "utf-8") as fh: 210 | db_credentials: t.Dict[str, t.Any] = json.load(fh) 211 | return MySQLCredentials( 212 | user=db_credentials["mysql_user"], 213 | password=db_credentials["mysql_password"], 214 | database=db_credentials["mysql_database"], 215 | host=db_credentials["mysql_host"], 216 | port=db_credentials["mysql_port"], 217 | ) 218 | 219 | port: int = pytestconfig.getoption("mysql_port") or 3306 220 | if pytestconfig.getoption("use_docker"): 221 | while is_port_in_use(port, pytestconfig.getoption("mysql_host")): 222 | if port >= 2**16 - 1: 223 | pytest.fail(f'No ports appear to be available on the host {pytestconfig.getoption("mysql_host")}') 224 | port += 1 225 | 226 | return MySQLCredentials( 227 | user=pytestconfig.getoption("mysql_user") or "tester", 228 | password=pytestconfig.getoption("mysql_password") or "testpass", 229 | database=pytestconfig.getoption("mysql_database") or "test_db", 230 | host=pytestconfig.getoption("mysql_host") or "0.0.0.0", 231 | port=port, 232 | ) 233 | 234 | 235 | @pytest.fixture(scope="session") 236 | def mysql_instance(mysql_credentials: MySQLCredentials, pytestconfig: Config) -> t.Iterator[MySQLConnection]: 237 | container: t.Optional[Container] = None 238 | mysql_connection: t.Optional[t.Union[PooledMySQLConnection, MySQLConnection, CMySQLConnection]] = None 239 | mysql_available: bool = False 240 | mysql_connection_retries: int = 15 # failsafe 241 | 242 | db_credentials_file = abspath(join(dirname(__file__), "db_credentials.json")) 243 | if isfile(db_credentials_file): 244 | use_docker = False 245 | else: 246 | use_docker = pytestconfig.getoption("use_docker") 247 | 248 | if use_docker: 249 | """Connecting to a MySQL server within a Docker container is quite tricky :P 250 | Read more on the issue here https://hub.docker.com/_/mysql#no-connections-until-mysql-init-completes 251 | """ 252 | try: 253 | client = docker.from_env() 254 | except Exception as err: 255 | pytest.fail(str(err)) 256 | 257 | docker_mysql_image = pytestconfig.getoption("docker_mysql_image") or "mysql:latest" 258 | 259 | if not any(docker_mysql_image in image.tags for image in client.images.list()): 260 | print(f"Attempting to download Docker image {docker_mysql_image}'") 261 | try: 262 | client.images.pull(docker_mysql_image) 263 | except (HTTPError, NotFound) as err: 264 | pytest.fail(str(err)) 265 | 266 | container = client.containers.run( 267 | image=docker_mysql_image, 268 | name="pytest_sqlite3_to_mysql", 269 | ports={ 270 | "3306/tcp": ( 271 | mysql_credentials.host, 272 | f"{mysql_credentials.port}/tcp", 273 | ) 274 | }, 275 | environment={ 276 | "MYSQL_RANDOM_ROOT_PASSWORD": "yes", 277 | "MYSQL_USER": mysql_credentials.user, 278 | "MYSQL_PASSWORD": mysql_credentials.password, 279 | "MYSQL_DATABASE": mysql_credentials.database, 280 | }, 281 | command=[ 282 | "--character-set-server=utf8mb4", 283 | "--collation-server=utf8mb4_unicode_ci", 284 | ], 285 | detach=True, 286 | auto_remove=True, 287 | ) 288 | 289 | while not mysql_available and mysql_connection_retries > 0: 290 | try: 291 | mysql_connection = mysql.connector.connect( 292 | user=mysql_credentials.user, 293 | password=mysql_credentials.password, 294 | host=mysql_credentials.host, 295 | port=mysql_credentials.port, 296 | charset="utf8mb4", 297 | collation="utf8mb4_unicode_ci", 298 | ) 299 | except mysql.connector.Error as err: 300 | if err.errno == errorcode.CR_SERVER_LOST: 301 | # sleep for two seconds and retry the connection 302 | sleep(2) 303 | else: 304 | raise 305 | finally: 306 | mysql_connection_retries -= 1 307 | if mysql_connection and mysql_connection.is_connected(): 308 | mysql_available = True 309 | mysql_connection.close() 310 | else: 311 | if not mysql_available and mysql_connection_retries <= 0: 312 | raise ConnectionAbortedError("Maximum MySQL connection retries exhausted! Are you sure MySQL is running?") 313 | 314 | yield # type: ignore[misc] 315 | 316 | if use_docker and container is not None: 317 | container.kill() 318 | 319 | 320 | @pytest.fixture() 321 | def mysql_database(mysql_instance: t.Generator, mysql_credentials: MySQLCredentials) -> t.Iterator[Engine]: 322 | yield # type: ignore[misc] 323 | 324 | engine: Engine = create_engine( 325 | f"mysql+pymysql://{mysql_credentials.user}:{mysql_credentials.password}@{mysql_credentials.host}:{mysql_credentials.port}/{mysql_credentials.database}" 326 | ) 327 | 328 | if database_exists(engine.url): 329 | drop_database(engine.url) 330 | 331 | 332 | @pytest.fixture() 333 | def cli_runner() -> t.Iterator[CliRunner]: 334 | yield CliRunner() 335 | -------------------------------------------------------------------------------- /tests/unit/mysql_utils_test.py: -------------------------------------------------------------------------------- 1 | from unittest.mock import MagicMock, patch 2 | 3 | import pytest 4 | from packaging.version import Version 5 | 6 | from sqlite3_to_mysql.mysql_utils import ( 7 | CharSet, 8 | check_mysql_current_timestamp_datetime_support, 9 | check_mysql_expression_defaults_support, 10 | check_mysql_fractional_seconds_support, 11 | check_mysql_fulltext_support, 12 | check_mysql_json_support, 13 | check_mysql_values_alias_support, 14 | get_mysql_version, 15 | mysql_supported_character_sets, 16 | safe_identifier_length, 17 | ) 18 | 19 | 20 | class TestMySQLUtils: 21 | @pytest.mark.parametrize( 22 | "version_string,expected", 23 | [ 24 | ("5.7.7", Version("5.7.7")), 25 | ("5.7.8", Version("5.7.8")), 26 | ("8.0.0", Version("8.0.0")), 27 | ("9.0.0", Version("9.0.0")), 28 | ("10.2.6-mariadb", Version("10.2.6")), 29 | ("10.2.7-mariadb", Version("10.2.7")), 30 | ("11.4.0-mariadb", Version("11.4.0")), 31 | ], 32 | ) 33 | def test_get_mysql_version(self, version_string: str, expected: Version) -> None: 34 | assert get_mysql_version(version_string) == expected 35 | 36 | @pytest.mark.parametrize( 37 | "version_string,expected", 38 | [ 39 | ("5.7.7", False), 40 | ("5.7.8", True), 41 | ("8.0.0", True), 42 | ("9.0.0", True), 43 | ("10.2.6-mariadb", False), 44 | ("10.2.7-mariadb", True), 45 | ("11.4.0-mariadb", True), 46 | ], 47 | ) 48 | def test_check_mysql_json_support(self, version_string: str, expected: bool) -> None: 49 | assert check_mysql_json_support(version_string) == expected 50 | 51 | @pytest.mark.parametrize( 52 | "version_string,expected", 53 | [ 54 | ("5.7.8", False), 55 | ("8.0.0", False), 56 | ("8.0.18", False), 57 | ("8.0.19", True), 58 | ("9.0.0", True), 59 | ("10.2.6-mariadb", False), 60 | ("10.2.7-mariadb", False), 61 | ("11.4.0-mariadb", False), 62 | ], 63 | ) 64 | def test_check_mysql_values_alias_support(self, version_string: str, expected: bool) -> None: 65 | assert check_mysql_values_alias_support(version_string) == expected 66 | 67 | @pytest.mark.parametrize( 68 | "version_string,expected", 69 | [ 70 | ("5.0.0", False), 71 | ("5.5.0", False), 72 | ("5.6.0", True), 73 | ("8.0.0", True), 74 | ("10.0.4-mariadb", False), 75 | ("10.0.5-mariadb", True), 76 | ("10.2.6-mariadb", True), 77 | ("11.4.0-mariadb", True), 78 | ], 79 | ) 80 | def test_check_mysql_fulltext_support(self, version_string: str, expected: bool) -> None: 81 | assert check_mysql_fulltext_support(version_string) == expected 82 | 83 | @pytest.mark.parametrize( 84 | "identifier,expected", 85 | [ 86 | ("a" * 67, "a" * 64), 87 | ("a" * 66, "a" * 64), 88 | ("a" * 65, "a" * 64), 89 | ("a" * 64, "a" * 64), 90 | ("a" * 63, "a" * 63), 91 | ], 92 | ) 93 | def test_safe_identifier_length(self, identifier: str, expected: str) -> None: 94 | assert safe_identifier_length(identifier) == expected 95 | 96 | def test_mysql_supported_character_sets_with_charset(self) -> None: 97 | """Test mysql_supported_character_sets function with a specific charset.""" 98 | # Mock the MYSQL_CHARACTER_SETS list 99 | mock_character_sets = [ 100 | None, # Index 0 101 | ("utf8", "utf8_general_ci", True), # Index 1 102 | ("latin1", "latin1_swedish_ci", True), # Index 2 103 | ] 104 | 105 | with patch("sqlite3_to_mysql.mysql_utils.MYSQL_CHARACTER_SETS", mock_character_sets): 106 | # Test with a charset that exists 107 | result = list(mysql_supported_character_sets(charset="utf8")) 108 | assert len(result) == 1 109 | assert result[0] == CharSet(1, "utf8", "utf8_general_ci") 110 | 111 | # Test with a charset that doesn't exist 112 | result = list(mysql_supported_character_sets(charset="unknown")) 113 | assert len(result) == 0 114 | 115 | def test_mysql_supported_character_sets_without_charset(self) -> None: 116 | """Test mysql_supported_character_sets function without a specific charset.""" 117 | # Mock the MYSQL_CHARACTER_SETS list 118 | mock_character_sets = [ 119 | None, # Index 0 120 | ("utf8", "utf8_general_ci", True), # Index 1 121 | ("latin1", "latin1_swedish_ci", True), # Index 2 122 | ] 123 | 124 | # Mock the CharacterSet().get_supported() method 125 | mock_get_supported = MagicMock(return_value=["utf8", "latin1"]) 126 | 127 | with patch("sqlite3_to_mysql.mysql_utils.MYSQL_CHARACTER_SETS", mock_character_sets): 128 | with patch("mysql.connector.CharacterSet.get_supported", mock_get_supported): 129 | result = list(mysql_supported_character_sets()) 130 | # The function yields a CharSet for each combination of charset and index 131 | assert len(result) == 4 132 | assert CharSet(1, "utf8", "utf8_general_ci") in result 133 | assert CharSet(2, "utf8", "latin1_swedish_ci") in result 134 | assert CharSet(1, "latin1", "utf8_general_ci") in result 135 | assert CharSet(2, "latin1", "latin1_swedish_ci") in result 136 | 137 | def test_mysql_supported_character_sets_with_keyerror(self) -> None: 138 | """Test mysql_supported_character_sets function with KeyError.""" 139 | # Mock the MYSQL_CHARACTER_SETS list with an entry that will cause a KeyError 140 | mock_character_sets = [ 141 | None, # Index 0 142 | ("utf8", "utf8_general_ci", True), # Index 1 143 | None, # Index 2 - This will cause a KeyError when accessed 144 | ] 145 | 146 | with patch("sqlite3_to_mysql.mysql_utils.MYSQL_CHARACTER_SETS", mock_character_sets): 147 | # Test with a charset that exists but will cause a KeyError 148 | result = list(mysql_supported_character_sets(charset="utf8")) 149 | assert len(result) == 1 150 | assert result[0] == CharSet(1, "utf8", "utf8_general_ci") 151 | 152 | # Mock for testing without charset 153 | mock_get_supported = MagicMock(return_value=["utf8"]) 154 | 155 | with patch("sqlite3_to_mysql.mysql_utils.MYSQL_CHARACTER_SETS", mock_character_sets): 156 | with patch("mysql.connector.CharacterSet.get_supported", mock_get_supported): 157 | result = list(mysql_supported_character_sets()) 158 | assert len(result) == 1 159 | assert CharSet(1, "utf8", "utf8_general_ci") in result 160 | 161 | def test_mysql_supported_character_sets_with_keyerror_in_info_access(self) -> None: 162 | """Test mysql_supported_character_sets function with KeyError when accessing info elements.""" 163 | 164 | # Create a mock tuple that will raise KeyError when accessed with index 165 | class MockTuple: 166 | def __getitem__(self, key): 167 | if key == 0: 168 | return "utf8" 169 | raise KeyError("Mock KeyError") 170 | 171 | # Mock the MYSQL_CHARACTER_SETS list with an entry that will cause a KeyError 172 | mock_character_sets = [ 173 | None, # Index 0 174 | MockTuple(), # Index 1 - This will cause a KeyError when accessing index 1 175 | ] 176 | 177 | # Test with a specific charset 178 | with patch("sqlite3_to_mysql.mysql_utils.MYSQL_CHARACTER_SETS", mock_character_sets): 179 | result = list(mysql_supported_character_sets(charset="utf8")) 180 | # The function should skip the KeyError and return an empty list 181 | assert len(result) == 0 182 | 183 | # Test without a specific charset 184 | mock_get_supported = MagicMock(return_value=["utf8"]) 185 | with patch("sqlite3_to_mysql.mysql_utils.MYSQL_CHARACTER_SETS", mock_character_sets): 186 | with patch("mysql.connector.CharacterSet.get_supported", mock_get_supported): 187 | result = list(mysql_supported_character_sets()) 188 | # The function should skip the KeyError and return an empty list 189 | assert len(result) == 0 190 | 191 | def test_mysql_supported_character_sets_with_keyerror_in_charset_match(self) -> None: 192 | """Test mysql_supported_character_sets function with KeyError when matching charset.""" 193 | 194 | # Create a mock tuple that will raise KeyError when checking if info[0] == charset 195 | class MockTuple: 196 | def __getitem__(self, key): 197 | if key == 0: 198 | raise KeyError("Mock KeyError in charset match") 199 | if key == 1: 200 | return "utf8_general_ci" 201 | return None 202 | 203 | # Mock the MYSQL_CHARACTER_SETS list with an entry that will cause a KeyError 204 | mock_character_sets = [ 205 | None, # Index 0 206 | MockTuple(), # Index 1 - This will cause a KeyError when accessing info[0] 207 | ] 208 | 209 | # Test with a specific charset 210 | with patch("sqlite3_to_mysql.mysql_utils.MYSQL_CHARACTER_SETS", mock_character_sets): 211 | result = list(mysql_supported_character_sets(charset="utf8")) 212 | # The function should skip the KeyError and return an empty list 213 | assert len(result) == 0 214 | 215 | # ----------------------------- 216 | # Expression defaults (MySQL 8.0.13+, MariaDB 10.2.0+) 217 | # ----------------------------- 218 | @pytest.mark.parametrize( 219 | "ver, expected", 220 | [ 221 | ("8.0.12", False), 222 | ("8.0.13", True), 223 | ("8.0.13-8ubuntu1", True), 224 | ("5.7.44", False), 225 | ], 226 | ) 227 | def test_expr_defaults_mysql(self, ver: str, expected: bool) -> None: 228 | assert check_mysql_expression_defaults_support(ver) is expected 229 | 230 | @pytest.mark.parametrize( 231 | "ver, expected", 232 | [ 233 | ("10.1.99-MariaDB", False), 234 | ("10.2.0-MariaDB", True), 235 | ("10.2.7-MariaDB-1~deb10u1", True), 236 | ("10.1.2-mArIaDb", False), # case-insensitive detection 237 | ], 238 | ) 239 | def test_expr_defaults_mariadb(self, ver: str, expected: bool) -> None: 240 | assert check_mysql_expression_defaults_support(ver) is expected 241 | 242 | # ----------------------------- 243 | # CURRENT_TIMESTAMP for DATETIME (MySQL 5.6.5+, MariaDB 10.0.1+) 244 | # ----------------------------- 245 | @pytest.mark.parametrize( 246 | "ver, expected", 247 | [ 248 | ("5.6.4", False), 249 | ("5.6.5", True), 250 | ("5.6.5-ps-log", True), 251 | ("5.5.62", False), 252 | ], 253 | ) 254 | def test_current_timestamp_datetime_mysql(self, ver: str, expected: bool) -> None: 255 | assert check_mysql_current_timestamp_datetime_support(ver) is expected 256 | 257 | @pytest.mark.parametrize( 258 | "ver, expected", 259 | [ 260 | ("10.0.0-MariaDB", False), 261 | ("10.0.1-MariaDB", True), 262 | ("10.3.39-MariaDB-1:10.3.39+maria~focal", True), 263 | ], 264 | ) 265 | def test_current_timestamp_datetime_mariadb(self, ver: str, expected: bool) -> None: 266 | assert check_mysql_current_timestamp_datetime_support(ver) is expected 267 | 268 | # ----------------------------- 269 | # Fractional seconds (fsp) (MySQL 5.6.4+, MariaDB 10.1.2+) 270 | # ----------------------------- 271 | @pytest.mark.parametrize( 272 | "ver, expected", 273 | [ 274 | ("5.6.3", False), 275 | ("5.6.4", True), 276 | ("5.7.44-0ubuntu0.18.04.1", True), 277 | ], 278 | ) 279 | def test_fractional_seconds_mysql(self, ver: str, expected: bool) -> None: 280 | assert check_mysql_fractional_seconds_support(ver) is expected 281 | 282 | @pytest.mark.parametrize( 283 | "ver, expected", 284 | [ 285 | ("10.1.1-MariaDB", False), 286 | ("10.1.2-MariaDB", True), 287 | ("10.6.16-MariaDB-1:10.6.16+maria~jammy", True), 288 | ("10.1.2-mArIaDb", True), # case-insensitive detection 289 | ], 290 | ) 291 | def test_fractional_seconds_mariadb(self, ver: str, expected: bool) -> None: 292 | assert check_mysql_fractional_seconds_support(ver) is expected 293 | -------------------------------------------------------------------------------- /.github/workflows/test.yml: -------------------------------------------------------------------------------- 1 | name: Test 2 | 3 | on: 4 | push: 5 | branches: 6 | - master 7 | pull_request: 8 | branches: 9 | - master 10 | workflow_call: 11 | 12 | defaults: 13 | run: 14 | shell: bash 15 | 16 | permissions: read-all 17 | 18 | concurrency: 19 | group: test-${{ github.ref }} 20 | cancel-in-progress: true 21 | 22 | jobs: 23 | analyze: 24 | name: "Analyze" 25 | runs-on: ubuntu-latest 26 | steps: 27 | - uses: actions/checkout@v6 28 | - name: Set up Python 29 | uses: actions/setup-python@v6 30 | with: 31 | python-version: "3.x" 32 | - name: Install dependencies 33 | run: | 34 | python3 -m pip install --upgrade pip 35 | pip install -r requirements_dev.txt 36 | - name: Run static analysis 37 | run: tox -e linters 38 | test: 39 | needs: analyze 40 | name: "Test" 41 | runs-on: ubuntu-latest 42 | strategy: 43 | fail-fast: false 44 | max-parallel: 8 45 | matrix: 46 | include: 47 | - toxenv: "python3.9" 48 | db: "mariadb:5.5" 49 | legacy_db: 1 50 | experimental: true 51 | py: "3.9" 52 | 53 | - toxenv: "python3.10" 54 | db: "mariadb:5.5" 55 | legacy_db: 1 56 | experimental: true 57 | py: "3.10" 58 | 59 | - toxenv: "python3.11" 60 | db: "mariadb:5.5" 61 | legacy_db: 1 62 | experimental: true 63 | py: "3.11" 64 | 65 | - toxenv: "python3.12" 66 | db: "mariadb:5.5" 67 | legacy_db: 1 68 | experimental: true 69 | py: "3.12" 70 | 71 | - toxenv: "python3.13" 72 | db: "mariadb:5.5" 73 | legacy_db: 1 74 | experimental: true 75 | py: "3.13" 76 | 77 | - toxenv: "python3.14" 78 | db: "mariadb:5.5" 79 | legacy_db: 1 80 | experimental: true 81 | py: "3.14" 82 | 83 | - toxenv: "python3.9" 84 | db: "mariadb:10.0" 85 | legacy_db: 1 86 | experimental: true 87 | py: "3.9" 88 | 89 | - toxenv: "python3.10" 90 | db: "mariadb:10.0" 91 | legacy_db: 1 92 | experimental: true 93 | py: "3.10" 94 | 95 | - toxenv: "python3.11" 96 | db: "mariadb:10.0" 97 | legacy_db: 1 98 | experimental: true 99 | py: "3.11" 100 | 101 | - toxenv: "python3.12" 102 | db: "mariadb:10.0" 103 | legacy_db: 1 104 | experimental: true 105 | py: "3.12" 106 | 107 | - toxenv: "python3.13" 108 | db: "mariadb:10.0" 109 | legacy_db: 1 110 | experimental: true 111 | py: "3.13" 112 | 113 | - toxenv: "python3.14" 114 | db: "mariadb:10.0" 115 | legacy_db: 1 116 | experimental: true 117 | py: "3.14" 118 | 119 | - toxenv: "python3.9" 120 | db: "mariadb:10.6" 121 | legacy_db: 0 122 | experimental: false 123 | py: "3.9" 124 | 125 | - toxenv: "python3.10" 126 | db: "mariadb:10.6" 127 | legacy_db: 0 128 | experimental: false 129 | py: "3.10" 130 | 131 | - toxenv: "python3.11" 132 | db: "mariadb:10.6" 133 | legacy_db: 0 134 | experimental: false 135 | py: "3.11" 136 | 137 | - toxenv: "python3.12" 138 | db: "mariadb:10.6" 139 | legacy_db: 0 140 | experimental: false 141 | py: "3.12" 142 | 143 | - toxenv: "python3.13" 144 | db: "mariadb:10.6" 145 | legacy_db: 0 146 | experimental: false 147 | py: "3.13" 148 | 149 | - toxenv: "python3.14" 150 | db: "mariadb:10.6" 151 | legacy_db: 0 152 | experimental: false 153 | py: "3.14" 154 | 155 | - toxenv: "python3.9" 156 | db: "mariadb:10.11" 157 | legacy_db: 0 158 | experimental: false 159 | py: "3.9" 160 | 161 | - toxenv: "python3.10" 162 | db: "mariadb:10.11" 163 | legacy_db: 0 164 | experimental: false 165 | py: "3.10" 166 | 167 | - toxenv: "python3.11" 168 | db: "mariadb:10.11" 169 | legacy_db: 0 170 | experimental: false 171 | py: "3.11" 172 | 173 | - toxenv: "python3.12" 174 | db: "mariadb:10.11" 175 | legacy_db: 0 176 | experimental: false 177 | py: "3.12" 178 | 179 | - toxenv: "python3.13" 180 | db: "mariadb:10.11" 181 | legacy_db: 0 182 | experimental: false 183 | py: "3.13" 184 | 185 | - toxenv: "python3.14" 186 | db: "mariadb:10.11" 187 | legacy_db: 0 188 | experimental: false 189 | py: "3.14" 190 | 191 | - toxenv: "python3.9" 192 | db: "mariadb:11.4" 193 | legacy_db: 0 194 | experimental: false 195 | py: "3.9" 196 | 197 | - toxenv: "python3.10" 198 | db: "mariadb:11.4" 199 | legacy_db: 0 200 | experimental: false 201 | py: "3.10" 202 | 203 | - toxenv: "python3.11" 204 | db: "mariadb:11.4" 205 | legacy_db: 0 206 | experimental: false 207 | py: "3.11" 208 | 209 | - toxenv: "python3.12" 210 | db: "mariadb:11.4" 211 | legacy_db: 0 212 | experimental: false 213 | py: "3.12" 214 | 215 | - toxenv: "python3.13" 216 | db: "mariadb:11.4" 217 | legacy_db: 0 218 | experimental: false 219 | py: "3.13" 220 | 221 | - toxenv: "python3.14" 222 | db: "mariadb:11.4" 223 | legacy_db: 0 224 | experimental: false 225 | py: "3.14" 226 | 227 | - toxenv: "python3.9" 228 | db: "mariadb:11.8" 229 | legacy_db: 0 230 | experimental: false 231 | py: "3.9" 232 | 233 | - toxenv: "python3.10" 234 | db: "mariadb:11.8" 235 | legacy_db: 0 236 | experimental: false 237 | py: "3.10" 238 | 239 | - toxenv: "python3.11" 240 | db: "mariadb:11.8" 241 | legacy_db: 0 242 | experimental: false 243 | py: "3.11" 244 | 245 | - toxenv: "python3.12" 246 | db: "mariadb:11.8" 247 | legacy_db: 0 248 | experimental: false 249 | py: "3.12" 250 | 251 | - toxenv: "python3.13" 252 | db: "mariadb:11.8" 253 | legacy_db: 0 254 | experimental: false 255 | py: "3.13" 256 | 257 | - toxenv: "python3.14" 258 | db: "mariadb:11.8" 259 | legacy_db: 0 260 | experimental: false 261 | py: "3.14" 262 | 263 | - toxenv: "python3.9" 264 | db: "mysql:5.5" 265 | legacy_db: 1 266 | experimental: true 267 | py: "3.9" 268 | 269 | - toxenv: "python3.10" 270 | db: "mysql:5.5" 271 | legacy_db: 1 272 | experimental: true 273 | py: "3.10" 274 | 275 | - toxenv: "python3.11" 276 | db: "mysql:5.5" 277 | legacy_db: 1 278 | experimental: true 279 | py: "3.11" 280 | 281 | - toxenv: "python3.12" 282 | db: "mysql:5.5" 283 | legacy_db: 1 284 | experimental: true 285 | py: "3.12" 286 | 287 | - toxenv: "python3.13" 288 | db: "mysql:5.5" 289 | legacy_db: 1 290 | experimental: true 291 | py: "3.13" 292 | 293 | - toxenv: "python3.14" 294 | db: "mysql:5.5" 295 | legacy_db: 1 296 | experimental: true 297 | py: "3.14" 298 | 299 | - toxenv: "python3.9" 300 | db: "mysql:5.6" 301 | legacy_db: 1 302 | experimental: true 303 | py: "3.9" 304 | 305 | - toxenv: "python3.10" 306 | db: "mysql:5.6" 307 | legacy_db: 1 308 | experimental: true 309 | py: "3.10" 310 | 311 | - toxenv: "python3.11" 312 | db: "mysql:5.6" 313 | legacy_db: 1 314 | experimental: true 315 | py: "3.11" 316 | 317 | - toxenv: "python3.12" 318 | db: "mysql:5.6" 319 | legacy_db: 1 320 | experimental: true 321 | py: "3.12" 322 | 323 | - toxenv: "python3.13" 324 | db: "mysql:5.6" 325 | legacy_db: 1 326 | experimental: true 327 | py: "3.13" 328 | 329 | - toxenv: "python3.14" 330 | db: "mysql:5.6" 331 | legacy_db: 1 332 | experimental: true 333 | py: "3.14" 334 | 335 | - toxenv: "python3.9" 336 | db: "mysql:5.7" 337 | legacy_db: 0 338 | experimental: true 339 | py: "3.9" 340 | 341 | - toxenv: "python3.10" 342 | db: "mysql:5.7" 343 | legacy_db: 0 344 | experimental: true 345 | py: "3.10" 346 | 347 | - toxenv: "python3.11" 348 | db: "mysql:5.7" 349 | legacy_db: 0 350 | experimental: true 351 | py: "3.11" 352 | 353 | - toxenv: "python3.12" 354 | db: "mysql:5.7" 355 | legacy_db: 0 356 | experimental: true 357 | py: "3.12" 358 | 359 | - toxenv: "python3.13" 360 | db: "mysql:5.7" 361 | legacy_db: 0 362 | experimental: true 363 | py: "3.13" 364 | 365 | - toxenv: "python3.14" 366 | db: "mysql:5.7" 367 | legacy_db: 0 368 | experimental: true 369 | py: "3.14" 370 | 371 | - toxenv: "python3.9" 372 | db: "mysql:8.0" 373 | legacy_db: 0 374 | experimental: false 375 | py: "3.9" 376 | 377 | - toxenv: "python3.10" 378 | db: "mysql:8.0" 379 | legacy_db: 0 380 | experimental: false 381 | py: "3.10" 382 | 383 | - toxenv: "python3.11" 384 | db: "mysql:8.0" 385 | legacy_db: 0 386 | experimental: false 387 | py: "3.11" 388 | 389 | - toxenv: "python3.12" 390 | db: "mysql:8.0" 391 | legacy_db: 0 392 | experimental: false 393 | py: "3.12" 394 | - toxenv: "python3.13" 395 | db: "mysql:8.0" 396 | legacy_db: 0 397 | experimental: false 398 | py: "3.13" 399 | 400 | - toxenv: "python3.14" 401 | db: "mysql:8.0" 402 | legacy_db: 0 403 | experimental: false 404 | py: "3.14" 405 | 406 | - toxenv: "python3.9" 407 | db: "mysql:8.4" 408 | legacy_db: 0 409 | experimental: true 410 | py: "3.9" 411 | 412 | - toxenv: "python3.10" 413 | db: "mysql:8.4" 414 | legacy_db: 0 415 | experimental: true 416 | py: "3.10" 417 | 418 | - toxenv: "python3.11" 419 | db: "mysql:8.4" 420 | legacy_db: 0 421 | experimental: true 422 | py: "3.11" 423 | 424 | - toxenv: "python3.12" 425 | db: "mysql:8.4" 426 | legacy_db: 0 427 | experimental: true 428 | py: "3.12" 429 | 430 | - toxenv: "python3.13" 431 | db: "mysql:8.4" 432 | legacy_db: 0 433 | experimental: true 434 | py: "3.13" 435 | 436 | - toxenv: "python3.14" 437 | db: "mysql:8.4" 438 | legacy_db: 0 439 | experimental: true 440 | py: "3.14" 441 | continue-on-error: ${{ matrix.experimental }} 442 | services: 443 | mysql: 444 | image: ${{ matrix.db }} 445 | ports: 446 | - 3306:3306 447 | env: 448 | MYSQL_ALLOW_EMPTY_PASSWORD: yes 449 | MARIADB_ALLOW_EMPTY_ROOT_PASSWORD: yes 450 | options: >- 451 | --name=mysqld 452 | --health-start-period=60s 453 | --health-cmd="command -v healthcheck.sh >/dev/null 2>&1 && healthcheck.sh --connect --innodb_initialized || mysqladmin ping -h 127.0.0.1 --silent" 454 | --health-interval=10s 455 | --health-timeout=5s 456 | --health-retries=30 457 | steps: 458 | - uses: actions/checkout@v6 459 | - name: Set up Python ${{ matrix.py }} 460 | uses: actions/setup-python@v6 461 | with: 462 | python-version: ${{ matrix.py }} 463 | cache: "pip" 464 | cache-dependency-path: requirements_dev.txt 465 | - name: Install dependencies 466 | run: | 467 | python -m pip install --upgrade pip 468 | python -m pip install -U tox-gh-actions 469 | pip install -r requirements_dev.txt 470 | - name: Install MySQL client 471 | run: | 472 | set -e 473 | sudo apt-get update 474 | sudo apt-get install -y mysql-client 475 | - name: Set up MySQL 476 | env: 477 | DB: ${{ matrix.db }} 478 | MYSQL_USER: tester 479 | MYSQL_PASSWORD: testpass 480 | MYSQL_DATABASE: test_db 481 | MYSQL_HOST: 0.0.0.0 482 | MYSQL_PORT: 3306 483 | run: | 484 | set -e 485 | 486 | case "$DB" in 487 | 'mysql:8.0'|'mysql:8.4') 488 | mysql -h127.0.0.1 -uroot -e "SET GLOBAL local_infile=on" 489 | docker cp mysqld:/var/lib/mysql/public_key.pem "${HOME}" 490 | docker cp mysqld:/var/lib/mysql/ca.pem "${HOME}" 491 | docker cp mysqld:/var/lib/mysql/server-cert.pem "${HOME}" 492 | docker cp mysqld:/var/lib/mysql/client-key.pem "${HOME}" 493 | docker cp mysqld:/var/lib/mysql/client-cert.pem "${HOME}" 494 | ;; 495 | esac 496 | 497 | USER_CREATION_COMMANDS='' 498 | WITH_PLUGIN='' 499 | 500 | if [ "$DB" == 'mysql:8.0' ]; then 501 | WITH_PLUGIN='with mysql_native_password' 502 | USER_CREATION_COMMANDS=' 503 | CREATE USER 504 | user_sha256 IDENTIFIED WITH "sha256_password" BY "pass_sha256", 505 | nopass_sha256 IDENTIFIED WITH "sha256_password", 506 | user_caching_sha2 IDENTIFIED WITH "caching_sha2_password" BY "pass_caching_sha2", 507 | nopass_caching_sha2 IDENTIFIED WITH "caching_sha2_password" 508 | PASSWORD EXPIRE NEVER; 509 | GRANT RELOAD ON *.* TO user_caching_sha2;' 510 | elif [ "$DB" == 'mysql:8.4' ]; then 511 | WITH_PLUGIN='with caching_sha2_password' 512 | USER_CREATION_COMMANDS=' 513 | CREATE USER 514 | user_caching_sha2 IDENTIFIED WITH "caching_sha2_password" BY "pass_caching_sha2", 515 | nopass_caching_sha2 IDENTIFIED WITH "caching_sha2_password" 516 | PASSWORD EXPIRE NEVER; 517 | GRANT RELOAD ON *.* TO user_caching_sha2;' 518 | fi 519 | 520 | if [ ! -z "$USER_CREATION_COMMANDS" ]; then 521 | mysql -uroot -h127.0.0.1 -e "$USER_CREATION_COMMANDS" 522 | fi 523 | 524 | mysql -h127.0.0.1 -uroot -e "create database $MYSQL_DATABASE DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci" 525 | mysql -h127.0.0.1 -uroot -e "create user $MYSQL_USER identified $WITH_PLUGIN by '${MYSQL_PASSWORD}'; grant all on ${MYSQL_DATABASE}.* to ${MYSQL_USER};" 526 | mysql -h127.0.0.1 -uroot -e "create user ${MYSQL_USER}@localhost identified $WITH_PLUGIN by '${MYSQL_PASSWORD}'; grant all on ${MYSQL_DATABASE}.* to ${MYSQL_USER}@localhost;" 527 | - name: Create db_credentials.json 528 | env: 529 | MYSQL_USER: tester 530 | MYSQL_PASSWORD: testpass 531 | MYSQL_DATABASE: test_db 532 | MYSQL_HOST: 0.0.0.0 533 | MYSQL_PORT: 3306 534 | run: | 535 | set -e 536 | jq -n \ 537 | --arg mysql_user "$MYSQL_USER" \ 538 | --arg mysql_password "$MYSQL_PASSWORD" \ 539 | --arg mysql_database "$MYSQL_DATABASE" \ 540 | --arg mysql_host "$MYSQL_HOST" \ 541 | --arg mysql_port $MYSQL_PORT \ 542 | '$ARGS.named' > tests/db_credentials.json 543 | - name: Test with tox 544 | env: 545 | LEGACY_DB: ${{ matrix.legacy_db }} 546 | TOXENV: ${{ matrix.toxenv }} 547 | run: tox 548 | - name: Upload coverage to Codecov 549 | env: 550 | OS: ubuntu-latest 551 | PYTHON: ${{ matrix.py }} 552 | uses: codecov/codecov-action@v5 553 | continue-on-error: true 554 | with: 555 | token: ${{ secrets.CODECOV_TOKEN }} 556 | slug: techouse/sqlite3-to-mysql 557 | files: ./coverage.xml 558 | env_vars: OS,PYTHON 559 | verbose: true 560 | - name: Cleanup 561 | if: ${{ always() }} 562 | run: | 563 | rm -rf tests/db_credentials.json 564 | -------------------------------------------------------------------------------- /tests/func/sqlite3_to_mysql_test.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import re 3 | import typing as t 4 | from collections import namedtuple 5 | from itertools import chain 6 | from pathlib import Path 7 | from random import choice, sample 8 | 9 | import mysql.connector 10 | import pytest 11 | import simplejson as json 12 | from _pytest._py.path import LocalPath 13 | from _pytest.capture import CaptureFixture 14 | from _pytest.logging import LogCaptureFixture 15 | from faker import Faker 16 | from mysql.connector import MySQLConnection, errorcode 17 | from mysql.connector.connection_cext import CMySQLConnection 18 | from mysql.connector.pooling import PooledMySQLConnection 19 | from pytest_mock import MockFixture 20 | from sqlalchemy import MetaData, Table, create_engine, inspect, select, text 21 | from sqlalchemy.engine import Connection, CursorResult, Engine, Inspector, Row 22 | from sqlalchemy.engine.interfaces import ReflectedIndex 23 | from sqlalchemy.sql import Select 24 | from sqlalchemy.sql.elements import TextClause 25 | 26 | from sqlite3_to_mysql import SQLite3toMySQL 27 | from tests.conftest import Helpers, MySQLCredentials 28 | 29 | 30 | @pytest.mark.usefixtures("sqlite_database", "mysql_instance") 31 | class TestSQLite3toMySQL: 32 | @pytest.mark.init 33 | @pytest.mark.parametrize("quiet", [False, True]) 34 | def test_no_sqlite_file_raises_exception(self, quiet: bool) -> None: 35 | with pytest.raises(ValueError) as excinfo: 36 | SQLite3toMySQL(quiet=quiet) # type: ignore 37 | assert "Please provide an SQLite file" in str(excinfo.value) 38 | 39 | @pytest.mark.init 40 | @pytest.mark.parametrize("quiet", [False, True]) 41 | def test_invalid_sqlite_file_raises_exception(self, faker: Faker, quiet: bool) -> None: 42 | with pytest.raises((FileNotFoundError, IOError)) as excinfo: 43 | SQLite3toMySQL(sqlite_file=faker.file_path(depth=1, extension=".sqlite3"), quiet=quiet) # type: ignore[call-arg] 44 | assert "SQLite file does not exist" in str(excinfo.value) 45 | 46 | @pytest.mark.init 47 | @pytest.mark.parametrize("quiet", [False, True]) 48 | def test_missing_mysql_user_raises_exception(self, sqlite_database: str, quiet: bool) -> None: 49 | with pytest.raises(ValueError) as excinfo: 50 | SQLite3toMySQL(sqlite_file=sqlite_database, quiet=quiet) # type: ignore[call-arg] 51 | assert "Please provide a MySQL user" in str(excinfo.value) 52 | 53 | @pytest.mark.init 54 | @pytest.mark.parametrize("quiet", [False, True]) 55 | def test_valid_sqlite_file_and_valid_mysql_credentials( 56 | self, 57 | sqlite_database: str, 58 | mysql_database: Engine, 59 | mysql_credentials: MySQLCredentials, 60 | helpers: Helpers, 61 | quiet: bool, 62 | ) -> None: 63 | with helpers.not_raises(FileNotFoundError): 64 | SQLite3toMySQL( # type: ignore 65 | sqlite_file=sqlite_database, 66 | mysql_user=mysql_credentials.user, 67 | mysql_password=mysql_credentials.password, 68 | mysql_host=mysql_credentials.host, 69 | mysql_port=mysql_credentials.port, 70 | mysql_database=mysql_credentials.database, 71 | chunk=10, 72 | quiet=quiet, 73 | ) 74 | 75 | @pytest.mark.init 76 | @pytest.mark.xfail 77 | @pytest.mark.parametrize("quiet", [False, True]) 78 | def test_valid_sqlite_file_and_invalid_mysql_credentials_raises_access_denied_exception( 79 | self, 80 | sqlite_database: str, 81 | mysql_database: Engine, 82 | mysql_credentials: MySQLCredentials, 83 | faker: Faker, 84 | quiet: bool, 85 | ) -> None: 86 | with pytest.raises(mysql.connector.Error) as excinfo: 87 | SQLite3toMySQL( # type: ignore[call-arg] 88 | sqlite_file=sqlite_database, 89 | mysql_user=faker.first_name().lower(), 90 | mysql_password=faker.password(length=16), 91 | mysql_host=mysql_credentials.host, 92 | mysql_port=mysql_credentials.port, 93 | mysql_database=mysql_credentials.database, 94 | quiet=quiet, 95 | ) 96 | assert "Access denied for user" in str(excinfo.value) 97 | 98 | @pytest.mark.init 99 | @pytest.mark.xfail 100 | @pytest.mark.parametrize("quiet", [False, True]) 101 | def test_unspecified_mysql_error( 102 | self, 103 | sqlite_database: str, 104 | mysql_credentials: MySQLCredentials, 105 | mocker: MockFixture, 106 | caplog: LogCaptureFixture, 107 | quiet: bool, 108 | ) -> None: 109 | mocker.patch.object( 110 | mysql.connector, 111 | "connect", 112 | side_effect=mysql.connector.Error( 113 | msg="Error Code: 2000. Unknown MySQL error", 114 | errno=errorcode.CR_UNKNOWN_ERROR, 115 | ), 116 | ) 117 | caplog.set_level(logging.DEBUG) 118 | with pytest.raises(mysql.connector.Error) as excinfo: 119 | SQLite3toMySQL( # type: ignore[call-arg] 120 | sqlite_file=sqlite_database, 121 | mysql_user=mysql_credentials.user, 122 | mysql_password=mysql_credentials.password, 123 | mysql_host=mysql_credentials.host, 124 | mysql_port=mysql_credentials.port, 125 | mysql_database=mysql_credentials.database, 126 | chunk=10, 127 | quiet=quiet, 128 | ) 129 | assert str(errorcode.CR_UNKNOWN_ERROR) in str(excinfo.value) 130 | assert any(str(errorcode.CR_UNKNOWN_ERROR) in message for message in caplog.messages) 131 | 132 | @pytest.mark.init 133 | @pytest.mark.parametrize("quiet", [False, True]) 134 | def test_bad_database_error( 135 | self, 136 | sqlite_database: str, 137 | mysql_credentials: MySQLCredentials, 138 | mocker: MockFixture, 139 | caplog: LogCaptureFixture, 140 | quiet: bool, 141 | ) -> None: 142 | class FakeMySQLConnection(MySQLConnection): 143 | @property 144 | def database(self): 145 | return self._database 146 | 147 | @database.setter 148 | def database(self, value): 149 | self._database = value 150 | # raise a fake exception 151 | raise mysql.connector.Error(msg="This is a test", errno=errorcode.ER_UNKNOWN_ERROR) 152 | 153 | def is_connected(self): 154 | return True 155 | 156 | def cursor( 157 | self, 158 | buffered=None, 159 | raw=None, 160 | prepared=None, 161 | cursor_class=None, 162 | dictionary=None, 163 | named_tuple=None, 164 | ): 165 | return True 166 | 167 | mocker.patch.object(mysql.connector, "connect", return_value=FakeMySQLConnection()) 168 | with pytest.raises(mysql.connector.Error): 169 | caplog.set_level(logging.DEBUG) 170 | SQLite3toMySQL( # type: ignore[call-arg] 171 | sqlite_file=sqlite_database, 172 | mysql_user=mysql_credentials.user, 173 | mysql_password=mysql_credentials.password, 174 | mysql_host=mysql_credentials.host, 175 | mysql_port=mysql_credentials.port, 176 | mysql_database=mysql_credentials.database, 177 | chunk=10, 178 | quiet=quiet, 179 | ) 180 | 181 | @pytest.mark.init 182 | @pytest.mark.xfail 183 | @pytest.mark.parametrize("quiet", [False, True]) 184 | def test_bad_mysql_connection( 185 | self, sqlite_database: str, mysql_credentials: MySQLCredentials, mocker: MockFixture, quiet: bool 186 | ) -> None: 187 | FakeConnector = namedtuple("FakeConnector", ["is_connected"]) 188 | mocker.patch.object( 189 | mysql.connector, 190 | "connect", 191 | return_value=FakeConnector(is_connected=lambda: False), 192 | ) 193 | with pytest.raises((ConnectionError, IOError)) as excinfo: 194 | SQLite3toMySQL( # type: ignore[call-arg] 195 | sqlite_file=sqlite_database, 196 | mysql_user=mysql_credentials.user, 197 | mysql_password=mysql_credentials.password, 198 | mysql_host=mysql_credentials.host, 199 | mysql_port=mysql_credentials.port, 200 | mysql_database=mysql_credentials.database, 201 | chunk=10, 202 | quiet=quiet, 203 | ) 204 | assert "Unable to connect to MySQL" in str(excinfo.value) 205 | 206 | @pytest.mark.init 207 | @pytest.mark.parametrize("quiet", [False, True]) 208 | def test_mysql_skip_create_tables_and_transfer_data( 209 | self, 210 | sqlite_database: str, 211 | mysql_credentials: MySQLCredentials, 212 | mocker: MockFixture, 213 | quiet: bool, 214 | ) -> None: 215 | mocker.patch.object( 216 | SQLite3toMySQL, 217 | "transfer", 218 | return_value=None, 219 | ) 220 | with pytest.raises(ValueError) as excinfo: 221 | SQLite3toMySQL( # type: ignore[call-arg] 222 | sqlite_file=sqlite_database, 223 | mysql_user=mysql_credentials.user, 224 | mysql_password=mysql_credentials.password, 225 | mysql_host=mysql_credentials.host, 226 | mysql_port=mysql_credentials.port, 227 | mysql_database=mysql_credentials.database, 228 | mysql_create_tables=False, 229 | mysql_transfer_data=False, 230 | quiet=quiet, 231 | ) 232 | assert "Unable to continue without transferring data or creating tables!" in str(excinfo.value) 233 | 234 | @pytest.mark.xfail 235 | @pytest.mark.init 236 | @pytest.mark.parametrize("quiet", [False, True]) 237 | def test_log_to_file( 238 | self, 239 | sqlite_database: str, 240 | mysql_database: Engine, 241 | mysql_credentials: MySQLCredentials, 242 | faker: Faker, 243 | caplog: LogCaptureFixture, 244 | tmpdir: LocalPath, 245 | quiet: bool, 246 | ): 247 | log_file: LocalPath = tmpdir.join(Path("db.log")) 248 | with pytest.raises(mysql.connector.Error): 249 | caplog.set_level(logging.DEBUG) 250 | SQLite3toMySQL( # type: ignore[call-arg] 251 | sqlite_file=sqlite_database, 252 | mysql_user=faker.first_name().lower(), 253 | mysql_password=faker.password(length=16), 254 | mysql_host=mysql_credentials.host, 255 | mysql_port=mysql_credentials.port, 256 | mysql_database=mysql_credentials.database, 257 | log_file=str(log_file), 258 | quiet=quiet, 259 | ) 260 | assert any("Access denied for user" in message for message in caplog.messages) 261 | with log_file.open("r") as log_fh: 262 | log: str = log_fh.read() 263 | if len(caplog.messages) > 1: 264 | assert caplog.messages[1] in log 265 | else: 266 | assert caplog.messages[0] in log 267 | assert re.match(r"^\d{4,}-\d{2,}-\d{2,}\s+\d{2,}:\d{2,}:\d{2,}\s+\w+\s+", log) is not None 268 | 269 | @pytest.mark.transfer 270 | @pytest.mark.parametrize( 271 | "chunk, with_rowid, mysql_insert_method, ignore_duplicate_keys", 272 | [ 273 | (None, False, "IGNORE", False), 274 | (None, False, "IGNORE", True), 275 | (None, False, "UPDATE", True), 276 | (None, False, "UPDATE", False), 277 | (None, False, "DEFAULT", True), 278 | (None, False, "DEFAULT", False), 279 | (None, True, "IGNORE", False), 280 | (None, True, "IGNORE", True), 281 | (None, True, "UPDATE", True), 282 | (None, True, "UPDATE", False), 283 | (None, True, "DEFAULT", True), 284 | (None, True, "DEFAULT", False), 285 | (10, False, "IGNORE", False), 286 | (10, False, "IGNORE", True), 287 | (10, False, "UPDATE", True), 288 | (10, False, "UPDATE", False), 289 | (10, False, "DEFAULT", True), 290 | (10, False, "DEFAULT", False), 291 | (10, True, "IGNORE", False), 292 | (10, True, "IGNORE", True), 293 | (10, True, "UPDATE", True), 294 | (10, True, "UPDATE", False), 295 | (10, True, "DEFAULT", True), 296 | (10, True, "DEFAULT", False), 297 | ], 298 | ) 299 | def test_transfer_transfers_all_tables_in_sqlite_file( 300 | self, 301 | sqlite_database: str, 302 | mysql_database: Engine, 303 | mysql_credentials: MySQLCredentials, 304 | helpers: Helpers, 305 | capsys: CaptureFixture, 306 | caplog: LogCaptureFixture, 307 | chunk: t.Optional[int], 308 | with_rowid: bool, 309 | mysql_insert_method: str, 310 | ignore_duplicate_keys: bool, 311 | ): 312 | proc: SQLite3toMySQL = SQLite3toMySQL( # type: ignore[call-arg] 313 | sqlite_file=sqlite_database, 314 | mysql_user=mysql_credentials.user, 315 | mysql_password=mysql_credentials.password, 316 | mysql_host=mysql_credentials.host, 317 | mysql_port=mysql_credentials.port, 318 | mysql_database=mysql_credentials.database, 319 | chunk=chunk, 320 | with_rowid=with_rowid, 321 | mysql_insert_method=mysql_insert_method, 322 | ignore_duplicate_keys=ignore_duplicate_keys, 323 | ) 324 | caplog.set_level(logging.DEBUG) 325 | proc.transfer() 326 | assert all(record.levelname == "INFO" for record in caplog.records) 327 | assert not any(record.levelname == "ERROR" for record in caplog.records) 328 | out, err = capsys.readouterr() 329 | 330 | sqlite_engine: Engine = create_engine( 331 | f"sqlite:///{sqlite_database}", 332 | json_serializer=json.dumps, 333 | json_deserializer=json.loads, 334 | ) 335 | sqlite_cnx: Connection = sqlite_engine.connect() 336 | sqlite_inspect: Inspector = inspect(sqlite_engine) 337 | sqlite_tables: t.List[str] = sqlite_inspect.get_table_names() 338 | mysql_engine: Engine = create_engine( 339 | f"mysql+pymysql://{mysql_credentials.user}:{mysql_credentials.password}@{mysql_credentials.host}:{mysql_credentials.port}/{mysql_credentials.database}", 340 | json_serializer=json.dumps, 341 | json_deserializer=json.loads, 342 | ) 343 | mysql_cnx: Connection = mysql_engine.connect() 344 | mysql_inspect: Inspector = inspect(mysql_engine) 345 | mysql_tables: t.List[str] = mysql_inspect.get_table_names() 346 | 347 | mysql_connector_connection: t.Union[PooledMySQLConnection, MySQLConnection, CMySQLConnection] = ( 348 | mysql.connector.connect( 349 | user=mysql_credentials.user, 350 | password=mysql_credentials.password, 351 | host=mysql_credentials.host, 352 | port=mysql_credentials.port, 353 | database=mysql_credentials.database, 354 | charset="utf8mb4", 355 | collation="utf8mb4_unicode_ci", 356 | ) 357 | ) 358 | server_version: t.Tuple[int, ...] = mysql_connector_connection.get_server_version() 359 | 360 | """ Test if both databases have the same table names """ 361 | assert sqlite_tables == mysql_tables 362 | 363 | """ Test if all the tables have the same column names """ 364 | for table_name in sqlite_tables: 365 | column_names: t.List[str] = [column["name"] for column in sqlite_inspect.get_columns(table_name)] 366 | if with_rowid: 367 | column_names.insert(0, "rowid") 368 | assert column_names == [column["name"] for column in mysql_inspect.get_columns(table_name)] 369 | 370 | """ Test if all the tables have the same indices """ 371 | index_keys: t.Tuple[str, ...] = ("name", "column_names", "unique") 372 | mysql_indices: t.Tuple[ReflectedIndex, ...] = tuple( 373 | t.cast(ReflectedIndex, {key: index[key] for key in index_keys}) # type: ignore[literal-required] 374 | for index in (chain.from_iterable(mysql_inspect.get_indexes(table_name) for table_name in mysql_tables)) 375 | ) 376 | 377 | for table_name in sqlite_tables: 378 | sqlite_indices: t.List[ReflectedIndex] = sqlite_inspect.get_indexes(table_name) 379 | if with_rowid: 380 | sqlite_indices.insert( 381 | 0, 382 | ReflectedIndex( 383 | name=f"{table_name}_rowid", 384 | column_names=["rowid"], 385 | unique=True, 386 | ), 387 | ) 388 | for sqlite_index in sqlite_indices: 389 | sqlite_index["unique"] = bool(sqlite_index["unique"]) 390 | if "dialect_options" in sqlite_index: 391 | sqlite_index.pop("dialect_options", None) 392 | assert sqlite_index in mysql_indices 393 | 394 | """ Test if all the tables have the same foreign keys """ 395 | for table_name in sqlite_tables: 396 | mysql_fk_stmt: TextClause = text( 397 | """ 398 | SELECT k.REFERENCED_TABLE_NAME AS `table`, k.COLUMN_NAME AS `from`, k.REFERENCED_COLUMN_NAME AS `to` 399 | FROM information_schema.TABLE_CONSTRAINTS AS i 400 | {JOIN} information_schema.KEY_COLUMN_USAGE AS k ON i.CONSTRAINT_NAME = k.CONSTRAINT_NAME 401 | WHERE i.TABLE_SCHEMA = :table_schema 402 | AND i.TABLE_NAME = :table_name 403 | AND i.CONSTRAINT_TYPE = :constraint_type 404 | """.format( 405 | # MySQL 8.0.19 still works with "LEFT JOIN" everything above requires "JOIN" 406 | JOIN="JOIN" if (server_version[0] == 8 and server_version[2] > 19) else "LEFT JOIN" 407 | ) 408 | ).bindparams( 409 | table_schema=mysql_credentials.database, 410 | table_name=table_name, 411 | constraint_type="FOREIGN KEY", 412 | ) 413 | mysql_fk_result: CursorResult = mysql_cnx.execute(mysql_fk_stmt) 414 | mysql_foreign_keys: t.List[t.Dict[str, t.Any]] = [ 415 | { 416 | "table": fk["table"], 417 | "from": fk["from"], 418 | "to": fk["to"], 419 | } 420 | for fk in mysql_fk_result.mappings() 421 | ] 422 | 423 | sqlite_fk_stmt: TextClause = text(f'PRAGMA foreign_key_list("{table_name}")') 424 | sqlite_fk_result = sqlite_cnx.execute(sqlite_fk_stmt) 425 | if sqlite_fk_result.returns_rows: 426 | for fk in sqlite_fk_result.mappings(): 427 | assert { 428 | "table": fk["table"], 429 | "from": fk["from"], 430 | "to": fk["to"], 431 | } in mysql_foreign_keys 432 | 433 | """ Check if all the data was transferred correctly """ 434 | sqlite_results: t.List[t.Tuple[t.Tuple[t.Any, ...], ...]] = [] 435 | mysql_results: t.List[t.Tuple[t.Tuple[t.Any, ...], ...]] = [] 436 | 437 | meta: MetaData = MetaData() 438 | for table_name in sqlite_tables: 439 | sqlite_table: Table = Table(table_name, meta, autoload_with=sqlite_engine) 440 | sqlite_stmt: Select = select(sqlite_table) 441 | sqlite_result: t.List[Row[t.Any]] = list(sqlite_cnx.execute(sqlite_stmt).fetchall()) 442 | sqlite_result.sort() 443 | sqlite_results.append(tuple(tuple(data for data in row) for row in sqlite_result)) 444 | 445 | for table_name in mysql_tables: 446 | mysql_table: Table = Table(table_name, meta, autoload_with=mysql_engine) 447 | mysql_stmt: Select = select(mysql_table) 448 | mysql_result: t.List[Row[t.Any]] = list(mysql_cnx.execute(mysql_stmt).fetchall()) 449 | mysql_result.sort() 450 | mysql_results.append(tuple(tuple(data for data in row) for row in mysql_result)) 451 | 452 | assert sqlite_results == mysql_results 453 | 454 | mysql_cnx.close() 455 | sqlite_cnx.close() 456 | mysql_engine.dispose() 457 | sqlite_engine.dispose() 458 | 459 | @pytest.mark.transfer 460 | @pytest.mark.parametrize( 461 | "chunk, with_rowid, mysql_insert_method, ignore_duplicate_keys, exclude_tables", 462 | [ 463 | (None, False, "IGNORE", False, False), 464 | (None, False, "IGNORE", False, True), 465 | (None, False, "IGNORE", True, False), 466 | (None, False, "IGNORE", True, True), 467 | (None, False, "UPDATE", True, False), 468 | (None, False, "UPDATE", True, True), 469 | (None, False, "UPDATE", False, False), 470 | (None, False, "UPDATE", False, True), 471 | (None, False, "DEFAULT", True, False), 472 | (None, False, "DEFAULT", True, True), 473 | (None, False, "DEFAULT", False, False), 474 | (None, False, "DEFAULT", False, True), 475 | (None, True, "IGNORE", False, False), 476 | (None, True, "IGNORE", False, True), 477 | (None, True, "IGNORE", True, False), 478 | (None, True, "IGNORE", True, True), 479 | (None, True, "UPDATE", True, False), 480 | (None, True, "UPDATE", True, True), 481 | (None, True, "UPDATE", False, False), 482 | (None, True, "UPDATE", False, True), 483 | (None, True, "DEFAULT", True, False), 484 | (None, True, "DEFAULT", True, True), 485 | (None, True, "DEFAULT", False, False), 486 | (None, True, "DEFAULT", False, True), 487 | (10, False, "IGNORE", False, False), 488 | (10, False, "IGNORE", False, True), 489 | (10, False, "IGNORE", True, False), 490 | (10, False, "IGNORE", True, True), 491 | (10, False, "UPDATE", True, False), 492 | (10, False, "UPDATE", True, True), 493 | (10, False, "UPDATE", False, False), 494 | (10, False, "UPDATE", False, True), 495 | (10, False, "DEFAULT", True, False), 496 | (10, False, "DEFAULT", True, True), 497 | (10, False, "DEFAULT", False, False), 498 | (10, False, "DEFAULT", False, True), 499 | (10, True, "IGNORE", False, False), 500 | (10, True, "IGNORE", False, True), 501 | (10, True, "IGNORE", True, False), 502 | (10, True, "IGNORE", True, True), 503 | (10, True, "UPDATE", True, False), 504 | (10, True, "UPDATE", True, True), 505 | (10, True, "UPDATE", False, False), 506 | (10, True, "UPDATE", False, True), 507 | (10, True, "DEFAULT", True, False), 508 | (10, True, "DEFAULT", True, True), 509 | (10, True, "DEFAULT", False, False), 510 | (10, True, "DEFAULT", False, True), 511 | ], 512 | ) 513 | def test_transfer_specific_tables_transfers_only_specified_tables_from_sqlite_file( 514 | self, 515 | sqlite_database: str, 516 | mysql_database: Engine, 517 | mysql_credentials: MySQLCredentials, 518 | helpers: Helpers, 519 | capsys: CaptureFixture, 520 | caplog: LogCaptureFixture, 521 | chunk: t.Optional[int], 522 | with_rowid: bool, 523 | mysql_insert_method: str, 524 | ignore_duplicate_keys: bool, 525 | exclude_tables: bool, 526 | ) -> None: 527 | sqlite_engine: Engine = create_engine( 528 | f"sqlite:///{sqlite_database}", 529 | json_serializer=json.dumps, 530 | json_deserializer=json.loads, 531 | ) 532 | sqlite_cnx: Connection = sqlite_engine.connect() 533 | sqlite_inspect: Inspector = inspect(sqlite_engine) 534 | sqlite_tables: t.List[str] = sqlite_inspect.get_table_names() 535 | 536 | table_number: int = choice(range(1, len(sqlite_tables))) 537 | 538 | random_sqlite_tables: t.List[str] = sample(sqlite_tables, table_number) 539 | random_sqlite_tables.sort() 540 | 541 | remaining_tables: t.List[str] = list(set(sqlite_tables) - set(random_sqlite_tables)) 542 | remaining_tables.sort() 543 | 544 | proc: SQLite3toMySQL = SQLite3toMySQL( # type: ignore[call-arg] 545 | sqlite_file=sqlite_database, 546 | sqlite_tables=None if exclude_tables else random_sqlite_tables, 547 | exclude_sqlite_tables=random_sqlite_tables if exclude_tables else None, 548 | mysql_user=mysql_credentials.user, 549 | mysql_password=mysql_credentials.password, 550 | mysql_host=mysql_credentials.host, 551 | mysql_port=mysql_credentials.port, 552 | mysql_database=mysql_credentials.database, 553 | chunk=chunk, 554 | with_rowid=with_rowid, 555 | mysql_insert_method=mysql_insert_method, 556 | ignore_duplicate_keys=ignore_duplicate_keys, 557 | ) 558 | caplog.set_level(logging.DEBUG) 559 | proc.transfer() 560 | assert all( 561 | message in [record.message for record in caplog.records] 562 | for message in set( 563 | [ 564 | f"Transferring table {table}" 565 | for table in (remaining_tables if exclude_tables else random_sqlite_tables) 566 | ] 567 | + ["Done!"] 568 | ) 569 | ) 570 | assert all(record.levelname == "INFO" for record in caplog.records) 571 | assert not any(record.levelname == "ERROR" for record in caplog.records) 572 | out, err = capsys.readouterr() 573 | 574 | mysql_engine: Engine = create_engine( 575 | f"mysql+pymysql://{mysql_credentials.user}:{mysql_credentials.password}@{mysql_credentials.host}:{mysql_credentials.port}/{mysql_credentials.database}", 576 | json_serializer=json.dumps, 577 | json_deserializer=json.loads, 578 | ) 579 | mysql_cnx: Connection = mysql_engine.connect() 580 | mysql_inspect: Inspector = inspect(mysql_engine) 581 | mysql_tables: t.List[str] = mysql_inspect.get_table_names() 582 | 583 | """ Test if both databases have the same table names """ 584 | if exclude_tables: 585 | assert remaining_tables == mysql_tables 586 | else: 587 | assert random_sqlite_tables == mysql_tables 588 | 589 | """ Test if all the tables have the same column names """ 590 | for table_name in remaining_tables if exclude_tables else random_sqlite_tables: 591 | column_names: t.List[t.Any] = [column["name"] for column in sqlite_inspect.get_columns(table_name)] 592 | if with_rowid: 593 | column_names.insert(0, "rowid") 594 | assert column_names == [column["name"] for column in mysql_inspect.get_columns(table_name)] 595 | 596 | """ Test if all the tables have the same indices """ 597 | index_keys: t.Tuple[str, ...] = ("name", "column_names", "unique") 598 | mysql_indices: t.Tuple[ReflectedIndex, ...] = tuple( 599 | t.cast(ReflectedIndex, {key: index[key] for key in index_keys}) # type: ignore[literal-required] 600 | for index in (chain.from_iterable(mysql_inspect.get_indexes(table_name) for table_name in mysql_tables)) 601 | ) 602 | 603 | for table_name in remaining_tables if exclude_tables else random_sqlite_tables: 604 | sqlite_indices: t.List[ReflectedIndex] = sqlite_inspect.get_indexes(table_name) 605 | if with_rowid: 606 | sqlite_indices.insert( 607 | 0, 608 | ReflectedIndex( 609 | name=f"{table_name}_rowid", 610 | column_names=["rowid"], 611 | unique=True, 612 | ), 613 | ) 614 | for sqlite_index in sqlite_indices: 615 | sqlite_index["unique"] = bool(sqlite_index["unique"]) 616 | if "dialect_options" in sqlite_index: 617 | sqlite_index.pop("dialect_options", None) 618 | assert sqlite_index in mysql_indices 619 | 620 | """ Check if all the data was transferred correctly """ 621 | sqlite_results: t.List[t.Tuple[t.Tuple[t.Any, ...], ...]] = [] 622 | mysql_results: t.List[t.Tuple[t.Tuple[t.Any, ...], ...]] = [] 623 | 624 | meta: MetaData = MetaData() 625 | for table_name in remaining_tables if exclude_tables else random_sqlite_tables: 626 | sqlite_table: Table = Table(table_name, meta, autoload_with=sqlite_engine) 627 | sqlite_stmt: Select = select(sqlite_table) 628 | sqlite_result: t.List[Row[t.Any]] = list(sqlite_cnx.execute(sqlite_stmt).fetchall()) 629 | sqlite_result.sort() 630 | sqlite_results.append(tuple(tuple(data for data in row) for row in sqlite_result)) 631 | 632 | for table_name in mysql_tables: 633 | mysql_table: Table = Table(table_name, meta, autoload_with=mysql_engine) 634 | mysql_stmt: Select = select(mysql_table) 635 | mysql_result: t.List[Row[t.Any]] = list(mysql_cnx.execute(mysql_stmt).fetchall()) 636 | mysql_result.sort() 637 | mysql_results.append(tuple(tuple(data for data in row) for row in mysql_result)) 638 | 639 | assert sqlite_results == mysql_results 640 | 641 | mysql_cnx.close() 642 | sqlite_cnx.close() 643 | mysql_engine.dispose() 644 | sqlite_engine.dispose() 645 | --------------------------------------------------------------------------------