├── .env ├── .gitignore ├── Readme.md ├── mongo_connector.py ├── mysql_to_mongo.py ├── requirements.txt ├── sql_connector.py └── table_to_collection.py /.env: -------------------------------------------------------------------------------- 1 | ## after add mongo uri add '/', example - "www.mongodb.com/" 2 | MONGO_URI="your_mongo_db_url" 3 | SQL_HOST="your_sql_host" 4 | SQL_DATABASE="your_sql_db_name" 5 | MONGO_DATABASE="your_mongodb_name" 6 | SQL_USER="your_sql_user_name" 7 | SQL_PASSWORD="your_sql_password" -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | __pycache__ 2 | cpy-errors.log 3 | -------------------------------------------------------------------------------- /Readme.md: -------------------------------------------------------------------------------- 1 | # MySQL to MongoDB Data Converter 2 | 3 | This Python script allows you to convert data from a MySQL database to a MongoDB database. 4 | 5 | ## Getting Started 6 | 7 | 1. **Clone the project repository:** 8 | 9 | ```bash 10 | git clone https://github.com/Laviru-Dilshan/mysql-to-mongodb-python.git 11 | cd mysql-to-mongodb-python 12 | ``` 13 | 14 | 2. **Install the required Python packages:** 15 | 16 | ```bash 17 | pip install -r requirements.txt 18 | ``` 19 | 20 | 3. **Create a `.env` file in the project directory and add your environment variables:** 21 | 22 | ``` 23 | SQL_HOST=your_sql_host 24 | SQL_USER=your_sql_user 25 | SQL_PASSWORD=your_sql_password 26 | SQL_DATABASE=your_sql_database 27 | MONGO_URI=your_mongo_uri 28 | MONGO_DATABASE=your_mongo_database 29 | ``` 30 | 31 | (important: after add mongo uri add '/', example - "www.mongodb.com/") 32 | 33 | ## Convert MySQL Database To MongoDB 34 | 35 | Using This You Can Convert Your Full MYSQL Database To MongoDB 36 | 37 | 1. **Open `mysql_to_mongo.py` and add your sql config ,mongo uri and call functions:** 38 | 39 | ```python 40 | # Create MYSQL Config 41 | sql_config = { 42 | "host": os.getenv('SQL_HOST'), 43 | "user": os.getenv('SQL_USER'), 44 | "password": os.getenv('SQL_PASSWORD'), 45 | "database": os.getenv('SQL_DATABASE'), 46 | } 47 | 48 | #add mondo db uri 49 | mongo_uri = os.getenv('MONGO_URI') 50 | 51 | # call functions 52 | converter = MySQLToMongoConverter(sql_config, mongo_uri) 53 | converter.convert() 54 | ``` 55 | 56 | 2. **Run the script:** 57 | 58 | ```bash 59 | python mysql_to_mongo.py 60 | ``` 61 | 62 | 63 | ## Convert MySql Single Table To Single Mongo Collection 64 | 65 | It retrieves data from a specified table in MySQL and inserts it into a specified collection in MongoDB, based on a mapping of fields between the two databases. 66 | 67 | 68 | 1. **Open `table_to_collection.py` and add your data mapping and collection name:** 69 | 70 | ```python 71 | # Define your field mappings for each table 72 | province_mapping = { 73 | "id": 0, 74 | "name_en": 1, 75 | "name_si": 2, 76 | "name_ta": 3, 77 | } 78 | 79 | # Example usage 80 | converter = MySQLToMongoConverter( 81 | { 82 | "host": os.getenv('SQL_HOST'), 83 | "user": os.getenv('SQL_USER'), 84 | "password": os.getenv('SQL_PASSWORD'), 85 | "database": os.getenv('SQL_DATABASE'), 86 | }, 87 | os.getenv('MONGO_URI'), 88 | os.getenv('MONGO_DATABASE'), 89 | 'provinces', 90 | province_mapping 91 | ) 92 | converter.convert() 93 | ``` 94 | 95 | 2. **Run the script:** 96 | 97 | ```bash 98 | python table_to_collection.py 99 | ``` 100 | 101 | # COPYRIGHT 102 | 103 | All rights reserved by Laviru Dilshan Jr. 2024 104 | 105 | Connect with me: 106 | - GitHub: [https://github.com/Laviru-Dilshan](https://github.com/Laviru-Dilshan) 107 | - Twitter: [https://x.com/laviru_dilshan](https://x.com/laviru_dilshan) 108 | - LinkedIn: [https://www.linkedin.com/in/laviru-dilshan](https://www.linkedin.com/in/laviru-dilshan) 109 | - Facebook: [https://www.facebook.com/LaviruD](https://www.facebook.com/LaviruD) 110 | - Instagram: [https://www.instagram.com/lavirudilshan](https://www.instagram.com/lavirudilshan) 111 | -------------------------------------------------------------------------------- /mongo_connector.py: -------------------------------------------------------------------------------- 1 | """ 2 | All rights reserved by Laviru Dilshan Jr. 2024 3 | 4 | Connect with me: 5 | - GitHub: https://github.com/Laviru-Dilshan 6 | - Twitter: https://x.com/laviru_dilshan 7 | - LinkedIn: https://www.linkedin.com/in/laviru-dilshan 8 | - Facebook: https://www.facebook.com/LaviruD 9 | - Instagram: https://www.instagram.com/lavirudilshan 10 | 11 | """ 12 | 13 | from pymongo import MongoClient 14 | 15 | class MongoDBConnector: 16 | def __init__(self, uri, db_name, collection_name=None): 17 | self.client = MongoClient(uri) 18 | self.db = self.client[db_name] 19 | self.collection = self.db[collection_name] if collection_name else None 20 | 21 | def set_collection(self, collection_name): 22 | self.collection = self.db[collection_name] 23 | 24 | def insert_data(self, data): 25 | if self.collection is not None: 26 | result = self.collection.insert_one(data) 27 | return result.inserted_id 28 | else: 29 | raise ValueError("Collection is not set") 30 | 31 | def find_data(self, query): 32 | if self.collection is not None: 33 | return self.collection.find(query) 34 | else: 35 | raise ValueError("Collection is not set") 36 | 37 | def update_data(self, query, new_data): 38 | if self.collection is not None: 39 | result = self.collection.update_one(query, {'$set': new_data}) 40 | return result.modified_count 41 | else: 42 | raise ValueError("Collection is not set") 43 | 44 | def delete_data(self, query): 45 | if self.collection is not None: 46 | result = self.collection.delete_one(query) 47 | return result.deleted_count 48 | else: 49 | raise ValueError("Collection is not set") 50 | 51 | def close_connection(self): 52 | self.client.close() 53 | -------------------------------------------------------------------------------- /mysql_to_mongo.py: -------------------------------------------------------------------------------- 1 | """ 2 | All rights reserved by Laviru Dilshan Jr. 2024 3 | 4 | Connect with me: 5 | - GitHub: https://github.com/Laviru-Dilshan 6 | - Twitter: https://x.com/laviru_dilshan 7 | - LinkedIn: https://www.linkedin.com/in/laviru-dilshan 8 | - Facebook: https://www.facebook.com/LaviruD 9 | - Instagram: https://www.instagram.com/lavirudilshan 10 | 11 | """ 12 | 13 | from sql_connector import connect_to_mysql 14 | from mongo_connector import MongoDBConnector 15 | from dotenv import load_dotenv 16 | import os 17 | 18 | load_dotenv() 19 | 20 | class MySQLToMongoConverter: 21 | def __init__(self, sql_config, mongo_uri): 22 | self.sql_config = sql_config 23 | self.mongo_uri = mongo_uri 24 | self.mongo_db_name = self.sql_config['database'] 25 | self.mongo_connector = MongoDBConnector(self.mongo_uri, self.mongo_db_name) 26 | 27 | def connect_mysql(self): 28 | return connect_to_mysql(self.sql_config, attempts=3) 29 | 30 | def get_mysql_table_names(self, cnx): 31 | with cnx.cursor() as cursor: 32 | cursor.execute("SHOW TABLES") 33 | tables = [table[0] for table in cursor.fetchall()] 34 | return tables 35 | 36 | def get_table_columns(self, cnx, table_name): 37 | with cnx.cursor() as cursor: 38 | cursor.execute(f"DESCRIBE {table_name}") 39 | columns = [column[0] for column in cursor.fetchall()] 40 | return columns 41 | 42 | def convert_table_to_collection(self, cnx, table_name): 43 | columns = self.get_table_columns(cnx, table_name) 44 | field_mapping = {column: index for index, column in enumerate(columns)} 45 | 46 | with cnx.cursor() as cursor: 47 | cursor.execute(f"SELECT * FROM {table_name}") 48 | rows = cursor.fetchall() 49 | items = [] 50 | for row in rows: 51 | item = {} 52 | for field, index in field_mapping.items(): 53 | item[field] = row[index] 54 | items.append(item) 55 | 56 | self.mongo_connector.set_collection(table_name) 57 | 58 | for item_data in items: 59 | inserted_id = self.mongo_connector.insert_data(item_data) 60 | print(f"Data inserted into {table_name} with id: {inserted_id}") 61 | 62 | def convert(self): 63 | cnx = self.connect_mysql() 64 | 65 | if cnx and cnx.is_connected(): 66 | table_names = self.get_mysql_table_names(cnx) 67 | 68 | for table_name in table_names: 69 | self.convert_table_to_collection(cnx, table_name) 70 | 71 | cnx.close() 72 | self.mongo_connector.close_connection() 73 | else: 74 | print("Could not connect") 75 | 76 | # Example usage 77 | sql_config = { 78 | "host": os.getenv('SQL_HOST'), 79 | "user": os.getenv('SQL_USER'), 80 | "password": os.getenv('SQL_PASSWORD'), 81 | "database": os.getenv('SQL_DATABASE'), 82 | } 83 | 84 | mongo_uri = os.getenv('MONGO_URI') 85 | 86 | converter = MySQLToMongoConverter(sql_config, mongo_uri) 87 | converter.convert() 88 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | pymongo 2 | mysql-connector-python 3 | python-dotenv -------------------------------------------------------------------------------- /sql_connector.py: -------------------------------------------------------------------------------- 1 | """ 2 | All rights reserved by Laviru Dilshan Jr. 2024 3 | 4 | Connect with me: 5 | - GitHub: https://github.com/Laviru-Dilshan 6 | - Twitter: https://x.com/laviru_dilshan 7 | - LinkedIn: https://www.linkedin.com/in/laviru-dilshan 8 | - Facebook: https://www.facebook.com/LaviruD 9 | - Instagram: https://www.instagram.com/lavirudilshan 10 | 11 | """ 12 | 13 | import logging 14 | import time 15 | import mysql.connector 16 | 17 | # Set up logger 18 | logger = logging.getLogger(__name__) 19 | logger.setLevel(logging.INFO) 20 | formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s") 21 | 22 | # Log to console 23 | handler = logging.StreamHandler() 24 | handler.setFormatter(formatter) 25 | logger.addHandler(handler) 26 | 27 | # Also log to a file 28 | file_handler = logging.FileHandler("cpy-errors.log") 29 | file_handler.setFormatter(formatter) 30 | logger.addHandler(file_handler) 31 | 32 | def connect_to_mysql(config, attempts=3, delay=2): 33 | attempt = 1 34 | # Implement a reconnection routine 35 | while attempt < attempts + 1: 36 | try: 37 | return mysql.connector.connect(**config) 38 | except (mysql.connector.Error, IOError) as err: 39 | if (attempts is attempt): 40 | # Attempts to reconnect failed; returning None 41 | logger.info("Failed to connect, exiting without a connection: %s", err) 42 | return None 43 | logger.info( 44 | "Connection failed: %s. Retrying (%d/%d)...", 45 | err, 46 | attempt, 47 | attempts-1, 48 | ) 49 | # progressive reconnect delay 50 | time.sleep(delay ** attempt) 51 | attempt += 1 52 | return None -------------------------------------------------------------------------------- /table_to_collection.py: -------------------------------------------------------------------------------- 1 | """ 2 | All rights reserved by Laviru Dilshan Jr. 2024 3 | 4 | Connect with me: 5 | - GitHub: https://github.com/Laviru-Dilshan 6 | - Twitter: https://x.com/laviru_dilshan 7 | - LinkedIn: https://www.linkedin.com/in/laviru-dilshan 8 | - Facebook: https://www.facebook.com/LaviruD 9 | - Instagram: https://www.instagram.com/lavirudilshan 10 | 11 | """ 12 | 13 | from sql_connector import connect_to_mysql 14 | from mongo_connector import MongoDBConnector 15 | from dotenv import load_dotenv 16 | import os 17 | load_dotenv() 18 | 19 | class MySQLToMongoConverter: 20 | def __init__(self, sql_config, mongo_uri, mongo_db, collection_name, field_mapping): 21 | self.sql_config = sql_config 22 | self.mongo_connector = MongoDBConnector(mongo_uri, mongo_db, collection_name) 23 | self.field_mapping = field_mapping 24 | self.collection_name = collection_name # Add this line 25 | 26 | def convert(self): 27 | cnx = connect_to_mysql(self.sql_config, attempts=3) 28 | 29 | if cnx and cnx.is_connected(): 30 | with cnx.cursor() as cursor: 31 | result = cursor.execute(f"SELECT * FROM {self.collection_name}") 32 | rows = cursor.fetchall() 33 | items = [] 34 | for row in rows: 35 | item = {} 36 | for field, index in self.field_mapping.items(): 37 | item[field] = row[index] 38 | items.append(item) 39 | 40 | for item_data in items: 41 | inserted_id = self.mongo_connector.insert_data(item_data) 42 | print(f"Data inserted with id: {inserted_id}") 43 | 44 | self.mongo_connector.close_connection() 45 | cnx.close() 46 | else: 47 | print("Could not connect") 48 | 49 | # Define your field mappings for each table 50 | province_mapping = { 51 | "id": 0, 52 | "district_id": 1, 53 | "name_en": 2, 54 | "name_si": 3, 55 | "name_ta": 4, 56 | "sub_name_en": 5, 57 | "sub_name_si": 6, 58 | "sub_name_ta": 7, 59 | "postcode": 8, 60 | "latitude": 9, 61 | "longitude": 10 62 | } 63 | 64 | # Example usage 65 | converter = MySQLToMongoConverter( 66 | { 67 | "host": os.getenv('SQL_HOST'), 68 | "user": os.getenv('SQL_USER'), 69 | "password": os.getenv('SQL_PASSWORD'), 70 | "database": os.getenv('SQL_DATABASE'), 71 | }, 72 | os.getenv('MONGO_URI'), 73 | os.getenv('MONGO_DATABASE'), 74 | 'cities', 75 | province_mapping 76 | ) 77 | converter.convert() 78 | 79 | --------------------------------------------------------------------------------