├── .gitignore ├── .travis.yml ├── LICENSE ├── README.md ├── apps ├── __init__.py └── schema │ ├── __init__.py │ ├── admin.py │ ├── apps.py │ ├── filters.py │ ├── migrations │ ├── 0001_initial.py │ └── __init__.py │ ├── models.py │ ├── tables.py │ └── views.py ├── configs ├── newrelic.ini ├── nginx.conf ├── setup.sh ├── supervisor.conf └── uwsgi.ini ├── dbhub ├── __init__.py ├── settings │ ├── __init__.py │ ├── common.py │ └── dev.py ├── urls.py └── wsgi.py ├── manage.py ├── requirements.txt ├── runtests.sh ├── screenshoot.png ├── scripts ├── __init__.py ├── check.py ├── db.py ├── parser.py └── sync.py ├── templates └── columns.html ├── test-requirements.txt └── tests ├── __init__.py └── test_parser.py /.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | !static/.gitkeep 3 | static/ 4 | env/ 5 | db.sqlite3 6 | prod.py 7 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: python 2 | env: 3 | - DJANGO=1.11 4 | python: 5 | - "2.7" 6 | # command to install dependencies 7 | install: 8 | - pip install -r test-requirements.txt 9 | # command to run tests 10 | script: ./runtests.sh 11 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2019 会分期 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # DBHub 2 | 3 | DBHub is a free database schema automatic discovery and easily commenting tool. 4 | DBHub now can support mysql, mongodb and other 6 databases, it is easy to extend. 5 | DBHub has a web page for viewing all schema with comments and has an admin system for managing. 6 | 7 | [![Build Status](https://travis-ci.org/huifenqi/dbhub.png?branch=master)](https://travis-ci.org/huifenqi/dbhub) 8 | 9 | ![demo](./screenshoot.png) 10 | 11 | ## Structure: 12 | 13 | * apps (web and admin page) 14 | * scripts (sync and check) 15 | * configs (supervisor, newrelic and uwsgi) 16 | 17 | ## Installation 18 | 19 | To set up a development environment quickly, install Python 2.x first. It 20 | comes with virtualenv built-in. so create a virtual environment with: 21 | 22 | `virtualenv -p python2 env` 23 | 24 | Install dependencies: 25 | 26 | `pip install -r requirements.txt` 27 | 28 | ## Create default user 29 | 30 | `python manage.py createsuperuser` 31 | 32 | ## Run server 33 | 34 | `python manage.py runserver --settings=dbhub.settings.dev` 35 | 36 | ## Add database 37 | 38 | * name: database name 39 | * config: whole url for connect with database 40 | * for MySQL: mysql://{username}:{password}@{database-url}:3306/{database-name}?charset=utf8 41 | * for SQLite: sqlite:////{absolute-path-to-db-file} 42 | * for MongoDB: mongodb://{username}:{password}@{database-url}:27017/{database-name} 43 | 44 | ## Sync databases' schema and check columns' enumeration 45 | 46 | `python manage.py runscript sync` 47 | 48 | `python manage.py runscript check --script--args [db_name] [table_name] ` 49 | 50 | ## How to write comments with enumeration 51 | 52 | 1. write description first; 53 | 2. write enumerations below with {enum}: {description}. 54 | 55 | ``` 56 | charset with description, blah, blah, blah 57 | 58 | utf8: A UTF-8 encoding of the Unicode character set using one to three bytes per character. default utf8 of mysql, max length is 3 bytes, not support characters, such as emoji. 59 | 60 | utf8mb4: A UTF-8 encoding of the Unicode character set using one to four bytes per character. 61 | 62 | ``` 63 | 64 | ## Supported dialects 65 | 66 | * MySQL 67 | * MongoDB 68 | * PostgreSQL 69 | * Oracle 70 | * SQLite 71 | * Microsoft SQL Server 72 | * Firebird 73 | * Sybase 74 | -------------------------------------------------------------------------------- /apps/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/huifenqi/dbhub/825ce53f0b1c0e3c2e033ebe2c92bfcfbf51bd9e/apps/__init__.py -------------------------------------------------------------------------------- /apps/schema/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/huifenqi/dbhub/825ce53f0b1c0e3c2e033ebe2c92bfcfbf51bd9e/apps/schema/__init__.py -------------------------------------------------------------------------------- /apps/schema/admin.py: -------------------------------------------------------------------------------- 1 | from django.contrib import admin 2 | from django.forms import Textarea 3 | from django.db import models 4 | from reversion.admin import VersionAdmin 5 | 6 | from models import Database, Table, Column, Index 7 | 8 | 9 | class DatabaseAdmin(VersionAdmin): 10 | list_display = ('name', 'config', 'engine', 'charset', 'comment', 'enable') 11 | list_editable = ('config', 'enable') 12 | readonly_fields = ('engine', 'charset') 13 | 14 | 15 | class TableAdmin(VersionAdmin): 16 | list_display = ('name', 'database', 'engine', 'charset', 'comment', 'is_deleted') 17 | search_fields = ('name', 'comment') 18 | readonly_fields = ('name', 'database', 'engine', 'charset') 19 | list_filter = ('database',) 20 | list_editable = ('comment',) 21 | 22 | formfield_overrides = { 23 | models.TextField: {'widget': Textarea(attrs={'rows': 3, 'cols': 40})}, 24 | } 25 | 26 | 27 | class ColumnAdmin(VersionAdmin): 28 | list_display = ('name', 'table', 'data_type', 'is_null', 'default_value', 'comment', 'other_enums', 'is_enum', 29 | 'is_comment_dirty', 'is_deleted') 30 | search_fields = ('name', 'table__name', 'comment') 31 | readonly_fields = ('name', 'table', 'data_type', 'is_null', 'default_value', 'other_enums') 32 | list_filter = ('is_comment_dirty', 'table') 33 | list_editable = ('comment', 'is_enum', 'is_deleted', 'is_comment_dirty') 34 | 35 | formfield_overrides = { 36 | models.TextField: {'widget': Textarea(attrs={'rows': 3, 'cols': 40})}, 37 | } 38 | 39 | 40 | class IndexAdmin(VersionAdmin): 41 | list_display = ('name', 'table', 'type', 'include_columns') 42 | search_fields = ('name', 'table__name') 43 | readonly_fields = ('name', 'table', 'type', 'include_columns') 44 | 45 | 46 | admin.site.register(Database, DatabaseAdmin) 47 | admin.site.register(Table, TableAdmin) 48 | admin.site.register(Column, ColumnAdmin) 49 | admin.site.register(Index, IndexAdmin) 50 | -------------------------------------------------------------------------------- /apps/schema/apps.py: -------------------------------------------------------------------------------- 1 | from __future__ import unicode_literals 2 | 3 | from django.apps import AppConfig 4 | 5 | 6 | class SchemaConfig(AppConfig): 7 | name = 'schema' 8 | -------------------------------------------------------------------------------- /apps/schema/filters.py: -------------------------------------------------------------------------------- 1 | from django.db.models import Q 2 | from django.forms import TextInput 3 | from django_filters import FilterSet, CharFilter, ModelChoiceFilter 4 | 5 | from .models import Database, Table, Column 6 | 7 | 8 | class ColumnFilter(FilterSet): 9 | database = ModelChoiceFilter(queryset=Database.objects.filter(enable=True), method='database_filter', label='', 10 | empty_label='Choose a database') 11 | table = ModelChoiceFilter(queryset=Table.objects.all(), label='') 12 | word = CharFilter(method='word_filter', label='', 13 | widget=TextInput(attrs={'placeholder': 'Search a word'})) 14 | 15 | def database_filter(self, queryset, name, value): 16 | return queryset.filter(table__database=value) 17 | 18 | def word_filter(self, queryset, name, value): 19 | return queryset.filter( 20 | Q(name__icontains=value) | Q(comment__icontains=value) | Q(table__name__icontains=value) | Q( 21 | table__comment__icontains=value)) 22 | 23 | class Meta: 24 | model = Column 25 | fields = ['database', 'table', 'word'] 26 | -------------------------------------------------------------------------------- /apps/schema/migrations/0001_initial.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Generated by Django 1.11.14 on 2020-09-27 15:21 3 | from __future__ import unicode_literals 4 | 5 | from django.db import migrations, models 6 | import django.db.models.deletion 7 | 8 | 9 | class Migration(migrations.Migration): 10 | 11 | initial = True 12 | 13 | dependencies = [ 14 | ] 15 | 16 | operations = [ 17 | migrations.CreateModel( 18 | name='Column', 19 | fields=[ 20 | ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), 21 | ('name', models.CharField(help_text='\u5217\u540d', max_length=100)), 22 | ('data_type', models.CharField(blank=True, help_text='\u6570\u636e\u7c7b\u578b', max_length=100, null=True)), 23 | ('is_null', models.NullBooleanField(choices=[(True, 'NULL'), (False, 'NOT NULL')], help_text='\u53ef\u7a7a')), 24 | ('default_value', models.CharField(blank=True, help_text='\u9ed8\u8ba4\u503c', max_length=100, null=True)), 25 | ('comment', models.TextField(blank=True, help_text='\u6ce8\u91ca', max_length=5000, null=True)), 26 | ('is_comment_dirty', models.BooleanField(default=False)), 27 | ('is_enum', models.BooleanField(default=False)), 28 | ('other_enums', models.TextField(blank=True, default='', help_text='\u672a\u5339\u914d\u7684\u679a\u4e3e\u503c', max_length=2000, null=True)), 29 | ('is_deleted', models.BooleanField(default=False)), 30 | ], 31 | ), 32 | migrations.CreateModel( 33 | name='Database', 34 | fields=[ 35 | ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), 36 | ('name', models.CharField(help_text='\u6570\u636e\u5e93\u540d', max_length=100, unique=True)), 37 | ('config', models.CharField(help_text='\u914d\u7f6e', max_length=100, unique=True)), 38 | ('engine', models.CharField(blank=True, default='InnoDB', help_text='\u5f15\u64ce', max_length=10, null=True)), 39 | ('charset', models.CharField(blank=True, default='utf8', help_text='\u7f16\u7801', max_length=100, null=True)), 40 | ('comment', models.TextField(blank=True, default='TBD', help_text='\u6ce8\u91ca', max_length=5000, null=True)), 41 | ('enable', models.NullBooleanField(choices=[(True, 'on'), (False, 'off')], default=True, help_text='\u662f\u5426\u542f\u7528')), 42 | ], 43 | ), 44 | migrations.CreateModel( 45 | name='Index', 46 | fields=[ 47 | ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), 48 | ('name', models.CharField(help_text='\u7d22\u5f15\u540d', max_length=100)), 49 | ('type', models.CharField(blank=True, choices=[('KEY', 'KEY'), ('PRIMARY KEY', 'PRIMARY KEY'), ('UNIQUE KEY', 'UNIQUE KEY')], help_text='\u7c7b\u578b', max_length=100, null=True)), 50 | ('include_columns', models.CharField(blank=True, help_text='\u5305\u542b\u5b57\u6bb5', max_length=100, null=True)), 51 | ], 52 | ), 53 | migrations.CreateModel( 54 | name='Table', 55 | fields=[ 56 | ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), 57 | ('name', models.CharField(help_text='\u8868\u540d', max_length=100)), 58 | ('engine', models.CharField(blank=True, help_text='\u5f15\u64ce', max_length=10, null=True)), 59 | ('charset', models.CharField(blank=True, help_text='\u7f16\u7801', max_length=100, null=True)), 60 | ('comment', models.TextField(blank=True, help_text='\u6ce8\u91ca', max_length=5000, null=True)), 61 | ('is_deleted', models.BooleanField(default=False)), 62 | ('database', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='schema.Database')), 63 | ], 64 | ), 65 | migrations.AddField( 66 | model_name='index', 67 | name='table', 68 | field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='schema.Table'), 69 | ), 70 | migrations.AddField( 71 | model_name='column', 72 | name='table', 73 | field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='schema.Table'), 74 | ), 75 | migrations.AlterUniqueTogether( 76 | name='table', 77 | unique_together=set([('database', 'name')]), 78 | ), 79 | migrations.AlterUniqueTogether( 80 | name='index', 81 | unique_together=set([('table', 'name')]), 82 | ), 83 | migrations.AlterUniqueTogether( 84 | name='column', 85 | unique_together=set([('table', 'name')]), 86 | ), 87 | ] 88 | -------------------------------------------------------------------------------- /apps/schema/migrations/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/huifenqi/dbhub/825ce53f0b1c0e3c2e033ebe2c92bfcfbf51bd9e/apps/schema/migrations/__init__.py -------------------------------------------------------------------------------- /apps/schema/models.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | from __future__ import unicode_literals 4 | 5 | from django.db import models 6 | 7 | 8 | class Database(models.Model): 9 | NULL_TYPES = ( 10 | (True, 'on'), 11 | (False, 'off'), 12 | ) 13 | name = models.CharField(unique=True, max_length=100, help_text=u'数据库名') 14 | config = models.CharField(unique=True, max_length=100, help_text=u'配置') 15 | engine = models.CharField(max_length=10, help_text=u'引擎', default='InnoDB', null=True, blank=True) 16 | charset = models.CharField(max_length=100, help_text=u'编码', default='utf8', null=True, blank=True) 17 | comment = models.TextField(max_length=5000, help_text=u'注释', default='TBD', null=True, blank=True) 18 | enable = models.NullBooleanField(choices=NULL_TYPES, help_text=u'是否启用', default=True, null=True, blank=True) 19 | 20 | def __unicode__(self): 21 | return self.name 22 | 23 | 24 | class Table(models.Model): 25 | database = models.ForeignKey(Database) 26 | name = models.CharField(max_length=100, help_text=u'表名') 27 | engine = models.CharField(max_length=10, help_text=u'引擎', null=True, blank=True) 28 | charset = models.CharField(max_length=100, help_text=u'编码', null=True, blank=True) 29 | comment = models.TextField(max_length=5000, help_text=u'注释', null=True, blank=True) 30 | is_deleted = models.BooleanField(default=False) 31 | 32 | def __unicode__(self): 33 | return '{} ({})'.format(self.name, self.database.name) 34 | 35 | class Meta: 36 | unique_together = (('database', 'name'),) 37 | 38 | 39 | class Column(models.Model): 40 | NULL_TYPES = ( 41 | (True, 'NULL'), 42 | (False, 'NOT NULL'), 43 | ) 44 | 45 | table = models.ForeignKey(Table) 46 | name = models.CharField(max_length=100, help_text=u'列名') 47 | data_type = models.CharField(max_length=100, help_text=u'数据类型', null=True, blank=True) 48 | is_null = models.NullBooleanField(choices=NULL_TYPES, help_text=u'可空', null=True, blank=True) 49 | default_value = models.CharField(max_length=100, help_text=u'默认值', null=True, blank=True) 50 | comment = models.TextField(max_length=5000, help_text=u'注释', null=True, blank=True) 51 | is_comment_dirty = models.BooleanField(default=False) 52 | is_enum = models.BooleanField(default=False) 53 | other_enums = models.TextField(max_length=2000, help_text=u'未匹配的枚举值', default='', null=True, blank=True) 54 | is_deleted = models.BooleanField(default=False) 55 | 56 | def __unicode__(self): 57 | return self.name 58 | 59 | class Meta: 60 | unique_together = (('table', 'name'),) 61 | 62 | 63 | class Index(models.Model): 64 | KEY_TYPES = ( 65 | ('KEY', 'KEY'), 66 | ('PRIMARY KEY', 'PRIMARY KEY'), 67 | ('UNIQUE KEY', 'UNIQUE KEY'), 68 | ) 69 | 70 | table = models.ForeignKey(Table) 71 | name = models.CharField(max_length=100, help_text=u'索引名') 72 | type = models.CharField(max_length=100, choices=KEY_TYPES, help_text=u'类型', null=True, blank=True) 73 | include_columns = models.CharField(max_length=100, help_text=u'包含字段', null=True, blank=True) 74 | 75 | def __unicode__(self): 76 | return self.name 77 | 78 | class Meta: 79 | unique_together = (('table', 'name'),) 80 | -------------------------------------------------------------------------------- /apps/schema/tables.py: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | from __future__ import unicode_literals 3 | import django_tables2 as tables 4 | 5 | from models import Column 6 | 7 | 8 | TEMPLATE = """ 9 | {% if record.is_deleted %}Deleted{% endif %} 10 | {% if record.is_comment_dirty %} 11 | {% if record.other_enums|length > 20 %} 12 | Not Match:
{{record.other_enums|slice:":20"}}... 13 | {% else %} 14 | Not Match:
{{record.other_enums}}{% endif %}{% endif %} 15 | """ 16 | 17 | 18 | class ColumnTable(tables.Table): 19 | table_comment = tables.TemplateColumn('{{ value|linebreaks }}', verbose_name='Table Comment', 20 | accessor='table.comment') 21 | comment = tables.TemplateColumn('{{ value|linebreaks }}') 22 | warning_info = tables.TemplateColumn(TEMPLATE, verbose_name='Warning', orderable=False) 23 | database = tables.TemplateColumn('{{value}}', accessor='table.database') 24 | 25 | class Meta: 26 | model = Column 27 | sequence = ('name', 'table', 'data_type', 'is_null', 'default_value', 'comment', 'table_comment', 28 | 'warning_info') 29 | template_name = "django_tables2/semantic.html" 30 | exclude = ("id", "is_comment_dirty", "is_enum", "is_deleted", "other_enums") 31 | -------------------------------------------------------------------------------- /apps/schema/views.py: -------------------------------------------------------------------------------- 1 | from django_filters.views import FilterView 2 | from django_tables2 import SingleTableMixin 3 | from django.conf import settings 4 | from dal import autocomplete 5 | 6 | from .tables import ColumnTable 7 | from .models import Column, Table 8 | from .filters import ColumnFilter 9 | 10 | 11 | class ColumnListView(SingleTableMixin, FilterView): 12 | table_class = ColumnTable 13 | model = Column 14 | template_name = "columns.html" 15 | filterset_class = ColumnFilter 16 | 17 | def get_queryset(self): 18 | return super(ColumnListView, self).get_queryset().select_related("table") 19 | 20 | def get_context_data(self, **kwargs): 21 | ctx = super(ColumnListView, self).get_context_data(**kwargs) 22 | ctx['title'] = settings.TITLE 23 | ctx['enable_oauth'] = settings.ENABLE_OAUTH 24 | return ctx 25 | 26 | 27 | class TableAutocomplete(autocomplete.Select2QuerySetView): 28 | def dispatch(self, request, *args, **kwargs): 29 | self.database = request.GET.get('database', None) 30 | return super(TableAutocomplete, self).dispatch(request, *args, **kwargs) 31 | 32 | def get_queryset(self): 33 | qs = Table.objects.all() 34 | if self.database: 35 | qs = qs.filter(database=self.database) 36 | if self.q: 37 | qs = qs.filter(name__contains=self.q) 38 | return qs 39 | -------------------------------------------------------------------------------- /configs/newrelic.ini: -------------------------------------------------------------------------------- 1 | # --------------------------------------------------------------------------- 2 | 3 | # 4 | # This file configures the New Relic Python Agent. 5 | # 6 | # The path to the configuration file should be supplied to the function 7 | # newrelic.agent.initialize() when the agent is being initialized. 8 | # 9 | # The configuration file follows a structure similar to what you would 10 | # find for Microsoft Windows INI files. For further information on the 11 | # configuration file format see the Python ConfigParser documentation at: 12 | # 13 | # http://docs.python.org/library/configparser.html 14 | # 15 | # For further discussion on the behaviour of the Python agent that can 16 | # be configured via this configuration file see: 17 | # 18 | # http://newrelic.com/docs/python/python-agent-configuration 19 | # 20 | 21 | # --------------------------------------------------------------------------- 22 | 23 | # Here are the settings that are common to all environments. 24 | 25 | [newrelic] 26 | 27 | # You must specify the license key associated with your New 28 | # Relic account. This key binds the Python Agent's data to your 29 | # account in the New Relic service. 30 | license_key = eba6291ea8bd00cfa3729c86bb943097ab34d532 31 | 32 | # The application name. Set this to be the name of your 33 | # application as you would like it to show up in New Relic UI. 34 | # The UI will then auto-map instances of your application into a 35 | # entry on your home dashboard page. 36 | app_name = dbhub 37 | 38 | # When "true", the agent collects performance data about your 39 | # application and reports this data to the New Relic UI at 40 | # newrelic.com. This global switch is normally overridden for 41 | # each environment below. 42 | monitor_mode = true 43 | 44 | # Sets the name of a file to log agent messages to. Useful for 45 | # debugging any issues with the agent. This is not set by 46 | # default as it is not known in advance what user your web 47 | # application processes will run as and where they have 48 | # permission to write to. Whatever you set this to you must 49 | # ensure that the permissions for the containing directory and 50 | # the file itself are correct, and that the user that your web 51 | # application runs as can write to the file. If not able to 52 | # write out a log file, it is also possible to say "stderr" and 53 | # output to standard error output. This would normally result in 54 | # output appearing in your web server log. 55 | #log_file = /tmp/newrelic-python-agent.log 56 | 57 | # Sets the level of detail of messages sent to the log file, if 58 | # a log file location has been provided. Possible values, in 59 | # increasing order of detail, are: "critical", "error", "warning", 60 | # "info" and "debug". When reporting any agent issues to New 61 | # Relic technical support, the most useful setting for the 62 | # support engineers is "debug". However, this can generate a lot 63 | # of information very quickly, so it is best not to keep the 64 | # agent at this level for longer than it takes to reproduce the 65 | # problem you are experiencing. 66 | log_level = info 67 | 68 | # The Python Agent communicates with the New Relic service using 69 | # SSL by default. Note that this does result in an increase in 70 | # CPU overhead, over and above what would occur for a non SSL 71 | # connection, to perform the encryption involved in the SSL 72 | # communication. This work is though done in a distinct thread 73 | # to those handling your web requests, so it should not impact 74 | # response times. You can if you wish revert to using a non SSL 75 | # connection, but this will result in information being sent 76 | # over a plain socket connection and will not be as secure. 77 | ssl = true 78 | 79 | # High Security Mode enforces certain security settings, and 80 | # prevents them from being overridden, so that no sensitive data 81 | # is sent to New Relic. Enabling High Security Mode means that 82 | # SSL is turned on, request parameters are not collected, and SQL 83 | # can not be sent to New Relic in its raw form. To activate High 84 | # Security Mode, it must be set to 'true' in this local .ini 85 | # configuration file AND be set to 'true' in the server-side 86 | # configuration in the New Relic user interface. For details, see 87 | # https://docs.newrelic.com/docs/subscriptions/high-security 88 | high_security = false 89 | 90 | # The Python Agent will attempt to connect directly to the New 91 | # Relic service. If there is an intermediate firewall between 92 | # your host and the New Relic service that requires you to use a 93 | # HTTP proxy, then you should set both the "proxy_host" and 94 | # "proxy_port" settings to the required values for the HTTP 95 | # proxy. The "proxy_user" and "proxy_pass" settings should 96 | # additionally be set if proxy authentication is implemented by 97 | # the HTTP proxy. The "proxy_scheme" setting dictates what 98 | # protocol scheme is used in talking to the HTTP proxy. This 99 | # would normally always be set as "http" which will result in the 100 | # agent then using a SSL tunnel through the HTTP proxy for end to 101 | # end encryption. 102 | # proxy_scheme = http 103 | # proxy_host = hostname 104 | # proxy_port = 8080 105 | # proxy_user = 106 | # proxy_pass = 107 | 108 | # Capturing request parameters is off by default. To enable the 109 | # capturing of request parameters, first ensure that the setting 110 | # "attributes.enabled" is set to "true" (the default value), and 111 | # then add "request.parameters.*" to the "attributes.include" 112 | # setting. For details about attributes configuration, please 113 | # consult the documentation. 114 | # attributes.include = request.parameters.* 115 | 116 | # The transaction tracer captures deep information about slow 117 | # transactions and sends this to the UI on a periodic basis. The 118 | # transaction tracer is enabled by default. Set this to "false" 119 | # to turn it off. 120 | transaction_tracer.enabled = true 121 | 122 | # Threshold in seconds for when to collect a transaction trace. 123 | # When the response time of a controller action exceeds this 124 | # threshold, a transaction trace will be recorded and sent to 125 | # the UI. Valid values are any positive float value, or (default) 126 | # "apdex_f", which will use the threshold for a dissatisfying 127 | # Apdex controller action - four times the Apdex T value. 128 | transaction_tracer.transaction_threshold = apdex_f 129 | 130 | # When the transaction tracer is on, SQL statements can 131 | # optionally be recorded. The recorder has three modes, "off" 132 | # which sends no SQL, "raw" which sends the SQL statement in its 133 | # original form, and "obfuscated", which strips out numeric and 134 | # string literals. 135 | transaction_tracer.record_sql = obfuscated 136 | 137 | # Threshold in seconds for when to collect stack trace for a SQL 138 | # call. In other words, when SQL statements exceed this 139 | # threshold, then capture and send to the UI the current stack 140 | # trace. This is helpful for pinpointing where long SQL calls 141 | # originate from in an application. 142 | transaction_tracer.stack_trace_threshold = 0.5 143 | 144 | # Determines whether the agent will capture query plans for slow 145 | # SQL queries. Only supported in MySQL and PostgreSQL. Set this 146 | # to "false" to turn it off. 147 | transaction_tracer.explain_enabled = true 148 | 149 | # Threshold for query execution time below which query plans 150 | # will not not be captured. Relevant only when "explain_enabled" 151 | # is true. 152 | transaction_tracer.explain_threshold = 0.5 153 | 154 | # Space separated list of function or method names in form 155 | # 'module:function' or 'module:class.function' for which 156 | # additional function timing instrumentation will be added. 157 | transaction_tracer.function_trace = 158 | 159 | # The error collector captures information about uncaught 160 | # exceptions or logged exceptions and sends them to UI for 161 | # viewing. The error collector is enabled by default. Set this 162 | # to "false" to turn it off. 163 | error_collector.enabled = true 164 | 165 | # To stop specific errors from reporting to the UI, set this to 166 | # a space separated list of the Python exception type names to 167 | # ignore. The exception name should be of the form 'module:class'. 168 | error_collector.ignore_errors = 169 | 170 | # Browser monitoring is the Real User Monitoring feature of the UI. 171 | # For those Python web frameworks that are supported, this 172 | # setting enables the auto-insertion of the browser monitoring 173 | # JavaScript fragments. 174 | browser_monitoring.auto_instrument = true 175 | 176 | # A thread profiling session can be scheduled via the UI when 177 | # this option is enabled. The thread profiler will periodically 178 | # capture a snapshot of the call stack for each active thread in 179 | # the application to construct a statistically representative 180 | # call tree. 181 | thread_profiler.enabled = true 182 | 183 | # --------------------------------------------------------------------------- 184 | 185 | # 186 | # The application environments. These are specific settings which 187 | # override the common environment settings. The settings related to a 188 | # specific environment will be used when the environment argument to the 189 | # newrelic.agent.initialize() function has been defined to be either 190 | # "development", "test", "staging" or "production". 191 | # 192 | 193 | [newrelic:development] 194 | monitor_mode = false 195 | 196 | [newrelic:test] 197 | monitor_mode = false 198 | 199 | [newrelic:staging] 200 | app_name = Python Application (Staging) 201 | monitor_mode = true 202 | 203 | [newrelic:production] 204 | monitor_mode = true 205 | 206 | # --------------------------------------------------------------------------- 207 | -------------------------------------------------------------------------------- /configs/nginx.conf: -------------------------------------------------------------------------------- 1 | server { 2 | listen 80; 3 | server_name dbhub.wanggezhi.com; 4 | 5 | location /static { 6 | alias /data/www/dbhub/static; 7 | } 8 | 9 | location / { 10 | proxy_pass http://127.0.0.1:8005; 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /configs/setup.sh: -------------------------------------------------------------------------------- 1 | ln -s /data/www/dbhub/configs/supervisor.conf /etc/supervisor/conf.d/dbhub.conf 2 | ln -s /data/www/dbhub/configs/nginx.conf /etc/nginx/conf.d/dbhub.conf 3 | -------------------------------------------------------------------------------- /configs/supervisor.conf: -------------------------------------------------------------------------------- 1 | [program:dbhub] 2 | environment = NEW_RELIC_CONFIG_FILE=/data/www/%(program_name)s/configs/newrelic.ini 3 | directory = /data/www/%(program_name)s/configs 4 | command = /data/www/%(program_name)s/env/local/bin/uwsgi --ini uwsgi.ini 5 | autostart = true 6 | autorestart = true 7 | stopsignal = QUIT 8 | killasgroup = true 9 | buffer-size = 65535 10 | redirect_stderr = true 11 | stdout_logfile_maxbytes = 0 12 | stdout_logfile_backups = 0 13 | stdout_logfile = /data/logs/%(program_name)s/uwsgi.stdout.log 14 | stderr_logfile=/data/logs/%(program_name)s/uwsgi.stderr.log 15 | -------------------------------------------------------------------------------- /configs/uwsgi.ini: -------------------------------------------------------------------------------- 1 | [uwsgi] 2 | http = 127.0.0.1:8005 3 | chdir = /data/www/dbhub/ 4 | wsgi-file = dbhub/wsgi.py 5 | master = true 6 | processes = 2 7 | harakiri = 60 8 | limit-as = 1000 9 | max-requests = 5000 10 | single-interpreter = true 11 | enable-threads = true 12 | env = DJANGO_SETTINGS_MODULE=dbhub.settings.prod 13 | virtualenv = /data/www/dbhub/env/ 14 | -------------------------------------------------------------------------------- /dbhub/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/huifenqi/dbhub/825ce53f0b1c0e3c2e033ebe2c92bfcfbf51bd9e/dbhub/__init__.py -------------------------------------------------------------------------------- /dbhub/settings/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/huifenqi/dbhub/825ce53f0b1c0e3c2e033ebe2c92bfcfbf51bd9e/dbhub/settings/__init__.py -------------------------------------------------------------------------------- /dbhub/settings/common.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | Django settings for DBHub project. 4 | 5 | Generated by 'django-admin startproject' using Django 1.8.18. 6 | 7 | For more information on this file, see 8 | https://docs.djangoproject.com/en/1.8/topics/settings/ 9 | 10 | For the full list of settings and their values, see 11 | https://docs.djangoproject.com/en/1.8/ref/settings/ 12 | """ 13 | 14 | # Build paths inside the project like this: os.path.join(BASE_DIR, ...) 15 | import os 16 | 17 | BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) 18 | 19 | # Quick-start development settings - unsuitable for production 20 | # See https://docs.djangoproject.com/en/1.8/howto/deployment/checklist/ 21 | 22 | # SECURITY WARNING: keep the secret key used in production secret! 23 | SECRET_KEY = '7@$-=s^akdzw5$02*=q#9@2@7otk%d3u3j3cm7rt)mc=wg2g0!' 24 | 25 | # SECURITY WARNING: don't run with debug turned on in production! 26 | DEBUG = True 27 | 28 | ALLOWED_HOSTS = ['*'] 29 | 30 | # Application definition 31 | 32 | INSTALLED_APPS = ( 33 | 'django.contrib.admin', 34 | 'django.contrib.auth', 35 | 'django.contrib.contenttypes', 36 | 'django.contrib.sessions', 37 | 'django.contrib.messages', 38 | 'django.contrib.staticfiles', 39 | 'django_extensions', 40 | "django_tables2", 41 | "django_filters", 42 | "dal", 43 | "dal_select2", 44 | "oauthadmin", 45 | "reversion", 46 | 'apps.schema', 47 | ) 48 | 49 | MIDDLEWARE_CLASSES = ( 50 | 'django.contrib.sessions.middleware.SessionMiddleware', 51 | 'oauthadmin.middleware.OauthAdminSessionMiddleware', 52 | 'django.middleware.common.CommonMiddleware', 53 | 'django.middleware.csrf.CsrfViewMiddleware', 54 | 'django.contrib.auth.middleware.AuthenticationMiddleware', 55 | 'django.contrib.auth.middleware.SessionAuthenticationMiddleware', 56 | 'django.contrib.messages.middleware.MessageMiddleware', 57 | 'django.middleware.clickjacking.XFrameOptionsMiddleware', 58 | 'django.middleware.security.SecurityMiddleware', 59 | ) 60 | 61 | ROOT_URLCONF = 'dbhub.urls' 62 | 63 | TEMPLATES = [ 64 | { 65 | 'BACKEND': 'django.template.backends.django.DjangoTemplates', 66 | "DIRS": ["templates"], 67 | 'APP_DIRS': True, 68 | 'OPTIONS': { 69 | 'context_processors': [ 70 | 'django.template.context_processors.debug', 71 | 'django.template.context_processors.request', 72 | 'django.contrib.auth.context_processors.auth', 73 | 'django.contrib.messages.context_processors.messages', 74 | ], 75 | }, 76 | }, 77 | ] 78 | 79 | WSGI_APPLICATION = 'dbhub.wsgi.application' 80 | 81 | # Database 82 | # https://docs.djangoproject.com/en/1.8/ref/settings/#databases 83 | 84 | DATABASES = { 85 | 'default': { 86 | 'ENGINE': 'django.db.backends.sqlite3', 87 | 'NAME': os.path.join(BASE_DIR, 'db.sqlite3'), 88 | } 89 | } 90 | 91 | # Internationalization 92 | # https://docs.djangoproject.com/en/1.8/topics/i18n/ 93 | 94 | LANGUAGE_CODE = 'zh-hans' 95 | 96 | TIME_ZONE = 'Asia/Shanghai' 97 | 98 | USE_I18N = True 99 | 100 | USE_L10N = True 101 | 102 | USE_TZ = True 103 | 104 | # Static files (CSS, JavaScript, Images) 105 | # https://docs.djangoproject.com/en/1.8/howto/static-files/ 106 | 107 | STATIC_URL = '/static/' 108 | 109 | STATIC_ROOT = BASE_DIR + '/../static/' 110 | 111 | TITLE = '[DEMO] DBHub\'s self design' 112 | DB_INSTANCES = [] 113 | 114 | # django-admin-oauth2 115 | SESSION_SERIALIZER = "django.contrib.sessions.serializers.PickleSerializer" 116 | 117 | OAUTHADMIN_CLIENT_ID = '' 118 | OAUTHADMIN_CLIENT_SECRET = '' 119 | OAUTHADMIN_BASE_URL = "https://sso.huifenqi.com/sso/oauth/" 120 | OAUTHADMIN_AUTH_URL = 'https://sso.huifenqi.com/sso/authorize/' 121 | OAUTHADMIN_TOKEN_URL = 'https://sso.huifenqi.com/sso/token/' 122 | OAUTHADMIN_GROUPS = [] 123 | OAUTHADMIN_SCOPE = [] 124 | 125 | # replace admin login with oauth login 126 | ENABLE_OAUTH = False 127 | -------------------------------------------------------------------------------- /dbhub/settings/dev.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | from .common import * 4 | -------------------------------------------------------------------------------- /dbhub/urls.py: -------------------------------------------------------------------------------- 1 | """dbhub URL Configuration 2 | 3 | The `urlpatterns` list routes URLs to views. For more information please see: 4 | https://docs.djangoproject.com/en/1.8/topics/http/urls/ 5 | Examples: 6 | Function views 7 | 1. Add an import: from my_app import views 8 | 2. Add a URL to urlpatterns: url(r'^$', views.home, name='home') 9 | Class-based views 10 | 1. Add an import: from other_app.views import Home 11 | 2. Add a URL to urlpatterns: url(r'^$', Home.as_view(), name='home') 12 | Including another URLconf 13 | 1. Add a URL to urlpatterns: url(r'^blog/', include('blog.urls')) 14 | """ 15 | from django.conf.urls import include, url 16 | from django.contrib import admin 17 | from django.contrib.auth.decorators import login_required 18 | from django.conf import settings 19 | from apps.schema.views import ColumnListView, TableAutocomplete 20 | 21 | admin.site.site_header = 'DBHub' 22 | 23 | urlpatterns = [ 24 | url(r"^$", ColumnListView.as_view(), name="index"), 25 | url(r'^autocomplete/$', login_required(TableAutocomplete.as_view()), name='table-autocomplete'), 26 | ] 27 | 28 | if settings.ENABLE_OAUTH: 29 | old_admin_login = admin.site.login 30 | admin.site.login = login_required(admin.site.login) 31 | urlpatterns += [ 32 | url(r'^admin-login/', old_admin_login), 33 | url(r'^oauth/', include('oauthadmin.urls')), 34 | ] 35 | settings.LOGIN_URL = '/oauth/login/' 36 | settings.LOGOUT_REDIRECT_URL = '/oauth/logout_redirect/' 37 | 38 | urlpatterns += [ 39 | url(r'^admin/', include(admin.site.urls)), 40 | ] 41 | -------------------------------------------------------------------------------- /dbhub/wsgi.py: -------------------------------------------------------------------------------- 1 | """ 2 | WSGI config for dbhub project. 3 | 4 | It exposes the WSGI callable as a module-level variable named ``application``. 5 | 6 | For more information on this file, see 7 | https://docs.djangoproject.com/en/1.8/howto/deployment/wsgi/ 8 | """ 9 | 10 | import os 11 | 12 | from django.core.wsgi import get_wsgi_application 13 | 14 | os.environ.setdefault("DJANGO_SETTINGS_MODULE", "dbhub.settings") 15 | 16 | application = get_wsgi_application() 17 | -------------------------------------------------------------------------------- /manage.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import os 3 | import sys 4 | 5 | if __name__ == "__main__": 6 | os.environ.setdefault("DJANGO_SETTINGS_MODULE", "dbhub.settings.dev") 7 | 8 | from django.core.management import execute_from_command_line 9 | 10 | execute_from_command_line(sys.argv) 11 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | Django==1.11.29 2 | django-autocomplete-light==3.2.10 3 | django-extensions==2.1.0 4 | django-filter==1.1.0 5 | django-reversion==3.0.2 6 | django-tables2==1.21.2 7 | MySQL-python==1.2.5 8 | SQLAlchemy>=1.3.0 9 | sqlsoup>=0.9.1 10 | pymongo 11 | lark-parser 12 | uwsgi==2.0.17.1 13 | https://github.com/bastionhost/django-admin-oauth2/archive/master.zip 14 | https://github.com/pajachiet/pymongo-schema/archive/master.zip 15 | -------------------------------------------------------------------------------- /runtests.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | DJANGO_SETTINGS_MODULE=settings py.test $* 3 | -------------------------------------------------------------------------------- /screenshoot.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/huifenqi/dbhub/825ce53f0b1c0e3c2e033ebe2c92bfcfbf51bd9e/screenshoot.png -------------------------------------------------------------------------------- /scripts/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/huifenqi/dbhub/825ce53f0b1c0e3c2e033ebe2c92bfcfbf51bd9e/scripts/__init__.py -------------------------------------------------------------------------------- /scripts/check.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | import pymongo 3 | 4 | from apps.schema.models import Database 5 | from scripts.db import DB 6 | from scripts.parser import CommentParser 7 | 8 | 9 | class Checker(object): 10 | def __init__(self, database, t_name_list): 11 | self.database = database 12 | self.dialect = database.config.split(':')[0] 13 | if self.dialect == 'mongodb': 14 | parts1 = self.database.config.rsplit(':', 1) 15 | parts2 = parts1[1].split('/') 16 | host = parts1[0] 17 | port = int(parts2[0]) 18 | db_name = parts2[1] 19 | self.db = pymongo.MongoClient(host, port)[db_name] 20 | else: 21 | self.db = DB(self.database.config) 22 | if len(t_name_list) == 1 and t_name_list[0] == '': 23 | tables = database.table_set.all() 24 | else: 25 | tables = database.table_set.filter(name__in=t_name_list) 26 | if not tables: 27 | tables = database.table_set.all() 28 | self.tables = tables 29 | 30 | def get_enum_list(self, table_name, column_name): 31 | real_enums = [] 32 | if self.dialect == 'mongodb': 33 | pipeline = [ 34 | { 35 | u"$group": { 36 | u"_id": { 37 | column_name: u"${}".format(column_name) 38 | }, 39 | u"COUNT(*)": { 40 | u"$sum": 1 41 | } 42 | } 43 | }, 44 | { 45 | u"$project": { 46 | column_name: u"$_id.{}".format(column_name), 47 | u"COUNT(*)": u"$COUNT(*)", 48 | u"_id": 0 49 | } 50 | } 51 | ] 52 | cursor = self.db[table_name].aggregate(pipeline, allowDiskUse=True) 53 | real_enums = [r[column_name] for r in cursor] 54 | else: 55 | tb = getattr(self.db, table_name) 56 | enum_list = tb.group_by(column_name).all() 57 | for row in enum_list: 58 | tmp = getattr(row, column_name) 59 | if isinstance(tmp, unicode): 60 | real_enums.append(tmp.encode('utf-8')) 61 | else: 62 | real_enums.append(str(tmp)) 63 | return real_enums 64 | 65 | def run(self): 66 | for table in self.tables: 67 | for column in table.column_set.all(): 68 | # skip column which is dirty 69 | if column.is_comment_dirty or column.is_deleted: 70 | continue 71 | comment_enums = CommentParser.get_enums((column.comment or '').encode('utf-8')) 72 | # set is_enum as have comment_enums 73 | if comment_enums and not column.is_enum: 74 | column.is_enum = True 75 | # skip column which is not enum 76 | if not column.is_enum: 77 | continue 78 | real_enums = self.get_enum_list(table.name, column.name) 79 | if len(real_enums) > 50: 80 | column.other_enums = u'枚举值异常!' 81 | column.is_comment_dirty = True 82 | column.save() 83 | continue 84 | not_match_enums = (set(real_enums) - set(comment_enums)) 85 | if not_match_enums: 86 | print(self.database, table, column, comment_enums, real_enums) 87 | column.is_comment_dirty = True 88 | warning = ','.join(not_match_enums) 89 | column.other_enums = warning 90 | else: 91 | column.is_comment_dirty = False 92 | column.save() 93 | 94 | 95 | def run(db_list, t_list): 96 | db_name_list = db_list.split(',') 97 | if len(db_name_list) == 1 and db_name_list[0] == '': 98 | databases = Database.objects.filter(enable=True) 99 | else: 100 | databases = Database.objects.filter(enable=True, name__in=db_name_list) 101 | t_name_list = t_list.split(',') 102 | for database in databases: 103 | checker = Checker(database, t_name_list) 104 | checker.run() 105 | -------------------------------------------------------------------------------- /scripts/db.py: -------------------------------------------------------------------------------- 1 | import sqlsoup 2 | from sqlalchemy import create_engine 3 | from sqlalchemy.orm import scoped_session, sessionmaker 4 | 5 | 6 | class Engine(object): 7 | def __new__(cls, connect_url): 8 | engine = create_engine(connect_url, 9 | strategy='threadlocal', 10 | pool_size=5, 11 | pool_recycle=1800, 12 | encoding='utf-8', 13 | max_overflow=2) 14 | return engine 15 | 16 | 17 | class DB(object): 18 | def __new__(cls, connect_url): 19 | return sqlsoup.SQLSoup(Engine(connect_url), session=scoped_session(sessionmaker( 20 | autoflush=False, 21 | expire_on_commit=False, 22 | autocommit=True))) 23 | 24 | 25 | if __name__ == '__main__': 26 | # db = DB('mysql://root:123456@localhost:3306/test?charset=utf8') 27 | pass 28 | -------------------------------------------------------------------------------- /scripts/parser.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | import re 3 | 4 | 5 | class CommentParser(object): 6 | @classmethod 7 | def parse(cls, comment): 8 | pattern = r"([-\d\w]+)([:|：|-]{1})\s*[^:：,，-]?" 9 | objs = re.findall(pattern, comment, re.M | re.I) 10 | return objs 11 | 12 | @classmethod 13 | def parse_with_lark(cls, comment): 14 | from lark import Lark 15 | parser = Lark('''start: WORD "," WORD "!" 16 | LCASE_LETTER: "a".."z" 17 | UCASE_LETTER: "A".."Z" 18 | CN_ZH_LETTER: /[u"\u4e00-\u9fa5"]/ 19 | LETTER: UCASE_LETTER | LCASE_LETTER | CN_ZH_LETTER 20 | WORD: LETTER+ 21 | %import common.NUMBER // imports from terminal library 22 | %ignore " " // Disregard spaces in text 23 | ''', parser='lalr') 24 | 25 | print(parser.parse(comment).pretty()) 26 | # print(parser.parse(u'默认代扣银行卡 1:默认代扣 0:不默认代扣')) 27 | # not works as parsing library need accurate sentence 28 | 29 | @classmethod 30 | def get_enums(cls, comment): 31 | enums = list(set([obj[0] for obj in cls.parse(comment)])) 32 | return sorted(enums) 33 | -------------------------------------------------------------------------------- /scripts/sync.py: -------------------------------------------------------------------------------- 1 | import pymongo 2 | from django.conf import settings 3 | from sqlalchemy import create_engine, inspect 4 | from sqlalchemy import MetaData 5 | from pymongo_schema.extract import extract_pymongo_client_schema 6 | from pymongo_schema.tosql import mongo_schema_to_mapping 7 | 8 | from apps.schema.models import Database, Table, Column, Index 9 | 10 | 11 | class RelationalDBSync(object): 12 | @staticmethod 13 | def save_indexes(t, indexes): 14 | for index in indexes: 15 | i, created = Index.objects.get_or_create(table=t, name=index.name) 16 | i.type = 'UNIQUE KEY' if index.unique else 'KEY' 17 | i.include_columns = ', '.join([c.name for c in index.columns]) 18 | i.save() 19 | 20 | @staticmethod 21 | def save_primary_keys(t, primary_keys): 22 | for index in primary_keys: 23 | i, created = Index.objects.get_or_create(table=t, name=index.name) 24 | i.type = 'PRIMARY KEY' 25 | i.include_columns = index.name 26 | i.save() 27 | 28 | @staticmethod 29 | def save_columns(t, columns): 30 | for column in columns: 31 | default_value = column.server_default.arg if column.server_default else None 32 | c, created = Column.objects.get_or_create(table=t, name=column.name) 33 | try: 34 | c.data_type = str(column.type).split(' ')[0] 35 | except Exception: 36 | c.data_type = repr(column.type) 37 | c.is_null = column.nullable 38 | c.default_value = default_value 39 | if not c.comment and column.comment: 40 | c.comment = column.comment 41 | c.save() 42 | 43 | def build(self, database): 44 | engine = create_engine(database.config) 45 | m = MetaData() 46 | m.reflect(engine) 47 | if not database.charset: 48 | # fill database info 49 | database.charset = engine.dialect.encoding 50 | database.save() 51 | for table in m.sorted_tables: 52 | print(table.name) 53 | dialect = database.config.split(':')[0] 54 | table_info = table.dialect_options[dialect]._non_defaults 55 | t, created = Table.objects.get_or_create(database=database, name=table.name) 56 | t.engine = table_info.get('engine', '') 57 | t.charset = table_info.get('default charset', '') 58 | if not t.comment and table.comment: 59 | t.comment = table.comment 60 | t.save() 61 | self.save_columns(t, table.columns) 62 | self.save_primary_keys(t, table.primary_key.columns) 63 | self.save_indexes(t, table.indexes) 64 | 65 | 66 | class MongoDBSync(object): 67 | def __init__(self, database): 68 | self.database = database 69 | parts1 = database.config.rsplit(':', 1) 70 | parts2 = parts1[1].split('/') 71 | self.host = parts1[0] 72 | self.port = int(parts2[0]) 73 | self.db = parts2[1] 74 | 75 | def build(self): 76 | with pymongo.MongoClient(self.host, self.port) as client: 77 | for collection in client[self.db].list_collection_names(): 78 | print(collection) 79 | schema = extract_pymongo_client_schema(client, [self.db], [collection]) 80 | mapping = mongo_schema_to_mapping(schema) 81 | t, created = Table.objects.get_or_create(database=self.database, name=collection) 82 | if self.db not in mapping or collection not in mapping[self.db]: 83 | continue 84 | for column in mapping[self.db][collection].keys(): 85 | if column == 'pk': 86 | continue 87 | c, created = Column.objects.get_or_create(table=t, name=column) 88 | c.data_type = mapping[self.db][collection][column]['type'] 89 | c.is_null = True 90 | c.save() 91 | 92 | 93 | def init_databases(): 94 | for instance in settings.DB_INSTANCES: 95 | engine = create_engine(instance) 96 | insp = inspect(engine) 97 | db_list = insp.get_schema_names() 98 | dbs = set(db_list) - {'information_schema', 'performance_schema', 'mysql', 'sys'} 99 | for db in dbs: 100 | config = '{}/{}?charset=utf8'.format(instance.rstrip('/'), db) 101 | d, created = Database.objects.get_or_create(name=db) 102 | d.config = config 103 | d.save() 104 | 105 | 106 | def run(): 107 | init_databases() 108 | databases = Database.objects.filter(enable=True) 109 | for database in databases: 110 | if database.config.startswith('mongodb'): 111 | MongoDBSync(database).build() 112 | else: 113 | RelationalDBSync().build(database) 114 | 115 | 116 | if __name__ == '__main__': 117 | run() 118 | -------------------------------------------------------------------------------- /templates/columns.html: -------------------------------------------------------------------------------- 1 | {% load render_table from django_tables2 %} 2 | 3 | 4 | 5 | {{ title }} 6 | 8 | 9 | 10 | 12 | 13 | 14 | 15 | 16 |
17 | {% block body %} 18 |

{{ title }}

19 |
20 |
21 | {{ filter.form.as_p }} 22 | 23 |
24 |
25 | {% if user.is_authenticated %} 26 |

Welcome, {{ user.username }}

27 | {% else %} 28 | {% if enable_oauth %} 29 |

Login

30 | {% else %} 31 |

Login

32 | {% endif %} 33 | {% endif %} 34 | {% if user.is_authenticated %} 35 | {% render_table table %} 36 | {% else %} 37 |

Please login to view!

38 | {% endif %} 39 | {% endblock %} 40 | 41 | DBHUB is built with <3 using Django & maintained by iamsk. 42 | 43 |
44 | 45 | 88 | 114 | 115 | -------------------------------------------------------------------------------- /test-requirements.txt: -------------------------------------------------------------------------------- 1 | pytest==2.9.2 2 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/huifenqi/dbhub/825ce53f0b1c0e3c2e033ebe2c92bfcfbf51bd9e/tests/__init__.py -------------------------------------------------------------------------------- /tests/test_parser.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | import pytest 3 | 4 | from scripts.parser import CommentParser 5 | 6 | 7 | @pytest.mark.parametrize('comment, expected', [ 8 | (u""" 9 | 是否启用 10 | 0: 启用 11 | 1: 停用 12 | """, [u'0', u'1']), 13 | (u'默认代扣银行卡 1:默认代扣 0:不默认代扣', [u'0', u'1']), 14 | (u'是否在线，0-离线 1-在线', [u'0', u'1']), 15 | (u"""1:xxx 16 | 2:zzz""", [u'1', u'2']), 17 | (u"""银行账户开户行 PABC:平安银行 SPDB:浦发银行 HXB:华夏银行...参考程序枚举""", [u'HXB', u'PABC', u'SPDB']), 18 | # (u'xxx', 'xxx'), 19 | (u"""扣款状态,-1-未发起,0-已发起,1-扣款成功,2-扣款失败,3-处理中,4-不一致""", [u'-1', u'0', u'1', u'2', u'3', u'4']), 20 | (u'前置电签状态：-1-调用北银接口程序异常；0：成功，1：失败；系统异常则返回系统错误码', [u'-1']), 21 | (u'门店是否启用:0:否,1:是', [u'0', u'1']), 22 | (u""" 23 | charset with description, blah, blah, blah 24 | 25 | utf8: A UTF-8 encoding of the Unicode character set using one to three bytes per character. default utf8 of mysql, max length is 3 bytes, not support characters, such as emoji. 26 | 27 | utf8mb4: A UTF-8 encoding of the Unicode character set using one to four bytes per character. 28 | """, ['UTF', u'utf8', u'utf8mb4']), # error case 29 | ]) 30 | def test_get_enums(comment, expected): 31 | real = CommentParser.get_enums(comment) 32 | assert expected == real 33 | --------------------------------------------------------------------------------