├── DEVELOPERS.md ├── README.md ├── ispdb ├── TODO ├── __init__.py ├── apache │ ├── __init__.py │ ├── ispdb.wsgi │ ├── production.py │ ├── staging.py │ ├── trunk.py │ └── wsgi.conf ├── audit.py ├── config │ ├── __init__.py │ ├── admin.py │ ├── configChecks.py │ ├── forms.py │ ├── models.py │ ├── serializers.py │ ├── static │ │ ├── css │ │ │ ├── base.css │ │ │ ├── comments.css │ │ │ ├── details.css │ │ │ ├── enter_config.css │ │ │ ├── ispdb.css │ │ │ └── show_issue.css │ │ ├── img │ │ │ ├── ajax-loading.gif │ │ │ ├── error-icon.png │ │ │ ├── nav-bg-grabber.gif │ │ │ ├── nav-bg-reverse.gif │ │ │ ├── nav-bg.gif │ │ │ ├── sign_in_blue.png │ │ │ ├── thunderbird.png │ │ │ └── warning-icon.png │ │ └── js │ │ │ ├── jquery-1.7.2.min.js │ │ │ └── jquery.validate.min.js │ ├── templatetags │ │ ├── __init__.py │ │ └── custom_filters.py │ └── views.py ├── convert.py ├── fixtures │ ├── domain_testdata.json │ ├── issue_testdata.json │ ├── login_testdata.json │ ├── sanity.json │ └── xml_testdata.json ├── middleware │ ├── __init__.py │ └── x_ssl.py ├── settings.py ├── templates │ ├── 404.html │ ├── 500.html │ └── config │ │ ├── base.html │ │ ├── details.html │ │ ├── enter_config.html │ │ ├── export.xml │ │ ├── intro.html │ │ ├── list.html │ │ ├── login.html │ │ ├── queue.html │ │ ├── show_issue.html │ │ └── urls │ │ ├── desc_form.html │ │ ├── desc_list.html │ │ ├── desc_template_form.html │ │ ├── url_form.html │ │ ├── url_list.html │ │ └── url_template_form.html ├── tests │ ├── __init__.py │ ├── common.py │ ├── relaxng_schema.1.1.xml │ ├── relaxng_schema.xml │ ├── test_add.py │ ├── test_approve.py │ ├── test_cache.py │ ├── test_delete.py │ ├── test_edit.py │ ├── test_issue.py │ ├── test_sanity.py │ ├── test_view.py │ └── test_xml.py └── urls.py ├── manage.py ├── requirements.txt └── tools ├── MozAutoConfig.pm ├── convert.py ├── etld.py └── quickparse.py /DEVELOPERS.md: -------------------------------------------------------------------------------- 1 | # ISPDB 2 | 3 | ISPDB is a Django front-end to figure out workflow of ISP database info for the 4 | Thunderbird autoconfig database. 5 | 6 | This file contains useful informations to help developers. 7 | 8 | ## Documentation 9 | 10 | ### Models 11 | 12 | ISPDB has the following models: Domain, DomainRequest, Config, Issue, DocURL, 13 | DocURLDesc, EnableURL, EnableURLInst. 14 | 15 | Both Domain and DomainRequest represent a mail domain. The difference is that 16 | DomainRequest is not approved yet. It means that users can vote on it and their 17 | name attribute is not unique (we can have more than one DomainRequest with the 18 | same name). A Config holds the domain configuration. It can be related to more 19 | than one domain. It has zero or more documentation pages (DocURL) and zero or 20 | more enable pages (EnableURL). A DocURL has one or more descriptions 21 | (DocURLDesc) and a EnableURL has one or more instructions (EnableURLInst). 22 | Because those 4 classes are very similar, we have created common model and form 23 | classes. 24 | 25 | The Issue model represents a reported issue. It is related to the configuration 26 | and optionally to an user suggested configuration. 27 | 28 | ### Forms 29 | 30 | The complexity of the forms is in the form's classes. For example, ConfigForm 31 | class has domain_formset, docurl_formset and enableurl_formset attributes. So we 32 | can do things like the save_all method which save all of the forms (Domain, 33 | Config, DocURL, EnableURL), simplifying the views. 34 | 35 | Also there are two nested forms (for DocURL and EnableURL models). So each 36 | DocURLForm has a DocURLDesc formset and each EnableURLForm has a EnableURLInst 37 | formset. 38 | 39 | It is possible to add new forms dynamically in all of the formsets. There are 40 | two form classes to help with this task: DynamicModelForm and 41 | DynamicBaseModelFormSet. Basically they add a DELETE hidden boolean field and a 42 | overriden empty_form attribute (see below). 43 | 44 | #### Dynamic Forms 45 | 46 | enter_config.html template has hidden textarea elements which contains empty 47 | forms. These empty forms have on their attributes JQuery Validation tags, in our 48 | case "{1}-{0}" string, which are replaced with a form prefix and a index. 49 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | This repository is no longer active, current ISPDB is available at: https://github.com/thundernest/autoconfig 2 | 3 | experimental django front-end to figure out workflow of ISP database info 4 | for the Thunderbird autoconfig database 5 | 6 | ## Dependencies 7 | 8 | You can install all of the dependencies with: 9 | 10 | pip install -r requirements.txt 11 | 12 | Note: You may have to do an "easy_install pip" first. See requirements.txt for details on what we depend on. 13 | 14 | ## Getting Started 15 | 1. python ../manage.py syncdb 16 | 2. convert existing XML data to the DB: 17 | 18 | have http://svn.mozilla.org/mozillamessaging.com/sites/autoconfig.mozillamessaging.com/trunk checked out at ../autoconfig_data 19 | 20 | if autoconfig_data is somewhere else, you can set the env't var AUTOCONFIG_DATA to point to it 21 | 22 | echo 'import ispdb.convert;ispdb.convert.main()' | python manage.py shell 23 | 24 | 3. python manage.py runserver 25 | 4. then hit http://localhost:8000 26 | -------------------------------------------------------------------------------- /ispdb/TODO: -------------------------------------------------------------------------------- 1 | Work: 2 | 3 | P1 4 | * new config entry 5 | * validation: domain already in the DB? config wrong? 6 | * correct models (e.g. authentication types, etc.) 7 | * language list for popup 8 | * remove buttons for +1 and -1 9 | * command line script to export XML files w/ some sane convention 10 | * in review process, allow choice when marking invalid 11 | - "not available from our domain" 12 | - "incorrect data" 13 | - "other" 14 | (in addition to textfield) 15 | 16 | P2 17 | * allow reviewer-only comments: not visible to non reviewers 18 | * review process: have backend script to do telnet/ssh call to the ports, 19 | detect IMAP settinsgs, etc. 20 | * annotate results of backend script to the configuration 21 | * think about how to indicate when a record is out of date, the new record is in the db, but the old one is still being published (link to actual http:// data) 22 | * settings page language detection? 23 | * allow addition of domains to existing configurations 24 | 25 | Quality: 26 | * test convert -> export compare w/ original data 27 | * test suite 28 | 29 | Prettier:vi 30 | * CSS: make comment form prettier 31 | * django/ajax: fix comment submission (ajax baby!) 32 | -------------------------------------------------------------------------------- /ispdb/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mozilla/ispdb/3b87b2c797d0d56cdf5343cca831af721360ca53/ispdb/__init__.py -------------------------------------------------------------------------------- /ispdb/apache/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mozilla/ispdb/3b87b2c797d0d56cdf5343cca831af721360ca53/ispdb/apache/__init__.py -------------------------------------------------------------------------------- /ispdb/apache/ispdb.wsgi: -------------------------------------------------------------------------------- 1 | import os, sys 2 | 3 | #Calculate the path based on the location of the WSGI script. 4 | apache_configuration= os.path.dirname(__file__) 5 | project = os.path.dirname(apache_configuration) 6 | workspace = os.path.dirname(project) 7 | sys.path.append(workspace) 8 | 9 | print >> sys.stderr, "Path is %s" % sys.path 10 | 11 | #Add the path to 3rd party django application and to django itself. 12 | #sys.path.append('C:\\yml\\_myScript_\\dj_things\\web_development\\svn_views\\django_src\\trunk') 13 | #sys.path.append('C:\\yml\\_myScript_\\dj_things\\web_development\\svn_views\\django-registration') 14 | 15 | os.environ.setdefault("DJANGO_SETTINGS_MODULE", "ispdb.apache.production") 16 | 17 | # This application object is used by any WSGI server configured to use this 18 | # file. This includes Django's development server, if the WSGI_APPLICATION 19 | # setting points here. 20 | from django.core.wsgi import get_wsgi_application 21 | application = get_wsgi_application() 22 | -------------------------------------------------------------------------------- /ispdb/apache/production.py: -------------------------------------------------------------------------------- 1 | # Django settings for ispdb project. 2 | 3 | #Calculate the path based on the location of the WSGI script. 4 | import os 5 | apache_configuration = os.path.dirname(__file__) 6 | project = os.path.dirname(apache_configuration) 7 | workspace = os.path.dirname(project) 8 | 9 | DEBUG = False 10 | TEMPLATE_DEBUG = DEBUG 11 | 12 | ADMINS = ( 13 | # ('Your Name', 'your_email@domain.com'), 14 | ) 15 | 16 | MANAGERS = ADMINS 17 | 18 | DATABASES = { 19 | 'default': { 20 | 'ENGINE': 'django.db.backends.mysql', # Add 'postgresql_psycopg2', 'postgresql', 'mysql', 'sqlite3' or 'oracle'. 21 | 'NAME': 'production_ispdb', # Or path to database file if using sqlite3. 22 | 'USER': 'prod_ispdb', # Not used with sqlite3. 23 | 'PASSWORD': '', # Not used with sqlite3. 24 | 'HOST': 'db-write', # Set to empty string for localhost. Not used with sqlite3. 25 | 'HOST': '', # Set to empty string for localhost. Not used with sqlite3. 26 | 'PORT': '', # Set to empty string for default. Not used with sqlite3. 27 | } 28 | } 29 | 30 | # Local time zone for this installation. Choices can be found here: 31 | # http://en.wikipedia.org/wiki/List_of_tz_zones_by_name 32 | # although not all choices may be available on all operating systems. 33 | # If running in a Windows environment this must be set to the same as your 34 | # system time zone. 35 | TIME_ZONE = 'America/Chicago' 36 | 37 | # Language code for this installation. All choices can be found here: 38 | # http://www.i18nguy.com/unicode/language-identifiers.html 39 | LANGUAGE_CODE = 'en-us' 40 | 41 | SITE_ID = 1 42 | 43 | # If you set this to False, Django will make some optimizations so as not 44 | # to load the internationalization machinery. 45 | USE_I18N = False 46 | 47 | # If you set this to False, Django will not format dates, numbers and 48 | # calendars according to the current locale 49 | USE_L10N = True 50 | 51 | # If you set this to False, Django will not use timezone-aware datetimes. 52 | USE_TZ = True 53 | 54 | # Absolute filesystem path to the directory that will hold user-uploaded files. 55 | # Example: "/home/media/media.lawrence.com/media/" 56 | MEDIA_ROOT = '/var/www/ispdb/ispdb/media/' 57 | 58 | # URL that handles the media served from MEDIA_ROOT. Make sure to use a 59 | # trailing slash. 60 | # Examples: "http://media.lawrence.com/media/", "http://example.com/media/" 61 | MEDIA_URL = '/media/' 62 | 63 | # Absolute path to the directory static files should be collected to. 64 | # Don't put anything in this directory yourself; store your static files 65 | # in apps' "static/" subdirectories and in STATICFILES_DIRS. 66 | # Example: "/home/media/media.lawrence.com/static/" 67 | STATIC_ROOT = '/var/www/ispdb/ispdb/static/' 68 | 69 | # URL prefix for static files. 70 | # Example: "http://media.lawrence.com/static/" 71 | STATIC_URL = '/static/' 72 | 73 | # Additional locations of static files 74 | STATICFILES_DIRS = ( 75 | # Put strings here, like "/home/html/static" or "C:/www/django/static". 76 | # Always use forward slashes, even on Windows. 77 | # Don't forget to use absolute paths, not relative paths. 78 | ) 79 | 80 | # List of finder classes that know how to find static files in 81 | # various locations. 82 | STATICFILES_FINDERS = ( 83 | 'django.contrib.staticfiles.finders.FileSystemFinder', 84 | 'django.contrib.staticfiles.finders.AppDirectoriesFinder', 85 | # 'django.contrib.staticfiles.finders.DefaultStorageFinder', 86 | ) 87 | 88 | # Make this unique, and don't share it with anybody. 89 | SECRET_KEY = '02$7gj$t@dtd7um)d#3zlfg7ommedl#*ekt@@*ysy_fns(lp9+' 90 | 91 | # List of callables that know how to import templates from various sources. 92 | TEMPLATE_LOADERS = ( 93 | 'django.template.loaders.filesystem.Loader', 94 | 'django.template.loaders.app_directories.Loader', 95 | ) 96 | 97 | MIDDLEWARE_CLASSES = ( 98 | 'django.middleware.common.CommonMiddleware', 99 | 'django.contrib.sessions.middleware.SessionMiddleware', 100 | 'django.middleware.csrf.CsrfViewMiddleware', 101 | 'django.contrib.auth.middleware.AuthenticationMiddleware', 102 | 'django.contrib.messages.middleware.MessageMiddleware', 103 | # Uncomment the next line for simple clickjacking protection: 104 | # 'django.middleware.clickjacking.XFrameOptionsMiddleware', 105 | # 'ispdb.middleware.x_ssl', 106 | ) 107 | 108 | ROOT_URLCONF = 'ispdb.urls' 109 | 110 | TEMPLATE_DIRS = ( 111 | # Put strings here, like "/home/html/django_templates" or "C:/www/django/templates". 112 | # Always use forward slashes, even on Windows. 113 | # Don't forget to use absolute paths, not relative paths. 114 | "%s/templates" % project, 115 | ) 116 | 117 | INSTALLED_APPS = ( 118 | 'django.contrib.auth', 119 | 'django.contrib.contenttypes', 120 | 'django.contrib.sessions', 121 | 'django.contrib.sites', 122 | 'django.contrib.admin', 123 | 'django.contrib.comments', 124 | 'django.contrib.staticfiles', 125 | 'django_openid_auth', 126 | 'ispdb.config' 127 | ) 128 | 129 | AUTHENTICATION_BACKENDS = ( 130 | 'django_openid_auth.auth.OpenIDBackend', 131 | 'django.contrib.auth.backends.ModelBackend', 132 | ) 133 | 134 | OPENID_CREATE_USERS = True 135 | 136 | OPENID_UPDATE_DETAILS_FROM_SREG = True 137 | 138 | LOGIN_URL = '/openid/login' 139 | LOGIN_REDIRECT_URL = '/' 140 | DOCS_ROOT="/Users/davida/src/django/docs/" 141 | 142 | ROOT = '/' 143 | -------------------------------------------------------------------------------- /ispdb/apache/staging.py: -------------------------------------------------------------------------------- 1 | # Django settings for ispdb project. 2 | 3 | #Calculate the path based on the location of the WSGI script. 4 | import os 5 | apache_configuration = os.path.dirname(__file__) 6 | project = os.path.dirname(apache_configuration) 7 | workspace = os.path.dirname(project) 8 | 9 | DEBUG = True 10 | TEMPLATE_DEBUG = DEBUG 11 | 12 | ADMINS = ( 13 | # ('Your Name', 'your_email@domain.com'), 14 | ) 15 | 16 | MANAGERS = ADMINS 17 | 18 | DATABASE_ENGINE = 'mysql' # 'postgresql_psycopg2', 'postgresql', 'mysql', 'sqlite3' or 'oracle'. 19 | #DATABASE_NAME = "%s/ispdb.sqlite" % project 20 | DATABASE_NAME = 'stage_ispdb' 21 | DATABASE_USER = 'ispdb_user' # Not used with sqlite3. 22 | DATABASE_PASSWORD = '' # Not used with sqlite3. 23 | DATABASE_HOST = 'db-write' # Set to empty string for localhost. Not used with sqlite3. 24 | DATABASE_PORT = '' # Set to empty string for default. Not used with sqlite3. 25 | 26 | # Local time zone for this installation. Choices can be found here: 27 | # http://en.wikipedia.org/wiki/List_of_tz_zones_by_name 28 | # although not all choices may be available on all operating systems. 29 | # If running in a Windows environment this must be set to the same as your 30 | # system time zone. 31 | TIME_ZONE = 'America/Chicago' 32 | 33 | # Language code for this installation. All choices can be found here: 34 | # http://www.i18nguy.com/unicode/language-identifiers.html 35 | LANGUAGE_CODE = 'en-us' 36 | 37 | SITE_ID = 1 38 | 39 | # If you set this to False, Django will make some optimizations so as not 40 | # to load the internationalization machinery. 41 | USE_I18N = True 42 | 43 | # Absolute path to the directory that holds media. 44 | # Example: "/home/media/media.lawrence.com/" 45 | MEDIA_ROOT = '/var/www/ispdb/media' 46 | 47 | # URL that handles the media served from MEDIA_ROOT. Make sure to use a 48 | # trailing slash if there is a path component (optional in other cases). 49 | # Examples: "http://media.lawrence.com", "http://example.com/media/" 50 | MEDIA_URL = '/media/' 51 | 52 | # URL prefix for admin media -- CSS, JavaScript and images. Make sure to use a 53 | # trailing slash. 54 | # Examples: "http://foo.com/media/", "/media/". 55 | ADMIN_MEDIA_PREFIX = '/admin_media/' 56 | 57 | # Make this unique, and don't share it with anybody. 58 | SECRET_KEY = '02$7gj$t@dtd7um)d#3zlfg7ommedl#*ekt@@*ysy_fns(lp9+' 59 | 60 | # List of callables that know how to import templates from various sources. 61 | TEMPLATE_LOADERS = ( 62 | 'django.template.loaders.filesystem.load_template_source', 63 | 'django.template.loaders.app_directories.load_template_source', 64 | # 'django.template.loaders.eggs.load_template_source', 65 | ) 66 | 67 | MIDDLEWARE_CLASSES = ( 68 | 'django.middleware.common.CommonMiddleware', 69 | 'django.contrib.sessions.middleware.SessionMiddleware', 70 | 'django.contrib.auth.middleware.AuthenticationMiddleware', 71 | # 'ispdb.middleware.x_ssl', 72 | ) 73 | 74 | ROOT_URLCONF = 'ispdb.urls' 75 | 76 | TEMPLATE_DIRS = ( 77 | # Put strings here, like "/home/html/django_templates" or "C:/www/django/templates". 78 | # Always use forward slashes, even on Windows. 79 | # Don't forget to use absolute paths, not relative paths. 80 | "%s/templates" % project, 81 | ) 82 | 83 | INSTALLED_APPS = ( 84 | 'django.contrib.auth', 85 | 'django.contrib.contenttypes', 86 | 'django.contrib.sessions', 87 | 'django.contrib.sites', 88 | 'django.contrib.admin', 89 | 'django.contrib.comments', 90 | 'django_openid_auth', 91 | 'ispdb.config' 92 | ) 93 | 94 | AUTHENTICATION_BACKENDS = ( 95 | 'django_openid_auth.auth.OpenIDBackend', 96 | 'django.contrib.auth.backends.ModelBackend', 97 | ) 98 | 99 | OPENID_CREATE_USERS = True 100 | 101 | OPENID_UPDATE_DETAILS_FROM_SREG = True 102 | 103 | LOGIN_URL = '/openid/login' 104 | LOGIN_REDIRECT_URL = '/' 105 | DOCS_ROOT="/Users/davida/src/django/docs/" 106 | LOCAL_DEVELOPMENT = True 107 | USE_I18N = False 108 | 109 | ROOT = '/' 110 | -------------------------------------------------------------------------------- /ispdb/apache/trunk.py: -------------------------------------------------------------------------------- 1 | # Django settings for ispdb project. 2 | 3 | #Calculate the path based on the location of the WSGI script. 4 | import os 5 | apache_configuration = os.path.dirname(__file__) 6 | project = os.path.dirname(apache_configuration) 7 | workspace = os.path.dirname(project) 8 | 9 | DEBUG = True 10 | TEMPLATE_DEBUG = DEBUG 11 | 12 | ADMINS = ( 13 | # ('Your Name', 'your_email@domain.com'), 14 | ) 15 | 16 | MANAGERS = ADMINS 17 | 18 | DATABASE_ENGINE = 'sqlite3' # 'postgresql_psycopg2', 'postgresql', 'mysql', 'sqlite3' or 'oracle'. 19 | #DATABASE_NAME = "%s/ispdb.sqlite" % project 20 | DATABASE_NAME = '/var/ispdb/trunk/ispdb.sqlite' 21 | DATABASE_USER = '' # Not used with sqlite3. 22 | DATABASE_PASSWORD = '' # Not used with sqlite3. 23 | DATABASE_HOST = '' # Set to empty string for localhost. Not used with sqlite3. 24 | DATABASE_PORT = '' # Set to empty string for default. Not used with sqlite3. 25 | 26 | # Local time zone for this installation. Choices can be found here: 27 | # http://en.wikipedia.org/wiki/List_of_tz_zones_by_name 28 | # although not all choices may be available on all operating systems. 29 | # If running in a Windows environment this must be set to the same as your 30 | # system time zone. 31 | TIME_ZONE = 'America/Chicago' 32 | 33 | # Language code for this installation. All choices can be found here: 34 | # http://www.i18nguy.com/unicode/language-identifiers.html 35 | LANGUAGE_CODE = 'en-us' 36 | 37 | SITE_ID = 1 38 | 39 | # If you set this to False, Django will make some optimizations so as not 40 | # to load the internationalization machinery. 41 | USE_I18N = True 42 | 43 | # Absolute path to the directory that holds media. 44 | # Example: "/home/media/media.lawrence.com/" 45 | MEDIA_ROOT = '/var/www/ispdb/media' 46 | 47 | # URL that handles the media served from MEDIA_ROOT. Make sure to use a 48 | # trailing slash if there is a path component (optional in other cases). 49 | # Examples: "http://media.lawrence.com", "http://example.com/media/" 50 | MEDIA_URL = '/media/' 51 | 52 | # URL prefix for admin media -- CSS, JavaScript and images. Make sure to use a 53 | # trailing slash. 54 | # Examples: "http://foo.com/media/", "/media/". 55 | ADMIN_MEDIA_PREFIX = '/admin_media/' 56 | 57 | # Make this unique, and don't share it with anybody. 58 | SECRET_KEY = '02$7gj$t@dtd7um)d#3zlfg7ommedl#*ekt@@*ysy_fns(lp9+' 59 | 60 | # List of callables that know how to import templates from various sources. 61 | TEMPLATE_LOADERS = ( 62 | 'django.template.loaders.filesystem.load_template_source', 63 | 'django.template.loaders.app_directories.load_template_source', 64 | # 'django.template.loaders.eggs.load_template_source', 65 | ) 66 | 67 | MIDDLEWARE_CLASSES = ( 68 | 'django.middleware.common.CommonMiddleware', 69 | 'django.contrib.sessions.middleware.SessionMiddleware', 70 | 'django.contrib.auth.middleware.AuthenticationMiddleware', 71 | # 'ispdb.middleware.x_ssl', 72 | ) 73 | 74 | ROOT_URLCONF = 'ispdb.urls' 75 | 76 | TEMPLATE_DIRS = ( 77 | # Put strings here, like "/home/html/django_templates" or "C:/www/django/templates". 78 | # Always use forward slashes, even on Windows. 79 | # Don't forget to use absolute paths, not relative paths. 80 | "%s/templates" % project, 81 | ) 82 | 83 | INSTALLED_APPS = ( 84 | 'django.contrib.auth', 85 | 'django.contrib.contenttypes', 86 | 'django.contrib.sessions', 87 | 'django.contrib.sites', 88 | 'django.contrib.admin', 89 | 'django.contrib.comments', 90 | 'django_openid_auth', 91 | 'ispdb.config' 92 | ) 93 | 94 | AUTHENTICATION_BACKENDS = ( 95 | 'django_openid_auth.auth.OpenIDBackend', 96 | 'django.contrib.auth.backends.ModelBackend', 97 | ) 98 | 99 | OPENID_CREATE_USERS = True 100 | 101 | OPENID_UPDATE_DETAILS_FROM_SREG = True 102 | 103 | LOGIN_URL = '/openid/login' 104 | LOGIN_REDIRECT_URL = '/' 105 | DOCS_ROOT="/Users/davida/src/django/docs/" 106 | LOCAL_DEVELOPMENT = True 107 | USE_I18N = False 108 | 109 | ROOT = '/' 110 | -------------------------------------------------------------------------------- /ispdb/apache/wsgi.conf: -------------------------------------------------------------------------------- 1 | Alias /media/ "/var/www/ispdb/media/" 2 | Alias /static/ "/var/www/ispdb/static/" 3 | 4 | Order allow,deny 5 | Options Indexes 6 | Allow from all 7 | IndexOptions FancyIndexing 8 | 9 | 10 | WSGIDaemonProcess ispdb user=ispdb group=ispdb 11 | WSGIScriptAlias /ispdb /var/www/ispdb/apache/ispdb.wsgi 12 | WSGIPythonPath /var/www/ispdb/ 13 | WSGIProcessGroup ispdb 14 | 15 | 16 | Allow from all 17 | 18 | 19 | -------------------------------------------------------------------------------- /ispdb/audit.py: -------------------------------------------------------------------------------- 1 | # from http://code.djangoproject.com/wiki/AuditTrail 2 | 3 | import copy 4 | import re 5 | 6 | from django.contrib import admin 7 | from django.core.exceptions import ImproperlyConfigured 8 | from django.db import models 9 | try: 10 | import settings_audit 11 | except ImportError: 12 | settings_audit = None 13 | 14 | value_error_re = re.compile("^.+'(.+)'$") 15 | 16 | 17 | class AuditTrail(object): 18 | def __init__(self, show_in_admin=False, save_change_type=True, 19 | audit_deletes=True, track_fields=None): 20 | self.opts = {} 21 | self.opts['show_in_admin'] = show_in_admin 22 | self.opts['save_change_type'] = save_change_type 23 | self.opts['audit_deletes'] = audit_deletes 24 | if track_fields: 25 | self.opts['track_fields'] = track_fields 26 | else: 27 | self.opts['track_fields'] = [] 28 | 29 | def contribute_to_class(self, cls, name): 30 | # This should only get added once the class is otherwise complete 31 | def _contribute(sender, **kwargs): 32 | model = create_audit_model(sender, **self.opts) 33 | if self.opts['show_in_admin']: 34 | # Enable admin integration 35 | # If ModelAdmin needs options or different base class, find 36 | # some way to make the commented code work 37 | # cls_admin_name = cls.__name__ + 'Admin' 38 | # clsAdmin = type(cls_admin_name, (admin.ModelAdmin,),{}) 39 | # admin.site.register(cls, clsAdmin) 40 | # Otherwise, register class with default ModelAdmin 41 | admin.site.register(model) 42 | descriptor = AuditTrailDescriptor(model._default_manager, 43 | sender._meta.pk.attname) 44 | setattr(sender, name, descriptor) 45 | 46 | def _audit_track(instance, field_arr, **kwargs): 47 | field_name = field_arr[0] 48 | try: 49 | return getattr(instance, field_name) 50 | except: 51 | if len(field_arr) > 2: 52 | if callable(field_arr[2]): 53 | fn = field_arr[2] 54 | return fn(instance) 55 | else: 56 | return field_arr[2] 57 | 58 | def _audit(sender, instance, created, **kwargs): 59 | # Write model changes to the audit model. 60 | # instance is the current (non-audit) model. 61 | kwargs = {} 62 | for field in sender._meta.fields: 63 | #kwargs[field.attname] = getattr(instance, field.attname) 64 | kwargs[field.name] = getattr(instance, field.name) 65 | if self.opts['save_change_type']: 66 | if created: 67 | kwargs['_audit_change_type'] = 'I' 68 | else: 69 | kwargs['_audit_change_type'] = 'U' 70 | for field_arr in model._audit_track: 71 | kwargs[field_arr[0]] = _audit_track(instance, field_arr) 72 | model._default_manager.create(**kwargs) 73 | models.signals.post_save.connect(_audit, sender=cls, weak=False) 74 | 75 | if self.opts['audit_deletes']: 76 | def _audit_delete(sender, instance, **kwargs): 77 | # Write model changes to the audit model 78 | kwargs = {} 79 | for field in sender._meta.fields: 80 | kwargs[field.name] = getattr(instance, field.name) 81 | if self.opts['save_change_type']: 82 | kwargs['_audit_change_type'] = 'D' 83 | for field_arr in model._audit_track: 84 | kwargs[field_arr[0]] = _audit_track(instance, 85 | field_arr) 86 | model._default_manager.create(**kwargs) 87 | models.signals.pre_delete.connect(_audit_delete, sender=cls, 88 | weak=False) 89 | models.signals.class_prepared.connect(_contribute, sender=cls, 90 | weak=False) 91 | 92 | 93 | class AuditTrailDescriptor(object): 94 | def __init__(self, manager, pk_attribute): 95 | self.manager = manager 96 | self.pk_attribute = pk_attribute 97 | 98 | def __get__(self, instance=None, owner=None): 99 | if instance is None: 100 | return create_audit_manager_class(self.manager) 101 | else: 102 | return create_audit_manager_with_pk(self.manager, 103 | self.pk_attribute, instance._get_pk_val()) 104 | 105 | def __set__(self, instance, value): 106 | raise AttributeError("Audit trail may not be edited in this manner.") 107 | 108 | 109 | def create_audit_manager_with_pk(manager, pk_attribute, pk): 110 | """Create an audit trail manager based on the current object""" 111 | class AuditTrailWithPkManager(manager.__class__): 112 | def __init__(self): 113 | self.model = manager.model 114 | 115 | def get_query_set(self): 116 | return super(AuditTrailWithPkManager, self).get_query_set().filter( 117 | **{pk_attribute: pk}) 118 | return AuditTrailWithPkManager() 119 | 120 | 121 | def create_audit_manager_class(manager): 122 | """Create an audit trail manager based on the current object""" 123 | class AuditTrailManager(manager.__class__): 124 | def __init__(self): 125 | self.model = manager.model 126 | return AuditTrailManager() 127 | 128 | 129 | def create_audit_model(cls, **kwargs): 130 | """Create an audit model for the specific class""" 131 | name = cls.__name__ + 'Audit' 132 | 133 | class Meta: 134 | db_table = '%s_audit' % cls._meta.db_table 135 | verbose_name_plural = '%s audit trail' % cls._meta.verbose_name 136 | ordering = ['-_audit_timestamp'] 137 | 138 | # Set up a dictionary to simulate declarations within a class 139 | attrs = { 140 | '__module__': cls.__module__, 141 | 'Meta': Meta, 142 | '_audit_id': models.AutoField(primary_key=True), 143 | '_audit_timestamp': models.DateTimeField(auto_now_add=True, 144 | db_index=True), 145 | '_audit__str__': cls.__str__.im_func, 146 | '__str__': lambda self: '%s as of %s' % (self._audit__str__(), 147 | self._audit_timestamp), 148 | '_audit_track': _track_fields(track_fields=kwargs['track_fields'], 149 | unprocessed=True) 150 | } 151 | 152 | if 'save_change_type' in kwargs and kwargs['save_change_type']: 153 | attrs['_audit_change_type'] = models.CharField(max_length=1) 154 | 155 | # Copy the fields from the existing model to the audit model 156 | for field in cls._meta.fields: 157 | #if field.attname in attrs: 158 | if field.name in attrs: 159 | raise ImproperlyConfigured("%s cannot use %s as it is needed by " 160 | "AuditTrail." % (cls.__name__, 161 | field.attname)) 162 | if isinstance(field, models.AutoField): 163 | # Audit models have a separate AutoField 164 | attrs[field.name] = models.IntegerField(db_index=True, 165 | editable=False) 166 | else: 167 | attrs[field.name] = copy.copy(field) 168 | # If 'unique' is in there, we need to remove it, otherwise the 169 | # index is created and multiple audit entries for one item fail. 170 | attrs[field.name]._unique = False 171 | # If a model has primary_key = True, a second primary key would be 172 | # created in the audit model. Set primary_key to false. 173 | attrs[field.name].primary_key = False 174 | 175 | for track_field in _track_fields(kwargs['track_fields']): 176 | if track_field['name'] in attrs: 177 | raise NameError('Field named "%s" already exists in audit version ' 178 | 'of %s' % (track_field['name'], cls.__name__)) 179 | attrs[track_field['name']] = copy.copy(track_field['field']) 180 | 181 | return type(name, (models.Model,), attrs) 182 | 183 | 184 | def _build_track_field(track_item): 185 | track = {} 186 | track['name'] = track_item[0] 187 | if isinstance(track_item[1], models.Field): 188 | track['field'] = track_item[1] 189 | elif issubclass(track_item[1], models.Model): 190 | track['field'] = models.ForeignKey(track_item[1]) 191 | else: 192 | raise TypeError('Track fields only support items that are Fields or ' 193 | 'Models.') 194 | return track 195 | 196 | 197 | def _track_fields(track_fields=None, unprocessed=False): 198 | # Add in the fields from the Audit class "track" attribute. 199 | tracks_found = [] 200 | 201 | if settings_audit: 202 | global_track_fields = getattr(settings_audit, 'GLOBAL_TRACK_FIELDS', 203 | []) 204 | for track_item in global_track_fields: 205 | if unprocessed: 206 | tracks_found.append(track_item) 207 | else: 208 | tracks_found.append(_build_track_field(track_item)) 209 | 210 | if track_fields: 211 | for track_item in track_fields: 212 | if unprocessed: 213 | tracks_found.append(track_item) 214 | else: 215 | tracks_found.append(_build_track_field(track_item)) 216 | return tracks_found 217 | -------------------------------------------------------------------------------- /ispdb/config/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mozilla/ispdb/3b87b2c797d0d56cdf5343cca831af721360ca53/ispdb/config/__init__.py -------------------------------------------------------------------------------- /ispdb/config/admin.py: -------------------------------------------------------------------------------- 1 | from django.contrib import admin 2 | from ispdb.config.models import Config, Domain, DomainRequest 3 | 4 | 5 | class DomainInline(admin.TabularInline): 6 | model = Domain 7 | extra = 0 8 | 9 | 10 | class DomainRequestInline(admin.TabularInline): 11 | model = DomainRequest 12 | extra = 0 13 | 14 | 15 | class ConfigAdmin(admin.ModelAdmin): 16 | inlines = [ 17 | DomainRequestInline, 18 | ] 19 | radio_fields = {"incoming_type": admin.VERTICAL} 20 | list_display = ['display_name', 'status', 'list_domains'] 21 | list_filter = ['status'] 22 | search_fields = ['display_name', 'domains__name'] 23 | 24 | def change_view(self, request, obj_id): 25 | c = self.model.objects.get(pk=obj_id) 26 | if not c.domainrequests.all(): 27 | self.inlines = [DomainInline, ] 28 | return super(ConfigAdmin, self).change_view(request, obj_id) 29 | 30 | def list_domains(self, obj): 31 | if obj.domains.all(): 32 | return '
'.join(str(c) for c in obj.domains.all()) 33 | else: 34 | return '
'.join(str(c) for c in obj.domainrequests.all()) 35 | list_domains.allow_tags = True 36 | 37 | admin.site.register(Config, ConfigAdmin) 38 | -------------------------------------------------------------------------------- /ispdb/config/models.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | from django.conf import settings 4 | from django.contrib.auth.models import User 5 | from django.db import models 6 | 7 | import ispdb.audit as audit 8 | 9 | 10 | class Domain(models.Model): 11 | name = models.CharField(max_length=100, unique=True, 12 | verbose_name="Email domain", 13 | help_text="(e.g. \"gmail.com\")") 14 | config = models.ForeignKey('Config', related_name="domains", 15 | blank=True) # blank is for requests and rejects 16 | 17 | @staticmethod 18 | def create_from_domainrequest(domainrequest): 19 | """create domain from domainrequest 20 | """ 21 | d = Domain() 22 | d.name = domainrequest.name 23 | d.config = domainrequest.config 24 | return d 25 | 26 | def __str__(self): 27 | return str(self.name) 28 | 29 | def __unicode__(self): 30 | return self.name 31 | 32 | 33 | class DomainRequest(models.Model): 34 | name = models.CharField(max_length=100, verbose_name="Email domain", 35 | help_text="(e.g. \"gmail.com\")") 36 | config = models.ForeignKey('Config', related_name="domainrequests", 37 | blank=True, null=True) 38 | votes = models.IntegerField(default=1) 39 | 40 | def __str__(self): 41 | return str(self.name) 42 | 43 | def __unicode__(self): 44 | return self.name 45 | 46 | 47 | class Config(models.Model): 48 | """ 49 | A Config object contains all of the metadata about a configuration. 50 | It is 1-many mapped to Domain objects which apply to it. 51 | 52 | """ 53 | 54 | class Meta: 55 | permissions = ( 56 | ('can_approve', 'Can approve configurations'), 57 | ) 58 | 59 | def __str__(self): 60 | "for use in the admin UI" 61 | return str(self.display_name) 62 | 63 | def __unicode__(self): 64 | return self.display_name 65 | 66 | owner = models.ForeignKey(User, unique=False, blank=True, null=True, 67 | on_delete=models.SET_NULL) 68 | last_update_datetime = models.DateTimeField(auto_now=True) 69 | created_datetime = models.DateTimeField(auto_now_add=True) 70 | deleted_datetime = models.DateTimeField(null=True, blank=True) 71 | CONFIG_CHOICES = [ 72 | ("requested", "requested"), 73 | ("suggested", "suggested"), 74 | ("approved", "approved"), 75 | ("invalid", "invalid"), 76 | ("deleted", "deleted"), 77 | ] 78 | status = models.CharField(max_length=20, choices=CONFIG_CHOICES) 79 | last_status = models.CharField(max_length=20, choices=CONFIG_CHOICES, 80 | blank=True) 81 | email_provider_id = models.CharField(max_length=50, blank=True) 82 | display_name = models.CharField( 83 | max_length=100, 84 | verbose_name="The name of this ISP", 85 | help_text="e.g. \"Google's Gmail Service\"") 86 | display_short_name = models.CharField( 87 | max_length=100, 88 | verbose_name="A short version of the ISP name", 89 | help_text="e.g. \"Gmail\"") 90 | INCOMING_TYPE_CHOICES = ( 91 | ('imap', 'IMAP'), 92 | ('pop3', 'POP'), 93 | ) 94 | incoming_type = models.CharField(max_length=100, 95 | choices=INCOMING_TYPE_CHOICES) 96 | incoming_hostname = models.CharField(max_length=100) 97 | incoming_port = models.PositiveIntegerField() 98 | INCOMING_SOCKET_TYPE_CHOICES = ( 99 | ("plain", "No encryption"), 100 | ("SSL", "SSL/TLS"), 101 | ("STARTTLS", "STARTTLS"), 102 | ) 103 | incoming_socket_type = models.CharField( 104 | max_length=8, 105 | choices=INCOMING_SOCKET_TYPE_CHOICES) 106 | incoming_username_form = models.CharField(max_length=100, 107 | verbose_name="Username formula") 108 | INCOMING_AUTHENTICATION_CHOICES = ( 109 | ("password-cleartext", "Unencrypted Password"), 110 | ("password-encrypted", "Encrypted Password"), 111 | ("NTLM", "NTLM"), 112 | ("GSSAPI", "GSSAPI"), 113 | ) 114 | incoming_authentication = models.CharField( 115 | max_length=20, 116 | choices=INCOMING_AUTHENTICATION_CHOICES, 117 | default="password-encrypted", 118 | help_text="""Unencrypted Password: Send password 119 | unencrypted in the clear. Dangerous, if SSL isn't used 120 | either. PLAIN or LOGIN etc…
121 | 122 | Encrypted Password: Hashed password. 123 | Offers minimal protection for passwords. CRAM-MD5 or 124 | DIGEST-MD5. Not NTLM.
""" 125 | ) 126 | outgoing_hostname = models.CharField(max_length=100) 127 | outgoing_port = models.PositiveIntegerField() 128 | OUTGOING_SOCKET_TYPE_CHOICES = ( 129 | ("plain", "No encryption"), 130 | ("SSL", "SSL/TLS"), 131 | ("STARTTLS", "STARTTLS"), 132 | ) 133 | outgoing_socket_type = models.CharField( 134 | max_length=8, 135 | choices=OUTGOING_SOCKET_TYPE_CHOICES) 136 | outgoing_username_form = models.CharField(max_length=100, 137 | verbose_name="Username formula") 138 | OUTGOING_AUTHENTICATION_CHOICES = ( 139 | ("password-cleartext", "Unencrypted Password"), 140 | ("password-encrypted", "Encrypted Password"), 141 | ("none", "Client IP address"), 142 | ("smtp-after-pop", "SMTP-after-POP"), 143 | ("NTLM", "NTLM"), 144 | ("GSSAPI", "GSSAPI"), 145 | ) 146 | outgoing_authentication = models.CharField( 147 | max_length=20, 148 | choices=OUTGOING_AUTHENTICATION_CHOICES, 149 | default="password-encrypted", 150 | help_text="""Unencrypted Password: Send password 151 | unencrypted in the clear. Dangerous, if SSL isn't used 152 | either. PLAIN or LOGIN etc…
153 | 154 | Encrypted Password: Hashed password. 155 | Offers minimal protection for passwords. CRAM-MD5 or 156 | DIGEST-MD5. Not NTLM.
""" 157 | ) 158 | outgoing_add_this_server = models.BooleanField( 159 | verbose_name="Add this server to list???") 160 | outgoing_use_global_preferred_server = models.BooleanField( 161 | verbose_name="Use global server instead") 162 | 163 | locked = models.BooleanField() 164 | 165 | history = audit.AuditTrail() 166 | 167 | 168 | class Issue(models.Model): 169 | title = models.CharField(max_length=50) 170 | description = models.TextField(max_length=3000) 171 | created_datetime = models.DateTimeField(auto_now_add=True) 172 | config = models.ForeignKey(Config, related_name="reported_issues") 173 | updated_config = models.ForeignKey(Config, null=True, related_name="issue") 174 | owner = models.ForeignKey(User, unique=False, blank=True, null=True, 175 | on_delete=models.SET_NULL) 176 | STATUS_CHOICES = [ 177 | ("open", "open"), 178 | ("closed", "closed"), 179 | ] 180 | status = models.CharField(max_length=20, choices=STATUS_CHOICES, 181 | default="open") 182 | 183 | 184 | class CommonConfigURL(models.Model): 185 | url = models.URLField( 186 | verbose_name="URL of the page") 187 | config = models.ForeignKey(Config, related_name="%(class)s_set") 188 | 189 | class Meta: 190 | abstract = True 191 | 192 | def __str__(self): 193 | return str(self.url) 194 | 195 | def __unicode__(self): 196 | return self.url 197 | 198 | 199 | class CommonURLDesc(models.Model): 200 | description = models.TextField( 201 | max_length=100, 202 | verbose_name="Description") 203 | language = models.CharField( 204 | max_length=10, 205 | verbose_name="Language", 206 | choices=settings.LANGUAGES) 207 | 208 | class Meta: 209 | abstract = True 210 | 211 | def __str__(self): 212 | return str(self.description) 213 | 214 | def __unicode__(self): 215 | return self.description 216 | 217 | 218 | class DocURL(CommonConfigURL): 219 | pass 220 | DocURL._meta.get_field('url').verbose_name = ("URL of the page describing " 221 | "these settings") 222 | 223 | 224 | class DocURLDesc(CommonURLDesc): 225 | docurl = models.ForeignKey(DocURL, related_name="descriptions") 226 | DocURLDesc._meta.get_field('description').verbose_name = ('Description of the ' 227 | 'settings page') 228 | 229 | 230 | class EnableURL(CommonConfigURL): 231 | pass 232 | EnableURL._meta.get_field('url').verbose_name = ("URL of the page with enable " 233 | "instructions") 234 | 235 | 236 | class EnableURLInst(CommonURLDesc): 237 | enableurl = models.ForeignKey(EnableURL, related_name="instructions") 238 | EnableURLInst._meta.get_field('description').verbose_name = ('Instruction') 239 | -------------------------------------------------------------------------------- /ispdb/config/serializers.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | import lxml.etree as ET 4 | from StringIO import StringIO 5 | 6 | from ispdb.config.models import Domain, DomainRequest 7 | 8 | # The serializers in reverse order, newest at the top. 9 | 10 | 11 | def xmlOneDotOne(data): 12 | """ 13 | Return the configuration using the XML document that Thunderbird is 14 | expecting. 15 | """ 16 | desiredOutput = { 17 | "display_name": "displayName", 18 | "display_short_name": "displayShortName", 19 | "incoming_hostname": "hostname", 20 | "incoming_port": "port", 21 | "incoming_socket_type": "socketType", 22 | "incoming_username_form": "username", 23 | "incoming_authentication": "authentication", 24 | "outgoing_hostname": "hostname", 25 | "outgoing_port": "port", 26 | "outgoing_socket_type": "socketType", 27 | "outgoing_username_form": "username", 28 | "outgoing_authentication": "authentication", 29 | } 30 | incoming = None 31 | outgoing = None 32 | config = ET.Element("clientConfig") 33 | config.attrib["version"] = "1.1" 34 | emailProvider = ET.SubElement(config, "emailProvider") 35 | qs = Domain.objects.filter(config=data) or ( 36 | DomainRequest.objects.filter(config=data)) 37 | for domain in qs: 38 | if not data.email_provider_id: 39 | data.email_provider_id = domain.name 40 | ET.SubElement(emailProvider, "domain").text = domain.name 41 | emailProvider.attrib["id"] = data.email_provider_id 42 | for field in data._meta.fields: 43 | if field.name not in desiredOutput: 44 | continue 45 | if field.name.startswith("incoming"): 46 | if incoming is None: 47 | incoming = ET.SubElement(emailProvider, "incomingServer") 48 | incoming.attrib["type"] = data.incoming_type 49 | name = field.name 50 | currParent = incoming 51 | elif field.name.startswith("outgoing"): 52 | if outgoing is None: 53 | outgoing = ET.SubElement(emailProvider, "outgoingServer") 54 | outgoing.attrib["type"] = "smtp" 55 | name = field.name 56 | currParent = outgoing 57 | else: 58 | name = field.name 59 | currParent = emailProvider 60 | if (name == "incoming_username_form") and ( 61 | data.incoming_username_form == ""): 62 | data.incoming_username_form = data.outgoing_username_form 63 | name = desiredOutput[name] 64 | e = ET.SubElement(currParent, name) 65 | text = getattr(data, field.name) 66 | 67 | if type(text) is bool: 68 | # Force boolean values to use lowercase. 69 | text = unicode(text).lower() 70 | else: 71 | # Force other values to be converted into unicode strings. 72 | text = unicode(text) 73 | e.text = text 74 | # EnableURL 75 | for enableurl in data.enableurl_set.all(): 76 | enable = ET.SubElement(emailProvider, "enable") 77 | enable.attrib["visiturl"] = enableurl.url 78 | for inst in enableurl.instructions.all(): 79 | d = ET.SubElement(enable, "instruction") 80 | d.attrib["lang"] = inst.language 81 | d.text = unicode(inst.description) 82 | # DocURL 83 | for docurl in data.docurl_set.all(): 84 | doc = ET.SubElement(emailProvider, "documentation") 85 | doc.attrib["url"] = docurl.url 86 | for desc in docurl.descriptions.all(): 87 | d = ET.SubElement(doc, "descr") 88 | d.attrib["lang"] = desc.language 89 | d.text = unicode(desc.description) 90 | 91 | retval = StringIO("w") 92 | xml = ET.ElementTree(config) 93 | xml.write(retval, encoding="UTF-8", xml_declaration=True) 94 | return retval.getvalue() 95 | 96 | 97 | def xmlOneDotZero(data): 98 | """ 99 | Return the configuration using the XML document that Thunderbird is 100 | expecting. 101 | """ 102 | desiredOutput = { 103 | "display_name": "displayName", 104 | "display_short_name": "displayShortName", 105 | "incoming_hostname": "hostname", 106 | "incoming_port": "port", 107 | "incoming_socket_type": "socketType", 108 | "incoming_username_form": "username", 109 | "incoming_authentication": "authentication", 110 | "outgoing_hostname": "hostname", 111 | "outgoing_port": "port", 112 | "outgoing_socket_type": "socketType", 113 | "outgoing_username_form": "username", 114 | "outgoing_authentication": "authentication", 115 | "outgoing_add_this_server": "addThisServer", 116 | "outgoing_use_global_preferred_server": "useGlobalPreferredServer", 117 | } 118 | authentication_values = { 119 | "password-cleartext": "plain", 120 | "password-encrypted": "secure", 121 | "client-ip-address": "none", 122 | "smtp-after-pop": "none", 123 | } 124 | incoming = None 125 | outgoing = None 126 | config = ET.Element("clientConfig") 127 | config.attrib["version"] = "1.0" 128 | emailProvider = ET.SubElement(config, "emailProvider") 129 | for domain in Domain.objects.filter(config=data): 130 | ET.SubElement(emailProvider, "domain").text = domain.name 131 | if not data.email_provider_id: 132 | data.email_provider_id = domain.name 133 | emailProvider.attrib["id"] = data.email_provider_id 134 | for field in data._meta.fields: 135 | if field.name not in desiredOutput: 136 | continue 137 | if field.name.startswith("incoming"): 138 | if incoming is None: 139 | incoming = ET.SubElement(emailProvider, "incomingServer") 140 | incoming.attrib["type"] = data.incoming_type 141 | name = field.name 142 | currParent = incoming 143 | elif field.name.startswith("outgoing"): 144 | if outgoing is None: 145 | outgoing = ET.SubElement(emailProvider, "outgoingServer") 146 | outgoing.attrib["type"] = "smtp" 147 | name = field.name 148 | currParent = outgoing 149 | else: 150 | name = field.name 151 | currParent = emailProvider 152 | if (name == "incoming_username_form") and ( 153 | data.incoming_username_form == ""): 154 | data.incoming_username_form = data.outgoing_username_form 155 | name = desiredOutput[name] 156 | e = ET.SubElement(currParent, name) 157 | text = getattr(data, field.name) 158 | 159 | if type(text) is bool: 160 | # Force boolean values to use lowercase. 161 | text = unicode(text).lower() 162 | else: 163 | # Force other values to be converted into unicode strings. 164 | text = unicode(text) 165 | 166 | # Fix up the data to make it 1.0 compliant. 167 | if name == "authentication": 168 | text = authentication_values.get(text, text) 169 | 170 | e.text = text 171 | 172 | retval = StringIO("w") 173 | xml = ET.ElementTree(config) 174 | xml.write(retval, encoding="UTF-8", xml_declaration=True) 175 | return retval.getvalue() 176 | 177 | # This is the data format version, not the application version. 178 | # It only needs to change when the data format needs to change in a way 179 | # that breaks old clients, which are hopefully exceptional cases. 180 | # In other words, this does *not* change with every TB major release. 181 | _serializers = { 182 | "1.0": xmlOneDotZero, 183 | "1.1": xmlOneDotOne, 184 | } 185 | 186 | 187 | def get(version): 188 | # If there is no version requested, return the most recent version. 189 | if version is None: 190 | return xmlOneDotZero 191 | return _serializers.get(version, None) 192 | -------------------------------------------------------------------------------- /ispdb/config/static/css/base.css: -------------------------------------------------------------------------------- 1 | body { margin:0; padding:0; font-size:12px; font-family:"Lucida Grande","DejaVu Sans","Bitstream Vera Sans",Verdana,Arial,sans-serif; color:#333; background:#fff; } 2 | 3 | /* LINKS */ 4 | a:link, a:visited { color: #5b80b2; text-decoration:none; } 5 | a:hover { color: #036; } 6 | a img { border:none; } 7 | a.section:link, a.section:visited { color: white; text-decoration:none; } 8 | 9 | /* GLOBAL DEFAULTS */ 10 | p, ol, ul, dl { margin:.2em 0 .8em 0; } 11 | p { padding:0; line-height:140%; } 12 | 13 | h1,h2,h3,h4,h5 { font-weight:bold; } 14 | h1 { font-size:18px; color:#666; padding:0 6px 0 0; margin:0 0 .2em 0; } 15 | h2 { font-size:16px; margin:1em 0 .5em 0; } 16 | h2.subhead { font-weight:normal;margin-top:0; } 17 | h3 { font-size:14px; margin:.8em 0 .3em 0; color:#666; font-weight:bold; } 18 | h4 { font-size:12px; margin:1em 0 .8em 0; padding-bottom:3px; } 19 | h5 { font-size:10px; margin:1.5em 0 .5em 0; color:#666; text-transform:uppercase; letter-spacing:1px; } 20 | 21 | ul li { list-style-type:square; padding:1px 0; } 22 | ul.plainlist { margin-left:0 !important; } 23 | ul.plainlist li { list-style-type:none; } 24 | li ul { margin-bottom:0; } 25 | li, dt, dd { font-size:11px; line-height:14px; } 26 | dt { font-weight:bold; margin-top:4px; } 27 | dd { margin-left:0; } 28 | 29 | form { margin:0; padding:0; } 30 | fieldset { margin:0; padding:0; } 31 | 32 | blockquote { font-size:11px; color:#777; margin-left:2px; padding-left:10px; border-left:5px solid #ddd; } 33 | code, pre { font-family:"Bitstream Vera Sans Mono", Monaco, "Courier New", Courier, monospace; background:inherit; color:#666; font-size:11px; } 34 | pre.literal-block { margin:10px; background:#eee; padding:6px 8px; } 35 | code strong { color:#930; } 36 | hr { clear:both; color:#eee; background-color:#eee; height:1px; border:none; margin:0; padding:0; font-size:1px; line-height:1px; } 37 | 38 | /* TEXT STYLES & MODIFIERS */ 39 | .small { font-size:11px; } 40 | .tiny { font-size:10px; } 41 | p.tiny { margin-top:-2px; } 42 | .mini { font-size:9px; } 43 | p.mini { margin-top:-3px; } 44 | .help, p.help { font-size:10px !important; color:#999; } 45 | p img, h1 img, h2 img, h3 img, h4 img, td img { vertical-align:middle; } 46 | .quiet, a.quiet:link, a.quiet:visited { color:#999 !important;font-weight:normal !important; } 47 | .quiet strong { font-weight:bold !important; } 48 | .float-right { float:right; } 49 | .float-left { float:left; } 50 | .clear { clear:both; } 51 | .align-left { text-align:left; } 52 | .align-right { text-align:right; } 53 | .example { margin:10px 0; padding:5px 10px; background:#efefef; } 54 | .nowrap { white-space:nowrap; } 55 | 56 | /* TABLES */ 57 | table { border-collapse:collapse; border-color:#ccc; } 58 | td, th { font-size:11px; line-height:13px; border-bottom:1px solid #eee; vertical-align:top; padding:5px; font-family:"Lucida Grande", Verdana, Arial, sans-serif; } 59 | th { text-align:left; font-size:12px; font-weight:bold; } 60 | thead th, 61 | tfoot td { color:#666; padding:2px 5px; font-size:11px; background:#e1e1e1 url(../img/nav-bg.gif) top left repeat-x; border-left:1px solid #ddd; border-bottom:1px solid #ddd; } 62 | tfoot td { border-bottom:none; border-top:1px solid #ddd; } 63 | thead th:first-child, 64 | tfoot td:first-child { border-left:none !important; } 65 | thead th.optional { font-weight:normal !important; } 66 | fieldset table { border-right:1px solid #eee; } 67 | tr.row-label td { font-size:9px; padding-top:2px; padding-bottom:0; border-bottom:none; color:#666; margin-top:-1px; } 68 | tr.alt { background:#f6f6f6; } 69 | .row1 { background:#EDF3FE; } 70 | .row2 { background:white; } 71 | 72 | /* FORM DEFAULTS */ 73 | input, textarea, select { margin:2px 0; padding:2px 3px; vertical-align:middle; font-family:"Lucida Grande", Verdana, Arial, sans-serif; font-weight:normal; font-size:11px; } 74 | textarea { vertical-align:top !important; } 75 | input[type=text], input[type=password], textarea, select, .vTextField { border:1px solid #ccc; } 76 | 77 | /* FORM BUTTONS */ 78 | .button, input[type=submit], input[type=button], .submit-row input { background:white url(../img/nav-bg.gif) bottom repeat-x; padding:3px; color:black; border:1px solid #bbb; border-color:#ddd #aaa #aaa #ddd; } 79 | .button:active, input[type=submit]:active, input[type=button]:active { background-image:url(../img/nav-bg-reverse.gif); background-position:top; } 80 | .button.default, input[type=submit].default, .submit-row input.default { border:2px solid #5b80b2; background:#7CA0C7 url(../img/default-bg.gif) bottom repeat-x; font-weight:bold; color:white; float:right; } 81 | .button.default:active, input[type=submit].default:active { background-image:url(../img/default-bg-reverse.gif); background-position:top; } 82 | 83 | /* MODULES */ 84 | .module { border:1px solid #ccc; margin-bottom:5px; background:white; } 85 | .module p, .module ul, .module h3, .module h4, .module dl, .module pre { padding-left:10px; padding-right:10px; } 86 | .module blockquote { margin-left:12px; } 87 | .module ul, .module ol { margin-left:1.5em; } 88 | .module h3 { margin-top:.6em; } 89 | .module h2, .module caption, .inline-group h2 { margin:0; padding:2px 5px 3px 5px; font-size:11px; text-align:left; font-weight:bold; background:#7CA0C7 url(../img/default-bg.gif) top left repeat-x; color:white; } 90 | .module table { border-collapse: collapse; } 91 | 92 | /* MESSAGES & ERRORS */ 93 | ul.messagelist { padding:0 0 5px 0; margin:0; } 94 | ul.messagelist li { font-size:12px; display:block; padding:4px 5px 4px 25px; margin:0 0 3px 0; border-bottom:1px solid #ddd; color:#666; background:#ffc url(../img/icon_success.gif) 5px .3em no-repeat; } 95 | .errornote { font-size:12px !important; display:block; padding:4px 5px 4px 25px; margin:0 0 3px 0; border:1px solid red; color:red;background:#ffc url(../img/icon_error.gif) 5px .3em no-repeat; } 96 | ul.errorlist { margin:0 !important; padding:0 !important; } 97 | .errorlist li { font-size:12px !important; display:block; padding:4px 5px 4px 25px; margin:0 0 3px 0; border:1px solid red; color:white; background:red url(../img/icon_alert.gif) 5px .3em no-repeat; } 98 | td ul.errorlist { margin:0 !important; padding:0 !important; } 99 | td ul.errorlist li { margin:0 !important; } 100 | .errors { background:#ffc; } 101 | .errors input, .errors select { border:1px solid red; } 102 | div.system-message { background: #ffc; margin: 10px; padding: 6px 8px; font-size: .8em; } 103 | div.system-message p.system-message-title { padding:4px 5px 4px 25px; margin:0; color:red; background:#ffc url(../img/icon_error.gif) 5px .3em no-repeat; } 104 | .description { font-size:12px; padding:5px 0 0 12px; } 105 | #warning { 106 | display: block; margin: 20px auto 0px; width: 700px; background-color: #FFFFCC; color: red; border: medium thin black; padding-left:12px; 107 | } 108 | #warning a {color: DarkRed !important; text-decoration: underline !important} 109 | 110 | 111 | /* PAGE STRUCTURE */ 112 | #container { position:relative; width:100%; padding:0; } 113 | #content { margin:10px 15px; } 114 | #header { width:100%; } 115 | #content-main { float:left; width:100%; } 116 | #content-related { float:right; width:18em; position:relative; margin-right:-19em; } 117 | #footer { clear:both; padding:10px; } 118 | 119 | /* COLUMN TYPES */ 120 | .colMS { margin-right:20em !important; } 121 | .colSM { margin-left:20em !important; } 122 | .colSM #content-related { float:left; margin-right:0; margin-left:-19em; } 123 | .colSM #content-main { float:right; } 124 | .popup .colM { width:95%; } 125 | .subcol { float:left; width:46%; margin-right:15px; } 126 | .dashboard #content { width:500px; } 127 | 128 | /* HEADER */ 129 | #header { background:#394345; color:orange; overflow:hidden; } 130 | #header a:link, #header a:visited { color:white; } 131 | #header a:hover { text-decoration:underline; } 132 | #branding h1 { 133 | padding:0 10px; 134 | font-size:18px; 135 | margin:8px 0; 136 | font-weight:normal; 137 | color:white; 138 | } 139 | #branding h2 { padding:0 10px; font-size:14px; margin:-8px 0 8px 0; font-weight:normal; color:#ffc; } 140 | #user-tools { position:absolute; top:0; right:0; padding:1.2em 10px; font-size:11px; text-align:right; } 141 | 142 | #branding h1 { 143 | padding-top: 20px; 144 | } 145 | 146 | #branding { 147 | background: transparent url(../img/thunderbird.png) 0 0 no-repeat; 148 | margin-right: 200px; 149 | padding-left: 64px; 150 | min-height: 64px; 151 | } 152 | 153 | 154 | /* SIDEBAR */ 155 | #content-related h3 { font-size:12px; color:#666; margin-bottom:3px; } 156 | #content-related h4 { font-size:11px; } 157 | #content-related .module h2 { background:#eee url(../img/nav-bg.gif) bottom left repeat-x; color:#666; } 158 | 159 | div.breadcrumbs { 160 | background: white url(../img/nav-bg-reverse.gif) 0 -10px repeat-x; 161 | padding: 2px 8px 3px 8px; 162 | font-size: 11px; 163 | color: #999; 164 | border-top: 1px solid white; 165 | border-bottom: 1px solid #ccc; 166 | text-align: left; 167 | } 168 | 169 | /* useful for which don't have hrefs */ 170 | a.link { color: #5b80b2; text-decoration:none; } 171 | a.link:hover { 172 | color: #036; 173 | cursor: pointer; 174 | } 175 | 176 | 177 | -------------------------------------------------------------------------------- /ispdb/config/static/css/comments.css: -------------------------------------------------------------------------------- 1 | .comment { 2 | width: 50%; 3 | overflow: auto; 4 | word-wrap: break-word; 5 | padding-top: 10px; 6 | } 7 | 8 | .comment_author { 9 | font-weight: bold; 10 | border: 0px; 11 | font-size: 12px; 12 | } 13 | 14 | .comment_date { 15 | border: 0px; 16 | font-size: 11px; 17 | } 18 | 19 | .comment_body { 20 | padding-top: 10px; 21 | padding-left: 5px; 22 | padding-right: 5px; 23 | } 24 | 25 | .comment_list { 26 | padding-top: 2px; 27 | padding-bottom: 10px; 28 | padding-left:10px; 29 | } 30 | 31 | #showcomments { 32 | padding-top: 2px; 33 | } 34 | 35 | #add_a_comment { 36 | padding-top: 10px; 37 | padding-bottom: 10px; 38 | } 39 | 40 | .remove_comment { 41 | font-size: 10px; 42 | } 43 | -------------------------------------------------------------------------------- /ispdb/config/static/css/details.css: -------------------------------------------------------------------------------- 1 | #server_data { 2 | } 3 | 4 | #incoming_data { 5 | display: inline; 6 | } 7 | 8 | #outgoing_data { 9 | float: right; 10 | } 11 | 12 | #other_data { 13 | } 14 | 15 | #domains { 16 | padding: 1em; 17 | } 18 | 19 | .domain { 20 | -moz-border-radius: 3px; 21 | border: 1px solid grey; 22 | background: #FFEFAF; 23 | padding: 1ex; 24 | margin-right: 0.2ex; 25 | margin-bottom: 1ex; 26 | display: inline-block; 27 | } 28 | 29 | .configstatus { 30 | padding: 1.5em; 31 | margin: 1.5em; 32 | } 33 | 34 | .approved { 35 | border: 1px solid green; 36 | } 37 | 38 | .invalid { 39 | border: 1px solid red; 40 | } 41 | 42 | .deleted { 43 | border: 1px solid black; 44 | } 45 | 46 | .requested { 47 | border: 1px solid orange; 48 | } 49 | 50 | .float { 51 | float: left; 52 | clear: none; 53 | margin-right: 4px; 54 | } 55 | 56 | .errornote { 57 | margin-top: 10px; 58 | } 59 | 60 | #commenttext_div { 61 | padding: 8px; 62 | } 63 | 64 | #commenttext { 65 | font-size: 12px; 66 | } 67 | 68 | #sanity_checks{ 69 | margin-top: 15px; 70 | } 71 | 72 | .error { 73 | background:url(../img/error-icon.png); 74 | background-size:15px 15px; 75 | background-repeat: no-repeat; 76 | padding: 0px 0 3px 23px; 77 | font-size: 13px; 78 | line-height: 20px; 79 | } 80 | 81 | .warning { 82 | background:url(../img/warning-icon.png); 83 | background-size:15px 15px; 84 | background-repeat: no-repeat; 85 | padding: 2px 0 3px 23px; 86 | font-size: 13px; 87 | line-height: 20px; 88 | } 89 | 90 | #sanity_results { 91 | margin-top: 10px; 92 | } 93 | 94 | .tab_desc { 95 | margin-left: 10px; 96 | } 97 | -------------------------------------------------------------------------------- /ispdb/config/static/css/enter_config.css: -------------------------------------------------------------------------------- 1 | #domain_list { 2 | position: relative; 3 | max-width: 50em; 4 | } 5 | 6 | #domain_list table { 7 | max-width: 40em; 8 | } 9 | input[readonly=readonly] { 10 | color: grey; 11 | background-color:#F0F0F0; 12 | } 13 | 14 | #id_title { 15 | width: 30%; 16 | padding: 5px 5px; 17 | font-size: 14px; 18 | } 19 | 20 | #id_description { 21 | width: 30%; 22 | padding: 5px 5px; 23 | font-size: 14px; 24 | } 25 | 26 | #div_report_link { 27 | padding: 10px; 28 | } 29 | 30 | table.tab_descs td, table.tab_descs th, 31 | table.tab_insts td, table.tab_insts th, 32 | td.noborder, th.noborder { 33 | border-bottom: 0px; 34 | } 35 | -------------------------------------------------------------------------------- /ispdb/config/static/css/ispdb.css: -------------------------------------------------------------------------------- 1 | .hidden { 2 | display: none; 3 | } 4 | 5 | -------------------------------------------------------------------------------- /ispdb/config/static/css/show_issue.css: -------------------------------------------------------------------------------- 1 | .box { 2 | border: 1px solid #CACACA; 3 | border-radius: 10px; 4 | width: 70%; 5 | overflow: auto; 6 | word-wrap: break-word; 7 | } 8 | 9 | #author { 10 | color: #666; 11 | font-style: italic; 12 | padding: 0 10px; 13 | } 14 | 15 | #title { 16 | margin: -10px 0 0; 17 | padding: 0 10px; 18 | } 19 | 20 | #status { 21 | background-color: rgb(245, 245, 245); 22 | border-top: 1px solid #CACACA; 23 | border-bottom: 1px solid #CACACA; 24 | border-radius: 2px; 25 | padding: 5px 10px 0px 10px; 26 | color: #444; 27 | font-size: 11.5px; 28 | } 29 | 30 | #description { 31 | padding: 10px 40px 0 40px; 32 | } 33 | 34 | #buttons { 35 | padding: 5px 10px; 36 | border-top: 1px solid #CACACA; 37 | border-top:1px solid #eee; 38 | border-left:1px solid #eee; 39 | } 40 | 41 | #domains { 42 | line-height: 20px; 43 | padding: 0px 20px; 44 | } 45 | 46 | .domain_removed { 47 | text-decoration: line-through; 48 | } 49 | 50 | .domain_added { 51 | font-weight: bold; 52 | } 53 | 54 | #config { 55 | padding: 0 10px; 56 | } 57 | 58 | .config_changed { 59 | font-weight: bold; 60 | } 61 | 62 | #edit { 63 | float: right; 64 | } 65 | 66 | .tab_desc { 67 | margin-left: 10px; 68 | } 69 | -------------------------------------------------------------------------------- /ispdb/config/static/img/ajax-loading.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mozilla/ispdb/3b87b2c797d0d56cdf5343cca831af721360ca53/ispdb/config/static/img/ajax-loading.gif -------------------------------------------------------------------------------- /ispdb/config/static/img/error-icon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mozilla/ispdb/3b87b2c797d0d56cdf5343cca831af721360ca53/ispdb/config/static/img/error-icon.png -------------------------------------------------------------------------------- /ispdb/config/static/img/nav-bg-grabber.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mozilla/ispdb/3b87b2c797d0d56cdf5343cca831af721360ca53/ispdb/config/static/img/nav-bg-grabber.gif -------------------------------------------------------------------------------- /ispdb/config/static/img/nav-bg-reverse.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mozilla/ispdb/3b87b2c797d0d56cdf5343cca831af721360ca53/ispdb/config/static/img/nav-bg-reverse.gif -------------------------------------------------------------------------------- /ispdb/config/static/img/nav-bg.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mozilla/ispdb/3b87b2c797d0d56cdf5343cca831af721360ca53/ispdb/config/static/img/nav-bg.gif -------------------------------------------------------------------------------- /ispdb/config/static/img/sign_in_blue.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mozilla/ispdb/3b87b2c797d0d56cdf5343cca831af721360ca53/ispdb/config/static/img/sign_in_blue.png -------------------------------------------------------------------------------- /ispdb/config/static/img/thunderbird.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mozilla/ispdb/3b87b2c797d0d56cdf5343cca831af721360ca53/ispdb/config/static/img/thunderbird.png -------------------------------------------------------------------------------- /ispdb/config/static/img/warning-icon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mozilla/ispdb/3b87b2c797d0d56cdf5343cca831af721360ca53/ispdb/config/static/img/warning-icon.png -------------------------------------------------------------------------------- /ispdb/config/templatetags/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mozilla/ispdb/3b87b2c797d0d56cdf5343cca831af721360ca53/ispdb/config/templatetags/__init__.py -------------------------------------------------------------------------------- /ispdb/config/templatetags/custom_filters.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | from django import template 4 | from django.utils import timezone 5 | 6 | register = template.Library() 7 | 8 | 9 | @register.filter 10 | def data_verbose(boundField, attr_name="value"): 11 | """ 12 | Returns field's data or its verbose version 13 | for a field with choices defined. 14 | 15 | Usage:: 16 | 17 | {% load data_verbose %} 18 | {{form.some_field|data_verbose}} 19 | """ 20 | data = boundField[attr_name] 21 | field = boundField 22 | rv = data 23 | if 'choices' in field and dict(field['choices']).get(data, ''): 24 | rv = dict(field["choices"]).get(data, "") 25 | return rv 26 | 27 | 28 | @register.filter 29 | def data_verbose_field(boundField, attr_name="value"): 30 | """ 31 | Returns field's data or its verbose version 32 | for a field with choices defined. 33 | 34 | Usage:: 35 | 36 | {{form.some_field|data_verbose_field}} 37 | """ 38 | data = boundField.value() 39 | field = boundField.field 40 | return hasattr(field, 'choices') and dict(field.choices).get(data, '') or \ 41 | data 42 | 43 | 44 | @register.filter 45 | def is_undo_available(self): 46 | delta = timezone.now() - self.deleted_datetime 47 | if delta.days > 0: 48 | return False 49 | return True 50 | -------------------------------------------------------------------------------- /ispdb/convert.py: -------------------------------------------------------------------------------- 1 | import lxml.etree as ET 2 | import os 3 | 4 | from ispdb.config.models import Config, Domain 5 | 6 | 7 | def get_username(element): 8 | rv = element.find('username') 9 | if rv is None: 10 | rv = element.find('usernameForm') 11 | if rv is None: 12 | rv = "" 13 | else: 14 | rv = rv.text 15 | return rv 16 | 17 | 18 | def get_authentication(element): 19 | rv = element.find('authentication').text 20 | if (rv == "plain"): 21 | rv = "password-cleartext" 22 | elif (rv == "secure"): 23 | rv = "password-encrypted" 24 | return rv 25 | 26 | 27 | def main(): 28 | datadir = os.environ.get("AUTOCONFIG_DATA", "../autoconfig_data") 29 | for fname in os.listdir(datadir): 30 | fullpath = os.path.join(datadir, fname) 31 | if fname.startswith('.') or os.path.isdir(fullpath) or (fname == 32 | "README"): 33 | continue 34 | print "PROCESSING", fname 35 | et = ET.parse(fullpath) 36 | root = et.getroot() 37 | #print root 38 | incoming = root.find('.//incomingServer') 39 | outgoing = root.find('.//outgoingServer') 40 | id = root.find(".//emailProvider").attrib['id'] 41 | # if we have one for this id, skip it 42 | if Config.objects.filter(email_provider_id=id): 43 | continue 44 | 45 | # Handle the older forms of XML. 46 | incoming_username_form = get_username(incoming) 47 | outgoing_username_form = get_username(outgoing) 48 | incoming_authentication = get_authentication(incoming) 49 | outgoing_authentication = get_authentication(outgoing) 50 | 51 | if not incoming_authentication: 52 | continue 53 | 54 | try: 55 | addThisServer = outgoing.find('addThisServer').text == "true" 56 | except: 57 | addThisServer = False 58 | 59 | try: 60 | useGlobalPreferredServer = \ 61 | outgoing.find('useGlobalPreferredServer').text == "true" 62 | except: 63 | useGlobalPreferredServer = False 64 | 65 | c = Config(id=None, 66 | email_provider_id=id, 67 | display_name=root.find('.//displayName').text, 68 | display_short_name=root.find('.//displayShortName').text, 69 | incoming_type=incoming.attrib['type'], 70 | incoming_hostname=incoming.find('hostname').text, 71 | incoming_port=int(incoming.find('port').text), 72 | incoming_socket_type=incoming.find('socketType').text, 73 | incoming_authentication=incoming_authentication, 74 | incoming_username_form=incoming_username_form, 75 | outgoing_hostname=outgoing.find('hostname').text, 76 | outgoing_port=int(outgoing.find('port').text), 77 | outgoing_socket_type=outgoing.find('socketType').text, 78 | outgoing_username_form=outgoing_username_form, 79 | outgoing_authentication=outgoing_authentication, 80 | outgoing_add_this_server=addThisServer, 81 | outgoing_use_global_preferred_server=( 82 | useGlobalPreferredServer), 83 | status='approved', 84 | ) 85 | domains = root.findall('.//domain') 86 | c.save() 87 | ds = [Domain(id=None, name=d.text, config=c) for d in domains] 88 | for d in ds: 89 | d.save() 90 | 91 | 92 | if __name__ == "__main__": 93 | main() 94 | -------------------------------------------------------------------------------- /ispdb/fixtures/domain_testdata.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "pk": 1, 4 | "model": "config.domain", 5 | "fields": { 6 | "config": 2, 7 | "name": "validdomain.com" 8 | } 9 | }, 10 | { 11 | "pk": 2, 12 | "model": "config.domain", 13 | "fields": { 14 | "config": 4, 15 | "name": "invaliddomain.com" 16 | } 17 | }, 18 | { 19 | "pk": 1, 20 | "model": "config.domainrequest", 21 | "fields": { 22 | "votes": 1, 23 | "config": 1, 24 | "name": "domainrequest.com" 25 | } 26 | }, 27 | { 28 | "pk": 2, 29 | "model": "config.domainrequest", 30 | "fields": { 31 | "votes": 1, 32 | "config": 3, 33 | "name": "invaliddomainrequest.com" 34 | } 35 | }, 36 | { 37 | "pk": 1, 38 | "model": "config.config", 39 | "fields": { 40 | "outgoing_username_form": "test", 41 | "incoming_socket_type": "plain", 42 | "outgoing_port": 465, 43 | "incoming_authentication": "password-encrypted", 44 | "email_provider_id": "", 45 | "owner": 1, 46 | "display_name": "Domain Request", 47 | "outgoing_add_this_server": false, 48 | "outgoing_socket_type": "SSL", 49 | "created_datetime": "2012-05-31T21:06:47.613Z", 50 | "outgoing_authentication": "password-encrypted", 51 | "incoming_hostname": "test.com", 52 | "status": "requested", 53 | "display_short_name": "Domain Request", 54 | "outgoing_use_global_preferred_server": false, 55 | "incoming_port": 143, 56 | "last_update_datetime": "2012-05-31T21:06:47.613Z", 57 | "incoming_username_form": "", 58 | "outgoing_hostname": "test.com", 59 | "incoming_type": "imap", 60 | "deleted_datetime": null, 61 | "last_status": "", 62 | "locked": "False" 63 | } 64 | }, 65 | { 66 | "pk": 2, 67 | "model": "config.config", 68 | "fields": { 69 | "outgoing_username_form": "Valid Domain", 70 | "incoming_socket_type": "plain", 71 | "outgoing_port": 587, 72 | "incoming_authentication": "password-encrypted", 73 | "email_provider_id": "", 74 | "owner": 1, 75 | "display_name": "Valid Domain", 76 | "outgoing_add_this_server": false, 77 | "outgoing_socket_type": "plain", 78 | "created_datetime": "2012-05-31T21:07:24.426Z", 79 | "outgoing_authentication": "password-encrypted", 80 | "incoming_hostname": "Valid Domain", 81 | "status": "approved", 82 | "display_short_name": "Valid Domain", 83 | "outgoing_use_global_preferred_server": false, 84 | "incoming_port": 143, 85 | "last_update_datetime": "2012-05-31T21:07:27.421Z", 86 | "incoming_username_form": "", 87 | "outgoing_hostname": "Valid Domain", 88 | "incoming_type": "imap", 89 | "deleted_datetime": null, 90 | "last_status": "", 91 | "locked": "False" 92 | } 93 | }, 94 | { 95 | "pk": 3, 96 | "model": "config.config", 97 | "fields": { 98 | "outgoing_username_form": "test", 99 | "incoming_socket_type": "plain", 100 | "outgoing_port": 587, 101 | "incoming_authentication": "password-encrypted", 102 | "email_provider_id": "", 103 | "owner": 1, 104 | "display_name": "Invalid Domain", 105 | "outgoing_add_this_server": false, 106 | "outgoing_socket_type": "plain", 107 | "created_datetime": "2012-05-31T21:08:09.314Z", 108 | "outgoing_authentication": "password-encrypted", 109 | "incoming_hostname": "Invalid Domain", 110 | "status": "invalid", 111 | "display_short_name": "Invalid Domain", 112 | "outgoing_use_global_preferred_server": false, 113 | "incoming_port": 143, 114 | "last_update_datetime": "2012-05-31T21:08:11.305Z", 115 | "incoming_username_form": "", 116 | "outgoing_hostname": "test.com", 117 | "incoming_type": "imap", 118 | "last_status": "", 119 | "deleted_datetime": null, 120 | "locked": "False" 121 | } 122 | }, 123 | { 124 | "pk": 4, 125 | "model": "config.config", 126 | "fields": { 127 | "outgoing_username_form": "test", 128 | "incoming_socket_type": "plain", 129 | "outgoing_port": 587, 130 | "incoming_authentication": "password-encrypted", 131 | "email_provider_id": "", 132 | "owner": 1, 133 | "display_name": "Invalid Domain", 134 | "outgoing_add_this_server": false, 135 | "outgoing_socket_type": "plain", 136 | "created_datetime": "2012-05-31T21:08:09.314Z", 137 | "outgoing_authentication": "password-encrypted", 138 | "incoming_hostname": "Invalid Domain", 139 | "status": "invalid", 140 | "display_short_name": "Invalid Domain", 141 | "outgoing_use_global_preferred_server": false, 142 | "incoming_port": 143, 143 | "last_update_datetime": "2012-05-31T21:08:11.305Z", 144 | "incoming_username_form": "", 145 | "outgoing_hostname": "test.com", 146 | "incoming_type": "imap", 147 | "last_status": "", 148 | "deleted_datetime": null, 149 | "locked": "False" 150 | } 151 | }, 152 | { 153 | "pk": 1, 154 | "model": "config.docurl", 155 | "fields": { 156 | "url": "http://test.com/", 157 | "config": 2 158 | } 159 | }, 160 | { 161 | "pk": 1, 162 | "model": "config.docurldesc", 163 | "fields": { 164 | "docurl": 1, 165 | "description": "test", 166 | "language": "en" 167 | } 168 | } 169 | ] 170 | -------------------------------------------------------------------------------- /ispdb/fixtures/issue_testdata.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "pk": 1, 4 | "model": "config.domain", 5 | "fields": { 6 | "config": 1, 7 | "name": "test.com" 8 | } 9 | }, 10 | { 11 | "pk": 1, 12 | "model": "config.config", 13 | "fields": { 14 | "outgoing_username_form": "%EMAILLOCALPART%", 15 | "incoming_socket_type": "plain", 16 | "outgoing_port": 587, 17 | "incoming_authentication": "password-encrypted", 18 | "email_provider_id": "", 19 | "owner": 1, 20 | "display_name": "Test", 21 | "outgoing_add_this_server": false, 22 | "outgoing_socket_type": "plain", 23 | "created_datetime": "2012-05-31T21:06:47.613Z", 24 | "outgoing_authentication": "password-encrypted", 25 | "incoming_hostname": "test.com", 26 | "status": "approved", 27 | "display_short_name": "Domain Request", 28 | "outgoing_use_global_preferred_server": false, 29 | "incoming_port": 143, 30 | "last_update_datetime": "2012-05-31T21:06:47.613Z", 31 | "incoming_username_form": "%EMAILLOCALPART%", 32 | "outgoing_hostname": "test.com", 33 | "incoming_type": "imap", 34 | "deleted_datetime": null, 35 | "last_status": "" 36 | } 37 | }, 38 | { 39 | "pk": 1, 40 | "model": "config.docurl", 41 | "fields": { 42 | "url": "http://test.com/", 43 | "config": 1 44 | } 45 | }, 46 | { 47 | "pk": 1, 48 | "model": "config.docurldesc", 49 | "fields": { 50 | "docurl": 1, 51 | "description": "test", 52 | "language": "en" 53 | } 54 | }, 55 | { 56 | "pk": 1, 57 | "model": "config.enableurl", 58 | "fields": { 59 | "url": "http://test.com/", 60 | "config": 1 61 | } 62 | }, 63 | { 64 | "pk": 1, 65 | "model": "config.enableurlinst", 66 | "fields": { 67 | "enableurl": 1, 68 | "description": "test", 69 | "language": "en" 70 | } 71 | } 72 | ] 73 | -------------------------------------------------------------------------------- /ispdb/fixtures/login_testdata.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "pk": 1, 4 | "model": "auth.user", 5 | "fields": { 6 | "username": "test", 7 | "first_name": "", 8 | "last_name": "", 9 | "is_active": true, 10 | "is_superuser": false, 11 | "is_staff": false, 12 | "last_login": "2012-05-21T14:24:50.063Z", 13 | "groups": [], 14 | "user_permissions": [], 15 | "password": "pbkdf2_sha256$10000$1xx5mUXNECJX$8nPvbdDdEsSFIinfnyQueJdMwzU74XaDYqSTHySdvcw=", 16 | "email": "test@test.com", 17 | "date_joined": "2012-05-21T14:24:50.063Z" 18 | } 19 | }, 20 | { 21 | "pk": 2, 22 | "model": "auth.user", 23 | "fields": { 24 | "username": "test_admin", 25 | "first_name": "", 26 | "last_name": "", 27 | "is_active": true, 28 | "is_superuser": true, 29 | "is_staff": true, 30 | "last_login": "2012-05-21T14:24:50.063Z", 31 | "groups": [], 32 | "user_permissions": [], 33 | "password": "pbkdf2_sha256$10000$1xx5mUXNECJX$8nPvbdDdEsSFIinfnyQueJdMwzU74XaDYqSTHySdvcw=", 34 | "email": "testadmin@test.com", 35 | "date_joined": "2012-05-21T14:24:50.063Z" 36 | } 37 | }, 38 | { 39 | "pk": 3, 40 | "model": "auth.user", 41 | "fields": { 42 | "username": "test2", 43 | "first_name": "", 44 | "last_name": "", 45 | "is_active": true, 46 | "is_superuser": false, 47 | "is_staff": false, 48 | "last_login": "2012-05-21T14:24:50.063Z", 49 | "groups": [], 50 | "user_permissions": [], 51 | "password": "pbkdf2_sha256$10000$1xx5mUXNECJX$8nPvbdDdEsSFIinfnyQueJdMwzU74XaDYqSTHySdvcw=", 52 | "email": "test2@test.com", 53 | "date_joined": "2012-05-21T14:24:50.063Z" 54 | } 55 | } 56 | ] 57 | -------------------------------------------------------------------------------- /ispdb/fixtures/sanity.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "pk": 1, 4 | "model": "config.domain", 5 | "fields": { 6 | "config": 1, 7 | "name": "test.org" 8 | } 9 | }, 10 | { 11 | "pk": 2, 12 | "model": "config.domain", 13 | "fields": { 14 | "config": 1, 15 | "name": "test.com" 16 | } 17 | }, 18 | { 19 | "pk": 1, 20 | "model": "config.config", 21 | "fields": { 22 | "outgoing_username_form": "%EMAILLOCALPART%", 23 | "incoming_socket_type": "SSL", 24 | "outgoing_port": 465, 25 | "incoming_authentication": "password-cleartext", 26 | "email_provider_id": "", 27 | "owner": 2, 28 | "display_name": "Test", 29 | "outgoing_add_this_server": false, 30 | "outgoing_socket_type": "SSL", 31 | "created_datetime": "2012-05-31T21:06:47.613Z", 32 | "outgoing_authentication": "password-cleartext", 33 | "incoming_hostname": "mail.test.com", 34 | "status": "valid", 35 | "display_short_name": "Test", 36 | "outgoing_use_global_preferred_server": false, 37 | "incoming_port": 995, 38 | "last_update_datetime": "2012-05-31T21:06:47.613Z", 39 | "incoming_username_form": "%EMAILLOCALPART%", 40 | "outgoing_hostname": "mail.test.com", 41 | "incoming_type": "imap", 42 | "deleted_datetime": null, 43 | "last_status": "" 44 | } 45 | } 46 | ] 47 | -------------------------------------------------------------------------------- /ispdb/fixtures/xml_testdata.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "pk":1, 4 | "model":"config.domain", 5 | "fields": { 6 | "config": 1, 7 | "name": "test.com" 8 | } 9 | }, 10 | { 11 | "pk":2, 12 | "model":"config.domain", 13 | "fields": { 14 | "config": 2, 15 | "name": "test2.com" 16 | } 17 | }, 18 | { 19 | "pk":1, 20 | "model":"config.config", 21 | "fields": { 22 | "outgoing_use_global_preferred_server":false, 23 | "incoming_socket_type":"SSL", 24 | "outgoing_port":25, 25 | "email_provider_id":"test.com", 26 | "incoming_authentication":"password-cleartext", 27 | "display_name":"NetZero Email", 28 | "outgoing_add_this_server":false, 29 | "outgoing_socket_type":"SSL", 30 | "created_datetime": "2012-05-31T21:08:09.314Z", 31 | "outgoing_authentication":"password-cleartext", 32 | "incoming_hostname":"hostname_in", 33 | "outgoing_username_form":"%EMAILLOCALPART%", 34 | "display_short_name":"netzero", 35 | "status":"approved", 36 | "incoming_port":143, 37 | "last_update_datetime": "2012-05-31T21:08:11.305Z", 38 | "incoming_username_form":"", 39 | "outgoing_hostname":"hostname_out", 40 | "incoming_type":"imap", 41 | "locked": "False" 42 | } 43 | }, 44 | { 45 | "pk":2, 46 | "model":"config.config", 47 | "fields": { 48 | "outgoing_use_global_preferred_server":false, 49 | "incoming_socket_type":"SSL", 50 | "outgoing_port":25, 51 | "flagged_by_email":"", 52 | "flagged_as_incorrect":false, 53 | "email_provider_id":"test.com", 54 | "incoming_authentication":"password-cleartext", 55 | "display_name":"NetZero Email", 56 | "outgoing_add_this_server":false, 57 | "outgoing_socket_type":"SSL", 58 | "created_datetime": "2012-05-31T21:08:09.314Z", 59 | "outgoing_authentication":"password-cleartext", 60 | "incoming_hostname":"hostname_in", 61 | "outgoing_username_form":"%EMAILLOCALPART%", 62 | "display_short_name":"netzero", 63 | "status":"approved", 64 | "incoming_port":143, 65 | "last_update_datetime": "2012-04-30T21:08:11.305Z", 66 | "incoming_username_form":"", 67 | "outgoing_hostname":"hostname_out", 68 | "incoming_type":"imap", 69 | "locked": "False" 70 | } 71 | }, 72 | { 73 | "pk": 1, 74 | "model": "config.docurl", 75 | "fields": { 76 | "url": "http://test.com/", 77 | "config": 1 78 | } 79 | }, 80 | { 81 | "pk": 1, 82 | "model": "config.docurldesc", 83 | "fields": { 84 | "docurl": 1, 85 | "description": "test", 86 | "language": "en" 87 | } 88 | }, 89 | { 90 | "pk": 2, 91 | "model": "config.docurl", 92 | "fields": { 93 | "url": "http://test.com/", 94 | "config": 1 95 | } 96 | }, 97 | { 98 | "pk": 2, 99 | "model": "config.docurldesc", 100 | "fields": { 101 | "docurl": 2, 102 | "description": "test", 103 | "language": "en" 104 | } 105 | }, 106 | { 107 | "pk": 3, 108 | "model": "config.docurldesc", 109 | "fields": { 110 | "docurl": 2, 111 | "description": "test2", 112 | "language": "fr" 113 | } 114 | }, 115 | { 116 | "pk": 1, 117 | "model": "config.enableurl", 118 | "fields": { 119 | "url": "http://test.com/", 120 | "config": 1 121 | } 122 | }, 123 | { 124 | "pk": 1, 125 | "model": "config.enableurlinst", 126 | "fields": { 127 | "enableurl": 1, 128 | "description": "test", 129 | "language": "en" 130 | } 131 | }, 132 | { 133 | "pk": 2, 134 | "model": "config.enableurl", 135 | "fields": { 136 | "url": "http://test.com/", 137 | "config": 1 138 | } 139 | }, 140 | { 141 | "pk": 2, 142 | "model": "config.enableurlinst", 143 | "fields": { 144 | "enableurl": 2, 145 | "description": "test", 146 | "language": "en" 147 | } 148 | }, 149 | { 150 | "pk": 3, 151 | "model": "config.enableurlinst", 152 | "fields": { 153 | "enableurl": 2, 154 | "description": "test2", 155 | "language": "fr" 156 | } 157 | } 158 | ] 159 | -------------------------------------------------------------------------------- /ispdb/middleware/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mozilla/ispdb/3b87b2c797d0d56cdf5343cca831af721360ca53/ispdb/middleware/__init__.py -------------------------------------------------------------------------------- /ispdb/middleware/x_ssl.py: -------------------------------------------------------------------------------- 1 | from django.conf import settings 2 | from django.http import HttpResponseRedirect, HttpResponsePermanentRedirect, get_host 3 | 4 | SSL = 'SSL' 5 | 6 | def is_secure(request): 7 | if 'HTTP_X_SSL' in request.META: 8 | return request.META['HTTP_X_SSL'] == 'on' 9 | return False 10 | 11 | class x_ssl: 12 | def process_view(self, request, view_func, view_args, view_kwargs): 13 | if SSL in view_kwargs: 14 | secure = view_kwargs[SSL] 15 | del view_kwargs[SSL] 16 | else: 17 | secure = False 18 | 19 | if not secure == is_secure(request): 20 | return self._redirect(request, secure) 21 | 22 | def _redirect(self, request, secure): 23 | protocol = secure and "https" or "http" 24 | newurl = "%s://%s%s" % (protocol,get_host(request),request.get_full_path()) 25 | if settings.DEBUG and request.method == 'POST': 26 | raise RuntimeError, \ 27 | """Django can't perform a SSL redirect while maintaining POST data. 28 | Please structure your views so that redirects only occur during GETs.""" 29 | 30 | #return HttpResponseRedirect(newurl) 31 | return HttpResponsePermanentRedirect(newurl) #I have not had time to test this, but it appears to work better. 32 | 33 | -------------------------------------------------------------------------------- /ispdb/settings.py: -------------------------------------------------------------------------------- 1 | import os 2 | # Django settings for ispdb project. 3 | 4 | DEBUG = True 5 | TEMPLATE_DEBUG = DEBUG 6 | 7 | ISPDB_ROOT = os.path.dirname(os.path.abspath(__file__)) 8 | 9 | ADMINS = ( 10 | ) 11 | 12 | MANAGERS = ADMINS 13 | 14 | DATABASES = { 15 | 'default': { 16 | 'ENGINE': 'django.db.backends.sqlite3', # Add 'postgresql_psycopg2', 'postgresql', 'mysql', 'sqlite3' or 'oracle'. 17 | 'NAME': os.path.join(ISPDB_ROOT,'ispdb.sqlite'), # Or path to database file if using sqlite3. 18 | 'USER': '', # Not used with sqlite3. 19 | 'PASSWORD': '', # Not used with sqlite3. 20 | 'HOST': '', # Set to empty string for localhost. Not used with sqlite3. 21 | 'PORT': '', # Set to empty string for default. Not used with sqlite3. 22 | } 23 | } 24 | 25 | #Test Runner 26 | TEST_RUNNER = 'django_nose.NoseTestSuiteRunner' 27 | NOSE_ARGS = ['--with-doctest', '--doctest-extension=.doctest'] 28 | 29 | #Fixtures 30 | FIXTURE_DIRS = ( 31 | #TODO: Clean this up after eric's patch for local settings 32 | os.path.join(ISPDB_ROOT,"fixtures/"), 33 | ) 34 | 35 | # Local time zone for this installation. Choices can be found here: 36 | # http://en.wikipedia.org/wiki/List_of_tz_zones_by_name 37 | # although not all choices may be available on all operating systems. 38 | # If running in a Windows environment this must be set to the same as your 39 | # system time zone. 40 | TIME_ZONE = 'America/Chicago' 41 | 42 | # Language code for this installation. All choices can be found here: 43 | # http://www.i18nguy.com/unicode/language-identifiers.html 44 | LANGUAGE_CODE = 'en-us' 45 | 46 | SITE_ID = 1 47 | 48 | # If you set this to False, Django will make some optimizations so as not 49 | # to load the internationalization machinery. 50 | USE_I18N = False 51 | 52 | # If you set this to False, Django will not format dates, numbers and 53 | # calendars according to the current locale 54 | USE_L10N = True 55 | 56 | # If you set this to False, Django will not use timezone-aware datetimes. 57 | USE_TZ = True 58 | 59 | # Absolute filesystem path to the directory that will hold user-uploaded files. 60 | # Example: "/home/media/media.lawrence.com/media/" 61 | MEDIA_ROOT = '' 62 | 63 | # URL that handles the media served from MEDIA_ROOT. Make sure to use a 64 | # trailing slash. 65 | # Examples: "http://media.lawrence.com/media/", "http://example.com/media/" 66 | MEDIA_URL = '' 67 | 68 | # Absolute path to the directory static files should be collected to. 69 | # Don't put anything in this directory yourself; store your static files 70 | # in apps' "static/" subdirectories and in STATICFILES_DIRS. 71 | # Example: "/home/media/media.lawrence.com/static/" 72 | STATIC_ROOT = '' 73 | 74 | # URL prefix for static files. 75 | # Example: "http://media.lawrence.com/static/" 76 | STATIC_URL = '/static/' 77 | 78 | # Additional locations of static files 79 | STATICFILES_DIRS = ( 80 | # Put strings here, like "/home/html/static" or "C:/www/django/static". 81 | # Always use forward slashes, even on Windows. 82 | # Don't forget to use absolute paths, not relative paths. 83 | ) 84 | 85 | # List of finder classes that know how to find static files in 86 | # various locations. 87 | STATICFILES_FINDERS = ( 88 | 'django.contrib.staticfiles.finders.FileSystemFinder', 89 | 'django.contrib.staticfiles.finders.AppDirectoriesFinder', 90 | # 'django.contrib.staticfiles.finders.DefaultStorageFinder', 91 | ) 92 | 93 | # Make this unique, and don't share it with anybody. 94 | SECRET_KEY = '02$7gj$t@dtd7um)d#3zlfg7ommedl#*ekt@@*ysy_fns(lp9+' 95 | 96 | # List of callables that know how to import templates from various sources. 97 | TEMPLATE_LOADERS = ( 98 | 'django.template.loaders.filesystem.Loader', 99 | 'django.template.loaders.app_directories.Loader', 100 | ) 101 | 102 | MIDDLEWARE_CLASSES = ( 103 | 'django.middleware.cache.UpdateCacheMiddleware', 104 | 'django.middleware.common.CommonMiddleware', 105 | 'django.contrib.sessions.middleware.SessionMiddleware', 106 | 'django.middleware.csrf.CsrfViewMiddleware', 107 | 'django.contrib.auth.middleware.AuthenticationMiddleware', 108 | 'django.contrib.messages.middleware.MessageMiddleware', 109 | # Uncomment the next line for simple clickjacking protection: 110 | # 'django.middleware.clickjacking.XFrameOptionsMiddleware', 111 | 'django.middleware.cache.FetchFromCacheMiddleware', 112 | ) 113 | 114 | ROOT_URLCONF = 'ispdb.urls' 115 | 116 | TEMPLATE_DIRS = ( 117 | # Put strings here, like "/home/html/django_templates" or "C:/www/django/templates". 118 | # Always use forward slashes, even on Windows. 119 | # Don't forget to use absolute paths, not relative paths. 120 | os.path.join(ISPDB_ROOT, "templates/"), 121 | os.path.join(ISPDB_ROOT, "templates/config/"), 122 | ) 123 | 124 | 125 | INSTALLED_APPS = ( 126 | 'django.contrib.auth', 127 | 'django_browserid', 128 | 'django.contrib.contenttypes', 129 | 'django.contrib.sessions', 130 | 'django.contrib.sites', 131 | 'django.contrib.admin', 132 | 'django.contrib.comments', 133 | 'django.contrib.staticfiles', 134 | 'ispdb.config', 135 | 'django_nose', 136 | ) 137 | 138 | AUTHENTICATION_BACKENDS = ( 139 | 'django_browserid.auth.BrowserIDBackend', 140 | 'django.contrib.auth.backends.ModelBackend', 141 | ) 142 | 143 | SITE_URL = 'http://localhost:8000' 144 | LOGIN_URL = '/login' 145 | LOGIN_REDIRECT_URL = '/' 146 | LOGIN_REDIRECT_URL_FAILURE = '/' 147 | BROWSERID_CREATE_USER = True 148 | 149 | TEMPLATE_CONTEXT_PROCESSORS = ( 150 | "django.contrib.auth.context_processors.auth", 151 | "django.core.context_processors.debug", 152 | "django.core.context_processors.i18n", 153 | "django.core.context_processors.media", 154 | "django.core.context_processors.static", 155 | "django.core.context_processors.tz", 156 | "django.contrib.messages.context_processors.messages", 157 | "django.core.context_processors.request", 158 | "django_browserid.context_processors.browserid_form", 159 | ) 160 | 161 | CACHES = { 162 | 'default': { 163 | 'BACKEND': 'django.core.cache.backends.dummy.DummyCache', 164 | } 165 | } 166 | -------------------------------------------------------------------------------- /ispdb/templates/404.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 404 Not Found 4 | 5 | 6 |

Not Found

7 | 8 |

The requested URL {{ request.path|escape }} was not found on this 9 | server.

10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /ispdb/templates/500.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 500 Internal Server Error 4 | 5 | 6 |

Internal Server Error

7 | 8 |

I'm sorry, that page is currently unavailable due to a server 9 | misconfiguration.

10 | 11 |

The server administrator has been notified, and we apologise for any 12 | inconvenience.

13 | 14 | 15 | 16 | -------------------------------------------------------------------------------- /ispdb/templates/config/base.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | {% load i18n %} 4 | {% load url from future %} 5 | 6 | {% block title %}{% endblock %} 7 | 8 | 9 | 10 | {{ browserid_form.media }} 11 | 12 | {% block scripts %} 13 | {% endblock %} 14 | 15 | 16 | 17 | 18 | {% block bodystart %} 19 | {% endblock %} 20 | 21 |
22 | 46 |
47 | Home › 48 | List | 49 | Add new 50 | {% if perms.config.can_approve %} 51 | | Review Queue 52 | {% endif %} 53 |
54 |
55 | {% block content %}{% endblock %} 56 |
57 |
58 | 59 | 60 | -------------------------------------------------------------------------------- /ispdb/templates/config/details.html: -------------------------------------------------------------------------------- 1 | {% extends "config/base.html" %} 2 | {% load url from future %} 3 | {% load custom_filters %} 4 | {% block scripts %} 5 | 6 | 7 | {% endblock %} 8 | 9 | {% block title %}Mozilla ISP Database{% endblock %} 10 | {% block bodystart %} 11 | 12 | 81 | {% endblock %} 82 | 83 | {% block heading %}Configuration: {{config.display_short_name}} {% endblock %} 84 | 85 | {% block content %} 86 | 87 |
88 |

{{config.display_name}} 89 | {% ifnotequal config.display_name config.display_short_name %} ({{config.display_short_name}} for short) {% endifnotequal %} 90 |

91 |
92 | {% if perms.is_superuser or config.status == 'requested' and config.owner == user %} 93 |
[Edit]
94 | {% endif %} 95 | 96 | {% if error %} 97 |
98 |

{{ error|linebreaks }}

99 |
100 | {% endif %} 101 | 102 |
103 | {% if config.status == 'approved' %} 104 |

This configuration is APPROVED.

105 | {% if perms.config.can_approve %} 106 |

Is there a reason it shouldn't be?

107 |
{% csrf_token %} 108 |
109 | 110 | Not available from our domain
111 | 112 | Incorrect data
113 | 114 | Other 115 |
116 |
117 | 118 |
119 |
120 | 121 |
122 |
123 | {% endif %} 124 | {% else %} 125 | {% if config.status == 'invalid' %} 126 |

127 | This configuration has been determined INVALID. We've looked at the 128 | data, and either it's not correct, or it can't be verified from outside 129 | the ISP's firewall, or the ISP has asked not to publish it. 130 |

131 | {% if perms.config.can_approve %} 132 |

Is there a reason it shouldn't be?

133 |
{% csrf_token %} 134 |
135 | 136 |
137 |
138 |
{% csrf_token %} 139 |
140 | 141 |
142 |
143 |
144 | Run sanity checks. 145 | 146 |
147 |
148 | {% endif %} 149 | {% else %} 150 | {% if config.status == 'deleted' %} 151 |

This configuration has been DELETED.

152 | {% if config|is_undo_available %} 153 | {% if perms.config.can_approve or config.owner == user and config.last_status == 'requested' %} 154 |
{% csrf_token %} 155 |
156 | 157 |
158 |
159 | {% endif %} 160 | {% endif %} 161 | {% else %} 162 | {% if perms.config.can_approve %} 163 |

Can you confirm that this is a good configuration according to our policies?

164 |
{% csrf_token %} 165 | 173 |
174 | 175 |
176 |
177 | 178 | 179 |
180 |
181 |
{% csrf_token %} 182 |
183 | 184 |
185 |
186 |
187 | Run sanity checks. 188 | 189 |
190 |
191 | {% else %} 192 |

This configuration is PENDING REVIEW. Our team of crack volunteers will review 193 | it, and assuming the data checks out, we'll publish it.

194 | {% if user == config.owner %} 195 |
{% csrf_token %} 196 |
197 | 198 |
199 |
200 | {% endif %} 201 | {% endif %} 202 | {% endif %} 203 | {% endif %} 204 | {% endif %} 205 |
206 | 207 |

Domains Served

208 |
209 | {% if config.domains.all %} 210 | {% for domain in config.domains.all %} 211 |
{{domain}}
212 | {% endfor %} 213 | {% else %} 214 | {% for domain in config.domainrequests.all %} 215 |
{{domain}}
216 | {% endfor %} 217 | {% endif %} 218 |
219 | 220 |
221 | 222 | 223 | 235 | 249 | 250 |
224 |
225 |

Incoming Server

226 | 227 | {% for field in incoming %} 228 | 229 | 230 | 231 | {% endfor %} 232 |
{{ field.verbose_name|capfirst }}:{{field|data_verbose}}
233 |
234 | 		236 | 237 |
238 |

Outgoing Server

239 | 240 | {% for field in outgoing %} 241 | 242 | 243 | 244 | 245 | {% endfor %} 246 |
{{ field.verbose_name|capfirst }}:{{field|data_verbose}}
247 |
248 |
251 |
252 | 253 | {% if docurls %} 254 |
255 |

Documentation

256 | {% for form in docurls %} 257 | {% include "config/urls/url_list.html" %} 258 | {% for desc_form in form.desc_formset %} 259 | {% include "config/urls/desc_list.html" %} 260 | {% endfor %} 261 | {% endfor %} 262 |
263 | {% endif %} 264 | 265 | {% if enableurls %} 266 |
267 |

Enable Instructions

268 | {% for form in enableurls %} 269 | {% include "config/urls/url_list.html" %} 270 | {% for desc_form in form.inst_formset %} 271 | {% include "config/urls/desc_list.html" %} 272 | {% endfor %} 273 | {% endfor %} 274 |
275 | {% endif %} 276 | 277 |
278 |

Other Fields

279 | 280 | {% for field in other_fields %} 281 | 282 | 283 | 284 | {% endfor %} 285 |
{{ field.verbose_name|capfirst }}:{{field.value}}
286 |
287 | 288 | 289 | 290 |

View XML data

291 | 292 |

See XML version (used by Thunderbird).

293 | 294 | {% if config.status == "approved" %} 295 |

Reported Issues

296 | 297 | {% for issue in issues %} 298 | 301 | {% empty %} 302 |

No reported issues.

303 | {% endfor %} 304 | 305 |

Accurate?

306 | 307 |

If you know the data above is not accurate, do let us know: 308 | these parameters are out of date! 309 |

310 | {% endif %} 311 | 312 |

Comments

313 | 314 | {% load comments %} 315 | 316 | {% get_comment_list for config as comment_list %} 317 | {% if comment_list %} 318 |
319 |
Show {{ comment_list|length }} existing comment{{comment_list|length|pluralize}}
320 | 343 |
344 | {% endif %} 345 | 346 |
347 |
Add a comment
348 | 366 |
367 | 368 | {% endblock %} 369 | -------------------------------------------------------------------------------- /ispdb/templates/config/export.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | {{ data }} 5 | 6 | -------------------------------------------------------------------------------- /ispdb/templates/config/intro.html: -------------------------------------------------------------------------------- 1 | {% extends "config/base.html" %} 2 | {% load url from future %} 3 | 4 | {% block scripts %} 5 | 6 | {% endblock %} 7 | 8 | {% block title %}Mozilla ISP Database{% endblock %} 9 | 10 | {% block bodystart %} 11 | 12 | 59 | {% endblock %} 60 | 61 | {% block heading %}Welcome to Mozilla's ISP database site{% endblock %} 62 | 63 | {% block content %} 64 | 65 |

This site lets you contribute data to Mozilla's database of 66 | ISP email configuration files. These files will be used by 67 | Thunderbird 3 to facilitate account configuration. 68 |

69 | 70 |

Check whether your email domain is registered

71 | 72 |
73 |

74 |
75 | 76 | 86 | 87 | 90 | 91 |

Browse our database

92 | 93 |

Show all configurations

94 | 95 | {% endblock %} 96 | -------------------------------------------------------------------------------- /ispdb/templates/config/list.html: -------------------------------------------------------------------------------- 1 | {% extends "config/base.html" %} 2 | {% load url from future %} 3 | 4 | {% block title %}Show known ISP configurations{% endblock %} 5 | {% block bodystart %} 6 | 10 | {% endblock %} 11 | {% block heading %}ISP Configurations: All configurations{% endblock %} 12 | 13 | {% block content %} 14 | 15 |

Approved configurations

16 | {% for config in configs %} 17 | {% if config.status == 'approved' %} 18 |
19 |

{{config.display_name}} ({{ config.incoming_type }})

20 | {% for domain in config.domains.all %} 21 |
{{ domain }}
22 | {% endfor %} 23 |
24 | {% endif %} 25 | {% endfor %} 26 | 27 |

Configurations pending review

28 | {% for config in configs %} 29 | {% if config.status == 'requested' %} 30 |
31 |

{{config.display_name}} ({{ config.incoming_type }})

32 | {% for domain in config.domainrequests.all %} 33 |
{{ domain }}
34 | {% endfor %} 35 |
36 | {% endif %} 37 | {% endfor %} 38 | 39 | {% endblock %} 40 | -------------------------------------------------------------------------------- /ispdb/templates/config/login.html: -------------------------------------------------------------------------------- 1 | {% extends "config/base.html" %} 2 | {% load url from future %} 3 | 4 | {% block scripts %} 5 | 6 | 15 | {% endblock %} 16 | 17 | {% block title %}Login{% endblock %} 18 | 19 | {% block bodystart %} 20 | 21 | {% endblock %} 22 | 23 | {% block heading %}Login{% endblock %} 24 | 25 | {% block content %} 26 |

You don't have permission to access this page. 27 | {% if not user.is_authenticated %} 28 | Log in.

29 | 30 | {% endif %} 31 | {% endblock %} 32 | -------------------------------------------------------------------------------- /ispdb/templates/config/queue.html: -------------------------------------------------------------------------------- 1 | {% extends "config/base.html" %} 2 | {% load url from future %} 3 | {% block scripts %} 4 | 5 | 6 | {% endblock %} 7 | 8 | 9 | {% block title %}ISP Configuration Review Queue{% endblock %} 10 | {% block heading %}Mozilla Messaging ISP Configuration Management: Review Queue{% endblock %} 11 | 12 | {% block bodystart %} 13 | 16 | {% endblock %} 17 | 18 | {% block content %} 19 |

Configurations pending approval

20 | 21 | {% for config in pending_configs %} 22 |
{{ config.display_name }} ({{config.domainrequests.all|length}} domain{{config.domainrequests.all|length|pluralize}})
23 | {% endfor %} 24 | 25 |

Most asked for domains

26 | 27 | {% for domain in domains %} 28 |
{{ domain.name }} ({{domain.votes}} vote{{domain.votes|pluralize}})
29 | {% endfor %} 30 | 31 |

Configurations that were marked invalid

32 | 33 | {% for config in invalid_configs %} 34 |
{{ config.display_name }} 35 | {% if config.domains.all %} 36 | ({{config.domains.all|length}} domain{{config.domains.all|length|pluralize}})
37 | {% else %} 38 | ({{config.domainrequests.all|length}} domain{{config.domainrequests.all|length|pluralize}}) 39 | {% endif %} 40 | {% endfor %} 41 | 42 | {% endblock %} 43 | 44 | -------------------------------------------------------------------------------- /ispdb/templates/config/show_issue.html: -------------------------------------------------------------------------------- 1 | {% extends "config/base.html" %} 2 | {% load url from future %} 3 | {% load custom_filters %} 4 | {% block scripts %} 5 | 6 | 7 | {% endblock %} 8 | 9 | {% block title %}Mozilla ISP Database{% endblock %} 10 | {% block bodystart %} 11 | 12 | 26 | {% endblock %} 27 | 28 | {% block heading %}Issue: {{ issue.title }} {% endblock %} 29 | 30 | {% block content %} 31 | 32 |

{{ issue.config.display_name }}

33 | 34 | {% if error %} 35 |

{{ error }}

36 | {% endif %} 37 | 38 |
39 |

opened by {{ issue.owner.email }} {{ issue.created_datetime|timesince }} ago.

40 |

{{ issue.title }}

41 |

This issue is {{ issue.status }}

42 |

{{ issue.description }}

43 | {% if perms.config.can_approve and issue.status == "open" %} 44 |
{% csrf_token %} 45 |
46 | 47 | {% if issue.updated_config and issue.config.status == 'approved' %} 48 | 49 | {% endif %} 50 |
51 |
52 | {% endif %} 53 |
54 | 55 |
56 | {% if issue.updated_config %} 57 |
58 |
59 |
60 | {% if issue.status == 'open' and issue.owner == user %} 61 | Edit 62 | {% endif %} 63 |

Suggested configuration

64 |
65 |

Domains Served

66 |
67 | {% for domain in non_modified_domains %} 68 |
{{ domain }}
69 | {% endfor %} 70 |
71 | {% for domain in removed_domains %} 72 |
{{ domain }}
73 | {% endfor %} 74 |
75 |
76 | {% for domain in added_domains %} 77 |
{{ domain }}
78 | {% endfor %} 79 |
80 |
81 | 82 |
83 |

Base data

84 | 85 | {% for field in base %} 86 | 87 | {% if field.new_value %} 88 | 89 | 90 | 91 | {% else %} 92 | 93 | {% endif %} 94 | 95 | {% endfor %} 96 |
{{ field.verbose_name|capfirst }}:{{field|data_verbose}}{{field|data_verbose:"new_value"}}{{ field.verbose_name|capfirst }}:{{field|data_verbose}}
97 |
98 | 99 |
100 |

Incoming Server

101 | 102 | {% for field in incoming %} 103 | 104 | {% if field.new_value %} 105 | 106 | 107 | 108 | {% else %} 109 | 110 | {% endif %} 111 | 112 | {% endfor %} 113 |
{{ field.verbose_name|capfirst }}:{{field|data_verbose}}{{field|data_verbose:"new_value"}}{{ field.verbose_name|capfirst }}:{{field|data_verbose}}
114 |
115 | 116 |
117 |

Outgoing Server

118 | 119 | {% for field in outgoing %} 120 | 121 | {% if field.new_value %} 122 | 123 | 124 | 125 | {% else %} 126 | 127 | 128 | {% endif %} 129 | 130 | {% endfor %} 131 |
{{ field.verbose_name|capfirst }}:{{field|data_verbose}}{{field|data_verbose:"new_value"}}{{ field.verbose_name|capfirst }}:{{field|data_verbose}}
132 |
133 | 134 |
135 |

Documentation (only the suggested is shown)

136 | {% for form in new_docurl_formset %} 137 | {% include "config/urls/url_list.html" %} 138 | {% for desc_form in form.desc_formset %} 139 | {% include "config/urls/desc_list.html" %} 140 | {% endfor %} 141 | {% endfor %} 142 |
143 | 144 |
145 |

Enable Instructions (only the suggested is shown)

146 | {% for form in new_enableurl_formset %} 147 | {% include "config/urls/url_list.html" %} 148 | {% for desc_form in form.inst_formset %} 149 | {% include "config/urls/desc_list.html" %} 150 | {% endfor %} 151 | {% endfor %} 152 |
153 | 154 |
155 |
156 | {% endif %} 157 | 158 |

Comments

159 | 160 | {% load comments %} 161 | 162 | {% get_comment_list for issue as comment_list %} 163 | {% if comment_list %} 164 |
165 |
166 | {% for comment in comment_list %} 167 |
168 |
169 | {% if comment.user %} 170 | {{ comment.user.email }} 171 | {% else %} 172 | {{ comment.user_name }} 173 | {% endif %} 174 | at {{ comment.submit_date }} 175 | {% if perms.comments.can_moderate %} 176 | Remove 177 | {% endif %} 178 |
179 | {% if comment.user_name == 'ISPDB System' and comment.user == None %} 180 |
{{comment.comment|safe|linebreaks}}
181 | {% else %} 182 |
{{comment.comment|linebreaks}}
183 | {% endif %} 184 |
185 |
186 | {% endfor %} 187 |
188 |
189 | {% endif %} 190 | 191 |
192 |
Add a comment
193 | 211 |
212 | 213 |
214 |

Go back to configuration

215 |
216 | 217 | {% endblock %} 218 | -------------------------------------------------------------------------------- /ispdb/templates/config/urls/desc_form.html: -------------------------------------------------------------------------------- 1 | {{ desc_form.id }} 2 | 3 | {{ desc_form.description.label_tag }}: 4 | {{ desc_form.description.errors }}{{ desc_form.description }} 5 | 6 | {{ desc_form.language.errors }}{{ desc_form.language }} 7 | {{ desc_form.DELETE }} 8 | {% if desc_form.DELETE.value == "True" %} 9 | Undo 10 | {% else %} 11 | Remove 12 | {% endif %} 13 | 14 | 15 | -------------------------------------------------------------------------------- /ispdb/templates/config/urls/desc_list.html: -------------------------------------------------------------------------------- 1 | {% load custom_filters %} 2 | 3 | 4 | 5 | 6 |
{{ desc_form.description.label }}:{{ desc_form.description.value }} {{ desc_form.language|data_verbose_field }}
7 | -------------------------------------------------------------------------------- /ispdb/templates/config/urls/desc_template_form.html: -------------------------------------------------------------------------------- 1 | 2 | {{ form.description.label_tag }}: 3 | {{ form.description }} 4 | 5 | {{ form.language }} 6 | {{ form.DELETE }} 7 | Remove 8 | 9 | 10 | -------------------------------------------------------------------------------- /ispdb/templates/config/urls/url_form.html: -------------------------------------------------------------------------------- 1 | {{ form.non_field_errors }} 2 | {{ form.id }} 3 | 4 | {{ form.url.label_tag }}: 5 | 6 | {{ form.url.errors }} 7 | {{ form.url }} 8 | {{ form.DELETE }} 9 | {% if form.DELETE.value == "True" %} 10 | Undo 11 | {% else %} 12 | Remove 13 | {% endif %} 14 | 15 | 16 | -------------------------------------------------------------------------------- /ispdb/templates/config/urls/url_list.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 |
{{ form.url.label}}:{{ form.url.value}}
6 | -------------------------------------------------------------------------------- /ispdb/templates/config/urls/url_template_form.html: -------------------------------------------------------------------------------- 1 | 2 | {{ form.url.label_tag }}: 3 | 4 | {{ form.url }} 5 | {{ form.DELETE }} 6 | Remove 7 | 8 | 9 | -------------------------------------------------------------------------------- /ispdb/tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mozilla/ispdb/3b87b2c797d0d56cdf5343cca831af721360ca53/ispdb/tests/__init__.py -------------------------------------------------------------------------------- /ispdb/tests/common.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | import httplib 4 | 5 | # Redirect to /add/ on success 6 | success_code = httplib.FOUND 7 | # Return with form errors if form is invalid 8 | fail_code = httplib.OK 9 | 10 | 11 | def asking_domain_form(): 12 | return { 13 | "asking_or_adding": "asking", 14 | "domain-TOTAL_FORMS": "1", 15 | "domain-INITIAL_FORMS": "0", 16 | "domain-0-id": "", 17 | "domain-0-name": "test.com", 18 | "domain-0-DELETE": "False", 19 | "docurl-INITIAL_FORMS": "0", 20 | "docurl-TOTAL_FORMS": "1", 21 | "docurl-MAX_NUM_FORMS": "", 22 | "docurl-0-id": "", 23 | "docurl-0-DELETE": "False", 24 | "docurl-0-url": "http://test.com/", 25 | "desc_0-INITIAL_FORMS": "0", 26 | "desc_0-TOTAL_FORMS": "1", 27 | "desc_0-MAX_NUM_FORMS": "", 28 | "desc_0-0-id": "", 29 | "desc_0-0-DELETE": "False", 30 | "desc_0-0-language": "en", 31 | "desc_0-0-description": "test", 32 | "enableurl-INITIAL_FORMS": "0", 33 | "enableurl-TOTAL_FORMS": "1", 34 | "enableurl-MAX_NUM_FORMS": "", 35 | "enableurl-0-id": "", 36 | "enableurl-0-DELETE": "False", 37 | "enableurl-0-url": "http://test.com/", 38 | "inst_0-INITIAL_FORMS": "0", 39 | "inst_0-TOTAL_FORMS": "1", 40 | "inst_0-MAX_NUM_FORMS": "", 41 | "inst_0-0-id": "", 42 | "inst_0-0-DELETE": "False", 43 | "inst_0-0-language": "en", 44 | "inst_0-0-description": "test" 45 | } 46 | 47 | 48 | def adding_domain_form(): 49 | return { 50 | "asking_or_adding": "adding", 51 | "domain-TOTAL_FORMS": "1", 52 | "domain-INITIAL_FORMS": "0", 53 | "domain-MAX_NUM_FORMS": "10", 54 | "domain-0-id": "", 55 | "domain-0-name": "test.com", 56 | "domain-0-DELETE": "False", 57 | "display_name": "test", 58 | "display_short_name": "test", 59 | "incoming_type": "imap", 60 | "incoming_hostname": "foo", 61 | "incoming_port": "333", 62 | "incoming_socket_type": "plain", 63 | "incoming_authentication": "password-cleartext", 64 | "incoming_username_form": "%25EMAILLOCALPART%25", 65 | "outgoing_hostname": "bar", 66 | "outgoing_port": "334", 67 | "outgoing_socket_type": "STARTTLS", 68 | "outgoing_username_form": "%25EMAILLOCALPART%25", 69 | "outgoing_authentication": "password-cleartext", 70 | "docurl-INITIAL_FORMS": "0", 71 | "docurl-TOTAL_FORMS": "1", 72 | "docurl-MAX_NUM_FORMS": "", 73 | "docurl-0-id": "", 74 | "docurl-0-DELETE": "False", 75 | "docurl-0-url": "http://test.com/", 76 | "desc_0-INITIAL_FORMS": "0", 77 | "desc_0-TOTAL_FORMS": "1", 78 | "desc_0-MAX_NUM_FORMS": "", 79 | "desc_0-0-id": "", 80 | "desc_0-0-DELETE": "False", 81 | "desc_0-0-language": "en", 82 | "desc_0-0-description": "test", 83 | "enableurl-INITIAL_FORMS": "0", 84 | "enableurl-TOTAL_FORMS": "1", 85 | "enableurl-MAX_NUM_FORMS": "", 86 | "enableurl-0-id": "", 87 | "enableurl-0-DELETE": "False", 88 | "enableurl-0-url": "http://test.com/", 89 | "inst_0-INITIAL_FORMS": "0", 90 | "inst_0-TOTAL_FORMS": "1", 91 | "inst_0-MAX_NUM_FORMS": "", 92 | "inst_0-0-id": "", 93 | "inst_0-0-DELETE": "False", 94 | "inst_0-0-language": "en", 95 | "inst_0-0-description": "test" 96 | } 97 | -------------------------------------------------------------------------------- /ispdb/tests/relaxng_schema.1.1.xml: -------------------------------------------------------------------------------- 1 | 2 | 4 | 5 | 6 | 7 | 1.1 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | imap 29 | pop3 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | true 44 | false 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | smtp 63 | 64 | 65 | 66 | 67 | 68 | SMTP-after-POP 69 | client-IP-address 70 | 71 | 72 | 73 | 74 | 75 | client-IP-address 76 | 77 | 78 | 79 | 80 | 81 | 82 | 83 | 84 | 85 | 86 | 87 | 88 | 89 | 90 | 91 | 92 | 93 | 94 | 95 | 96 | 97 | 98 | 99 | 100 | 101 | 102 | 103 | 104 | 105 | 106 | 107 | 108 | 109 | 110 | 111 | 112 | 113 | 114 | 115 | 116 | 117 | 118 | 119 | 120 | 121 | 122 | 123 | 124 | 125 | 126 | 127 | 128 | plain 129 | SSL 130 | STARTTLS 131 | 132 | 133 | 134 | 135 | 1 136 | 137 | 138 | 139 | 140 | 141 | 142 | password-cleartext 143 | password-encrypted 144 | NTLM 145 | GSSAPI 146 | none 147 | 148 | 149 | 150 | 151 | -------------------------------------------------------------------------------- /ispdb/tests/relaxng_schema.xml: -------------------------------------------------------------------------------- 1 | 2 | 4 | 5 | 6 | 7 | 8 | 1.0 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | imap 29 | pop3 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | plain 41 | SSL 42 | STARTTLS 43 | 44 | 45 | 46 | 47 | 48 | 49 | plain 50 | secure 51 | 52 | 53 | 54 | 55 | 56 | 57 | true 58 | false 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | smtp 69 | 70 | 71 | 72 | 73 | 74 | 75 | 76 | 77 | 78 | 79 | plain 80 | SSL 81 | STARTTLS 82 | 83 | 84 | 85 | 86 | 87 | 88 | plain 89 | secure 90 | smtp-after-pop 91 | 92 | 93 | 94 | 95 | true 96 | false 97 | 98 | 99 | 100 | 101 | true 102 | false 103 | 104 | 105 | 106 | 107 | 108 | 109 | 110 | 111 | 112 | 113 | 114 | 115 | 116 | 117 | 118 | 119 | 120 | 121 | 1 122 | 123 | 124 | 125 | 126 | 127 | 128 | -------------------------------------------------------------------------------- /ispdb/tests/test_approve.py: -------------------------------------------------------------------------------- 1 | 2 | from urllib import quote_plus 3 | 4 | from django.contrib.comments.models import Comment 5 | from django.test import TestCase 6 | from nose.tools import assert_equal, assert_true 7 | 8 | from ispdb.config.models import Config 9 | 10 | 11 | class ApproveTest(TestCase): 12 | fixtures = ['xml_testdata', 'login_testdata'] 13 | 14 | def test_unauthenticated_user(self): 15 | result = self.client.post("/approve/1/", { 16 | "approved": True, 17 | }, follow=True) 18 | config = Config.objects.get(id=1) 19 | # Should not have changed the config 20 | assert_equal(config.status, 'requested') 21 | # Make sure it redirects to login page 22 | goodRedirect = "/login/?next=%s" % (quote_plus("/approve/1/")) 23 | self.assertRedirects(result, goodRedirect) 24 | 25 | def test_locked_config(self): 26 | user_info = {"username": "test_admin", "password": "test"} 27 | self.client.login(**user_info) 28 | config = Config.objects.get(id=1) 29 | config.locked = True 30 | config.save() 31 | res = self.client.post("/approve/1/", {"approved": True}, follow=True) 32 | config = Config.objects.get(id=1) 33 | # Should not have changed the config 34 | assert_equal(config.status, 'requested') 35 | assert_true("This configuration is locked. Only admins can unlock it." 36 | in res.content) 37 | 38 | def test_authenticated_user(self): 39 | user_info = {"username": "test_admin", "password": "test"} 40 | self.client.login(**user_info) 41 | result = self.client.post("/approve/1/", { 42 | "approved": True, 43 | "comment": "Test", 44 | }, follow=True) 45 | config = Config.objects.get(id=1) 46 | # Should have changed the config 47 | assert_equal(config.status, 'approved') 48 | # Make sure it redirects to details page 49 | goodRedirect = "/details/1/" 50 | self.assertRedirects(result, goodRedirect) 51 | comment = Comment.objects.get(pk=1) 52 | assert_equal(int(comment.object_pk), config.pk) 53 | 54 | def test_authenticated_user_deny_no_comment(self): 55 | user_info = {"username": "test_admin", "password": "test"} 56 | self.client.login(**user_info) 57 | self.client.post("/approve/1/", { 58 | "denied": True, 59 | "comment": 'Other - invalid', 60 | "commenttext": "", 61 | }, follow=True) 62 | 63 | config = Config.objects.get(id=1) 64 | # Should not have changed the config status because no comment was 65 | # supplied 66 | assert_equal(config.status, 'requested') 67 | 68 | def test_authenticated_user_deny_comment(self): 69 | user_info = {"username": "test_admin", "password": "test"} 70 | self.client.login(**user_info) 71 | self.client.post("/approve/1/", { 72 | "denied": True, 73 | "comment": 'Other - invalid', 74 | "commenttext": "Test", 75 | }, follow=True) 76 | 77 | config = Config.objects.get(id=1) 78 | assert_equal(config.status, 'invalid') 79 | -------------------------------------------------------------------------------- /ispdb/tests/test_cache.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | from datetime import datetime 4 | from nose.tools import assert_true 5 | 6 | from django.core.urlresolvers import reverse 7 | from django.test import TestCase 8 | from django.utils.timezone import utc 9 | 10 | from ispdb.config.models import Config 11 | 12 | 13 | class CacheTest(TestCase): 14 | 15 | fixtures = ['login_testdata', 'xml_testdata'] 16 | 17 | def test_default_cache(self): 18 | res = self.client.get(reverse("ispdb_list")) 19 | assert_true('Cache-Control' in res) 20 | assert_true(res['Cache-Control'] == 'max-age=600') 21 | 22 | def test_list_xml_cache(self): 23 | res = self.client.get(reverse("ispdb_list", args=['xml'])) 24 | assert_true('Cache-Control' in res) 25 | assert_true(res['Cache-Control'] == 'no-cache, max-age=600') 26 | config = Config.objects.get(pk=1) 27 | assert_true('Last-Modified' in res) 28 | d = datetime.strptime(res['Last-Modified'], "%a, %d %b %Y %H:%M:%S " 29 | "%Z").replace(tzinfo=utc) 30 | c = config.last_update_datetime.replace(microsecond=0) 31 | assert_true(d == c) 32 | 33 | def test_export_xml_cache(self): 34 | res = self.client.get(reverse("ispdb_export_xml", args=[2])) 35 | assert_true('Cache-Control' in res) 36 | assert_true(res['Cache-Control'] == 'no-cache, max-age=600') 37 | config = Config.objects.get(pk=2) 38 | assert_true('Last-Modified' in res) 39 | d = datetime.strptime(res['Last-Modified'], "%a, %d %b %Y %H:%M:%S " 40 | "%Z").replace(tzinfo=utc) 41 | c = config.last_update_datetime.replace(microsecond=0) 42 | assert_true(d == c) 43 | -------------------------------------------------------------------------------- /ispdb/tests/test_delete.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | import httplib 4 | 5 | from django.core.urlresolvers import reverse 6 | from django.test import TestCase 7 | from django.utils import timezone 8 | from nose.tools import assert_equal, assert_true 9 | 10 | from ispdb.config import models 11 | 12 | # Redirect to /add/ on success 13 | success_code = httplib.FOUND 14 | # Return with form errors if form is invalid 15 | fail_code = httplib.OK 16 | 17 | 18 | class DeleteTest(TestCase): 19 | fixtures = ['login_testdata', 'domain_testdata'] 20 | 21 | def delete_domain(self, id): 22 | self.client.login(username='test_admin', password='test') 23 | config = models.Config.objects.get(pk=id) 24 | assert_true(config.status != 'deleted') 25 | res = self.client.post(reverse("ispdb_delete", args=[id]), 26 | {'delete': 'delete'}) 27 | assert_equal(res.status_code, success_code) 28 | self.client.logout() 29 | 30 | def test_delete_no_superuser_domainrequest(self): 31 | self.client.login(username='test', password='test') 32 | res = self.client.post(reverse("ispdb_delete", args=[1]), 33 | {'delete': 'delete'}) 34 | assert_equal(res.status_code, success_code) 35 | config = models.Config.objects.get(pk=1) 36 | assert_equal(config.status, 'deleted') 37 | self.client.logout() 38 | 39 | def test_delete_no_superuser_domain(self): 40 | self.client.login(username='test', password='test') 41 | res = self.client.post(reverse("ispdb_delete", args=[2]), 42 | {'delete': 'delete'}, follow=True) 43 | # Make sure it redirects to login page 44 | goodRedirect = "/login/" 45 | self.assertRedirects(res, goodRedirect) 46 | 47 | def test_delete_locked_config(self): 48 | config = models.Config.objects.get(pk=1) 49 | config.locked = True 50 | config.save() 51 | self.client.login(username='test_admin', password='test') 52 | config = models.Config.objects.get(pk=1) 53 | assert_true(config.status != 'deleted') 54 | res = self.client.post(reverse("ispdb_delete", args=[1]), 55 | {'delete': 'delete'}) 56 | self.client.logout() 57 | config = models.Config.objects.get(pk=1) 58 | assert_true(config.status != 'deleted') 59 | assert_true("This configuration is locked. Only admins can unlock it." 60 | in res.content) 61 | 62 | def test_delete_domainrequest(self): 63 | self.delete_domain(1) 64 | config = models.Config.objects.get(pk=1) 65 | assert_equal(config.status, 'deleted') 66 | 67 | def test_delete_valid_domain(self): 68 | self.delete_domain(2) 69 | config = models.Config.objects.get(pk=2) 70 | assert_equal(config.status, 'approved') 71 | 72 | def test_delete_invalid_domain(self): 73 | self.delete_domain(3) 74 | config = models.Config.objects.get(pk=3) 75 | assert_equal(config.status, 'deleted') 76 | 77 | def test_undo_invalid_domain_normal_user(self): 78 | self.delete_domain(4) 79 | config = models.Config.objects.get(pk=4) 80 | assert_equal(config.status, 'deleted') 81 | self.client.login(username='test', password='test') 82 | res = self.client.post(reverse("ispdb_delete", args=[4]), 83 | {'delete': 'undo'}, 84 | follow=True) 85 | # Make sure it redirects to login page 86 | goodRedirect = "/login/" 87 | self.assertRedirects(res, goodRedirect) 88 | 89 | def test_undo_request_normal_user(self): 90 | self.delete_domain(1) 91 | config = models.Config.objects.get(pk=1) 92 | assert_equal(config.status, 'deleted') 93 | self.client.login(username='test', password='test') 94 | self.client.post(reverse("ispdb_delete", args=[1]), 95 | {'delete': 'undo'}, 96 | follow=True) 97 | config = models.Config.objects.get(pk=1) 98 | assert_equal(config.status, 'requested') 99 | 100 | def test_undo_invalid_request_superuser(self): 101 | self.delete_domain(3) 102 | config = models.Config.objects.get(pk=3) 103 | assert_equal(config.status, 'deleted') 104 | self.client.login(username='test_admin', password='test') 105 | self.client.post(reverse("ispdb_delete", args=[3]), 106 | {'delete': 'undo'}, 107 | follow=True) 108 | config = models.Config.objects.get(pk=3) 109 | assert_equal(config.status, 'invalid') 110 | 111 | def test_undo_old_config(self): 112 | self.delete_domain(4) 113 | config = models.Config.objects.get(pk=4) 114 | assert_equal(config.status, 'deleted') 115 | config.deleted_datetime = timezone.datetime(2000, 10, 10, 116 | tzinfo=timezone.utc) 117 | config.save() 118 | self.client.login(username='test_admin', password='test') 119 | self.client.post(reverse("ispdb_delete", args=[4]), 120 | {'delete': 'undo'}, 121 | follow=True) 122 | config = models.Config.objects.get(pk=4) 123 | assert_equal(config.status, 'deleted') 124 | -------------------------------------------------------------------------------- /ispdb/tests/test_edit.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | from urllib import quote_plus 4 | 5 | from django.core.urlresolvers import reverse 6 | from django.test import TestCase 7 | from nose.tools import assert_equal, assert_false, assert_true 8 | 9 | from ispdb.config import models 10 | from ispdb.tests.common import adding_domain_form 11 | from ispdb.tests.common import success_code, fail_code 12 | 13 | 14 | class EditTest(TestCase): 15 | fixtures = ['login_testdata.json'] 16 | 17 | def add_domain(self, name='test.com'): 18 | self.client.login(username='test', password='test') 19 | domain = models.DomainRequest.objects.filter(name=name) 20 | assert_false(domain) 21 | form = adding_domain_form() 22 | form["domain-0-name"] = name 23 | res = self.client.post(reverse("ispdb_add"), form) 24 | assert_equal(res.status_code, success_code) 25 | domain = models.DomainRequest.objects.get(name=name) 26 | assert isinstance(domain, models.DomainRequest) 27 | self.client.logout() 28 | 29 | def test_edit_no_login(self): 30 | self.add_domain() 31 | res = self.client.post(reverse("ispdb_edit", args=[1]), 32 | adding_domain_form(), 33 | follow=True) 34 | # Make sure it redirects to login page 35 | goodRedirect = "/login/?next=%s" % (quote_plus("/edit/1/")) 36 | self.assertRedirects(res, goodRedirect) 37 | 38 | def test_edit_invalid_user(self): 39 | self.add_domain() 40 | self.client.login(username='test2', password='test') 41 | res = self.client.post(reverse("ispdb_edit", args=[1]), 42 | adding_domain_form(), 43 | follow=True) 44 | # Make sure it redirects to login page 45 | goodRedirect = "/login/" 46 | self.assertRedirects(res, goodRedirect) 47 | 48 | def test_edit_locked_config(self): 49 | self.add_domain() 50 | config = models.Config.objects.get(pk=1) 51 | config.locked = True 52 | config.save() 53 | form = adding_domain_form() 54 | form["display_name"] = "testing2" 55 | self.client.login(username='test', password='test') 56 | res = self.client.post(reverse("ispdb_edit", args=[1]), 57 | form, 58 | follow=True) 59 | config = models.Config.objects.get(pk=1) 60 | assert_true(config.display_name != "testing2") 61 | assert_true("This configuration is locked. Only admins can unlock it." 62 | in res.content) 63 | 64 | def test_edit_same_user(self): 65 | self.add_domain() 66 | self.client.login(username='test', password='test') 67 | form = adding_domain_form() 68 | form["domain-INITIAL_FORMS"] = "1" 69 | form["domain-0-id"] = "1" 70 | form["domain-0-name"] = "test1.com" 71 | form["display_name"] = "testing2" 72 | form["docurl-INITIAL_FORMS"] = "1" 73 | form["docurl-0-id"] = "1" 74 | form["docurl-0-url"] = "http://test1.com/" 75 | form["desc_0-INITIAL_FORMS"] = "1" 76 | form["desc_0-0-id"] = "1" 77 | form["desc_0-0-description"] = "test1" 78 | form["enableurl-INITIAL_FORMS"] = "1" 79 | form["enableurl-0-id"] = "1" 80 | form["enableurl-0-url"] = "http://test1.com/" 81 | form["inst_0-INITIAL_FORMS"] = "1" 82 | form["inst_0-0-id"] = "1" 83 | form["inst_0-0-description"] = "test1" 84 | res = self.client.post(reverse("ispdb_edit", args=[1]), 85 | form, 86 | follow=True) 87 | goodRedirect = "/details/1/" 88 | self.assertRedirects(res, goodRedirect) 89 | assert_true(len(models.Config.objects.all()), 1) 90 | domain = models.DomainRequest.objects.get(pk=1) 91 | assert_true(domain) 92 | assert isinstance(domain, models.DomainRequest) 93 | assert_equal(domain.name, 'test1.com') 94 | assert_equal(domain.config.display_name, 'testing2') 95 | docurl = models.DocURL.objects.get(pk=1) 96 | assert_equal(docurl.url, 'http://test1.com/') 97 | desc = models.DocURLDesc.objects.get(pk=1) 98 | assert_equal(desc.description, 'test1') 99 | enableurl = models.EnableURL.objects.get(pk=1) 100 | assert_equal(enableurl.url, 'http://test1.com/') 101 | inst = models.EnableURLInst.objects.get(pk=1) 102 | assert_equal(inst.description, 'test1') 103 | 104 | def test_edit_staff_user(self): 105 | self.add_domain() 106 | self.client.login(username='test_admin', password='test') 107 | form = adding_domain_form() 108 | form["domain-INITIAL_FORMS"] = "1" 109 | form["domain-0-id"] = "1" 110 | form["domain-0-name"] = "test1.com" 111 | form["display_name"] = "testing2" 112 | form["docurl-INITIAL_FORMS"] = "1" 113 | form["docurl-0-id"] = "1" 114 | form["docurl-0-url"] = "http://test1.com/" 115 | form["desc_0-INITIAL_FORMS"] = "1" 116 | form["desc_0-0-id"] = "1" 117 | form["desc_0-0-description"] = "test1" 118 | form["enableurl-INITIAL_FORMS"] = "1" 119 | form["enableurl-0-id"] = "1" 120 | form["enableurl-0-url"] = "http://test1.com/" 121 | form["inst_0-INITIAL_FORMS"] = "1" 122 | form["inst_0-0-id"] = "1" 123 | form["inst_0-0-description"] = "test1" 124 | res = self.client.post(reverse("ispdb_edit", args=[1]), 125 | form, 126 | follow=True) 127 | goodRedirect = "/details/1/" 128 | self.assertRedirects(res, goodRedirect) 129 | assert_true(len(models.Config.objects.all()), 1) 130 | domain = models.DomainRequest.objects.get(pk=1) 131 | assert_true(domain) 132 | assert isinstance(domain, models.DomainRequest) 133 | assert_equal(domain.name, 'test1.com') 134 | assert_equal(domain.config.display_name, 'testing2') 135 | docurl = models.DocURL.objects.get(pk=1) 136 | assert_equal(docurl.url, 'http://test1.com/') 137 | desc = models.DocURLDesc.objects.get(pk=1) 138 | assert_equal(desc.description, 'test1') 139 | enableurl = models.EnableURL.objects.get(pk=1) 140 | assert_equal(enableurl.url, 'http://test1.com/') 141 | inst = models.EnableURLInst.objects.get(pk=1) 142 | assert_equal(inst.description, 'test1') 143 | 144 | def test_edit_duplicated_names(self): 145 | self.add_domain() 146 | self.client.login(username='test', password='test') 147 | form = adding_domain_form() 148 | form["domain-1-name"] = "test.com" 149 | form["domain-1-DELETE"] = "False" 150 | res = self.client.post(reverse("ispdb_edit", args=[1]), 151 | form, 152 | follow=True) 153 | assert_equal(res.status_code, fail_code) 154 | 155 | def test_add_existing_domain(self): 156 | self.add_domain(name='approved.com') 157 | self.add_domain() 158 | self.client.login(username='test_admin', password='test') 159 | self.client.post("/approve/1/", { 160 | "approved": True, }) 161 | config = models.Config.objects.get(id=1) 162 | assert_equal(config.status, 'approved') 163 | form = adding_domain_form() 164 | form["domain-1-name"] = "approved.com" 165 | form["domain-1-DELETE"] = "False" 166 | res = self.client.post(reverse("ispdb_edit", args=[1]), 167 | form, 168 | follow=True) 169 | assert_equal(res.status_code, fail_code) 170 | 171 | def test_owner_edit_approved_domain(self): 172 | self.add_domain(name='approved.com') 173 | self.client.login(username='test_admin', password='test') 174 | self.client.post("/approve/1/", { 175 | "approved": True, }) 176 | config = models.Config.objects.get(id=1) 177 | assert_equal(config.status, 'approved') 178 | self.client.logout() 179 | self.client.login(username='test', password='test') 180 | form = adding_domain_form() 181 | form["incoming_hostname"] = "bar" 182 | res = self.client.post(reverse("ispdb_edit", args=[1]), 183 | form, 184 | follow=True) 185 | goodRedirect = "/login/" 186 | self.assertRedirects(res, goodRedirect) 187 | -------------------------------------------------------------------------------- /ispdb/tests/test_issue.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | from urllib import quote_plus 4 | 5 | from django.core.urlresolvers import reverse 6 | from django.test import TestCase 7 | from nose.tools import assert_equal, assert_false, assert_true 8 | 9 | from ispdb.config import models 10 | from ispdb.tests.common import adding_domain_form 11 | from ispdb.tests.common import success_code 12 | 13 | 14 | def adding_issue_form(): 15 | form = adding_domain_form() 16 | form['show_form'] = 'False' 17 | form['title'] = 'Test' 18 | form['description'] = 'Test' 19 | return form 20 | 21 | 22 | class IssueTest(TestCase): 23 | fixtures = ['login_testdata', 'issue_testdata'] 24 | 25 | def add_issue(self, updated_config=False, config_id=1): 26 | self.client.login(username='test', password='test') 27 | issue = models.Issue.objects.filter(title='Test') 28 | assert_false(issue) 29 | form = adding_issue_form() 30 | form["domain-INITIAL_FORMS"] = "1" 31 | form["domain-0-id"] = "1" 32 | form["domain-0-name"] = "test1.com" 33 | form["display_name"] = "testing2" 34 | form["docurl-INITIAL_FORMS"] = "1" 35 | form["docurl-0-id"] = "1" 36 | form["docurl-0-url"] = "http://test1.com/" 37 | form["desc_0-INITIAL_FORMS"] = "1" 38 | form["desc_0-0-id"] = "1" 39 | form["desc_0-0-description"] = "test1" 40 | form["enableurl-INITIAL_FORMS"] = "1" 41 | form["enableurl-0-id"] = "1" 42 | form["enableurl-0-url"] = "http://test1.com/" 43 | form["inst_0-INITIAL_FORMS"] = "1" 44 | form["inst_0-0-id"] = "1" 45 | form["inst_0-0-description"] = "test1" 46 | if updated_config: 47 | form['show_form'] = 'True' 48 | res = self.client.post(reverse("ispdb_report", args=[config_id]), form) 49 | assert_equal(res.status_code, success_code) 50 | issue = models.Issue.objects.get(title='Test') 51 | assert isinstance(issue, models.Issue) 52 | assert_equal(issue.updated_config is not None, updated_config) 53 | self.client.logout() 54 | 55 | def test_add_issue_no_login(self): 56 | res = self.client.post(reverse("ispdb_report", args=[2]), 57 | adding_issue_form(), 58 | follow=True) 59 | # Make sure it redirects to login page 60 | goodRedirect = "/login/?next=%s" % (quote_plus("/report/2/")) 61 | self.assertRedirects(res, goodRedirect) 62 | 63 | def test_add_issue(self): 64 | self.add_issue() 65 | 66 | def test_add_issue_with_updated_config(self): 67 | self.add_issue(updated_config=True) 68 | 69 | def test_close_issue_normal_user(self): 70 | self.add_issue() 71 | self.client.login(username='test', password='test') 72 | res = self.client.post(reverse("ispdb_show_issue", args=[1]), 73 | {"action": "close"}, 74 | follow=True) 75 | issue = models.Issue.objects.get(title='Test') 76 | assert_equal(issue.status, "open") 77 | # Make sure it redirects to login page 78 | goodRedirect = "/login/" 79 | self.assertRedirects(res, goodRedirect) 80 | 81 | def test_close_issue_superuser(self): 82 | self.add_issue() 83 | self.client.login(username='test_admin', password='test') 84 | self.client.post(reverse("ispdb_show_issue", args=[1]), 85 | {"action": "close"}, 86 | follow=True) 87 | issue = models.Issue.objects.get(title='Test') 88 | assert_equal(issue.status, "closed") 89 | 90 | def test_merge_issue_normal_user(self): 91 | self.add_issue(updated_config=True) 92 | self.client.login(username='test', password='test') 93 | res = self.client.post(reverse("ispdb_show_issue", args=[1]), 94 | {"action": "merge"}, 95 | follow=True) 96 | issue = models.Issue.objects.get(title='Test') 97 | assert_equal(issue.status, "open") 98 | # Make sure it redirects to login page 99 | goodRedirect = "/login/" 100 | self.assertRedirects(res, goodRedirect) 101 | 102 | def test_merge_locked_config(self): 103 | self.add_issue(updated_config=True) 104 | issue = models.Issue.objects.get(title='Test') 105 | issue.config.locked = True 106 | issue.config.save() 107 | self.client.login(username='test_admin', password='test') 108 | res = self.client.post(reverse("ispdb_show_issue", args=[1]), 109 | {"action": "merge"}, 110 | follow=True) 111 | issue = models.Issue.objects.get(title='Test') 112 | assert_equal(issue.status, "open") 113 | assert_true(issue.config.display_name != "testing2") 114 | assert_true("This configuration is locked. Only admins can unlock it." 115 | in res.content) 116 | 117 | def test_merge_issue_superuser(self): 118 | self.add_issue(updated_config=True) 119 | self.client.login(username='test_admin', password='test') 120 | self.client.post(reverse("ispdb_show_issue", args=[1]), 121 | {"action": "merge"}, 122 | follow=True) 123 | issue = models.Issue.objects.get(title='Test') 124 | assert_equal(issue.status, "closed") 125 | assert_equal(issue.config.display_name, "testing2") 126 | assert_equal(1, len(issue.config.domains.all())) 127 | assert_equal("test1.com", issue.config.domains.all()[0].name) 128 | # doc url 129 | assert_equal(len(issue.config.docurl_set.all()), 1) 130 | docurl = issue.config.docurl_set.all()[0] 131 | assert_equal(docurl.url, 'http://test1.com/') 132 | assert_equal(len(docurl.descriptions.all()), 1) 133 | desc = docurl.descriptions.all()[0] 134 | assert_equal(desc.description, 'test1') 135 | # enable URL 136 | assert_equal(len(issue.config.enableurl_set.all()), 1) 137 | enableurl = issue.config.enableurl_set.all()[0] 138 | assert_equal(enableurl.url, 'http://test1.com/') 139 | assert_equal(len(enableurl.instructions.all()), 1) 140 | inst = enableurl.instructions.all()[0] 141 | assert_equal(inst.description, 'test1') 142 | -------------------------------------------------------------------------------- /ispdb/tests/test_view.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | import os 4 | from lxml import etree 5 | 6 | from django.core.urlresolvers import reverse 7 | from django.test import TestCase 8 | from nose.tools import assert_equal 9 | 10 | from ispdb.config import models 11 | 12 | 13 | # Utility Functions. 14 | 15 | def make_config(value): 16 | "Get the dictionary for a sample config." 17 | return { 18 | "asking_or_adding": "adding", 19 | "domain-TOTAL_FORMS": "1", 20 | "domain-INITIAL_FORMS": "0", 21 | "domain-0-id": "", 22 | "domain-0-name": "test%s.com" % value, 23 | "domain-0-DELETE": "False", 24 | "display_name": "test%s" % value, 25 | "display_short_name": "test%s" % value, 26 | "incoming_type": "imap", 27 | "incoming_hostname": "foo", 28 | "incoming_port": "22%s" % value, 29 | "incoming_socket_type": "plain", 30 | "incoming_authentication": "password-cleartext", 31 | "incoming_username_form": "%EMAILLOCALPART%", 32 | "outgoing_hostname": "bar", 33 | "outgoing_port": "22%s" % value, 34 | "outgoing_socket_type": "STARTTLS", 35 | "outgoing_username_form": "%EMAILLOCALPART%", 36 | "outgoing_authentication": "password-cleartext", 37 | "docurl-INITIAL_FORMS": "0", 38 | "docurl-TOTAL_FORMS": "1", 39 | "docurl-MAX_NUM_FORMS": "", 40 | "docurl-0-id": "", 41 | "docurl-0-DELETE": "False", 42 | "docurl-0-url": "http://test%s.com/" % value, 43 | "desc_0-INITIAL_FORMS": "0", 44 | "desc_0-TOTAL_FORMS": "1", 45 | "desc_0-MAX_NUM_FORMS": "", 46 | "desc_0-0-id": "", 47 | "desc_0-0-DELETE": "False", 48 | "desc_0-0-language": "en", 49 | "desc_0-0-description": "test%s" % value, 50 | "enableurl-INITIAL_FORMS": "0", 51 | "enableurl-TOTAL_FORMS": "1", 52 | "enableurl-MAX_NUM_FORMS": "", 53 | "enableurl-0-id": "", 54 | "enableurl-0-DELETE": "False", 55 | "enableurl-0-url": "http://test%s.com/" % value, 56 | "inst_0-INITIAL_FORMS": "0", 57 | "inst_0-TOTAL_FORMS": "1", 58 | "inst_0-MAX_NUM_FORMS": "", 59 | "inst_0-0-id": "", 60 | "inst_0-0-DELETE": "False", 61 | "inst_0-0-language": "en", 62 | "inst_0-0-description": "test%s" % value 63 | } 64 | 65 | 66 | def check_returned_xml(response, id_count): 67 | "Make sure the response xml has the right values." 68 | assert_equal(response.status_code, 200) 69 | assert_equal(response["Content-Type"], "text/xml") 70 | 71 | content = etree.XML(response.content) 72 | assert_equal(len(content.findall("provider")), id_count) 73 | 74 | ids = content.findall("provider/id") 75 | assert_equal(len(ids), id_count) 76 | for (n, i) in enumerate(ids): 77 | assert_equal(int(i.text), n + 1) 78 | 79 | exports = content.findall("provider/export") 80 | assert_equal(len(exports), id_count) 81 | for (n, i) in enumerate(exports): 82 | assert_equal(i.text, "/export_xml/%d/" % (n + 1)) 83 | 84 | 85 | def check_returned_html(response, id_count): 86 | assert_equal(response.template[0].name, "config/list.html") 87 | configs = response.context[0]["configs"] 88 | assert_equal(len(configs), id_count) 89 | 90 | 91 | class ListTest(TestCase): 92 | "A class to test the list view." 93 | 94 | test2dict = make_config("2") 95 | test3dict = make_config("3") 96 | 97 | fixtures = ['login_testdata.json'] 98 | 99 | def test_empty_xml_reponse(self): 100 | response = self.client.get(reverse("ispdb_list", args=["xml"]), {}) 101 | check_returned_xml(response, 0) 102 | 103 | def test_single_xml_reponse(self): 104 | self.client.login(username='test_admin', password='test') 105 | response = self.client.post(reverse("ispdb_add"), ListTest.test2dict) 106 | assert_equal(response.status_code, 302) 107 | domain = models.DomainRequest.objects.get(name="test2.com") 108 | assert isinstance(domain, models.DomainRequest) 109 | response = self.client.post( 110 | reverse("ispdb_approve", kwargs={"id": domain.config.id}), 111 | {"approved": "mark valid", "comment": "I liked this domain"}) 112 | domain = models.Domain.objects.get(name="test2.com") 113 | assert isinstance(domain, models.Domain) 114 | response = self.client.get(reverse("ispdb_list", args=["xml"]), {}) 115 | check_returned_xml(response, 1) 116 | 117 | def test_two_xml_reponses(self): 118 | self.client.login(username='test_admin', password='test') 119 | response = self.client.post(reverse("ispdb_add"), ListTest.test2dict) 120 | domain = models.DomainRequest.objects.get(name="test2.com") 121 | response = self.client.post( 122 | reverse("ispdb_approve", kwargs={"id": domain.config.id}), 123 | {"approved": "mark valid", "comment": "I liked this domain"}) 124 | domain = models.Domain.objects.get(name="test2.com") 125 | assert isinstance(domain, models.Domain) 126 | response = self.client.post(reverse("ispdb_add"), ListTest.test3dict) 127 | domain = models.DomainRequest.objects.get(name="test3.com") 128 | response = self.client.post( 129 | reverse("ispdb_approve", kwargs={"id": domain.config.id}), 130 | {"approved": "mark valid", "comment": "I liked this domain"}) 131 | domain = models.Domain.objects.get(name="test3.com") 132 | assert isinstance(domain, models.Domain) 133 | response = self.client.get(reverse("ispdb_list", args=["xml"]), {}) 134 | check_returned_xml(response, 2) 135 | 136 | def test_xml_reponse_invalid_domain(self): 137 | self.client.login(username='test_admin', password='test') 138 | response = self.client.post(reverse("ispdb_add"), ListTest.test2dict) 139 | assert_equal(response.status_code, 302) 140 | domain = models.DomainRequest.objects.get(name="test2.com") 141 | assert isinstance(domain, models.DomainRequest) 142 | response = self.client.post( 143 | reverse("ispdb_approve", kwargs={"id": domain.config.id}), 144 | {"denied": "mark invalid", "comment": "I didn't like this domain"}) 145 | response = self.client.get(reverse("ispdb_list", args=["xml"]), {}) 146 | check_returned_xml(response, 0) 147 | 148 | def test_empty_reponse(self): 149 | response = self.client.get(reverse("ispdb_list"), {}) 150 | check_returned_html(response, 0) 151 | 152 | def test_single_reponse(self): 153 | self.client.login(username='test_admin', password='test') 154 | response = self.client.post(reverse("ispdb_add"), ListTest.test2dict) 155 | assert_equal(response.status_code, 302) 156 | domain = models.DomainRequest.objects.get(name="test2.com") 157 | assert isinstance(domain, models.DomainRequest) 158 | 159 | response = self.client.post( 160 | reverse("ispdb_approve", kwargs={"id": domain.config.id}), 161 | {"approved": "mark valid", 162 | "comment": "Always enter a comment here" 163 | } 164 | ) 165 | response = self.client.get(reverse("ispdb_list"), {}) 166 | check_returned_html(response, 1) 167 | 168 | def test_two_reponses(self): 169 | self.client.login(username='test_admin', password='test') 170 | response = self.client.post(reverse("ispdb_add"), ListTest.test2dict) 171 | domain = models.DomainRequest.objects.get(name="test2.com") 172 | response = self.client.post( 173 | reverse("ispdb_approve", kwargs={"id": domain.config.id}), 174 | {"approved": "mark valid", "comment": "I liked this domain"}) 175 | 176 | response = self.client.post(reverse("ispdb_add"), ListTest.test3dict) 177 | domain = models.DomainRequest.objects.get(name="test3.com") 178 | response = self.client.post( 179 | reverse("ispdb_approve", kwargs={"id": domain.config.id}), 180 | {"denied": "mark invalid", "comment": "I didn't like this domain"}) 181 | response = self.client.get(reverse("ispdb_list"), {}) 182 | check_returned_html(response, 2) 183 | 184 | def test_one_dot_zero_xml_response(self): 185 | self.client.login(username='test', password='test') 186 | response = self.client.post(reverse("ispdb_add"), ListTest.test2dict) 187 | domain = models.DomainRequest.objects.get(name="test2.com") 188 | assert isinstance(domain, models.DomainRequest) 189 | 190 | response = self.client.post(reverse("ispdb_export_xml", 191 | kwargs={"id": domain.config.id})) 192 | etree.XML(response.content) 193 | etree.RelaxNG(file=os.path.join(os.path.dirname(__file__), 194 | 'relaxng_schema.xml')) 195 | 196 | def test_one_dot_one_xml_response(self): 197 | self.client.login(username='test', password='test') 198 | response = self.client.post(reverse("ispdb_add"), ListTest.test2dict) 199 | domain = models.DomainRequest.objects.get(name="test2.com") 200 | assert isinstance(domain, models.DomainRequest) 201 | response = self.client.post(reverse("ispdb_export_xml", 202 | kwargs={"version": "1.1", "id": domain.config.id})) 203 | doc = etree.XML(response.content) 204 | 205 | xml_schema = etree.RelaxNG(file=os.path.join(os.path.dirname(__file__), 206 | 'relaxng_schema.1.1.xml')) 207 | xml_schema.assertValid(doc) 208 | -------------------------------------------------------------------------------- /ispdb/tests/test_xml.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | import os 4 | from lxml import etree 5 | 6 | from django.core.urlresolvers import reverse 7 | from django.test import TestCase 8 | from nose.tools import assert_equal 9 | 10 | from ispdb.config import models, serializers 11 | 12 | 13 | def check_content(config_xml): 14 | expected = {"domain": ["test.com", "aim.com"], 15 | "displayName": "NetZero Email", 16 | "displayShortName": "netzero"} 17 | expected_incoming = {"hostname": "hostname_in", "port": "143", 18 | "socketType": "SSL", "username": "%EMAILLOCALPART%", 19 | "authentication": "plain"} 20 | expected_outgoing = {"hostname": "hostname_out", 21 | "port": "25", "socketType": "SSL", 22 | "username": "%EMAILLOCALPART%", "authentication": "plain", 23 | "addThisServer": "false", "useGlobalPreferredServer": "false"} 24 | doc = etree.XML(config_xml) 25 | #gets incoming/outgoing server element root 26 | root = doc.getiterator() 27 | incoming = doc.find("emailProvider/incomingServer") 28 | outgoing = doc.find("emailProvider/outgoingServer") 29 | for element in root: 30 | if element.tag in expected: 31 | if type(expected[element.tag]) == list: 32 | assert element.text in expected[element.tag] 33 | else: 34 | assert_equal(element.text, expected[element.tag]) 35 | for element in incoming: 36 | assert_equal(element.text, expected_incoming[element.tag]) 37 | for element in outgoing: 38 | assert_equal(element.text, expected_outgoing[element.tag]) 39 | 40 | 41 | class XMLTest(TestCase): 42 | 43 | fixtures = ['xml_testdata'] 44 | 45 | def test_validated_export_xml_one_zero(self): 46 | domain = models.Domain.objects.get(name="test.com") 47 | config_xml = serializers.xmlOneDotZero(domain.config) 48 | doc = etree.XML(config_xml) 49 | 50 | xml_schema = etree.RelaxNG(file=os.path.join(os.path.dirname(__file__), 51 | 'relaxng_schema.xml')) 52 | xml_schema.assertValid(doc) 53 | 54 | def test_validated_export_xml_one_one(self): 55 | domain = models.Domain.objects.get(name="test.com") 56 | config_xml = serializers.xmlOneDotOne(domain.config) 57 | doc = etree.XML(config_xml) 58 | xml_schema = etree.RelaxNG(file=os.path.join(os.path.dirname(__file__), 59 | 'relaxng_schema.1.1.xml')) 60 | xml_schema.assertValid(doc) 61 | 62 | def test_xml_content(self): 63 | domain = models.Domain.objects.get(name="test.com") 64 | config_xml = serializers.xmlOneDotZero(domain.config) 65 | check_content(config_xml) 66 | 67 | def test_export_xml_view_no_version(self): 68 | response = self.client.get(reverse("ispdb_export_xml", 69 | args=["1"]), {}) 70 | check_content(response.content) 71 | 72 | def test_export_xml_view_valid_version(self): 73 | response = self.client.get(reverse("ispdb_export_xml", 74 | args=["1.0", "1"]), {}) 75 | check_content(response.content) 76 | 77 | def test_export_xml_view_invalid_version(self): 78 | response = self.client.get(reverse("ispdb_export_xml", 79 | args=["10.0", "1"]), {}) 80 | assert_equal(response.status_code, 404) 81 | -------------------------------------------------------------------------------- /ispdb/urls.py: -------------------------------------------------------------------------------- 1 | from django.conf.urls import patterns, include, url 2 | from django.http import HttpResponse 3 | from django.contrib.staticfiles.urls import staticfiles_urlpatterns 4 | 5 | # Uncomment the next two lines to enable the admin: 6 | from django.contrib import admin 7 | admin.autodiscover() 8 | 9 | urlpatterns = patterns('', 10 | (r'^robots\.txt$', lambda r: HttpResponse("User-agent: *\nDisallow: /", 11 | mimetype="text/plain")), 12 | (r'^admin/', include(admin.site.urls)), 13 | (r'^comments/post/', 'ispdb.config.views.comment_post_wrapper'), 14 | url(r'^comments/delete/(\d+)/$', 'ispdb.config.views.delete_comment', 15 | name='ispdb_comment_delete'), 16 | (r'^comments/', include('django.contrib.comments.urls')), 17 | (r'^browserid/', include('django_browserid.urls')), 18 | url(r'^login/$', 'ispdb.config.views.login', name='ispdb_login'), 19 | url(r'^logout/$', 'ispdb.config.views.logout_view', name='ispdb_logout'), 20 | url(r'^$', 'ispdb.config.views.intro', name='ispdb_index'), 21 | url(r'^list/(?P\w+)/$', 'ispdb.config.views.list', 22 | name='ispdb_list'), 23 | url(r'^list/$', 'ispdb.config.views.list', name='ispdb_list'), 24 | url(r'^details/(?P\d+)/$', 'ispdb.config.views.details', 25 | name='ispdb_details'), 26 | url(r'^export_xml/(?P\d+)/$', 'ispdb.config.views.export_xml', 27 | name='ispdb_export_xml'), 28 | url(r'^export_xml/v(?P\d+\.\d+)/(?P\d+)/$', 29 | 'ispdb.config.views.export_xml', name='ispdb_export_xml'), 30 | url(r'^export_xml/(?P[^\/]+)/$', 31 | 'ispdb.config.views.export_xml', name='ispdb_export_xml'), 32 | url(r'^export_xml/v(?P\d+\.\d+)/(?P.+)/$', 33 | 'ispdb.config.views.export_xml', name='ispdb_export_xml'), 34 | url(r'^add/(?P[\w\d\-\.]+)/$', 'ispdb.config.views.add', 35 | name='ispdb_add'), 36 | url(r'^add/$', 'ispdb.config.views.add', name='ispdb_add'), 37 | url(r'^edit/(?P\d+)/$', 'ispdb.config.views.edit', 38 | name='ispdb_edit'), 39 | url(r'^queue/$', 'ispdb.config.views.queue', name='ispdb_queue'), 40 | url(r'^policy/$', 'ispdb.config.views.policy', name='ispdb_policy'), 41 | url(r'^approve/(?P\d+)/$', 'ispdb.config.views.approve', 42 | name='ispdb_approve'), 43 | url(r'^delete/(?P\d+)/$', 'ispdb.config.views.delete', 44 | name='ispdb_delete'), 45 | url(r'^report/(?P\d+)/$', 'ispdb.config.views.report', 46 | name='ispdb_report'), 47 | url(r'^show_issue/(?P\d+)/$', 'ispdb.config.views.show_issue', 48 | name='ispdb_show_issue'), 49 | url(r'^sanity/(?P\d+)/$', 'ispdb.config.views.sanity', 50 | name='ispdb_sanity'), 51 | ) 52 | 53 | urlpatterns += staticfiles_urlpatterns() 54 | -------------------------------------------------------------------------------- /manage.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import os 3 | import sys 4 | 5 | if __name__ == "__main__": 6 | os.environ.setdefault("DJANGO_SETTINGS_MODULE", "ispdb.settings") 7 | 8 | from django.core.management import execute_from_command_line 9 | 10 | execute_from_command_line(sys.argv) 11 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | Django==1.4 2 | django-browserid>=0.5 3 | django-nose==1.0 4 | lxml==2.3.4 5 | nose==1.1.2 6 | pytz 7 | dnspython==1.7.1 8 | mox 9 | tldextract 10 | -------------------------------------------------------------------------------- /tools/MozAutoConfig.pm: -------------------------------------------------------------------------------- 1 | package MozAutoConfig; 2 | 3 | use strict; 4 | use warnings; 5 | use Apache2::RequestRec (); 6 | use Apache2::RequestUtil (); 7 | use Apache2::RequestIO (); 8 | use APR::Table (); 9 | use Apache2::Const -compile => qw(OK HTTP_NOT_FOUND); 10 | use Net::DNS; 11 | use IO::Select; 12 | use Time::HiRes qw(time); 13 | use File::Path qw(mkpath); 14 | use File::Temp qw(mkstemps); 15 | use LWP::UserAgent; 16 | use HTTP::Request; 17 | 18 | use constant DEBUG => 0; 19 | 20 | # Domain matching RE (use our own, since _ is technically invalid, but seems to be used) 21 | my $domain_re = qr/[a-z0-9_\-]+(\.[a-z0-9_\-]+)+/i; 22 | 23 | # Some known primary MX exchange server -> domain mappings 24 | my %known_domains_mx = ( 25 | 'ASPMX.L.GOOGLE.COM' => 'gmail.com', 26 | 'IN1.SMTP.MESSAGINGENGINE.COM' => 'fastmail.fm', 27 | ); 28 | 29 | sub handler { 30 | my $r = shift; 31 | 32 | # Don't need request body (we only handle GET requests) 33 | $r->discard_request_body(); 34 | 35 | # Get config known dirs and cache dirs 36 | my $known_domains_path = $r->dir_config('known_domains_path') 37 | || die 'Must set config option "known_domains_path"'; 38 | my $cache_domains_path = $r->dir_config('cache_domains_path') 39 | || warn 'No config option "cache_domains_path" specified, so no domains being cached. Are you sure you want that?'; 40 | 41 | # Should we're only a single level directory, and the last part 42 | # is always the domain 43 | my $uri = $r->uri; 44 | 45 | if ($r->prev) { 46 | my $prev = $r->prev; 47 | $uri = $prev->uri; 48 | } 49 | 50 | warn "handler called. uri=$uri" if DEBUG; 51 | 52 | my ($domain) = $uri =~ m#/([a-z0-9_\-\.]*)$#; 53 | warn "extract domain. domain=$domain" if DEBUG; 54 | 55 | # Create simple object for state storage 56 | my $self = bless { 57 | r => $r, 58 | errors => [], 59 | depth => 0, 60 | seen_domains => {}, 61 | domain => $domain, 62 | known_domains_path => $known_domains_path, 63 | cache_domains_path => $cache_domains_path, 64 | }, 'MozAutoConfig'; 65 | 66 | # If we're given a domain, lookup the data for it 67 | if ($domain) { 68 | 69 | # Run full lookup and return data for domain if found 70 | if ($self->return_domain_data($domain)) { 71 | return Apache2::Const::OK; 72 | } 73 | 74 | } else { 75 | $self->add_error("Need a valid domain in the path"); 76 | } 77 | 78 | $self->add_error("Could not find domain autoconfig details"); 79 | 80 | $self->print_errors(); 81 | 82 | return Apache2::Const::OK; 83 | } 84 | 85 | sub return_domain_data { 86 | my ($self, $domain) = @_; 87 | 88 | # Avoid deep recursive lookups 89 | if ($self->{depth}++ > 10) { 90 | $self->add_error("Recursive lookup too deep"); 91 | return undef; 92 | } 93 | # Avoid lookup loops 94 | if ($self->{seen_domains}->{$domain}++) { 95 | $self->add_error("Recursive domain loop"); 96 | return undef; 97 | } 98 | 99 | # Check if it's from the pre-known list 100 | if ($self->return_known_domain($domain)) { 101 | # Found it and returned the data 102 | return 1; 103 | } 104 | 105 | # Or in our cache 106 | if ($self->return_cache_domain($domain)) { 107 | return 1; 108 | } 109 | 110 | # Resolve mx and txt records 111 | my $dns_results = $self->resolve_dns( 112 | [ "mx", $domain, "MX"], 113 | [ "txt", "mozautoconfig." . $domain, "TXT" ] 114 | ); 115 | if (!$dns_results) { 116 | $self->add_error("Timeout on dns lookup"); 117 | return undef; 118 | } 119 | 120 | # Check for valid TXT record 121 | my $dns_txt = $dns_results->{txt}; 122 | if ($dns_txt && $self->return_domain_txt($domain, $dns_txt)) { 123 | return 1; 124 | } 125 | 126 | # Check for known primary MX records 127 | my $dns_mx = $dns_results->{mx}; 128 | if ($dns_mx && $self->return_domain_mx($domain, $dns_mx)) { 129 | return 1; 130 | } 131 | 132 | return undef; 133 | } 134 | 135 | sub return_known_domain { 136 | my ($self, $domain) = @_; 137 | 138 | # Look for file in the known domains path 139 | my $known_domains_file = $self->known_domains_path($domain); 140 | warn "checking for known domain. path=$known_domains_file" if DEBUG; 141 | 142 | if (-f $known_domains_file) { 143 | warn "found known domain, returning data. path=$known_domains_file" if DEBUG; 144 | my $r = $self->{r}; 145 | 146 | # TODO: Set Expires/Etag headers? 147 | 148 | $r->content_type('text/xml'); 149 | $r->sendfile($known_domains_file); 150 | 151 | return 1; 152 | } 153 | 154 | return undef; 155 | } 156 | 157 | sub return_cache_domain { 158 | my ($self, $domain) = @_; 159 | 160 | # Look for file in the known domains path 161 | my $cache_domains_file = $self->cache_domains_path($domain); 162 | warn "checking cache domain. path=$cache_domains_file" if DEBUG; 163 | 164 | if ($cache_domains_file && -f($cache_domains_file)) { 165 | warn "found cache domain, returning data. path=$cache_domains_file" if DEBUG; 166 | 167 | my $r = $self->{r}; 168 | 169 | # TODO: Set Expires/Etag headers? 170 | 171 | $r->headers_out()->add("X-MozAutoConfig", "UsingCached"); 172 | $r->content_type('text/xml'); 173 | $r->sendfile($cache_domains_file); 174 | 175 | return 1; 176 | } 177 | 178 | return undef; 179 | } 180 | 181 | sub return_domain_mx { 182 | my ($self, $domain, $dns_mx) = @_; 183 | 184 | if ($dns_mx && @$dns_mx) { 185 | 186 | # Pull out on MX results and sort by priority 187 | my @mx_exchanges = grep { $_->type() eq 'MX' } @$dns_mx; 188 | @mx_exchanges = sort { $a->preference() <=> $b->preference() } @mx_exchanges; 189 | 190 | # Use the lowest priority 191 | my $primary_mx = @mx_exchanges ? uc($mx_exchanges[0]->exchange()) : ''; 192 | 193 | warn "checking primary mx. mx=$primary_mx" if DEBUG; 194 | 195 | # Primary MX record -> domain mapping 196 | if (my $alt_domain = $known_domains_mx{$primary_mx}) { 197 | warn "found known mx. mx=$primary_mx" if DEBUG; 198 | 199 | return $self->return_domain_data($alt_domain); 200 | } 201 | } 202 | 203 | return undef; 204 | } 205 | 206 | sub return_domain_txt { 207 | my ($self, $domain, $dns_txt) = @_; 208 | 209 | if ($dns_txt && @$dns_txt) { 210 | for (@$dns_txt) { 211 | if ($_->type() eq 'TXT') { 212 | my $txt_data = $_->txtdata(); 213 | 214 | warn "found txt record. txt=$txt_data" if DEBUG; 215 | 216 | # Use an SPF like syntax that starts with "v=mozautoconfig1 " 217 | # and then has lookup data 218 | 219 | # For now we support two formats: 220 | # domain:otherdomain.com - repeat lookup for otherdomain.com 221 | # url:http... - retrieve data from given url 222 | 223 | if ($txt_data =~ s#^v=mozautoconfig1 ##) { 224 | if ($txt_data =~ m#^domain:(${domain_re})$#) { 225 | my $domain = $1; 226 | 227 | warn "found txt domain reference. domain=$domain" if DEBUG; 228 | 229 | # Try and find domain details 230 | if ($self->return_domain_data($domain)) { 231 | return 1; 232 | } 233 | 234 | $self->add_error("Could not find $domain details"); 235 | 236 | } elsif ($txt_data =~ m#^url:(https?://${domain_re}[\x21-\x7f]*)$#i) { 237 | my $url = $1; 238 | 239 | warn "found txt url reference. url=$url" if DEBUG; 240 | 241 | if ($self->return_domain_httpfetch($domain, $url)) { 242 | return 1; 243 | } 244 | 245 | $self->add_error("Fetch from $url failed"); 246 | } 247 | } 248 | } 249 | } 250 | } 251 | 252 | return undef; 253 | } 254 | 255 | sub return_domain_httpfetch { 256 | my ($self, $domain, $url) = @_; 257 | 258 | $url =~ m#^https?://${domain_re}#i || die "URL sanity failure"; 259 | 260 | my $ua = LWP::UserAgent->new( 261 | max_size => 32768, 262 | timeout => 10, 263 | parse_head => 0, 264 | protocols_allowed => [ 'http', 'https' ], 265 | ); 266 | 267 | # Replace %%domain%% in url with original domain 268 | $url =~ s#\%\%domain\%\%#$self->{domain}#ge; 269 | 270 | warn "fetching url data. url=$url" if DEBUG; 271 | 272 | my $request = HTTP::Request->new(GET => $url); 273 | 274 | # Force a true 10 second timeout (see LWP docs about timeout param) 275 | my $result; 276 | eval { 277 | local $SIG{ALRM} = sub { die "timeout"; }; 278 | my $old_alarm = alarm(10); 279 | $result = $ua->request($request); 280 | alarm($old_alarm); 281 | }; 282 | if ($@ =~ /timeout/) { 283 | $self->add_error("A timeout occured fetching $url"); 284 | return 0; 285 | 286 | } elsif ($@) { 287 | $self->add_error("An unexpected error occured feteching $url"); 288 | return 0; 289 | } 290 | 291 | # Sanity check result and basic content expectation 292 | 293 | my $code = $result->code(); 294 | if ($code != 200) { 295 | $self->add_error("An error occured fetching $url, response code $code"); 296 | return 0; 297 | } 298 | 299 | my $content_type = $result->header('content-type'); 300 | if ($content_type !~ m#^text/xml(?:$|;)#i) { 301 | $self->add_error("Fetched $url, but not text/xml content type"); 302 | return 0; 303 | } 304 | 305 | my $content = $result->content(); 306 | if ($content !~ m#^\s*<\?xml#) { 307 | $self->add_error("Fetched $url, but doesn't seem to start with an section"); 308 | return 0; 309 | } 310 | if ($content !~ m#.*?#s) { 311 | $self->add_error("Fetched $url, but doesn't seem to have a block"); 312 | return 0; 313 | } 314 | if ($content =~ m#<(?:html|head|title|meta|body|div|p)\b#i) { 315 | $self->add_error("Fetched $url, but appears to contain html like content"); 316 | return 0; 317 | } 318 | 319 | # TODO: actually listen to Expires header, rather than out own 320 | # arbitrary caching 321 | 322 | # Ok, data looks sane, lets cache it (original domain requested) locally, then return it to the client 323 | $self->store_cache_domain($self->{domain}, $content); 324 | 325 | # TODO: Set Expires/Etag headers? 326 | 327 | warn "url fetch success, returning data. url=$url" if DEBUG; 328 | 329 | my $r = $self->{r}; 330 | $r->content_type('text/xml'); 331 | $r->print($content); 332 | 333 | return 1; 334 | } 335 | 336 | sub store_cache_domain { 337 | my ($self, $domain, $content) = @_; 338 | 339 | my $cache_domains_file = $self->cache_domains_path($domain); 340 | 341 | warn "storing cache file. domain=$domain, path=$cache_domains_file" if DEBUG; 342 | 343 | # Use standard unix create tmp, rename into place for atomicity 344 | 345 | my ($tmp_fh, $tmp_file) = mkstemps($cache_domains_file . ".XXXXXX", "tmp"); 346 | 347 | # Create to tmp file, and rename into place atomically 348 | if (!$tmp_fh || !$tmp_file) { 349 | warn "Open of tmp file failed, can't cache $domain data: $!"; 350 | return 0; 351 | } 352 | 353 | print $tmp_fh $content; 354 | close $tmp_fh; 355 | 356 | if (!rename($tmp_file, $cache_domains_file)) { 357 | warn "Rename of tmp file $tmp_file to $cache_domains_file failed, can't cache $domain data: $!"; 358 | unlink($tmp_file); 359 | return 0; 360 | } 361 | 362 | return 1; 363 | } 364 | 365 | sub resolve_dns { 366 | my ($self, @to_resolve) = @_; 367 | 368 | # Resolve records at the same time, so 369 | # use Net::Resolve background query mode 370 | 371 | my $timeout = 10; 372 | my $resolver = Net::DNS::Resolver->new; 373 | 374 | my (%sockets, %reverse_sockets); 375 | 376 | # Save key => socket lookup, and reverse socket => key (stringify socket ref) 377 | for (@to_resolve) { 378 | my ($key, $domain, $type) = @$_; 379 | 380 | my $socket = $resolver->bgsend($domain, $type); 381 | $sockets{$key} = $socket; 382 | $reverse_sockets{"$socket"} = $key; 383 | } 384 | 385 | my $select = IO::Select->new(values %sockets); 386 | 387 | # Keep going until we have everything or we timeout 388 | my %results; 389 | while (keys %sockets && $timeout > 0.1) { 390 | 391 | # Find sockets with data, track timeout remaining 392 | my $start = time; 393 | my @ready = $select->can_read($timeout); 394 | $timeout -= time - $start; 395 | 396 | foreach my $socket (@ready) { 397 | my $packet = $resolver->bgread($socket); 398 | my $key = $reverse_sockets{"$socket"}; 399 | 400 | $results{$key} = [ $packet->answer() ]; 401 | 402 | # Cleanup all socket references so it auto-closes 403 | $select->remove($socket); 404 | delete $sockets{delete $reverse_sockets{"$socket"}}; 405 | } 406 | } 407 | 408 | unless ($timeout > 0.1) { 409 | return undef; 410 | } 411 | 412 | return \%results; 413 | } 414 | 415 | sub known_domains_path { 416 | my ($self, $domain) = @_; 417 | 418 | # Sanity check domain in a few ways (should never be a problem, but be proactive) 419 | $domain !~ m#/# || die "Domain contains /"; 420 | $domain !~ m#\.\.# || die "Domain contains .."; 421 | $domain =~ m#^${domain_re}$# || die "Unexpected invalid domain '$domain'"; 422 | 423 | return $self->{known_domains_path} . "/" . $domain; 424 | } 425 | 426 | sub cache_domains_path { 427 | my ($self, $domain) = @_; 428 | 429 | # Sanity check domain in a few ways (should never be a problem, but be proactive) 430 | $domain !~ m#/# || die "Domain contains /"; 431 | $domain !~ m#\.\.# || die "Domain contains .."; 432 | $domain =~ m#^${domain_re}$# || die "Unexpected invalid domain"; 433 | 434 | my $cache_domains_path = $self->{cache_domains_path} 435 | || return undef; 436 | 437 | # Use 2 level deep dir on domain name 438 | $cache_domains_path .= "/" . substr($domain, 0, 1) . "/" . substr($domain, 1, 1); 439 | 440 | -d($cache_domains_path) || mkpath($cache_domains_path); 441 | 442 | return $cache_domains_path . "/" . $domain; 443 | } 444 | 445 | sub add_error { 446 | my $self = shift; 447 | 448 | push @{$self->{errors}}, @_; 449 | warn "adding errors. errors=@_" if DEBUG; 450 | } 451 | 452 | sub print_errors { 453 | my $self = shift; 454 | 455 | my ($r, $errors) = @$self{qw(r errors)}; 456 | $r->status(Apache2::Const::HTTP_NOT_FOUND); 457 | $r->content_type('text/plain'); 458 | print map { $_ . "\n" } @$errors; 459 | } 460 | 461 | 462 | 1; 463 | 464 | -------------------------------------------------------------------------------- /tools/convert.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | 3 | # ***** BEGIN LICENSE BLOCK ***** 4 | # Version: MPL 1.1 5 | # 6 | # The contents of this file are subject to the Mozilla Public License Version 7 | # 1.1 (the "License"); you may not use this file except in compliance with 8 | # the License. You may obtain a copy of the License at 9 | # http://www.mozilla.org/MPL/ 10 | # 11 | # Software distributed under the License is distributed on an "AS IS" basis, 12 | # WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License 13 | # for the specific language governing rights and limitations under the 14 | # License. 15 | # 16 | # The Original Code is Thunderbird autoconfig conversion tool. 17 | # 18 | # The Initial Developer of the Original Code is 19 | # Mogens Isager . 20 | # 21 | # Portions created by the Initial Developer are Copyright (C) 2010 22 | # the Initial Developer. All Rights Reserved. 23 | # 24 | # Contributor(s): 25 | # Blake Winton 26 | # 27 | # ***** END LICENSE BLOCK ***** 28 | 29 | import argparse 30 | import codecs 31 | import os.path 32 | import stat 33 | import sys 34 | import lxml.etree as ET 35 | 36 | 37 | def read_config(file, convertTime): 38 | return (ET.parse(file), max(os.stat(file).st_mtime, convertTime)) 39 | 40 | 41 | def print_config(doc): 42 | print doc.toxml(encoding="UTF-8") 43 | 44 | 45 | def write_config(outData, time, filename=None): 46 | if os.path.exists(filename) and os.stat(filename).st_mtime >= time: 47 | return 48 | print "Writing %s" % filename 49 | file = codecs.open(filename, "w") 50 | file.write(outData) 51 | file.close() 52 | 53 | 54 | def write_domains(doc, time, output_dir="."): 55 | outData = ET.tostring(doc.getroot(), encoding="UTF-8") + "\n" 56 | for d in doc.findall("//domain"): 57 | write_config(outData, time, output_dir + "/" + d.text) 58 | 59 | 60 | def convert_11_to_10(doc): 61 | # Mark that we're writing a 1.0 client config. 62 | doc.getroot().attrib["version"] = "1.0" 63 | 64 | # Change password-cleartext to plain and 65 | # password-encrypted to secure (from bug 525238). 66 | for a in doc.findall("//authentication"): 67 | if "password-cleartext" in a.text: 68 | a.text = "plain" 69 | if "password-encrypted" in a.text: 70 | a.text = "secure" 71 | 72 | # Remove all but the first of the incoming and outgoing servers. 73 | def removeRest(parent, tagname): 74 | first = True 75 | for a in parent.findall(tagname): 76 | if first: 77 | first = False 78 | else: 79 | parent.remove(a) 80 | parent = doc.find("//emailProvider") 81 | removeRest(parent, "incomingServer") 82 | removeRest(parent, "outgoingServer") 83 | outgoingServer = parent.find("outgoingServer") 84 | for a in outgoingServer.findall("restriction"): 85 | outgoingServer.remove(a) 86 | for a in parent.findall("documentation"): 87 | parent.remove(a) 88 | 89 | 90 | def main(): 91 | # parse command line options 92 | parser = argparse.ArgumentParser() 93 | parser.add_argument("-v", choices=["1.0"], 94 | help="convert input files to version 1.0") 95 | parser.add_argument("-d", metavar="dir", 96 | help="output directory") 97 | parser.add_argument("-a", action="store_true", 98 | help="write configuration files for all domains") 99 | parser.add_argument("file", nargs="*", 100 | help="input file(s) to process, wildcards allowed") 101 | args = parser.parse_args(sys.argv[1:]) 102 | 103 | # process arguments 104 | convertTime = os.stat(sys.argv[0]).st_mtime 105 | is_dir = stat.S_ISDIR 106 | 107 | for f in args.file: 108 | if is_dir(os.stat(f).st_mode): 109 | continue 110 | 111 | if f == "README": 112 | continue 113 | 114 | doc, time = read_config(f, convertTime) 115 | 116 | if args.v == "1.0": 117 | convert_11_to_10(doc) 118 | 119 | if args.a: 120 | if args.d: 121 | write_domains(doc, time, args.d) 122 | else: 123 | print "When you want to write domain files you", 124 | print "should also specify an output directory", 125 | print "using -d dir" 126 | parser.print_usage() 127 | exit(2) 128 | elif args.d: 129 | write_config(doc, time, args.d + "/" + os.path.basename(f)) 130 | else: 131 | print_config(doc) 132 | 133 | 134 | if __name__ == "__main__": 135 | main() 136 | -------------------------------------------------------------------------------- /tools/etld.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | 3 | # Copyright (c) 2009 Michael Still 4 | # Released under the terms of the GNU GPL v2 5 | 6 | # Mozilla publishes a rule file which may be used to calculate effective TLDs 7 | # at: 8 | # 9 | # http://mxr.mozilla.org/mozilla-central/source/netwerk/dns/src/ 10 | # effective_tld_names.dat?raw=1 11 | # 12 | # Use that file to take a domain name and return a (domain, etld) tuple. 13 | # Documentation for the rule file format is at: 14 | # 15 | # https://wiki.mozilla.org/Gecko:Effective_TLD_Service 16 | 17 | import re 18 | import sys 19 | import time 20 | 21 | class EtldException(Exception): 22 | pass 23 | 24 | 25 | class etld(object): 26 | """Helper to determine the effective TLD portion of a domain name.""" 27 | 28 | def __init__(self, datafile='effective_tld_names.dat'): 29 | """Load the data file ready for lookups.""" 30 | 31 | self.rules = {} 32 | 33 | file = open(datafile) 34 | line = file.readline() 35 | while line: 36 | line = line.rstrip() 37 | if line and not line.startswith('//'): 38 | tld = line.split('.')[-1] 39 | self.rules.setdefault(tld, []) 40 | self.rules[tld].append(re.compile(self.regexpize(line))) 41 | 42 | line = file.readline() 43 | file.close() 44 | 45 | def regexpize(self, line): 46 | """Convert a rule to regexp syntax.""" 47 | 48 | line = line[::-1].replace('.', '\\.').replace('*', '[^\\.]*').replace('!', '') 49 | return '^(%s)\.(.*)$' % line 50 | 51 | def parse(self, hostname): 52 | """Parse a hostanme into domain and etld portions.""" 53 | 54 | hostname = hostname.lower() 55 | tld = hostname.split('.')[-1] 56 | hostname = hostname[::-1] 57 | domain = '' 58 | etld = '' 59 | 60 | if tld in self.rules: 61 | for rule in self.rules[tld]: 62 | m = rule.match(hostname) 63 | if m and m.group(1) > etld: 64 | domain = m.group(2)[::-1] 65 | etld = m.group(1)[::-1] 66 | 67 | if not etld: 68 | raise EtldException('Parse failed') 69 | 70 | return (domain, etld) 71 | 72 | 73 | if __name__ == '__main__': 74 | e = etld() 75 | 76 | f = open(sys.argv[1]) 77 | l = f.readline() 78 | start_time = time.time() 79 | 80 | while l: 81 | try: 82 | l = l.rstrip() 83 | print '%s -> %s' %(l, e.parse(l)) 84 | except Exception, ex: 85 | print ex 86 | 87 | l = f.readline() 88 | 89 | print 'Took %f seconds' % (time.time() - start_time) 90 | f.close() 91 | -------------------------------------------------------------------------------- /tools/quickparse.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | from datetime import date, timedelta 5 | import DNS 6 | import etld # 7 | from optparse import OptionParser 8 | import os 9 | import pickle 10 | import re 11 | import sys 12 | 13 | 14 | # Constants. 15 | 16 | # Utility functions. 17 | 18 | def readPrevious(pickleName): 19 | """Read the previous data from the specified pickle.""" 20 | prevHits = None 21 | prevMisses = None 22 | mxs = {} 23 | if os.path.exists(pickleName): 24 | data = open(pickleName) 25 | prevHits = pickle.load(data) 26 | prevMisses = pickle.load(data) 27 | try: 28 | mxs = pickle.load(data) 29 | except EOFError: 30 | pass 31 | return prevHits, prevMisses, mxs 32 | 33 | def writeNext(pickleName, hits, misses, mxs): 34 | """Write the current data to the specified pickle.""" 35 | out = open(pickleName, "w") 36 | pickle.dump(hits, out, -1) 37 | pickle.dump(misses, out, -1) 38 | pickle.dump(mxs, out, -1) 39 | out.close() 40 | 41 | def gatherData(files, mxs): 42 | """Gather all the data.""" 43 | domains = [] 44 | domain2count = {} 45 | countsperIP = {} 46 | regex = re.compile(r"(\d+.\d+\d.\d+).*?live.mozillamessaging.com/autoconfig/(\S*) HTTP/1.\d\" (\d*)") 47 | for infile in files: 48 | for line in open(infile).readlines(): 49 | if not line: continue 50 | found = regex.search(line) 51 | if not found: continue 52 | ip, domain, code = found.groups() 53 | if ip not in countsperIP: 54 | countsperIP[ip] = 1 55 | else: 56 | countsperIP[ip] += 1 57 | if not domain: continue 58 | domain = domain.split("?")[0] 59 | if (domain,code) in domain2count: 60 | domain2count[(domain,code)] += 1 61 | else: 62 | domains.append(dict(domain=domain, code=code)) 63 | domain2count[(domain,code)] = 1 64 | for domain in domains: 65 | domain["count"] = domain2count[(domain["domain"],domain["code"])] 66 | 67 | # So, now we've got all the lines, but some of the failures will actually 68 | # be hits on the MX, so let's try to remove those. 69 | domainsDict = dictify(domains) 70 | mx_hits = []; 71 | for domain in domains: 72 | if domain["code"] == "404": 73 | # We've got a missing domain, so let's check for the MX record. 74 | values = domain["domain"].split("/") 75 | prefix = "" 76 | name = values[-1] 77 | if len(values) > 1: 78 | prefix = values[-2] 79 | mx = getMX(mxs, name) 80 | if domain["domain"] == mx or domain["domain"] == (prefix + "/" + mx): 81 | continue 82 | if mx and (mx in domainsDict or (prefix + "/" + mx) in domainsDict): 83 | mx_hits.append(domain["count"]) 84 | domains.remove(domain) 85 | 86 | ip_histogram = {} 87 | for ip, count in countsperIP.items(): 88 | ip_histogram[count] = ip_histogram.setdefault(count, 0) + 1 89 | 90 | counts = ip_histogram.keys() 91 | counts.sort() 92 | return domains, counts, mx_hits, ip_histogram 93 | 94 | def dictify(data): 95 | """Change a list of domains, counts, and codes into a dict of the same.""" 96 | retval = {} 97 | for d in data: 98 | domain = d["domain"] 99 | count = d["count"] 100 | retval[domain] = retval.get(domain, 0) + count 101 | return retval 102 | 103 | def printDetails(data, total_queries, hitPrefix=""): 104 | """Print the details lines for each domain in data.""" 105 | retval = 0 106 | for domain,count in data: 107 | percent = 100.0*count/total_queries 108 | retval += percent 109 | print " %s (%s%d hits, aka %3.1f%%)" % (domain, hitPrefix, count, percent) 110 | return retval 111 | 112 | def rank_by_count(a,b): 113 | return -(a[1] - b[1]) 114 | 115 | def calculateDiffs(prevData, data): 116 | """For each domain in data, calculate the delta between it and the 117 | domain in prevData.""" 118 | retval = [] 119 | for domain,count in data.iteritems(): 120 | prevValue = prevData.get(domain, 0) 121 | count -= prevValue 122 | retval.append((domain,count)) 123 | retval.sort(rank_by_count) 124 | return retval 125 | 126 | etldService = etld.etld() 127 | 128 | def getSLD(domain): 129 | """Get the "second level domain", e.g. "mozilla.org" or "bbc.co.uk" """ 130 | try: 131 | sp = etldService.parse(domain) # returns ("5.4.bbc", "co.uk") 132 | sld = sp[0].rsplit(".", 1)[-1] 133 | tld = sp[1] 134 | return sld + "." + tld 135 | except etld.EtldException: 136 | return domain 137 | 138 | 139 | mx_queries = 0 140 | mx_cache_hit = 0 141 | 142 | def getMX(mxs, name): 143 | """ You pass in domain |name| and it returns the hostname of the MX server. 144 | It either uses the cache |mxs| or does a lookup via DNS over the Internet 145 | (and populates the cache).""" 146 | global mx_queries 147 | global mx_cache_hit 148 | if name not in mxs: 149 | possible_mxs = [] 150 | try: 151 | possible_mxs = DNS.mxlookup(name.encode("utf-8")) 152 | mx_queries += 1 153 | except DNS.DNSError: 154 | pass 155 | except UnicodeError: 156 | pass 157 | if len(possible_mxs) < 1: 158 | possible_mxs = [(0, "")] 159 | mxs[name] = getSLD(possible_mxs[0][1]) 160 | else: 161 | mx_cache_hit += 1 162 | 163 | return mxs[name] 164 | 165 | class Usage(Exception): 166 | def __init__(self, msg): 167 | self.msg = msg 168 | 169 | 170 | # Main method. 171 | 172 | def main(argv=None): 173 | if argv is None: 174 | argv = sys.argv 175 | 176 | usage = """%prog [options] logfile [...] 177 | logfile Apache logfile""" 178 | parser = OptionParser(usage=usage) 179 | parser.add_option("-p", "--previous", dest="previous", 180 | default=None, #set below 181 | help="Where to get the cache data of the previous run" 182 | "of this script esp. DNS MX lookups, which can" 183 | "take hours. .pickle by default.") 184 | parser.add_option("-n", "--next", dest="next", 185 | default=None, # set below 186 | help="Where to write the new cache data of this run of" 187 | "this script. .pickle by default.") 188 | (options, logfiles) = parser.parse_args() 189 | 190 | if len(logfiles) < 1: 191 | parser.print_usage() 192 | exit(1) 193 | 194 | if not options.next: 195 | options.next = os.path.splitext(logfiles[0])[0]+".pickle" 196 | if not options.previous: 197 | try: 198 | name = os.path.splitext(logfiles[0])[0] 199 | d = date(int(name[0:4]), int(name[4:6]), int(name[6:8])) 200 | d -= timedelta(days=1) 201 | name = d.strftime("%Y%m%d") 202 | options.previous = name+".pickle" 203 | except: 204 | options.previous = options.next 205 | 206 | prevHits, prevMisses, mxs = readPrevious(options.previous) 207 | 208 | domains, counts, mx_hits, ip_histogram = gatherData(logfiles, mxs) 209 | 210 | print "# of requests per single IP:" 211 | for c in counts[:9]: 212 | print "%4d %d" %(c, ip_histogram[c]) 213 | if len(counts) > 9: 214 | print "%3d+" %(counts[9],), sum([ip_histogram[i] for i in counts[9:]]) 215 | 216 | print "" 217 | 218 | hits = dictify(d for d in domains if d["code"] in ("200","304")) 219 | misses = dictify(d for d in domains if d["code"] == "404") 220 | weirdos = sorted(d for d in domains if d["code"] not in ("200","304","404")) 221 | 222 | miss_total = sum(misses.values()) 223 | weirdo_total = sum(x["count"] for x in weirdos) 224 | hit_total = sum(hits.values()) 225 | mx_total = sum(mx_hits) 226 | total_queries = miss_total + hit_total + weirdo_total 227 | if total_queries != sum([d["count"] for d in domains]): 228 | print "Error: total_queries (%d) != sum of domain counts (%d)." % ( 229 | total_queries, sum([d["count"] for d in domains])) 230 | 231 | 232 | print "HITS: %d domains, accounting for %d successes, or %3.1f%% success rate" % (len(hits), hit_total, 100.*hit_total/total_queries) 233 | print " MX: %d domains, accounting for %d hits." % (len(mx_hits), mx_total) 234 | print "MISSES: %d domains, accounting for %d failures, or %3.1f%% fail rate" % (len(misses), miss_total, 100.*miss_total/total_queries) 235 | print "WEIRDOS: %d domains, accounting for %d oddities, or %3.1f%% strangeness rate" % (len(weirdos), weirdo_total, 100.*weirdo_total/total_queries) 236 | print "\n".join(" %(domain)s (%(count)s hits, returned %(code)s)" % x for x in weirdos) 237 | print 238 | 239 | 240 | # Print the details. 241 | 242 | hitlist = sorted(hits.iteritems(), rank_by_count) 243 | misslist = sorted(misses.iteritems(), rank_by_count) 244 | 245 | print "Top 10 hits:" 246 | printDetails(hitlist[:10], total_queries) 247 | print 248 | 249 | print "Top 20 misses:" 250 | top_20_percent = printDetails(misslist[:20], total_queries) 251 | print "adding all 20 would boost our HIT rate by %3.1f%%" % top_20_percent 252 | 253 | print "Next 50 misses:" 254 | next_50_percent = printDetails(misslist[20:70], total_queries) 255 | print "adding the next 50 would boost our HIT rate by %3.1f%%" % next_50_percent 256 | print 257 | 258 | 259 | # Calculate the fastest rising misses, and the fastest falling hits. 260 | 261 | if prevHits: 262 | print "Fastest falling hits:" 263 | prevHits = calculateDiffs(prevHits, hits) 264 | prevHits.reverse() 265 | printDetails(prevHits[:10], total_queries) 266 | print 267 | 268 | if prevMisses: 269 | print "Fastest rising misses:" 270 | prevMisses = calculateDiffs(prevMisses, misses) 271 | printDetails(prevMisses[:10], total_queries, "+") 272 | print 273 | 274 | print "# DNS Statistics lookups/cached (hit ratio)" 275 | dns_ratio = 100.0 * mx_cache_hit / (mx_queries + mx_cache_hit) 276 | print "%d/%d (%3.1f%%)" % (mx_queries, mx_cache_hit, dns_ratio) 277 | 278 | writeNext(options.next, hits, misses, mxs) 279 | return 0 280 | 281 | if __name__ == "__main__": 282 | sys.exit(main()) 283 | 284 | --------------------------------------------------------------------------------