├── .gitignore ├── AUTHORS ├── INSTALL ├── LICENSE ├── README.md ├── README.rst ├── contactdb ├── README.md ├── contactdb │ ├── __init__.py │ ├── settings.py │ ├── urls.py │ └── wsgi.py ├── contacts │ ├── __init__.py │ ├── admin.py │ ├── apps.py │ ├── migrations │ │ ├── 0001_initial.py │ │ └── __init__.py │ ├── models.py │ ├── tests.py │ └── views.py └── manage.py ├── db ├── README-ripe-import.md ├── defaults.sql ├── initdb.sql ├── ripe_data.py └── ripe_diff.py ├── doc ├── .gitignore ├── 20140603-meeting-notes.mkd ├── Makefile ├── README.md ├── abuse-lookups.md ├── common │ ├── cipherStringB.tex │ ├── commands.tex │ ├── style.tex │ └── system.tex ├── contactDB-pgp support.pptx ├── datasets.mkd ├── gitHeadInfo.gin ├── img │ ├── cert.png │ ├── contact-lookup-flow.png │ ├── contact-lookup-flow.pptx │ └── draft.png ├── parkingspace.mkd ├── structure.mkd ├── styleheader.tex └── update-metadata-for-gitinfo ├── old2 ├── REQUIREMENTS.rst ├── TODO.rst ├── __init__.py ├── certdir │ ├── __init__.py │ ├── settings.py │ ├── urls.py │ └── wsgi.py ├── client │ ├── NOTES.rst │ ├── __init__.py │ ├── api.py │ ├── cc_import.py │ ├── get_API_key.py │ ├── get_PGP_Key.py │ ├── sources_import.py │ └── ti_import.py ├── contactdb │ ├── __init__.py │ ├── admin.py │ ├── api │ │ ├── __init__.py │ │ └── resources.py │ ├── fields │ │ ├── __init__.py │ │ └── jsonfield.py │ ├── filters.py │ ├── forms │ │ ├── __init__.py │ │ ├── fields.py │ │ └── widgets.py │ ├── inetnum.py │ ├── inetnumadmin.py │ ├── migrations │ │ ├── 0001_initial.py │ │ ├── 0002_initial.py │ │ └── __init__.py │ ├── models.py │ ├── netobjects.py │ ├── permissions.py │ ├── serializers.py │ ├── templates │ │ ├── add_asn.html │ │ ├── add_source.html │ │ ├── index.html │ │ └── search.html │ └── views.py ├── doc │ ├── .gitignore │ ├── 20140603-meeting-notes.mkd │ ├── Makefile │ ├── README.md │ ├── abuse-lookups.mkd │ ├── common │ │ ├── cipherStringB.tex │ │ ├── commands.tex │ │ ├── style.tex │ │ └── system.tex │ ├── contactDB-pgp support.pptx │ ├── datasets.mkd │ ├── gitHeadInfo.gin │ ├── img │ │ ├── cert.png │ │ ├── contact-lookup-flow.png │ │ ├── contact-lookup-flow.pptx │ │ └── draft.png │ ├── parkingspace.mkd │ ├── structure.mkd │ ├── styleheader.tex │ └── update-metadata-for-gitinfo ├── install_deps.sh ├── manage.py ├── old │ ├── API │ ├── contrib │ │ ├── README.rst │ │ ├── TI-import.py │ │ ├── geolocation.py │ │ ├── kmlcertmap.py │ │ └── pgpwrapper.py │ ├── db │ │ └── initialize │ │ │ ├── README.rst │ │ │ ├── import-TI-pgp.py │ │ │ └── pgpImporter.py │ ├── html │ │ ├── README │ │ ├── certmap.kml │ │ ├── lib │ │ │ ├── jquery-1.9.0.min.js │ │ │ └── oms.min.js │ │ ├── out.json │ │ ├── page.html │ │ └── pagekml.html │ ├── installation │ │ ├── contactdb.macosx.install.txt │ │ └── contactdb.ubuntu.install.sh │ └── scripts │ │ ├── drop_and_load.sh │ │ ├── gen-api.sh │ │ └── get-all-classes.sh ├── requirements.txt └── ti │ ├── .keepdir │ ├── README.rst │ ├── fetch.sh │ └── prepare_certificate.sh └── requirements.txt /.gitignore: -------------------------------------------------------------------------------- 1 | *.py[cod] 2 | *~ 3 | 4 | # TI 5 | *.p12 6 | *.pem 7 | ti/*csv 8 | ti/*asc 9 | 10 | # C extensions 11 | *.so 12 | 13 | # Packages 14 | *.egg 15 | *.egg-info 16 | dist 17 | build 18 | eggs 19 | parts 20 | sdist 21 | develop-eggs 22 | .installed.cfg 23 | lib 24 | lib64 25 | __pycache__ 26 | 27 | # Installer logs 28 | pip-log.txt 29 | 30 | # Unit test / coverage reports 31 | .coverage 32 | .tox 33 | nosetests.xml 34 | 35 | # Translations 36 | *.mo 37 | 38 | # Mr Developer 39 | .mr.developer.cfg 40 | .project 41 | .pydevproject*.pyc 42 | 43 | # client api key 44 | client/keys.py 45 | 46 | # virtialenv 47 | virtenv/ 48 | -------------------------------------------------------------------------------- /AUTHORS: -------------------------------------------------------------------------------- 1 | L. Aaron Kaplan 2 | David Durvaux 3 | Mauro Silva 4 | Tomás Lima 5 | Romain Bourgue 6 | Stefan Lenzhofer 7 | 8 | History 9 | ======= 10 | This project started in our spare time. 11 | It is closely related to the IHAP project, [intelmq](http://intelmq.org) and incident handling topics in general. 12 | 13 | It then added code done by [Intevation GmbH for BSI in 2016](https://github.com/Intevation/intelmq/tree/certbund-contact). 14 | Furthermore this code was enhanced by change requests from CERT.at and added specific fields for personal contact data. 15 | 16 | 17 | 18 | 19 | 20 | 21 | -------------------------------------------------------------------------------- /INSTALL: -------------------------------------------------------------------------------- 1 | Contributors: 2 | L. Aaron Kaplan 3 | Tomás Lima 4 | 5 | Last modified: 6 | 2013/09/19 7 | 8 | 9 | 10 | PREREQUISITES 11 | ============= 12 | 13 | Database: 14 | - PostgreSQL 9 (or higher) 15 | 16 | Python: 17 | - Python2.7 18 | 19 | Python Modules: 20 | - Psycopg2 21 | - Django 22 | - South 23 | - Tastypie Django 24 | - GnuPG 25 | - python imaging library "pil": http://www.pythonware.com/products/pil/ 26 | - python-geopy (has a dependency to python-beautifulsoup) 27 | - python gnupg library 28 | 29 | Optional Python Modules: 30 | - PyYAML (used for YAML RESTful output) 31 | - lxml (used for XML RESTful output) 32 | 33 | 34 | 35 | INSTALL 36 | ======= 37 | 38 | OS X installation: see installation/contactdb.macosx.install.txt 39 | Ubuntu 12.04 installation: run installation/contactdb.ubuntu.install.sh 40 | 41 | 42 | NOTE on old versions of tastypie 43 | =============================== 44 | 45 | There seems to be an issue with tastypie together with django 1.5x. 46 | Just simply go and edit resources.py in tastypie 47 | 48 | Old: 49 | from django.db.models.sql.constants import QUERY_TERMS, LOOKUP_SEP 50 | 51 | New: 52 | from django.db.models.constants import LOOKUP_SEP 53 | from django.db.models.sql.constants import QUERY_TERMS 54 | 55 | See also this post: http://stackoverflow.com/questions/15408255/cannot-import-name-lookup-sep 56 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # ARCHIVED PROJECT! 2 | 3 | **This project is archived. Development focus moved to other, similar projects such as [tuency](https://gitlab.com/intevation/tuency/tuency). Also have a look at [contacts.cert.at](https://github.com/aaronkaplan/contacts.cert.at) which does a nice mapping between IP 2 country code -> national CERT lookup.** 4 | 5 | # Contact DB 6 | 7 | ## Database Setup 8 | 9 | The following commands assume that PostgreSQL is running and listening on the 10 | default port. They create a database called "contactdb" which matches the 11 | default configuration of the bot. 12 | 13 | ``` 14 | su - postgres 15 | 16 | createdb --encoding=UTF8 --template=template0 contactdb 17 | psql -f db/initdb.sql contactdb 18 | psql -f db/defaults.sql contactdb 19 | ``` 20 | 21 | A database user with the right to select the data in the Contact DB 22 | must be created. This is the account, which will be used in the bot's 23 | configuration for accessing the database. 24 | 25 | ``` 26 | createuser contactdbuser --pwprompt 27 | psql -c "GRANT SELECT ON ALL TABLES IN SCHEMA public TO contactdbuser;" contactdb 28 | 29 | ``` 30 | 31 | ## Adding New Contacts 32 | 33 | This contactDB allows you to either add contacts manually or via so called "shadow tables". 34 | A shadow table is basically a table which contains the same fields as the main table. However, 35 | the data there is inserted automatically and may be dropped again at will. 36 | A shadow table can be used for lookups (e.g. do we have any contact for network xyz). If nothing is found, the main tables are searched. 37 | 38 | The most prominent use of shadow tables is to import the RIPE database (abuse_c contacts from the RIPE DB). 39 | 40 | ### Manual contacts 41 | 42 | 43 | Contacts can be added to the database directly using SQL. These 44 | manually configured contacts will take precedence over contacts which 45 | were imported automatically, i.e. by `ripe_import.py`. 46 | 47 | Connect to the database: 48 | 49 | ``` 50 | su - postgres 51 | psql contactdb 52 | 53 | ``` 54 | Add a contact: 55 | 56 | ```pgsql 57 | 58 | -- 1. Prepare contact information 59 | 60 | \set asn 3320 61 | -- unique name of the organization: 62 | \set org_name 'org1' 63 | \set org_comment 'Example comment on organization.' 64 | \set contact_email 'test@example.com' 65 | \set contact_comment 'Test comment on contact.' 66 | -- set the notification interval in seconds 67 | -- an interval of -1 means no notifications will be generated 68 | \set notification_interval 0 69 | 70 | -- 2. Add new contact 71 | 72 | BEGIN TRANSACTION; 73 | INSERT INTO autonomous_system (number) VALUES (:asn); 74 | WITH new_org AS ( 75 | INSERT INTO organisation (name,comment) 76 | VALUES (:'org_name',:'org_comment') 77 | RETURNING id 78 | ), 79 | new_contact AS ( 80 | INSERT INTO contact (email,format_id,comment) 81 | VALUES (:'contact_email', 2, :'contact_comment') 82 | RETURNING id 83 | ), 84 | new_ota AS ( 85 | INSERT INTO organisation_to_asn (organisation_id,asn_id,notification_interval) 86 | VALUES ( 87 | (SELECT id from new_org), :asn, :notification_interval 88 | ) 89 | ) 90 | INSERT INTO role (organisation_id,contact_id) VALUES ( 91 | (SELECT id from new_org), 92 | (SELECT id from new_contact) 93 | ) 94 | ; 95 | COMMIT TRANSACTION; 96 | 97 | ``` 98 | 99 | ### RIPE database (shadow tables) 100 | 101 | Please see the [README](db/README-ripe-import.md) in the db/ directory. 102 | 103 | 104 | 105 | # Generating a graphic which visualizes the schema of the ContactDB 106 | 107 | You can use `postgresql-autodoc` to do this. PG autodoc is available on both 108 | debian and ubuntu via apt. 109 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | 2 | ======= 3 | About 4 | ======= 5 | 6 | The ContactDB project was initiated to cover the need for a tool 7 | to maintain contacts for CSIRT teams. The first POC was designed 8 | based on specification of a few CERT team including CERT.at, CIRCL, 9 | CERT.pt and CERT.be. 10 | 11 | 12 | =========================== 13 | (Expected) Features 14 | =========================== 15 | 16 | * Secure implementation 17 | * Easy and modular web interface 18 | * Integration with 3rd party tools like AbuseHelper 19 | * Support for GPG public key storage 20 | * Delegation (an organisation can keep his contact info up-to-date) 21 | * ... 22 | 23 | 24 | =========================== 25 | How to get the source code 26 | =========================== 27 | 28 | You can get the source code at 29 | 30 | :: 31 | 32 | $ git clone git@github.com:certtools/contactdb.git 33 | 34 | The private version is available at:: 35 | 36 | $ git clone git@git.lo-res.org:/home/git/contactdb.git 37 | 38 | Note: the private repo contains importers and data which is not public at the moment. 39 | That's why we have a private repo as well. The source code is the same however. 40 | 41 | 42 | If you want to make some changes do it like this: 43 | 44 | :: 45 | 46 | $ git clone git@github.com:certtools/contactdb.git 47 | $ vim README.rst 48 | $ git commit -am 'fix for the README file' 49 | $ git push origin master 50 | 51 | 52 | In case it does not work, contact aaron@lo-res.org 53 | 54 | 55 | ========================== 56 | Starting the contactdb 57 | ========================== 58 | 59 | First read the INSTALL.rst file and follow the instructions. 60 | 61 | # Here is how to start the contactdb:: 62 | 63 | cd 64 | 65 | export CONTACTDB_HOME=$(pwd) 66 | mkdir $CONTACTDB_HOME/.gnupg/ && chmod 700 $CONTACTDB_HOME/.gnupg 67 | echo export CONTACTDB_HOME=$(pwd) >> ./virtenv/bin/activate 68 | echo export GNUPGHOME=$(pwd)/.gnupg >> ./virtenv/bin/activate 69 | 70 | . ./virtenv/bin/activate 71 | pip install -r requirements.txt --upgrade 72 | 73 | python ./manage.py syncdb 74 | python ./manage.py runserver 75 | 76 | Then you connect your browser to http://127.0.0.1:8000 and log in. 77 | 78 | 79 | 80 | 81 | -------------------------------------------------------------------------------- /contactdb/README.md: -------------------------------------------------------------------------------- 1 | This is the main Django contactDB directory 2 | -------------------------------------------------------------------------------- /contactdb/contactdb/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/certtools/contactdb/51c19fc235a73a447e388c9778b34255eb420221/contactdb/contactdb/__init__.py -------------------------------------------------------------------------------- /contactdb/contactdb/settings.py: -------------------------------------------------------------------------------- 1 | """ 2 | Django settings for contactdb project. 3 | 4 | Generated by 'django-admin startproject' using Django 1.9.4. 5 | 6 | For more information on this file, see 7 | https://docs.djangoproject.com/en/1.9/topics/settings/ 8 | 9 | For the full list of settings and their values, see 10 | https://docs.djangoproject.com/en/1.9/ref/settings/ 11 | """ 12 | 13 | import os 14 | 15 | # Build paths inside the project like this: os.path.join(BASE_DIR, ...) 16 | BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) 17 | 18 | 19 | # Quick-start development settings - unsuitable for production 20 | # See https://docs.djangoproject.com/en/1.9/howto/deployment/checklist/ 21 | 22 | # SECURITY WARNING: keep the secret key used in production secret! 23 | SECRET_KEY = 'm*r9f#fb!o%i*h!$%gfukzrreg%mkr%=%__n+_*w#-e@+!af%!' 24 | 25 | # SECURITY WARNING: don't run with debug turned on in production! 26 | DEBUG = True 27 | 28 | ALLOWED_HOSTS = [] 29 | 30 | 31 | # Application definition 32 | 33 | INSTALLED_APPS = [ 34 | 'django.contrib.admin', 35 | 'django.contrib.auth', 36 | 'django.contrib.contenttypes', 37 | 'django.contrib.sessions', 38 | 'django.contrib.messages', 39 | 'django.contrib.staticfiles', 40 | 'contacts', 41 | ] 42 | 43 | MIDDLEWARE_CLASSES = [ 44 | 'django.middleware.security.SecurityMiddleware', 45 | 'django.contrib.sessions.middleware.SessionMiddleware', 46 | 'django.middleware.common.CommonMiddleware', 47 | 'django.middleware.csrf.CsrfViewMiddleware', 48 | 'django.contrib.auth.middleware.AuthenticationMiddleware', 49 | 'django.contrib.auth.middleware.SessionAuthenticationMiddleware', 50 | 'django.contrib.messages.middleware.MessageMiddleware', 51 | 'django.middleware.clickjacking.XFrameOptionsMiddleware', 52 | ] 53 | 54 | ROOT_URLCONF = 'contactdb.urls' 55 | 56 | TEMPLATES = [ 57 | { 58 | 'BACKEND': 'django.template.backends.django.DjangoTemplates', 59 | 'DIRS': [], 60 | 'APP_DIRS': True, 61 | 'OPTIONS': { 62 | 'context_processors': [ 63 | 'django.template.context_processors.debug', 64 | 'django.template.context_processors.request', 65 | 'django.contrib.auth.context_processors.auth', 66 | 'django.contrib.messages.context_processors.messages', 67 | ], 68 | }, 69 | }, 70 | ] 71 | 72 | WSGI_APPLICATION = 'contactdb.wsgi.application' 73 | 74 | 75 | # Database 76 | # https://docs.djangoproject.com/en/1.9/ref/settings/#databases 77 | 78 | DATABASES = { 79 | 'default': { 80 | 'ENGINE': 'django.db.backends.postgresql', 81 | 'NAME': 'contactdb', 82 | 'USER': 'aaron' 83 | } 84 | } 85 | 86 | 87 | # Password validation 88 | # https://docs.djangoproject.com/en/1.9/ref/settings/#auth-password-validators 89 | 90 | AUTH_PASSWORD_VALIDATORS = [ 91 | { 92 | 'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator', 93 | }, 94 | { 95 | 'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator', 96 | }, 97 | { 98 | 'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator', 99 | }, 100 | { 101 | 'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator', 102 | }, 103 | ] 104 | 105 | 106 | # Internationalization 107 | # https://docs.djangoproject.com/en/1.9/topics/i18n/ 108 | 109 | LANGUAGE_CODE = 'en-us' 110 | 111 | TIME_ZONE = 'UTC' 112 | 113 | USE_I18N = True 114 | 115 | USE_L10N = True 116 | 117 | USE_TZ = True 118 | 119 | 120 | # Static files (CSS, JavaScript, Images) 121 | # https://docs.djangoproject.com/en/1.9/howto/static-files/ 122 | 123 | STATIC_URL = '/static/' 124 | -------------------------------------------------------------------------------- /contactdb/contactdb/urls.py: -------------------------------------------------------------------------------- 1 | """contactdb URL Configuration 2 | 3 | The `urlpatterns` list routes URLs to views. For more information please see: 4 | https://docs.djangoproject.com/en/1.9/topics/http/urls/ 5 | Examples: 6 | Function views 7 | 1. Add an import: from my_app import views 8 | 2. Add a URL to urlpatterns: url(r'^$', views.home, name='home') 9 | Class-based views 10 | 1. Add an import: from other_app.views import Home 11 | 2. Add a URL to urlpatterns: url(r'^$', Home.as_view(), name='home') 12 | Including another URLconf 13 | 1. Import the include() function: from django.conf.urls import url, include 14 | 2. Add a URL to urlpatterns: url(r'^blog/', include('blog.urls')) 15 | """ 16 | from django.conf.urls import url 17 | from django.contrib import admin 18 | 19 | urlpatterns = [ 20 | url(r'^admin/', admin.site.urls), 21 | ] 22 | -------------------------------------------------------------------------------- /contactdb/contactdb/wsgi.py: -------------------------------------------------------------------------------- 1 | """ 2 | WSGI config for contactdb project. 3 | 4 | It exposes the WSGI callable as a module-level variable named ``application``. 5 | 6 | For more information on this file, see 7 | https://docs.djangoproject.com/en/1.9/howto/deployment/wsgi/ 8 | """ 9 | 10 | import os 11 | 12 | from django.core.wsgi import get_wsgi_application 13 | 14 | os.environ.setdefault("DJANGO_SETTINGS_MODULE", "contactdb.settings") 15 | 16 | application = get_wsgi_application() 17 | -------------------------------------------------------------------------------- /contactdb/contacts/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/certtools/contactdb/51c19fc235a73a447e388c9778b34255eb420221/contactdb/contacts/__init__.py -------------------------------------------------------------------------------- /contactdb/contacts/admin.py: -------------------------------------------------------------------------------- 1 | from django.contrib import admin 2 | 3 | # Register your models here. 4 | 5 | 6 | from .models import * 7 | 8 | admin.site.register(Address) 9 | admin.site.register(AutonomousSystem) 10 | admin.site.register(AutonomousSystemAutomatic) 11 | # 12 | #admin.site.register(ClassificationIdentifier) 13 | #admin.site.register(ClassificationType) 14 | admin.site.register(Contact) 15 | #admin.site.register(ContactAutomatic) 16 | admin.site.register(Format) 17 | admin.site.register(Fqdn) 18 | #admin.site.register(FqdnAutomatic) 19 | #admin.site.register(Inhibition) 20 | admin.site.register(Network) 21 | #admin.site.register(NetworkAutomatic) 22 | admin.site.register(OrgSector) 23 | admin.site.register(Organisation) 24 | #admin.site.register(OrganisationAutomatic) 25 | admin.site.register(OrganisationToAsn) 26 | admin.site.register(OrganisationToAsnAutomatic) 27 | #admin.site.register(OrganisationToFqdn) 28 | #admin.site.register(OrganisationToFqdnAutomatic) 29 | admin.site.register(OrganisationToNetwork) 30 | #admin.site.register(OrganisationToNetworkAutomatic) 31 | #admin.site.register(OrganisationToTemplate) 32 | #admin.site.register(OrganisationToTemplateAutomatic) 33 | admin.site.register(Role) 34 | #admin.site.register(RoleAutomatic) 35 | admin.site.register(Sector) 36 | admin.site.register(Template) 37 | -------------------------------------------------------------------------------- /contactdb/contacts/apps.py: -------------------------------------------------------------------------------- 1 | from django.apps import AppConfig 2 | 3 | 4 | class ContactsConfig(AppConfig): 5 | name = 'contacts' 6 | -------------------------------------------------------------------------------- /contactdb/contacts/migrations/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/certtools/contactdb/51c19fc235a73a447e388c9778b34255eb420221/contactdb/contacts/migrations/__init__.py -------------------------------------------------------------------------------- /contactdb/contacts/models.py: -------------------------------------------------------------------------------- 1 | # This is an auto-generated Django model module. 2 | # You'll have to do the following manually to clean this up: 3 | # * Rearrange models' order 4 | # * Make sure each model has one field with primary_key=True 5 | # * Make sure each ForeignKey has `on_delete` set to the desired behavior. 6 | # * Remove `managed = False` lines if you wish to allow Django to create, modify, and delete the table 7 | # Feel free to rename the models, but don't rename db_table values or field names. 8 | from __future__ import unicode_literals 9 | 10 | from django.db import models 11 | 12 | 13 | class Address(models.Model): 14 | id = models.IntegerField(primary_key=True), 15 | is_postal_address = models.BooleanField(null=True, blank=True), 16 | postal_address = models.CharField(max_length=1000) 17 | zip = models.IntegerField() 18 | country = models.CharField(max_length=2) 19 | tel = models.CharField(max_length=200) 20 | comment = models.TextField() 21 | 22 | def __str__(self): 23 | return self.postal_address + ", " + str(self.zip) + " (" + self.country + ")" 24 | 25 | class Meta: 26 | managed = False 27 | db_table = 'address' 28 | 29 | 30 | class AutonomousSystem(models.Model): 31 | number = models.BigIntegerField(primary_key=True) 32 | ripe_aut_num = models.CharField(max_length=100, blank=True, null=True, help_text='ASN in the RIPE DB') 33 | comment = models.TextField() 34 | 35 | def __str__(self): 36 | return str(self.number) 37 | 38 | class Meta: 39 | managed = False 40 | db_table = 'autonomous_system' 41 | 42 | 43 | class AutonomousSystemAutomatic(models.Model): 44 | number = models.BigIntegerField(primary_key=True) 45 | import_source = models.CharField(max_length=500) 46 | import_time = models.DateTimeField() 47 | ripe_aut_num = models.CharField(max_length=100, blank=True, null=True) 48 | comment = models.TextField() 49 | 50 | class Meta: 51 | managed = False 52 | db_table = 'autonomous_system_automatic' 53 | 54 | 55 | class ClassificationIdentifier(models.Model): 56 | name = models.TextField(unique=True, primary_key=True) 57 | 58 | class Meta: 59 | managed = False 60 | db_table = 'classification_identifier' 61 | 62 | 63 | class ClassificationType(models.Model): 64 | name = models.CharField(unique=True, max_length=100, primary_key=True) 65 | 66 | class Meta: 67 | managed = False 68 | db_table = 'classification_type' 69 | 70 | 71 | class Contact(models.Model): 72 | active = models.BooleanField() 73 | firstname = models.CharField(max_length=500) 74 | lastname = models.CharField(max_length=500) 75 | tel = models.CharField(max_length=500, blank=True , null=True) 76 | fax = models.CharField(max_length=500, blank=True , null=True) 77 | mobile = models.CharField(max_length=500, blank=True , null=True) 78 | tel_priv = models.CharField(max_length=500, blank=True , null=True) 79 | mobile_priv = models.CharField(max_length=500, blank=True , null=True) 80 | pgp_key_id = models.CharField(max_length=128, blank=True , null=True) 81 | email = models.CharField(max_length=100) 82 | email_priv = models.CharField(max_length=100, blank=True, null=True) 83 | title = models.CharField(max_length=500, blank=True , null=True) 84 | birthdate = models.DateTimeField(blank=True, null=True) 85 | format = models.ForeignKey('Format', models.DO_NOTHING, blank=True , null=True) 86 | organisation = models.ForeignKey('Organisation', models.DO_NOTHING, blank=True, null=True) 87 | comment = models.TextField(blank=True, null=True) 88 | picture = models.BinaryField(blank=True, null=True) 89 | smime_certificate = models.BinaryField(blank=True, null=True) 90 | address = models.ForeignKey(Address, models.DO_NOTHING, blank=True, null=True) 91 | vouched_by = models.ForeignKey('self', models.DO_NOTHING, db_column='vouched_by', related_name='vouched_by1', null=True, blank=True) 92 | maintained_by = models.ForeignKey('self', models.DO_NOTHING, db_column='maintained_by', blank=True, null=True, related_name='maintained_by1') 93 | 94 | def __str__(self): 95 | return self.lastname + "," + self.firstname 96 | 97 | class Meta: 98 | managed = False 99 | db_table = 'contact' 100 | 101 | 102 | 103 | class ContactAutomatic(models.Model): 104 | import_source = models.CharField(max_length=500) 105 | import_time = models.DateTimeField() 106 | active = models.BooleanField() 107 | firstname = models.CharField(max_length=500) 108 | lastname = models.CharField(max_length=500) 109 | tel = models.CharField(max_length=500) 110 | fax = models.CharField(max_length=500) 111 | mobile = models.CharField(max_length=500) 112 | tel_priv = models.CharField(max_length=500) 113 | mobile_priv = models.CharField(max_length=500) 114 | pgp_key_id = models.CharField(max_length=128) 115 | email = models.CharField(max_length=100) 116 | email_priv = models.CharField(max_length=100, blank=True, null=True) 117 | title = models.CharField(max_length=500) 118 | birthdate = models.DateTimeField(blank=True, null=True) 119 | format = models.ForeignKey('Format', models.DO_NOTHING) 120 | organisation_id = models.IntegerField(blank=True, null=True) 121 | comment = models.TextField() 122 | picture = models.BinaryField(blank=True, null=True) 123 | smime_certificate = models.BinaryField(blank=True, null=True) 124 | address_id = models.IntegerField(blank=True, null=True) 125 | vouched_by = models.IntegerField() 126 | maintained_by = models.IntegerField(blank=True, null=True) 127 | 128 | class Meta: 129 | managed = False 130 | db_table = 'contact_automatic' 131 | 132 | 133 | class Format(models.Model): 134 | name = models.CharField(unique=True, max_length=80) 135 | 136 | class Meta: 137 | managed = False 138 | db_table = 'format' 139 | 140 | 141 | class Fqdn(models.Model): 142 | fqdn = models.TextField(unique=True) 143 | comment = models.TextField() 144 | 145 | class Meta: 146 | managed = False 147 | db_table = 'fqdn' 148 | 149 | 150 | class FqdnAutomatic(models.Model): 151 | import_source = models.CharField(max_length=500) 152 | import_time = models.DateTimeField() 153 | fqdn = models.TextField(unique=True) 154 | comment = models.TextField() 155 | 156 | class Meta: 157 | managed = False 158 | db_table = 'fqdn_automatic' 159 | 160 | 161 | class Inhibition(models.Model): 162 | asn = models.ForeignKey(AutonomousSystem, models.DO_NOTHING, blank=True, null=True) 163 | net = models.ForeignKey('Network', models.DO_NOTHING, blank=True, null=True) 164 | classification_type = models.ForeignKey(ClassificationType, models.DO_NOTHING, blank=True, null=True) 165 | classification_identifier = models.ForeignKey(ClassificationIdentifier, models.DO_NOTHING, blank=True, null=True) 166 | comment = models.TextField() 167 | 168 | class Meta: 169 | managed = False 170 | db_table = 'inhibition' 171 | 172 | 173 | class Network(models.Model): 174 | address = models.TextField(unique=True) # This field type is a guess. 175 | comment = models.TextField() 176 | 177 | def __str__(self): 178 | return str(self.address) 179 | 180 | class Meta: 181 | managed = False 182 | db_table = 'network' 183 | 184 | 185 | class NetworkAutomatic(models.Model): 186 | import_source = models.CharField(max_length=500) 187 | import_time = models.DateTimeField() 188 | address = models.TextField(unique=True) # This field type is a guess. 189 | comment = models.TextField() 190 | 191 | class Meta: 192 | managed = False 193 | db_table = 'network_automatic' 194 | 195 | 196 | class OrgSector(models.Model): 197 | organisation = models.ForeignKey('Organisation', models.DO_NOTHING, blank=True, null=True) 198 | sector = models.ForeignKey('Sector', models.DO_NOTHING, blank=True, null=True) 199 | 200 | class Meta: 201 | managed = False 202 | db_table = 'org_sector' 203 | 204 | 205 | class Organisation(models.Model): 206 | parent_id = models.IntegerField(blank=True, null=True) 207 | name = models.CharField(max_length=500) 208 | sector = models.ForeignKey('Sector', models.DO_NOTHING, blank=True, null=True) 209 | comment = models.TextField() 210 | ripe_org_hdl = models.CharField(max_length=100, blank=True, null=True) 211 | ti_handle = models.CharField(max_length=500, blank=True, null=True) 212 | first_handle = models.CharField(max_length=500, blank=True, null=True) 213 | address = models.ForeignKey(Address, models.DO_NOTHING, blank=True, null=True) 214 | 215 | def __str__(self): 216 | return self.name 217 | 218 | class Meta: 219 | managed = False 220 | db_table = 'organisation' 221 | 222 | 223 | class OrganisationAutomatic(models.Model): 224 | import_source = models.CharField(max_length=500) 225 | import_time = models.DateTimeField() 226 | parent_id = models.IntegerField(blank=True, null=True) 227 | name = models.CharField(max_length=500) 228 | sector = models.ForeignKey('Sector', models.DO_NOTHING, blank=True, null=True) 229 | comment = models.TextField() 230 | ripe_org_hdl = models.CharField(max_length=100, blank=True, null=True) 231 | ti_handle = models.CharField(max_length=500, blank=True, null=True) 232 | first_handle = models.CharField(max_length=500, blank=True, null=True) 233 | address_id = models.IntegerField(blank=True, null=True) 234 | 235 | class Meta: 236 | managed = False 237 | db_table = 'organisation_automatic' 238 | 239 | 240 | class OrganisationToAsn(models.Model): 241 | organisation = models.ForeignKey(Organisation, models.DO_NOTHING) 242 | asn = models.ForeignKey(AutonomousSystem, models.DO_NOTHING) 243 | notification_interval = models.IntegerField() 244 | 245 | class Meta: 246 | managed = False 247 | db_table = 'organisation_to_asn' 248 | unique_together = (('organisation', 'asn'),) 249 | 250 | 251 | class OrganisationToAsnAutomatic(models.Model): 252 | import_source = models.CharField(max_length=500) 253 | import_time = models.DateTimeField() 254 | organisation = models.ForeignKey(OrganisationAutomatic, models.DO_NOTHING) 255 | asn = models.ForeignKey(AutonomousSystemAutomatic, models.DO_NOTHING) 256 | notification_interval = models.IntegerField() 257 | 258 | class Meta: 259 | managed = False 260 | db_table = 'organisation_to_asn_automatic' 261 | unique_together = (('organisation', 'asn'),) 262 | 263 | 264 | class OrganisationToFqdn(models.Model): 265 | organisation = models.ForeignKey(Organisation, models.DO_NOTHING) 266 | fqdn = models.ForeignKey(Fqdn, models.DO_NOTHING) 267 | notification_interval = models.IntegerField() 268 | 269 | class Meta: 270 | managed = False 271 | db_table = 'organisation_to_fqdn' 272 | unique_together = (('organisation', 'fqdn'),) 273 | 274 | 275 | class OrganisationToFqdnAutomatic(models.Model): 276 | import_source = models.CharField(max_length=500) 277 | import_time = models.DateTimeField() 278 | organisation = models.ForeignKey(OrganisationAutomatic, models.DO_NOTHING) 279 | fqdn = models.ForeignKey(FqdnAutomatic, models.DO_NOTHING) 280 | notification_interval = models.IntegerField() 281 | 282 | class Meta: 283 | managed = False 284 | db_table = 'organisation_to_fqdn_automatic' 285 | unique_together = (('organisation', 'fqdn'),) 286 | 287 | 288 | class OrganisationToNetwork(models.Model): 289 | organisation = models.ForeignKey(Organisation, models.DO_NOTHING) 290 | net = models.ForeignKey(Network, models.DO_NOTHING) 291 | notification_interval = models.IntegerField() 292 | 293 | class Meta: 294 | managed = False 295 | db_table = 'organisation_to_network' 296 | unique_together = (('organisation', 'net'),) 297 | 298 | 299 | class OrganisationToNetworkAutomatic(models.Model): 300 | import_source = models.CharField(max_length=500) 301 | import_time = models.DateTimeField() 302 | organisation = models.ForeignKey(OrganisationAutomatic, models.DO_NOTHING) 303 | net = models.ForeignKey(NetworkAutomatic, models.DO_NOTHING) 304 | notification_interval = models.IntegerField() 305 | 306 | class Meta: 307 | managed = False 308 | db_table = 'organisation_to_network_automatic' 309 | unique_together = (('organisation', 'net'),) 310 | 311 | 312 | class OrganisationToTemplate(models.Model): 313 | organisation = models.ForeignKey(Organisation, models.DO_NOTHING) 314 | template = models.ForeignKey('Template', models.DO_NOTHING) 315 | 316 | class Meta: 317 | managed = False 318 | db_table = 'organisation_to_template' 319 | 320 | 321 | class OrganisationToTemplateAutomatic(models.Model): 322 | import_source = models.CharField(max_length=500) 323 | import_time = models.DateTimeField() 324 | organisation = models.ForeignKey(OrganisationAutomatic, models.DO_NOTHING) 325 | template = models.ForeignKey('Template', models.DO_NOTHING) 326 | 327 | class Meta: 328 | managed = False 329 | db_table = 'organisation_to_template_automatic' 330 | 331 | 332 | class Role(models.Model): 333 | role_type = models.TextField(blank=True, null=True) # This field type is a guess. 334 | is_primary_contact = models.BooleanField() 335 | is_secondary_contact = models.BooleanField() 336 | organisation = models.ForeignKey(Organisation, models.DO_NOTHING) 337 | contact = models.ForeignKey(Contact, models.DO_NOTHING) 338 | 339 | class Meta: 340 | managed = False 341 | db_table = 'role' 342 | 343 | 344 | class RoleAutomatic(models.Model): 345 | import_source = models.CharField(max_length=500) 346 | import_time = models.DateTimeField() 347 | role_type = models.TextField(blank=True, null=True) # This field type is a guess. 348 | is_primary_contact = models.BooleanField() 349 | is_secondary_contact = models.BooleanField() 350 | organisation = models.ForeignKey(OrganisationAutomatic, models.DO_NOTHING) 351 | contact = models.ForeignKey(ContactAutomatic, models.DO_NOTHING) 352 | 353 | class Meta: 354 | managed = False 355 | db_table = 'role_automatic' 356 | 357 | 358 | class Sector(models.Model): 359 | name = models.CharField(max_length=100) 360 | 361 | def __str__(self): 362 | return self.name 363 | 364 | class Meta: 365 | managed = False 366 | db_table = 'sector' 367 | 368 | 369 | class Template(models.Model): 370 | path = models.CharField(max_length=200) 371 | classification_type = models.ForeignKey(ClassificationType, models.DO_NOTHING) 372 | 373 | class Meta: 374 | managed = False 375 | db_table = 'template' 376 | -------------------------------------------------------------------------------- /contactdb/contacts/tests.py: -------------------------------------------------------------------------------- 1 | from django.test import TestCase 2 | 3 | # Create your tests here. 4 | -------------------------------------------------------------------------------- /contactdb/contacts/views.py: -------------------------------------------------------------------------------- 1 | from django.shortcuts import render 2 | 3 | # Create your views here. 4 | -------------------------------------------------------------------------------- /contactdb/manage.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import os 3 | import sys 4 | 5 | if __name__ == "__main__": 6 | os.environ.setdefault("DJANGO_SETTINGS_MODULE", "contactdb.settings") 7 | 8 | from django.core.management import execute_from_command_line 9 | 10 | execute_from_command_line(sys.argv) 11 | -------------------------------------------------------------------------------- /db/README-ripe-import.md: -------------------------------------------------------------------------------- 1 | RIPE DB data import 2 | ========================== 3 | A set of tools to manage imports of ripe data into to the contact database. 4 | 5 | The following input files are required: 6 | 7 | * ripe.db.organisation.gz 8 | * ripe.db.role.gz 9 | * ripe.db.aut-num.gz 10 | 11 | They will be searching in the current working directory by default. 12 | The files can be downloaded 13 | from the RIPE website (ftp://ftp.ripe.net/ripe/dbase/split/). 14 | 15 | For each contact that is created by this script, the format `feed_specific` 16 | will be set as default. You can change this by using the parameter 17 | `--notification-format`. 18 | 19 | You can also set the notification intervall with `--notification-intervall`. 20 | Default is 0. The intervall is set in seconds. 0: Immediate notification, 21 | -1 No Notification, 60: 1 Minute, etc... 22 | 23 | It is also possible to provide a whitelist of ASNs to load. Use the ``--asn-whitelist-file`` 24 | parameter to pass a filename. The script expects one AS entry per line, with 25 | the AS-prefix, e.g. ``AS123``. 26 | 27 | Usage 28 | ===== 29 | 30 | Download data to a directory: 31 | 32 | ``` 33 | d=`date +%F` 34 | mkdir $d 35 | cd $d 36 | for db in ripe.db.organisation.gz ripe.db.role.gz ripe.db.aut-num.gz 37 | do 38 | curl -O "http://ftp.ripe.net/ripe/dbase/split/$db" 39 | done 40 | ``` 41 | 42 | Optionally construct an asn-whitelist for your country, for example for `DE`: 43 | ```shell 44 | curl -O ftp://ftp.ripe.net/ripe/stats/delegated-ripencc-latest 45 | cat delegated-ripencc-latest | \ 46 | gawk --field-separator='|' '{if ($2=="DE" && $3=="asn") print "AS"$4}' \ 47 | >asn-DE.txt 48 | ``` 49 | 50 | Call `ripe_import.py --help` or `ripe_diff.py --help` 51 | to see all command line options. 52 | 53 | Now import the data into your ContactDB, we assume you used `contactdb` as 54 | database name. 55 | 56 | You can use `ripe_diff.py` instead of `ripe_import.py` below 57 | to get shown what would be imported into the database by the import step. 58 | 59 | **Make sure the connection to the database is made 60 | with sufficient rights! Use the database superuser when in doubt.** 61 | 62 | The next step assumes you are currently in the same folder like the data you 63 | downloaded. 64 | 65 | ``` 66 | cd $d 67 | ripe_import.py --conninfo dbname=contactdb --asn-whitelist-file=asn-DE.txt -v 68 | ``` 69 | 70 | Here is a different example where the paths to the files is specified 71 | explicitly: 72 | 73 | ``` 74 | ripe_import.py --conninfo "host=localhost user=intelmqadm dbname=contactdb" \ 75 | --organisation-file=/tmp/ripe/ripe.db.organisation.gz \ 76 | --role-file=/tmp/ripe/ripe.db.role.gz \ 77 | --asn-file=/tmp/ripe/ripe.db.aut-num.gz \ 78 | --verbose 79 | ``` 80 | 81 | Also see the 82 | [documentation of the libpg conninfo string](https://www.postgresql.org/docs/current/static/libpq-connect.html#LIBPQ-CONNSTRING). 83 | The [documentaion on environment variables](https://www.postgresql.org/docs/current/static/libpq-envars.html) to the connection also 84 | points towards how to savely provide a password with a ~/.pgpass file. 85 | 86 | ### use as a module 87 | `check-ripe.py` is a simple example how to use the module 88 | ripe_data independently of intelmq to write a simple check 89 | that operates on ripe's dbsplit datafiles. Capabilities and limitations 90 | are documented with ripe_data.py. 91 | -------------------------------------------------------------------------------- /db/defaults.sql: -------------------------------------------------------------------------------- 1 | BEGIN; 2 | 3 | INSERT INTO format (name) VALUES ('csv'); 4 | INSERT INTO format (name) VALUES ('feed_specific'); 5 | 6 | 7 | -- classification_type values taken from the ClassificationType class in 8 | -- intelmq/lib/harmonization.py 9 | INSERT INTO classification_type (name) VALUES 10 | ('spam'), 11 | ('malware'), 12 | ('botnet drone'), 13 | ('ransomware'), 14 | ('malware configuration'), 15 | ('c&c'), 16 | ('scanner'), 17 | ('exploit'), 18 | ('brute-force'), 19 | ('ids alert'), 20 | ('defacement'), 21 | ('compromised'), 22 | ('backdoor'), 23 | ('ddos'), 24 | ('dropzone'), 25 | ('phishing'), 26 | ('vulnerable service'), 27 | ('blacklist'), 28 | ('other'), 29 | ('unknown'); 30 | 31 | 32 | INSERT INTO classification_identifier (name) VALUES 33 | ('openchargen'), 34 | ('opendns'), 35 | ('openelasticsearch'), 36 | ('openipmi'), 37 | ('openmdns'), 38 | ('openmemcached'), 39 | ('openmongodb'), 40 | ('openmssql'), 41 | ('opennetbios'), 42 | ('openntp'), 43 | ('openportmapper'), 44 | ('openqotd'), 45 | ('openredis'), 46 | ('openssdp'), 47 | ('opentftp'), 48 | ('snmp'); 49 | 50 | 51 | COMMIT; 52 | -------------------------------------------------------------------------------- /db/ripe_data.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | 4 | # This file is part of intelMQ RIPE importer. 5 | # 6 | # intelMQ RIPE importer is free software: you can redistribute it and/or modify 7 | # it under the terms of the GNU Affero General Public License as published by 8 | # the Free Software Foundation, either version 3 of the License, or 9 | # (at your option) any later version. 10 | # 11 | # intelMQ RIPE importer is distributed in the hope that it will be useful, 12 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | # GNU General Public License for more details. 15 | # 16 | # You should have received a copy of the GNU Affero General Public License 17 | 18 | import collections 19 | import gzip 20 | 21 | def add_db_args(parser): 22 | parser.add_argument("--conninfo", 23 | default='dbname=contactdb', 24 | help="Libpg connection string. E.g. 'host=localhost" 25 | " port=5432 user=intelmq dbname=connectdb'" 26 | " Default: 'dbname=contactdb'") 27 | 28 | def add_common_args(parser): 29 | parser.add_argument("-v", "--verbose", 30 | help="increase output verbosity", 31 | default=False, 32 | action="store_true") 33 | parser.add_argument("--organisation-file", 34 | default='ripe.db.organisation.gz', 35 | help="Specify the organisation data file. Default: ripe.db.organisation.gz") 36 | parser.add_argument("--role-file", 37 | default='ripe.db.role.gz', 38 | help="Specify the contact role data file. Default: ripe.db.role.gz") 39 | parser.add_argument("--asn-file", 40 | default='ripe.db.aut-num.gz', 41 | help="Specify the AS number data file. Default: ripe.db.aut-num.gz") 42 | parser.add_argument("--asn-whitelist-file", 43 | default='', 44 | help="A file name with a whitelist of ASNs. If this option is not set, all ASNs are imported") 45 | 46 | def load_ripe_files(options): 47 | '''Read ripe files as given in the command line options. 48 | 49 | :return: tuple of (asn_list, org_list, role_list, org_to_asn, abusec_to_org) 50 | ''' 51 | 52 | ## Step 1: read all files 53 | asn_whitelist = read_asn_whitelist(options.asn_whitelist_file, 54 | verbose=options.verbose) 55 | 56 | asn_list = parse_file(options.asn_file, 57 | ('aut-num', 'org', 'status'), 58 | verbose=options.verbose) 59 | organisation_list = parse_file(options.organisation_file, 60 | ('organisation', 'org-name', 'abuse-c'), 61 | verbose=options.verbose) 62 | role_list = parse_file(options.role_file, 63 | ('nic-hdl', 'abuse-mailbox', 'org'), 'role', 64 | verbose=options.verbose) 65 | 66 | ## Step 2: Prepare new data for insertion 67 | asn_list = sanitize_asn_list(asn_list, asn_whitelist) 68 | 69 | org_to_asn = org_to_asn_mapping(asn_list) 70 | 71 | organisation_list = sanitize_organisation_list(organisation_list, 72 | org_to_asn) 73 | if options.verbose: 74 | print('** Found {} orgs to be relevant.'.format(len(organisation_list))) 75 | 76 | abusec_to_org = role_to_org_mapping(organisation_list) 77 | 78 | role_list = sanitize_role_list(role_list, abusec_to_org) 79 | 80 | if options.verbose: 81 | print('** Found {} contacts to be relevant.'.format(len(role_list))) 82 | 83 | 84 | return (asn_list, organisation_list, role_list, org_to_asn, abusec_to_org) 85 | 86 | def read_asn_whitelist(filename, verbose=False): 87 | '''Reads a list of ASNs from file. 88 | 89 | Each line of the file being one ASN in the format "ASnnnnnn". 90 | 91 | :return: list of ASN strings (maybe empty) or None 92 | ''' 93 | if filename: 94 | out = [] 95 | with open(filename) as f: 96 | out = [line.strip() for line in f] 97 | 98 | if verbose and out: 99 | print('** Loaded {} entries from ' 100 | 'ASN whitelist {}'.format(len(out), filename)) 101 | return out 102 | else: 103 | return None 104 | 105 | def parse_file(filename, fields, index_field=None, verbose=False): 106 | '''Parses a file from the RIPE (split) database set. 107 | 108 | ftp://ftp.ripe.net/ripe/dbase/split/ 109 | :param filename: the gziped filename 110 | :param fields: the field names to read 111 | :param index_field: the field that marks the beginning of a dataset. 112 | If not provided, the first element of 'fields' will be used 113 | :return: returns a list of lists with the read out values 114 | 115 | NOTE: Does **not** handle "continuation lines" (see rfc2622 section 2). 116 | 117 | NOTE: Preserves the contents of the fields like lower and upper case 118 | characters, though the RPSL is case insensitive and ASCII only. 119 | Thus for some fields it makes sense to upper() them (before 120 | comparing). 121 | ''' 122 | if verbose: 123 | print('** Reading file {0}'.format(filename)) 124 | 125 | out = [] 126 | tmp = None 127 | 128 | f = gzip.open(filename, 'rt', encoding='latin1') 129 | if not index_field: 130 | index_field = fields[0] 131 | 132 | important_fields = list(fields) + [index_field] 133 | for line in f: 134 | 135 | # Comments and remarks 136 | if (line.startswith('%') or line.startswith('#') 137 | or line.startswith('remarks:')): 138 | continue 139 | 140 | if ":" in line: 141 | key, value = [x.strip() for x in line.split(":", 1)] 142 | 143 | # Fields we are interested in, plus the index 144 | if key not in important_fields: 145 | continue 146 | 147 | # If we reach the index again, we have reached the next dataset, add 148 | # the previous data and start again 149 | if key == index_field: 150 | if tmp: # template is filled, except on the first record 151 | out.append(tmp) 152 | 153 | tmp = {} 154 | for tmp_field in fields: 155 | if not tmp.get(tmp_field): 156 | tmp[tmp_field] = [] 157 | 158 | # Only add the fields we are interested in to the result set 159 | if key in fields: 160 | tmp[key].append(value) 161 | 162 | f.close() 163 | if verbose: 164 | print(' -> read {0} entries'.format(len(out))) 165 | return out 166 | 167 | 168 | def sanitize_asn_entry(entry): 169 | """Return a sanitized version of an ASN entry. 170 | The sanitized version always has an upper case org handle. 171 | The input entry must already have an org attribute. 172 | """ 173 | entry = entry.copy() 174 | entry["org"] = [handle.upper() for handle in entry["org"]] 175 | return entry 176 | 177 | 178 | def sanitize_asn_list(asn_list, whitelist=None): 179 | """Return a sanitized copy of the AS list read from a RIPE aut-num file. 180 | The returned list retains only those entries which have the 181 | attributes 'aut-num' and 'org'. Also, if the whitelist parameter is 182 | given and not None, the first of the aut-num values must be in 183 | whitelist. 184 | """ 185 | return [sanitize_asn_entry(entry) for entry in asn_list 186 | 187 | # keep only entries for which we have the minimal 188 | # necessary attributes 189 | if entry.get('aut-num') and entry.get('org') 190 | 191 | # when using a white-list, keep only AS in the whitelist: 192 | if whitelist is None or entry['aut-num'][0] in whitelist] 193 | 194 | 195 | def sanitize_role_entry(entry): 196 | """Return a sanitized version of a role entry. 197 | The sanitized version always has upper case nic-hdl values. 198 | The input entry must already have a nic-hdl attribute. 199 | """ 200 | entry = entry.copy() 201 | entry["nic-hdl"] = [handle.upper() for handle in entry["nic-hdl"]] 202 | return entry 203 | 204 | 205 | def sanitize_role_list(role_list, abuse_c_to_org=None): 206 | """Return a sanitized copy of the role list read from a RIPE role file. 207 | The returned list retains only those entries which have an 208 | 'abuse-mailbox' attribute. 209 | 210 | If abuse_c_to_org dict is given, only entries that are keys are returned. 211 | """ 212 | new_list = [sanitize_role_entry(entry) for entry in role_list 213 | # abuse-mailbox is mandatory for a role used in abuse-c 214 | if entry.get('abuse-mailbox')] 215 | 216 | if abuse_c_to_org is not None: 217 | new_list = [entry for entry in new_list 218 | if entry['nic-hdl'][0] in abuse_c_to_org] 219 | 220 | return new_list 221 | 222 | 223 | def sanitize_organisation_entry(entry): 224 | """Return a sanitized version of a organisation entry. 225 | The sanitized version always has upper case values for organisation 226 | and abuse-c. The input entry must already have a organisation 227 | and abuse-c attributes. 228 | """ 229 | entry = entry.copy() 230 | entry["organisation"] = [handle.upper() for handle in entry["organisation"]] 231 | entry["abuse-c"] = [handle.upper() for handle in entry["abuse-c"]] 232 | return entry 233 | 234 | 235 | def sanitize_organisation_list(organisation_list, org_to_asn=None): 236 | """Return a sanitized copy of the organisation list read from a RIPE file. 237 | The entries in the returned list have been sanitized with 238 | sanitize_organisation_entry. 239 | 240 | If org_to_asn dict is given, only entries that are keys are returned. 241 | """ 242 | new_list = [sanitize_organisation_entry(entry) 243 | for entry in organisation_list] 244 | 245 | if org_to_asn is not None: 246 | new_list = [org for org in new_list 247 | if org['organisation'][0] in org_to_asn] 248 | 249 | return new_list 250 | 251 | 252 | def org_to_asn_mapping(asn_list): 253 | """Return a dictionary mapping RIPE org handles to the corresponding ASNs. 254 | The parameter is an AS list as read by parse_file and is assumed to 255 | have been sanitized by sanitize_asn_list which makes sure that the 256 | relevant information is present in all entries in the list. 257 | """ 258 | org_to_asn = collections.defaultdict(list) 259 | for entry in asn_list: 260 | org_to_asn[entry['org'][0]].append(entry['aut-num'][0][2:]) 261 | return org_to_asn 262 | 263 | 264 | def role_to_org_mapping(organisation_list): 265 | """Return a dictionary mapping RIPE role handles to their organisations. 266 | """ 267 | mapping = collections.defaultdict(list) 268 | for entry in organisation_list: 269 | abuse_c = entry['abuse-c'][0] if entry['abuse-c'] else None 270 | if abuse_c: 271 | mapping[abuse_c].append(entry['organisation'][0]) 272 | return mapping 273 | -------------------------------------------------------------------------------- /db/ripe_diff.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | 4 | import sys 5 | import argparse 6 | 7 | import psycopg2 8 | 9 | import intelmq.bots.experts.certbund_contact.common as common 10 | import intelmq.bots.experts.certbund_contact.ripe_data as ripe_data 11 | 12 | 13 | SOURCE_NAME = "ripe" 14 | 15 | 16 | 17 | 18 | def extract_asn(aut_entry): 19 | return int(aut_entry['aut-num'][0][2:]) 20 | 21 | 22 | class Organisation: 23 | 24 | def __init__(self, handle, name, asns=(), contacts=()): 25 | self.handle = handle 26 | self.name = name 27 | self.asns = list(asns) 28 | self.contacts = list(contacts) 29 | 30 | 31 | def build_organisation_objects(asn_list, organisation_list, role_list, 32 | role_to_org): 33 | 34 | orgs = {entry["organisation"][0]: Organisation(entry["organisation"][0], 35 | entry['org-name'][0]) 36 | for entry in organisation_list} 37 | 38 | unattached_as = [] 39 | for aut in asn_list: 40 | org_handle = aut["org"][0] 41 | org = orgs.get(org_handle) 42 | if org is not None: 43 | org.asns.append(extract_asn(aut)) 44 | else: 45 | unattached_as.append(aut) 46 | 47 | unattached_roles = [] 48 | for role in role_list: 49 | role_orgs = role_to_org[role['nic-hdl'][0]] 50 | if not role_orgs: 51 | unattached_roles.append(role) 52 | else: 53 | for org_handle in role_orgs: 54 | orgs[org_handle].contacts.append(role['abuse-mailbox'][0]) 55 | 56 | return orgs, unattached_as, unattached_roles 57 | 58 | 59 | def build_organisation_objects_from_db(cur): 60 | cur.execute(""" 61 | SELECT o.ripe_org_hdl, o.name, 62 | ARRAY(SELECT oa.asn_id 63 | FROM organisation_to_asn_automatic oa 64 | WHERE oa.organisation_id = o.id), 65 | ARRAY(SELECT c.email 66 | FROM contact_automatic c 67 | JOIN role_automatic r ON r.contact_id = c.id 68 | WHERE r.organisation_id = o.id) 69 | FROM organisation_automatic o 70 | WHERE o.import_source = %s; 71 | """, (SOURCE_NAME,)) 72 | 73 | orgs = {} 74 | while True: 75 | row = cur.fetchone() 76 | if row is None: 77 | break 78 | org_handle, name, asns, contacts = row 79 | orgs[org_handle] = Organisation(org_handle, name, asns, contacts) 80 | 81 | return orgs 82 | 83 | 84 | def get_unattached_asns_from_db(cur): 85 | cur.execute(""" 86 | SELECT a.number 87 | FROM autonomous_system_automatic a 88 | WHERE a.import_source = %s 89 | AND NOT EXISTS (SELECT * FROM organisation_to_asn_automatic oa 90 | WHERE oa.asn_id = a.number); 91 | """, (SOURCE_NAME,)) 92 | return [row[0] for row in cur.fetchall()] 93 | 94 | 95 | def get_unattached_contacts_from_db(cur): 96 | cur.execute(""" 97 | SELECT c.email 98 | FROM contact_automatic c 99 | WHERE c.import_source = %s 100 | AND NOT EXISTS (SELECT * FROM role_automatic r 101 | WHERE r.contact_id = c.id); 102 | """, (SOURCE_NAME,)) 103 | return [row[0] for row in cur.fetchall()] 104 | 105 | 106 | def compare_sets(a, b): 107 | return (a - b, a & b, b - a) 108 | 109 | def compare_dicts(a, b): 110 | return compare_sets(a.keys(), b.keys()) 111 | 112 | def item_list_changes(plural_name, old, new): 113 | removed, both, added = compare_sets(set(old), set(new)) 114 | if removed: 115 | yield plural_name + " removed: " + ", ".join(map(str, removed)) 116 | if added: 117 | yield plural_name + " added: " + ", ".join(map(str, added)) 118 | 119 | 120 | def organisation_changes(handles, orgs_a, orgs_b): 121 | for handle in handles: 122 | a = orgs_a[handle] 123 | b = orgs_b[handle] 124 | 125 | changes = [] 126 | if a.name != b.name: 127 | changes.append("Name changed from %r to %r" % (a.name, b.name)) 128 | changes.extend(item_list_changes("ASNs", a.asns, b.asns)) 129 | changes.extend(item_list_changes("contacts", a.contacts, b.contacts)) 130 | if changes: 131 | yield handle, changes 132 | 133 | def find_overlaid_asns_db(cur, org): 134 | for asn in org.asns: 135 | results = common.lookup_by_asn_only(cur, '', asn) 136 | if results: 137 | print(" AS{} via manual db entries resolves to:".format(asn)) 138 | for result in results: 139 | print(" {}".format(result)) 140 | 141 | 142 | def compare_orgs(cur, old_orgs, new_orgs): 143 | removed, both, added = compare_dicts(old_orgs, new_orgs) 144 | 145 | if added: 146 | print("organisations to be added:") 147 | for handle in added: 148 | print(" %s: %r" % (handle, new_orgs[handle].name,)) 149 | find_overlaid_asns_db(cur, new_orgs[handle]) 150 | 151 | if removed: 152 | print("organisations to be deleted:") 153 | for handle in removed: 154 | print(" %s: %r" % (handle, old_orgs[handle].name,)) 155 | find_overlaid_asns_db(cur, old_orgs[handle]) 156 | 157 | if both: 158 | all_changes = list(organisation_changes(both, old_orgs, new_orgs)) 159 | if all_changes: 160 | print("Changed organisations:") 161 | for handle, changes in all_changes: 162 | print(" %r:" % (handle,)) 163 | for change in changes: 164 | print(" %s" % (change,)) 165 | find_overlaid_asns_db(cur, new_orgs[handle]) 166 | 167 | 168 | def compare_unattached(name, old, new): 169 | removed, both, added = compare_sets(set(old), set(new)) 170 | if removed: 171 | print("Unattached %s to be removed:" % (name,)) 172 | for item in sorted(removed): 173 | print(" ", item) 174 | if added: 175 | print("Unattached %s to be added:" % (name,)) 176 | for item in sorted(added): 177 | print(" ", item) 178 | 179 | 180 | def compare_orgs_with_db(cur, asn_list, organisation_list, role_list, 181 | abusec_to_org): 182 | orgs, unattached_as, unattached_roles = \ 183 | build_organisation_objects(asn_list, organisation_list, 184 | role_list, abusec_to_org) 185 | 186 | db_orgs = build_organisation_objects_from_db(cur) 187 | db_unattached_as = get_unattached_asns_from_db(cur) 188 | db_unattached_roles = get_unattached_contacts_from_db(cur) 189 | 190 | compare_orgs(cur, db_orgs, orgs) 191 | 192 | compare_unattached("AS", db_unattached_as, 193 | [extract_asn(a) for a in unattached_as]) 194 | compare_unattached("roles", db_unattached_roles, 195 | [r['abuse-mailbox'][0] for r in unattached_roles]) 196 | 197 | 198 | def main(): 199 | parser = argparse.ArgumentParser( 200 | description=("Show the differences between a set of RIPE DB files" 201 | " and the contents of the database.")) 202 | 203 | ripe_data.add_db_args(parser) 204 | ripe_data.add_common_args(parser) 205 | 206 | options = parser.parse_args() 207 | 208 | (asn_list, organisation_list, role_list, 209 | org_to_asn, abusec_to_org) = ripe_data.load_ripe_files(options) 210 | 211 | con = psycopg2.connect(dsn=options.conninfo) 212 | try: 213 | compare_orgs_with_db(con.cursor(), asn_list, organisation_list, 214 | role_list, abusec_to_org) 215 | finally: 216 | con.close() 217 | 218 | 219 | if __name__ == '__main__': 220 | main() 221 | -------------------------------------------------------------------------------- /doc/.gitignore: -------------------------------------------------------------------------------- 1 | contact-databases-for-abuse-handling.aux 2 | contact-databases-for-abuse-handling.log 3 | contact-databases-for-abuse-handling.pdf 4 | contact-databases-for-abuse-handling.tex 5 | contact-databases-for-abuse-handling.toc 6 | -------------------------------------------------------------------------------- /doc/20140603-meeting-notes.mkd: -------------------------------------------------------------------------------- 1 | 2 | 3 | # PGP key maintenance 4 | 5 | * the system needs to check the PGP keyring once per day. Find expired / revoked keys and inform the users that their key expires soon and send a link to upload a new key or edit user details 6 | * sync the keyring vs. the global key servers (pull to local only!) 7 | 8 | * invariant: if a fingerprint changes, then all the fingerprints in the corresponding Email Address object need to change as well 9 | (do that via script) 10 | 11 | * The model allows for multiple email address <-> to one fingerprint and one key (==fingerprint) per email address. A person can have only one (!) email 12 | * we decided to abandon the approach of having multiple email addresses per person/org in order to reduce complexity 13 | * remove cc3 from Country 14 | * remove TZ 15 | 16 | 17 | ``` 18 | 19 | is_a (entity) 20 | +----------+ 1 1 +---------+ m 1 +-------------+ 21 | | Person | ----- | Email | ------ | fingerprint | 22 | +----------+ +---------+ +-------------+ 23 | / 24 | 1 /---- 1 25 | +---------+ 26 | | Org | 27 | +---------+ 28 | is_a(entity) 29 | 30 | ``` 31 | 32 | 33 | # OTR fingerprints 34 | 35 | ``` 36 | 37 | +---------+ m 1 +-----------+ 1 1 +--------+ 38 | | OTRfpr | --------- | JabberID | ----- | Person | 39 | +---------+ +-----------+ +--------+ 40 | 41 | ``` 42 | -------------------------------------------------------------------------------- /doc/Makefile: -------------------------------------------------------------------------------- 1 | #pdf1: 2 | # ./update-metadata-for-gitinfo 3 | # #pandoc -f markdown_github -o notes.pdf notes.md 4 | 5 | all: datasets.pdf 6 | 7 | latex: datasets.tex 8 | 9 | datasets.tex: datasets.mkd 10 | pandoc -N --listings -T "Abuse Contact datasets for IT security incident handling" -s -f markdown_github+implicit_figures+pandoc_title_block+table_captions+multiline_tables --toc -t latex -o datasets.tex datasets.mkd 11 | #pandoc -N --listings -T "Contact database for IT security incident handling" -s -f markdown_github+pandoc_title_block+pipe_tables --toc -o datasets.pdf datasets.mkd 12 | 13 | datasets.pdf: datasets.tex 14 | # add the proper stylesheets 15 | cat styleheader.tex > r 16 | tail -n +2 datasets.tex >> r 17 | mv r datasets.tex 18 | # compute version info 19 | ./update-metadata-for-gitinfo 20 | # update the toc 21 | pdflatex datasets.tex 22 | pdflatex datasets.tex 23 | 24 | clean: 25 | rm -f datasets.pdf datasets.tex datasets.log datasets.out datasets.aux datasets.toc r 26 | -------------------------------------------------------------------------------- /doc/README.md: -------------------------------------------------------------------------------- 1 | 2 | What's here? 3 | ============ 4 | 5 | 6 | |File | Explanation | 7 | |:--------------|----------------------------------------------------------------------| 8 | |Makefile |Makefile for the markdown documents | 9 | |datasets.mkd |Part I: an overview of all datasets which contain abuse contat lookup data | 10 | |abuse-lookups.mkd|Part II: a proposed mechanism for generic abuse contact lookups | 11 | |common/ |supporting .tex templates for generating a nice PDF out of the markdown document | 12 | |img/ |supporting images for generating a nice PDF out of the markdown document | 13 | |contactDB-pgp support.pptx|slides describing the contact lookup for CERTs project. | 14 | 15 | 16 | How to build the document 17 | ========================= 18 | 19 | ``` 20 | make clean; make 21 | ``` 22 | 23 | Pre-requisites 24 | ============== 25 | 26 | * pandoc 27 | * macTeX (tested with MacTex as well as TexLive) 28 | or 29 | * TexLive (Linux) 30 | or a similar Tex distribution under Windows 31 | -------------------------------------------------------------------------------- /doc/abuse-lookups.md: -------------------------------------------------------------------------------- 1 | % Abuse Contact lookups for IT security incident handling 2 | % L. Aaron Kaplan ; Mirjam Kuehne ; Christian Teuschel ; Otmar Lendl 3 | % 2014/05/10 4 | 5 | 6 | # Address Lookup Algorithm for Abuse Handling 7 | Otmar Lendl, 12.9.2014 8 | 9 | ## Overview 10 | 11 | The daily processes of CERTs include the task of finding the right address to send events to: this can be based on various databases and keying data. 12 | 13 | This document specifies a generic algorithm and a class interface design to support the implementation of this algorithm. 14 | 15 | ### Problem Statement 16 | 17 | CERTs have been hardcoding these algorithms for a long time now. E.g. for an IP-Address contained in a sinkhole-log, do: 1) map to ASN using $DB, 2) map ASN to email-address using $DB2. Or for a phishing-site: 1) map URL to Domain, 2) map domain to owner … 18 | 19 | Enhancing these algorithms (e.g. adding exceptions based on local knowledge, or adding an alternative lookup for one step) requires an adaption of all scripts where these algorithms run. This degrades the flexibility of the CERT. Consistency across all use-cases is hard to achieve, too. 20 | 21 | ### Goal 22 | * Do away with the hard-coded lookup-sequences which are re-implemented in a multitude of scripts. 23 | * A processing script should invoke a generic algorithm. That generic algorithm needs to support sufficient parameterization to yield the desired result. 24 | * New data-sources/transformations can be added to the system and thus be used without changing either the generic algorithm, let alone the scripts using the system. 25 | * Optimizations (caching, bulk-lookups, loop-prevention …) should be centrally implemented. 26 | 27 | ### Idea 28 | If one considers 29 | * information elements (Addresses, domains, ASN, contacts …) as nodes in a graph 30 | * transformations (DB-lookups, …) as (directed) vertices in this graph, 31 | then the problem can be seen as an example of a graph-search algorithms. This is a well-researched topic in computer science. There are algorithms available to address this problem-space (keywords: depth-first vs. breadth-first searches …). 32 | 33 | Implementing this boils down to: 34 | 35 | 1. Implementing all the objects in well-defined class-hierarchy that provides a consistent interface exposing the graph (node, vertex) relationships. 36 | 2. Implementing the generic algorithm. 37 | 38 | ## Objects 39 | 40 | The following objects are used in modelling: 41 | 42 | ### Data-Source 43 | 44 | This is the abstract concept of some sort of database that contains information that can be helpful in this context. Examples: 45 | * CRM database 46 | * RIR databases 47 | * DNS 48 | * The global BGP routing table 49 | 50 | ### Resource 51 | 52 | These are the nodes in the graph. 53 | 54 | The algorithm is basically a mapping between some input data to a contact address. There might be some intermediate steps involved in the mapping. The following data-types are likely to be useful: 55 | 56 | * IP-Address (v4, v6) 57 | * IP Netblock (CIDR, v4, v6) 58 | * Autonomous system 59 | * URL 60 | * FQDN 61 | * Domain 62 | * Country 63 | * Registry 64 | * Registrar 65 | * Company 66 | * Role 67 | * Person 68 | 69 | ### Transformation 70 | 71 | A transformation is an Object capable of mapping an instance of a resource to (a set of) another resources. 72 | As successful call to a transformation returns one (or more) vertices in the graph. 73 | Examples: 74 | * FQDN -> IP-Address by A record lookup in the DNS 75 | * FQDN -> Domain: Procedural based on the public-suffix list 76 | * IP-Address -> Prefix: Routing table lookup 77 | * IP-Address -> Prefix: RIR database lookup 78 | * IP-Address -> AS: Routing table lookup 79 | * AS -> Role: RIR database lookup 80 | * IP-Addess -> FQDN: DNS (PTR query) 81 | * … 82 | 83 | Any Data-source can power more than just one transformation. 84 | 85 | The same type of transformation can be implemented using different Data-sources as the backend. 86 | 87 | **Properties:** 88 | * A transformation takes a single input object and returns a set of output objects. 89 | * Such an individual mapping can contain additional metadata: 90 | * Last update 91 | * Expiry 92 | * Confidence level 93 | * Granularity level 94 | * The transformation has other properties (independent of individual lookups) 95 | * Cost 96 | * Cache-able y/n 97 | * Address-focused vs. Domain-focused (to help the transformation-selection part of the overall algorithm 98 | 99 | ## Implementation Outline 100 | 101 | This framework can be implemented in any object-oriented language. As python seems to be the current trend in CERT tooling, an implementation in python seems to be the best choice. 102 | 103 | The following descriptions is not a full API design, it is purely for demonstration purposes. Expect changes. 104 | 105 | The “Data Source” concept is purely conceptual and will not be implemented. 106 | 107 | ### Resource 108 | 109 | Each type of Resource will have its own Resource class, derived from a generic class. Each of these classes basically wraps the native implementation of such a resource. That native data-type can be simple strings (e.g. company names), or more complex objects (e.g. CIDR for IP address ranges). 110 | 111 | In addition to this native description, a resource object can also store a string that describes the reason why this object exists 112 | 113 | All resource Objects need to implement the following interface. 114 | 115 | **Class interface** 116 | 117 | * new(string) 118 | Create new Object based on string description of the resource 119 | 120 | **Object interface** 121 | 122 | * asText 123 | Return a textual description of the object 124 | * a class-specific accessor for the wrapped object 125 | * isa 126 | Returns the class-name of this object 127 | * trace(text) 128 | Accessor/setter for the trace string 129 | 130 | ### Transformation 131 | 132 | There will be a parent class of all transformations which implements the following interface: 133 | 134 | **Class interface** 135 | 136 | * new(params, prioScheme) 137 | Instantiate a new “transformations” object. The params can override system defaults on which subclasses will be loaded. prioScheme defines the default sort order for transformations. 138 | 139 | **Object interface** 140 | * initialize() 141 | Load all transformations, initialize DB connections / network sockets / … 142 | * findMatchingTransformations(resource) 143 | Return an ordered (depending on prioScheme) list of transformation objects which take a resource of the type of the parameter. 144 | * runSearch(resource, destination_type, params …) 145 | Implement the graph search. Return a list of resulting resource objects (of type destination_type). The trace attributes will contain information on the search patch between the initial resource and the result. 146 | 147 | The subclasses (one for each possible transformation) implement the following API 148 | 149 | **Class interface** 150 | 151 | * new() 152 | Initialize transformation 153 | 154 | **Object interface** 155 | 156 | * inputType() 157 | Returns the class name of resource objects it takes as input 158 | * transform(resource, params) 159 | Execute the mapping. Returns an ordered list of resulting resource objects. Params can influence the mapping and/or the ordering of the results. The trace parameter of the results will be a combination of the trace parameter of the input and information about this transformation 160 | 161 | 162 | 163 | --- 164 | 165 | Original text follows 166 | 167 | --- 168 | 169 | # A unified model for abuse contact lookups 170 | 171 | 172 | ## Network resources 173 | 174 | 175 | Before looking at the proposed algorithm and framework for contact lookups, we need to define some terms. 176 | There are two types of lookup paths that can occur: based on problem at hand, either a number-resource lookup (AS number, IP address, netblock) or name-based resource lookup path (domain, URL) needs to be chosen. 177 | 178 | We define a network resource as being one of: 179 | 180 | * ASN (example: AS1901 or 1901) 181 | * netblock (CIDR notation) (example: 193.238.156.0/21) 182 | * domain, (example: ripe.net) 183 | * hostname (fqdn) (example: www.ripe.net), or 184 | * IP address (example: 193.1.1.0). 185 | 186 | A network resource can be any one of these (but not multiple at the same time). 187 | Should we need to specifically talk about a number-resource or a name-based resource, we will use these more specific termins. Otherwise, the term "netresource" will be used as a aggregated term. 188 | 189 | Based on the definition of a netresource we can now define functions on netresources which shall return the desired information. 190 | 191 | XXX @aaron: FIXME: add ABNF syntax description here XXX 192 | 193 | 194 | 195 | ## Functional specification 196 | 197 | There is an inherent hierarchy of asking first the most specific data sources 198 | and in case nothing is found, the algorithm shall move up to the least specific 199 | data sources. All contact lookups MAY be cached. Cache timeout values are TBD. 200 | 201 | ### Lookup by ASN 202 | 203 | * **Name**: contact\_by\_asn(asn) 204 | * **Input**: ASN 205 | * **Output**: best matching contact 206 | 207 | **Implementation**: 208 | 209 | 1. lookup most specific contact for this ASN in contactDB, if nothing found,... 210 | 2. lookup abuse@ for this ASN in whois, if nothing found,... 211 | 3. lookup general contact for this ASN in whois, if nothing found... 212 | 4. lookup the country of the ASN (team cymru or maxmind), then look up the national CERT of that country (for example via https://contacts.cert.at) 213 | 214 | ### Lookup by domain name 215 | * **Name**: contact\_by\_domain(domain) 216 | * **Input**: hostname (fqdn) or domain name 217 | * **Output**: best matching contact 218 | 219 | **Implementation (pseudo-code)**: 220 | 221 | lookup domain in specific internal contactDB, 222 | IF nothing found: 223 | IF (parameter == hostname): 224 | d = extract_domain(fqdn) 225 | ELSE d = parameter 226 | lookup domain_owner(d), and/or... 227 | lookup registrar_of(d) 228 | IF nothing_found and (parameter == hostname): 229 | i= lookup_ip_of_hostname(fqdn) 230 | lookup_hoster_of_ip(i) 231 | 232 | *NOTE*: this lookup function actually might be a bit too specific. Depending on the use case you might want to look up only the domain owner or the registrar only. 233 | 234 | ### Lookup by hostname 235 | * **Name**: contact\_by\_hostname(fqdn) 236 | * **Input**: hostname (fqdn) 237 | * **Output**: best matching contact 238 | 239 | **Implementation**: 240 | 241 | lookup domain in specific internal contactDB, 242 | IF nothing found: 243 | d = extract_domain(fqdn) 244 | lookup domain_owner(d), and/or... 245 | lookup registrar_of(d) and/or... 246 | i= lookup_ip(s)_of_hostname(fqdn) 247 | lookup_hoster_of_ip(i) 248 | 249 | optional: 250 | 251 | ds = lookup_other_domains_on_this_ip(i) 252 | lookup_domain_owners(ds) 253 | lookup_registrars(ds) 254 | 255 | NOTE: again, as above, this function might be doing too much already. Depending on the circumstances, a use-case might only need the registrar or the domain owner or the hoster. 256 | 257 | ### Lookup by netblock 258 | * **Name**: contact\_by\_netblock(netblock) 259 | * **Input**: netblock in CIDR notation (for example: 1.2.3.0/24 or 2a02:60:1:1::0/32) 260 | * **Output**: best matching contact 261 | 262 | **Implementation**: 263 | 264 | lookup contact for netblock in internal contactDB, 265 | IF nothing found: 266 | lookup abuse@ in whois for the contact block, 267 | IF nothing found: 268 | lookup whois contact for the netblock, 269 | IF nothing found: 270 | lookup ASN of netblock, call lookup_by_asn(ASN), 271 | IF nothing found: 272 | lookup country code of netblock, then look up the national CERT of that country (for example via https://contacts.cert.at) 273 | 274 | 275 | ###Lookup by IP address 276 | * **Name**: contact\_by\_ip(ip) 277 | * **Input**: ip address (v4 or v6) 278 | * **Output**: best matching contact 279 | 280 | **Implementation**: 281 | 282 | lookup contact for IP address in internal contactDB, 283 | IF nothing found: 284 | lookup most specific netblock of IP, call contact_by_netblock. 285 | IF nothing found: 286 | (optional: lookup less specific netblock of IP, call contact_by_netblock(). 287 | If nothing found... ) 288 | lookup ASN of IP address, call contact_by_asn. 289 | IF nothing found: 290 | lookup country code of IP address, lookup contact of national CERT 291 | of country (for example via https://contacts.cert.at) 292 | 293 | 294 | ### Lookup by country 295 | * **Name**: contact\_by\_country(country\_code) 296 | * **Input**: ISO 2 letter country code 297 | * **Output**: best matching contact 298 | 299 | **Implementation**: 300 | 301 | lookup(country code in national CERT DB), 302 | IF FOUND: 303 | return national CERT contact 304 | ELSE 305 | return undef 306 | 307 | ###Lookup by TLD 308 | * **Name**: contact\_by\_tld(domain or tld) 309 | * **Input**: domain name or TLD 310 | * **Output**: best matching contact 311 | 312 | **Implementation**: 313 | 314 | IF TLD == ccTLD: 315 | return contact_by_country(ccTLD) 316 | ELSE 317 | c = lookup(registrar of gTLD) 318 | IF nothing found : 319 | return undef 320 | ELSE return c 321 | 322 | 323 | 324 | # Other ideas 325 | 326 | ==> can we (CERTs) get bulk access to stat.ripe.net (and to it's source data). Why? 327 | Sometimes we want to look up things in bulk *quickly*. Like really fast. For 1 million records or a billion log records (think: conficker.C apache log files) 328 | Sometimes we have an APT case where we *can not* ask any external data source about specific IPs because the mere act of querying could already reveal something. 329 | (this mostly concerns CERTs which do ani-spying detection work) 330 | 331 | Christian: We are planned to develop an asynchronous request model which will make it possible that we serve requests that produce large result sets. 332 | 333 | The contactDB lookups should be documented as a RFC I-D (apps area ??) . Because it should be a standard / recommendation for everybody. 334 | 335 | 336 | 337 | 338 | # meta 339 | * this document helps in listing reqs for ripe ncc 340 | * RIPE document machen. Evtl. auch als I-D 341 | * ziele des documents: 342 | 343 | - best practice document fuer certs und datenlieferanten 344 | - weitere datasets? FIRST, TI, interne AS2email 345 | * issues: 346 | - ip2country geolocation - wie machen? lizenz problem mit maxmind. 347 | - google ?? 348 | - einwand wilfried: es geht darum, wo der abuse-c ist. nicht wo der end-user physikalisch sitzt. Es sind verschiedene use-cases 349 | - problem mit der datenqualitaet in der RIPE DB. Es geht nur: wo ist der urspruengliche contact fuers LIR 350 | - incentives?? warum soll ich was eintragen? 351 | 352 | -------------------------------------------------------------------------------- /doc/common/cipherStringB.tex: -------------------------------------------------------------------------------- 1 | \seqsplit{EDH+CAMELLIA:EDH+aRSA:EECDH+aRSA+AESGCM:EECDH+aRSA+SHA384:EECDH+aRSA+SHA256:EECDH:+CAMELLIA256:+AES256:+CAMELLIA128:+AES128:+SSLv3:!aNULL:!eNULL:!LOW:!3DES:!MD5:!EXP:!PSK:!DSS:!RC4:!SEED:!ECDSA:CAMELLIA256-SHA:AES256-SHA:CAMELLIA128-SHA:AES128-SHA} 2 | -------------------------------------------------------------------------------- /doc/common/commands.tex: -------------------------------------------------------------------------------- 1 | %%% 2 | %%% commands.tex 3 | %%% Document-specific commands 4 | %%% 5 | 6 | % Outputs red TODOs in the document. Requires \usepackage{color}. 7 | % 8 | % Usage: \todo{Document the TODO command.} 9 | % 10 | % Comment out second line to disable. 11 | \AtBeginDocument{\providecommand{\todo}[1]{}} 12 | \newcommand*{\todo}[1]{{\color{Red} TODO: {#1}}} 13 | 14 | % Creating a horizontal rule 15 | \newcommand*{\HorRule}{% 16 | \color{darkblue}% 17 | \rule{\linewidth}{1pt}% 18 | } 19 | 20 | %%% CIPHERSTRING 21 | \usepackage{seqsplit} % Use Sequence split. Basically it inserts between every character pair a box with zero width to allow linebreaks everywhere. Better solution wanted, but is there any better? 22 | \CatchFileDef{\cipherStringB}{common/cipherStringB.tex}{\endlinechar=-1 }% 23 | 24 | %%% Local Variables: 25 | %%% mode: latex 26 | %%% TeX-master: "../applied-crypto-hardening" 27 | %%% End: 28 | -------------------------------------------------------------------------------- /doc/common/style.tex: -------------------------------------------------------------------------------- 1 | %%% 2 | %%% style.tex 3 | %%% Stylistic configuration 4 | %%% 5 | 6 | % Colors 7 | \definecolor{green}{RGB}{32,113,10} 8 | \definecolor{orange}{RGB}{251,111,16} 9 | \definecolor{red}{RGB}{247,56,0} 10 | \definecolor{blue}{RGB}{0,28,128} 11 | \definecolor{lightgreen}{RGB}{187,218,216} 12 | \definecolor{intersectgreen}{RGB}{103,133,155} 13 | \definecolor{darkblue}{RGB}{76,87,117} 14 | \definecolor{Brown}{cmyk}{0,0.81,1,0.60} 15 | \definecolor{OliveGreen}{cmyk}{0.64,0,0.95,0.40} 16 | \definecolor{CadetBlue}{cmyk}{0.62,0.57,0.23,0} 17 | \definecolor{lightlightgray}{gray}{0.9} 18 | 19 | % Draft stuff 20 | \ifdraft{ 21 | % Ensure that everything is neatly set up for us 22 | \AtBeginDocument{ 23 | \def\draftInfo{% 24 | Draft revision\gitVtags: \gitAbbrevHash{} % 25 | (\gitCommitterIsoDate) \gitCommitterName} 26 | \sbox{\draftWatermark}{% 27 | \includegraphics[width=\paperwidth]{img/draft}} 28 | \sbox{\draftPageLine}{% 29 | \colorbox{black!10}{% 30 | % enlarge box vertically by 2/3 lines 31 | \raisebox{0pt}% 32 | [\dimexpr .33\baselineskip + \height]% 33 | [\dimexpr .33\baselineskip + \depth]{% 34 | \makebox[\paperwidth]{\color{black!50}\draftInfo}}}} 35 | } 36 | }{} 37 | 38 | 39 | %%% Fonts 40 | % Fonts are loaded, now is the right time for mictorype. 41 | \microtypesetup{stretch=9,shrink=15,step=3,tracking=smallcaps,letterspace=75} 42 | % makes default font sans-serif 43 | \renewcommand{\familydefault}{\sfdefault} 44 | 45 | % Captions 46 | \setcapindent{1em} 47 | \addtokomafont{caption}{\small\itshape} 48 | \addtokomafont{captionlabel}{\bfseries} 49 | 50 | % Section headers 51 | \addtokomafont{disposition}{\color{darkblue}\bfseries} 52 | 53 | % Page foot 54 | \setkomafont{pagefoot}{\normalfont\sffamily\footnotesize} 55 | \addtolength{\headheight}{2em} 56 | \ohead{% 57 | %\includegraphics[height=3em]{img/holistic.pdf} 58 | %\hspace{2em} 59 | %\includegraphics[height=3em]{img/cert.png} 60 | } 61 | \ifoot% 62 | [Contact databases for abuse handling\ifdraft{ % Space! 63 | \textbullet{} \draftInfo}{}]% 64 | {Contact databases for abuse handling \ifdraft{ % Space! 65 | \textbullet{} \draftInfo}{}} 66 | \cfoot[]{} 67 | \ofoot% 68 | [Seite \thepage\ ]% 69 | {Seite \thepage\ } 70 | %[page \thepage\ of \pageref{LastPage}]% 71 | %{page \thepage\ of \pageref{LastPage}} 72 | \pagestyle{scrheadings} 73 | % Epigraph / dictum 74 | \newcommand*{\epigraph}[3][]{\dictum[#3]{#2}\bigskip} 75 | \renewcommand*{\dictumrule}{} 76 | \renewcommand*{\dictumauthorformat}[1]{--- #1} 77 | \addtokomafont{dictumtext}{\itshape} 78 | \setkomafont{dictumauthor}{\normalfont} 79 | \renewcommand{\dictumwidth}{8cm} 80 | 81 | % Graphics 82 | %tell TeX where to look for graphics/logos 83 | \graphicspath{ {/img/} } 84 | 85 | % This block is for listings 86 | \usepackage[framemethod=TikZ]{mdframed} % mdframed is used to draw a grey box 87 | \mdfdefinestyle{listingstyle}{ 88 | backgroundcolor=black!10,outerlinewidth=0,outerlinecolor=black, 89 | innerleftmargin=9pt,innerrightmargin=0,innertopmargin=9pt,innerbottommargin=2pt 90 | } 91 | %\usepackage{amssymb}% for \curvearrowright 92 | % Insert a grey box behind the listing for uniform background color (The \cipherstring would the listing and the background would turn white) 93 | \BeforeBeginEnvironment{lstlisting}{\vspace{0.2cm}\begin{mdframed}[style=listingstyle]} 94 | \AfterEndEnvironment{lstlisting}{\end{mdframed}} 95 | % Listings 96 | \lstset{ 97 | basicstyle=\ttfamily, 98 | keywordstyle=\color{OliveGreen}, 99 | commentstyle=\color{gray}, 100 | backgroundcolor=\color{lightlightgray}, 101 | upquote=true, 102 | showstringspaces=false, 103 | tabsize=2, 104 | captionpos=b, 105 | breaklines=true, 106 | breakatwhitespace=false, 107 | inputencoding=utf8, 108 | breakatwhitespace=false, 109 | showspaces=false, 110 | columns=fullflexible, % Column format: no spaces are inserted for monospaced appearance 111 | breakindent=10pt, 112 | morekeywords={__global__, __device__},% 113 | escapechar=\`, 114 | escapeinside={\%*}{*)}, % Escape TeX commands inside %* and *) 115 | % prebreak=\mbox{$\curvearrowright$}, % Disply curved arrow before linebreak 116 | prebreak=\small\symbol{'134}, 117 | } 118 | 119 | % red warning box (for SSH section) 120 | \mdfdefinestyle{warningboxstyle}{ 121 | backgroundcolor=pink!20, 122 | innerleftmargin=9pt,innerrightmargin=9pt,innertopmargin=9pt,innerbottommargin=9pt 123 | } 124 | 125 | % Hyperref styles 126 | %\hypersetup{% 127 | % breaklinks,% 128 | % colorlinks,% 129 | % linkcolor=darkblue,citecolor=blue,urlcolor=blue,% 130 | % breaklinks=true,% 131 | % unicode,% 132 | % pdfnewwindow=true,% 133 | % final 134 | %} 135 | %\urlstyle{same} 136 | 137 | % Bibliography 138 | \bibliographystyle{alphalink} 139 | 140 | % Disable single lines at the start of a paragraph (Schusterjungen) 141 | \clubpenalty = 10000 142 | % Disable single lines at the end of a paragraph (Hurenkinder) 143 | \widowpenalty = 10000 144 | \displaywidowpenalty = 10000 % formulas 145 | \setlength{\textfloatsep}{\baselineskip} 146 | \setlength{\floatsep}{\baselineskip} 147 | 148 | \frenchspacing 149 | \raggedbottom 150 | 151 | % customized spaces between text and footnotes 152 | \setlength{\skip\footins}{2\baselineskip} 153 | 154 | % Better float parameters: (from the TeX FAQ) 155 | \renewcommand{\topfraction}{.85} 156 | \renewcommand{\bottomfraction}{.7} 157 | \renewcommand{\textfraction}{.15} 158 | \renewcommand{\floatpagefraction}{.66} 159 | \renewcommand{\dbltopfraction}{.66} 160 | \renewcommand{\dblfloatpagefraction}{.66} 161 | \setcounter{topnumber}{9} 162 | \setcounter{bottomnumber}{9} 163 | \setcounter{totalnumber}{20} 164 | \setcounter{dbltopnumber}{9} 165 | 166 | % Starred lists (\begin{itemize*}) for less space between items 167 | \usepackage{mdwlist} 168 | 169 | %%% Local Variables: 170 | %%% mode: latex 171 | %%% TeX-master: "../applied-crypto-hardening" 172 | %%% End: 173 | -------------------------------------------------------------------------------- /doc/common/system.tex: -------------------------------------------------------------------------------- 1 | %%% 2 | %%% system.tex 3 | %%% Necessary packages and sytem changes 4 | %%% 5 | % Document Encoding. important. 6 | \usepackage[utf8]{inputenc} 7 | 8 | % changes font encoding to T1 9 | \usepackage[T1]{fontenc} 10 | \usepackage{textcomp} 11 | 12 | % For searchable pdfs 13 | \input glyphtounicode 14 | \pdfgentounicode=1 15 | 16 | \usepackage{fixltx2e} 17 | 18 | % Setup KOMA script 19 | \usepackage{scrhack} 20 | \KOMAoptions{paper=a4% 21 | ,fontsize=10pt% 22 | ,DIV=12% 23 | ,parskip=true% 24 | } 25 | 26 | % Language 27 | \usepackage[english]{babel} 28 | 29 | % Color 30 | \usepackage[usenames,dvipsnames,svgnames,table]{xcolor} 31 | \usepackage{color} 32 | 33 | % Packages for fonts 34 | \usepackage{lmodern} 35 | \usepackage[defaultsans]{opensans} 36 | \usepackage[final,babel=true]{microtype}[2011/08/18] 37 | 38 | \usepackage{pifont} 39 | \newcommand{\yes}{\textcolor{green}{\ding{51}}} 40 | \newcommand{\no}{\textcolor{red}{\ding{55}}} 41 | 42 | % Figures and graphics 43 | \usepackage[final]{graphicx} 44 | \usepackage{epstopdf} 45 | \usepackage{float} 46 | \usepackage{subfig} 47 | \usepackage{placeins} 48 | \usepackage{wrapfig} 49 | \usepackage{tikz} 50 | \usetikzlibrary{shapes,arrows} 51 | 52 | % Tables 53 | \usepackage{longtable} 54 | \usepackage{booktabs} 55 | \renewcommand{\arraystretch}{1.25} 56 | \usepackage{multicol} 57 | 58 | % Verbatims and listings 59 | \usepackage{fancyvrb} 60 | \usepackage[final]{listings} 61 | 62 | % The page 63 | \usepackage[footsepline]{scrpage2} 64 | \usepackage{lastpage} 65 | 66 | % Misc 67 | \usepackage{gitinfo} 68 | \usepackage{catchfile} 69 | %\usepackage{hyperref} 70 | 71 | % for development 72 | \usepackage{ifdraft} 73 | \ifdraft{% 74 | %% Heavy debugging 75 | %\usepackage{showframe} 76 | \usepackage{blindtext} 77 | \usepackage{eso-pic} 78 | \newsavebox{\draftPageLine} 79 | \newsavebox{\draftWatermark} 80 | \AddToShipoutPicture{% 81 | \AtPageLowerLeft{\usebox{\draftWatermark}} 82 | \AtPageUpperLeft{% 83 | \raisebox{-\height}[\height][0pt]{\usebox{\draftPageLine}}}% 84 | \AtPageLowerLeft{% 85 | \raisebox{\depth}[\height][0pt]{\usebox{\draftPageLine}}}% 86 | } 87 | % \AtEndDocument{\listoftodos} 88 | }{ 89 | \let\blindtext\relax 90 | \let\Blindtext\relax 91 | \let\blinddocument\relax 92 | \let\Blinddocument\relax 93 | } 94 | 95 | %%% Local Variables: 96 | %%% mode: latex 97 | %%% TeX-master: "../applied-crypto-hardening" 98 | %%% End: 99 | -------------------------------------------------------------------------------- /doc/contactDB-pgp support.pptx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/certtools/contactdb/51c19fc235a73a447e388c9778b34255eb420221/doc/contactDB-pgp support.pptx -------------------------------------------------------------------------------- /doc/gitHeadInfo.gin: -------------------------------------------------------------------------------- 1 | \usepackage[% 2 | shash={5ba2de6}, 3 | lhash={5ba2de6bd3eb9ec0b8c38d559eaa1e37408ba60a}, 4 | authname={Aaron Kaplan}, 5 | authemail={aaron@lo-res.org}, 6 | authsdate={2014-09-26}, 7 | authidate={2014-09-26 17:08:15 +0200}, 8 | authudate={1411744095}, 9 | commname={Aaron Kaplan}, 10 | commemail={aaron@lo-res.org}, 11 | commsdate={2014-09-26}, 12 | commidate={2014-09-26 17:08:15 +0200}, 13 | commudate={1411744095}, 14 | refnames={ (HEAD, rest)} 15 | ]{gitsetinfo} -------------------------------------------------------------------------------- /doc/img/cert.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/certtools/contactdb/51c19fc235a73a447e388c9778b34255eb420221/doc/img/cert.png -------------------------------------------------------------------------------- /doc/img/contact-lookup-flow.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/certtools/contactdb/51c19fc235a73a447e388c9778b34255eb420221/doc/img/contact-lookup-flow.png -------------------------------------------------------------------------------- /doc/img/contact-lookup-flow.pptx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/certtools/contactdb/51c19fc235a73a447e388c9778b34255eb420221/doc/img/contact-lookup-flow.pptx -------------------------------------------------------------------------------- /doc/img/draft.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/certtools/contactdb/51c19fc235a73a447e388c9778b34255eb420221/doc/img/draft.png -------------------------------------------------------------------------------- /doc/structure.mkd: -------------------------------------------------------------------------------- 1 | ## $Name of the dataset 2 | 3 | ### Overview 4 | 5 | ### Data in the dataset 6 | 7 | ### Original or synthetic data? 8 | 9 | ### Where to find it? 10 | 11 | ### Who has access? 12 | 13 | ### APIs / automatic querying 14 | 15 | ### Maintainers 16 | 17 | ### Liveliness 18 | 19 | ### Known issues 20 | -------------------------------------------------------------------------------- /doc/styleheader.tex: -------------------------------------------------------------------------------- 1 | %%% 2 | %%% Salzburg-AG hack Dokument 3 | %%% 4 | %%% 5 | \RequirePackage{fix-cm} 6 | \documentclass[draft]{scrartcl} 7 | %\documentclass{scrreprt} % uncomment this if you want to make a final version (and remove the DRAFT watermark) 8 | \input{common/system} 9 | \input{common/style} 10 | \input{common/commands} 11 | -------------------------------------------------------------------------------- /doc/update-metadata-for-gitinfo: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | GITINFO_INDEX_FILENAME="gitHeadInfo.gin" 4 | GITINFO_GIT_LOG_PRETTY_FORMAT=$(cat <<'__EOS__' 5 | \usepackage[% 6 | shash={%h}, 7 | lhash={%H}, 8 | authname={%an}, 9 | authemail={%ae}, 10 | authsdate={%ad}, 11 | authidate={%ai}, 12 | authudate={%at}, 13 | commname={%an}, 14 | commemail={%ae}, 15 | commsdate={%ad}, 16 | commidate={%ai}, 17 | commudate={%at}, 18 | refnames={%d} 19 | ]{gitsetinfo} 20 | __EOS__ 21 | ) 22 | 23 | git log -1 --date=short \ 24 | --pretty=format:"${GITINFO_GIT_LOG_PRETTY_FORMAT}" \ 25 | HEAD > ./${GITINFO_INDEX_FILENAME} 26 | 27 | 28 | -------------------------------------------------------------------------------- /old2/REQUIREMENTS.rst: -------------------------------------------------------------------------------- 1 | Requirements 2 | ============== 3 | 4 | * Debian Sid or higher 5 | * Postgresql 9.3 or higher 6 | * Python 2.7 or higher 7 | * VirtualEnv 8 | 9 | Stuff which needs to be available in a virtual env 10 | -------------------------------------------------- 11 | django 12 | django-filter 13 | djangorestframework 14 | markdown 15 | psycopg2 16 | gnupg 17 | pillow 18 | 19 | # perso 20 | ipython 21 | 22 | # client 23 | requests 24 | 25 | # temp 26 | pygments 27 | 28 | -------------------------------------------------------------------------------- /old2/TODO.rst: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | ============= 5 | GENERAL 6 | ============= 7 | 8 | clean up the mess in scripts/* db/* contrib/* and unify it -> aaron 9 | 10 | ============= 11 | Model 12 | ============= 13 | 14 | * parse all TI fields and fill them into the DB 15 | * geolocate at Organisation INSERT time: use the google API? (==> for making a map) (Stefan Lenzhofer, based on klcertmap.py) 16 | 17 | PGP 18 | ----- 19 | * fetch unknown PGP keys from the keyserver or the TI keyring (Stefan Lenzhofer provides it as a function) 20 | 21 | ========== 22 | API 23 | ========== 24 | * define REST API in a document (needs discussion) 25 | * Queries: 26 | a) given an IP address,ASN, domain, netblock or country code, give me the best matching abuse email address 27 | b) given an IP address,ASN, domain, netblock or country code, give me all the data that you have in JSON format or CSV 28 | * implement best match search algo: given an IP address, ASN, domain, netblock or country code, give me best abuse email contact 29 | 30 | LATER: 31 | * permission system for the RESTful API? How to do it? ACLs or unix-style rwx-rwx-rwx for each path element of the RESTful URL? 32 | * think about who can see what part of the tree / DB? Permission system 33 | 34 | * think about federation: how can we make a tree of orgs that are federated? Should we re-use existing directory structures such as DNS? 35 | should we take some ideas from linkeddata.org ? 36 | 37 | 38 | ========== 39 | Exporters 40 | ========== 41 | * Person/user tables export to LDAP 42 | * -------- " ---------- to VCard 43 | * Test VCard export w. Mac Addressbook 44 | * download picture for person from { internet, ops-t, FB, xing, linked in, ...} 45 | * keyring download 46 | 47 | 48 | 49 | ============== 50 | Functionality 51 | ============== 52 | * Vouching: 53 | - document vouching mechanism 54 | - implement vouching mechanism 55 | * think what ppl need from the contactDB -> social network ---> that's a different app! not in this one ("CERTbook") 56 | * add bio/personal knowledge to contactDB ---> that's a different app! not in this one ("CERTbook") 57 | 58 | 59 | * write a whois server. output format: like cymru 60 | 61 | 62 | =============== 63 | User Interface 64 | =============== 65 | 66 | * Build the UI with Django templates with: 67 | - bootstrap 3 68 | - typeahead.js 69 | * make google map of all CERTs (c.f. contrib example) 70 | 71 | =============== 72 | Admin Interface 73 | =============== 74 | 75 | * admin.py: all classes which should be editable should be here 76 | * admin.py: list_fields 77 | 78 | 79 | ======================================= 80 | Supporting / External contact databases 81 | ======================================= 82 | 83 | List of external data sources: 84 | - TI DB: https://tiw.trusted-introducer.org/directory/index.html 85 | - FIRST DB: www.first.org/members/teams/ 86 | - National CSIRT db: https://nationalcsirts.cert.org/ 87 | - whois (and especially all kinds of whois abuse searches): 88 | - Perls' Net::Abuse service 89 | - ZCW (?): http://www.fr2.cyberabuse.org/whois/?page=downloads 90 | - RIPE stats JSON interface: https://stat.ripe.net/data/abuse-contact-finder/data.json?resource=AS16010 or 91 | https://stat.ripe.net/data/abuse-contact-finder/data.json?resource= 92 | - other registries: ARIN etc 93 | - ENISA Certmap? 94 | - ... extend... 95 | 96 | Initial DB import 97 | ----------------- 98 | * Supporting data: 99 | - iso country codes: make better initialize.sh script which downloads the ISO codes on it's own 100 | - patch list of countries: "UK" and "EU" 101 | 102 | * INitial basic data should be filled in at installation time: 103 | - sources table: TI and FIRST 104 | - list of countries 105 | * geolocate at Organisation INSERT time: use the google API? 106 | 107 | 108 | TI DB import 109 | --------------- 110 | * multilines are a problem: should we make lots of 1-n relationships out of it (email addr to org) or just have multiline entries? I prefer the latter. 111 | 112 | FIRST DB import 113 | --------------- 114 | * look at it 115 | 116 | whois proxy / whowas service 117 | --------------------------- 118 | It would be great to use the certdir project to also have a whowas service. Idea: 119 | query a whois object and the DB will do that for you as a proxy but also save the result, timestamp it and it can read from the cache if needed 120 | 121 | 122 | ================ 123 | Misc. 124 | ================ 125 | 126 | 127 | -------------------------------------------------------------------------------- /old2/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/certtools/contactdb/51c19fc235a73a447e388c9778b34255eb420221/old2/__init__.py -------------------------------------------------------------------------------- /old2/certdir/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/certtools/contactdb/51c19fc235a73a447e388c9778b34255eb420221/old2/certdir/__init__.py -------------------------------------------------------------------------------- /old2/certdir/settings.py: -------------------------------------------------------------------------------- 1 | # Django settings for certdir project. 2 | 3 | import os 4 | 5 | DEBUG = True 6 | TEMPLATE_DEBUG = DEBUG 7 | 8 | ADMINS = ( 9 | # ('Your Name', 'your_email@example.com'), 10 | ) 11 | 12 | MANAGERS = ADMINS 13 | 14 | DATABASES = { 15 | 'default': { 16 | 'ENGINE': 'django.db.backends.postgresql_psycopg2', # Or use an alternate database backend. 17 | 'NAME': 'contactdb', # Path to sqlite3 database file. 18 | 'USER': 'contactdb', # Not used with sqlite3. 19 | 'PASSWORD': '', # Not used with sqlite3. 20 | 'HOST': '', # Set to empty string for localhost. Not used with sqlite3. 21 | 'PORT': '', # Set to empty string for default. Not used with sqlite3. 22 | } 23 | } 24 | 25 | # Local time zone for this installation. Choices can be found here: 26 | # http://en.wikipedia.org/wiki/List_of_tz_zones_by_name 27 | # although not all choices may be available on all operating systems. 28 | # On Unix systems, a value of None will cause Django to use the same 29 | # timezone as the operating system. 30 | # If running in a Windows environment this must be set to the same as your 31 | # system time zone. 32 | TIME_ZONE = 'Europe/Brussels' 33 | 34 | # Language code for this installation. All choices can be found here: 35 | # http://www.i18nguy.com/unicode/language-identifiers.html 36 | LANGUAGE_CODE = 'en-us' 37 | 38 | SITE_ID = 1 39 | 40 | # If you set this to False, Django will make some optimizations so as not 41 | # to load the internationalization machinery. 42 | USE_I18N = True 43 | 44 | # If you set this to False, Django will not format dates, numbers and 45 | # calendars according to the current locale. 46 | USE_L10N = True 47 | 48 | # If you set this to False, Django will not use timezone-aware datetimes. 49 | USE_TZ = False 50 | 51 | # Absolute filesystem path to the directory that will hold user-uploaded files. 52 | # Example: "/home/media/media.lawrence.com/media/" 53 | MEDIA_ROOT = '' 54 | 55 | # URL that handles the media served from MEDIA_ROOT. Make sure to use a 56 | # trailing slash. 57 | # Examples: "http://media.lawrence.com/media/", "http://example.com/media/" 58 | MEDIA_URL = '' 59 | 60 | # Absolute path to the directory static files should be collected to. 61 | # Don't put anything in this directory yourself; store your static files 62 | # in apps' "static/" subdirectories and in STATICFILES_DIRS. 63 | # Example: "/home/media/media.lawrence.com/static/" 64 | STATIC_ROOT = os.environ['CONTACTDB_HOME'] + '/static/' 65 | 66 | # URL prefix for static files. 67 | # Example: "http://media.lawrence.com/static/" 68 | STATIC_URL = '/static/' 69 | 70 | # Additional locations of static files 71 | STATICFILES_DIRS = ( 72 | # Put strings here, like "/home/html/static" or "C:/www/django/static". 73 | # Always use forward slashes, even on Windows. 74 | # Don't forget to use absolute paths, not relative paths. 75 | ) 76 | 77 | # List of finder classes that know how to find static files in 78 | # various locations. 79 | STATICFILES_FINDERS = ( 80 | 'django.contrib.staticfiles.finders.FileSystemFinder', 81 | 'django.contrib.staticfiles.finders.AppDirectoriesFinder', 82 | # 'django.contrib.staticfiles.finders.DefaultStorageFinder', 83 | ) 84 | 85 | # Make this unique, and don't share it with anybody. 86 | SECRET_KEY = 'MxGyjwrHpxn26KW7f97eedHnOwqw7eTxWL97wJ.V7R8n73nBLfQk' 87 | 88 | # List of callables that know how to import templates from various sources. 89 | TEMPLATE_LOADERS = ( 90 | 'django.template.loaders.filesystem.Loader', 91 | 'django.template.loaders.app_directories.Loader', 92 | # 'django.template.loaders.eggs.Loader', 93 | ) 94 | 95 | MIDDLEWARE_CLASSES = ( 96 | 'django.middleware.common.CommonMiddleware', 97 | 'django.contrib.sessions.middleware.SessionMiddleware', 98 | 'django.middleware.csrf.CsrfViewMiddleware', 99 | 'django.contrib.auth.middleware.AuthenticationMiddleware', 100 | 'django.contrib.messages.middleware.MessageMiddleware', 101 | # Uncomment the next line for simple clickjacking protection: 102 | 'django.middleware.clickjacking.XFrameOptionsMiddleware', 103 | ) 104 | 105 | ROOT_URLCONF = 'certdir.urls' 106 | 107 | # Python dotted path to the WSGI application used by Django's runserver. 108 | WSGI_APPLICATION = 'certdir.wsgi.application' 109 | 110 | TEMPLATE_DIRS = ( 111 | # Put strings here, like "/home/html/django_templates" or "C:/www/django/templates". 112 | # Always use forward slashes, even on Windows. 113 | # Don't forget to use absolute paths, not relative paths. 114 | # os.environ['CONTACTDB_HOME'] + "/contactdb/templates" 115 | ) 116 | 117 | INSTALLED_APPS = ( 118 | 'django.contrib.auth', 119 | 'django.contrib.contenttypes', 120 | 'django.contrib.sessions', 121 | 'django.contrib.sites', 122 | 'django.contrib.messages', 123 | 'django.contrib.staticfiles', 124 | # Uncomment the next line to enable the admin: 125 | 'django.contrib.admin', 126 | # Uncomment the next line to enable admin documentation: 127 | # 'django.contrib.admindocs', 128 | 'rest_framework', 129 | 'rest_framework.authtoken', 130 | 'contactdb', 131 | ) 132 | 133 | # A sample logging configuration. The only tangible logging 134 | # performed by this configuration is to send an email to 135 | # the site admins on every HTTP 500 error when DEBUG=False. 136 | # See http://docs.djangoproject.com/en/dev/topics/logging for 137 | # more details on how to customize your logging configuration. 138 | LOGGING = { 139 | 'version': 1, 140 | 'disable_existing_loggers': False, 141 | 'filters': { 142 | 'require_debug_false': { 143 | '()': 'django.utils.log.RequireDebugFalse' 144 | } 145 | }, 146 | 'handlers': { 147 | 'mail_admins': { 148 | 'level': 'ERROR', 149 | 'filters': ['require_debug_false'], 150 | 'class': 'django.utils.log.AdminEmailHandler' 151 | } 152 | }, 153 | 'loggers': { 154 | 'django.request': { 155 | 'handlers': ['mail_admins'], 156 | 'level': 'ERROR', 157 | 'propagate': True, 158 | }, 159 | } 160 | } 161 | 162 | REST_FRAMEWORK = { 163 | # Use Django's standard `django.contrib.auth` permissions, 164 | # or allow read-only access for unauthenticated users. 165 | 'DEFAULT_PERMISSION_CLASSES': [ 166 | 'rest_framework.permissions.DjangoModelPermissionsOrAnonReadOnly', 167 | ], 168 | 'DEFAULT_AUTHENTICATION_CLASSES': ( 169 | 'rest_framework.authentication.SessionAuthentication', 170 | 'rest_framework.authentication.TokenAuthentication', 171 | ), 172 | 'DEFAULT_FILTER_BACKENDS': ('rest_framework.filters.DjangoFilterBackend',) 173 | } 174 | 175 | LOGIN_REDIRECT_URL = '/' 176 | -------------------------------------------------------------------------------- /old2/certdir/urls.py: -------------------------------------------------------------------------------- 1 | from contactdb import views 2 | from django.conf.urls import patterns, url, include 3 | from rest_framework.routers import DefaultRouter 4 | 5 | # Uncomment the next two lines to enable the admin: 6 | from django.contrib import admin 7 | admin.autodiscover() 8 | 9 | 10 | router = DefaultRouter() 11 | router.register(r'countrycodes', views.CountrycodeViewSet) 12 | router.register(r'sources', views.SourceViewSet) 13 | router.register(r'organisations', views.OrganisationViewSet) 14 | router.register(r'persons', views.PersonViewSet) 15 | router.register(r'users', views.UserViewSet) 16 | router.register(r'groups', views.GroupViewSet) 17 | router.register(r'tags', views.TagViewSet) 18 | router.register(r'asns', views.ASNViewSet) 19 | 20 | urlpatterns = patterns( 21 | '', 22 | url(r'^', include(router.urls)), 23 | url(r'^pgpkeys/(?P[a-zA-Z0-9]+)$', views.PGPKey), 24 | url(r'^api-auth/', include('rest_framework.urls', 25 | namespace='rest_framework')), 26 | url(r'^api-token-auth/', 27 | 'rest_framework.authtoken.views.obtain_auth_token'), 28 | url(r'^admin/', include(admin.site.urls)), 29 | ) 30 | -------------------------------------------------------------------------------- /old2/certdir/wsgi.py: -------------------------------------------------------------------------------- 1 | """ 2 | WSGI config for certdir project. 3 | 4 | This module contains the WSGI application used by Django's development server 5 | and any production WSGI deployments. It should expose a module-level variable 6 | named ``application``. Django's ``runserver`` and ``runfcgi`` commands discover 7 | this application via the ``WSGI_APPLICATION`` setting. 8 | 9 | Usually you will have the standard Django WSGI application here, but it also 10 | might make sense to replace the whole Django WSGI application with a custom one 11 | that later delegates to the Django one. For example, you could introduce WSGI 12 | middleware here, or combine a Django application with an application of another 13 | framework. 14 | 15 | """ 16 | import os 17 | 18 | os.environ.setdefault("DJANGO_SETTINGS_MODULE", "certdir.settings") 19 | 20 | # This application object is used by any WSGI server configured to use this 21 | # file. This includes Django's development server, if the WSGI_APPLICATION 22 | # setting points here. 23 | from django.core.wsgi import get_wsgi_application 24 | application = get_wsgi_application() 25 | 26 | # Apply WSGI middleware here. 27 | # from helloworld.wsgi import HelloWorldApplication 28 | # application = HelloWorldApplication(application) 29 | -------------------------------------------------------------------------------- /old2/client/NOTES.rst: -------------------------------------------------------------------------------- 1 | USAGE 2 | ===== 3 | 4 | To use the API, yopu need to get your API key: 5 | - Edit get_API_key.py with your user and password 6 | - Run the script 7 | 8 | TODO 9 | ==== 10 | 11 | UK and GB have to be merged at the API/search level 12 | 13 | 14 | 15 | -------------------------------------------------------------------------------- /old2/client/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/certtools/contactdb/51c19fc235a73a447e388c9778b34255eb420221/old2/client/__init__.py -------------------------------------------------------------------------------- /old2/client/api.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import json 4 | import requests 5 | 6 | 7 | class PyContactBD(object): 8 | """ 9 | Python API for ContactDB 10 | """ 11 | 12 | def __init__(self, url, key): 13 | self.url = url 14 | self.key = key 15 | 16 | def __prepare_session(self): 17 | """ 18 | Prepare the headers of the session 19 | """ 20 | session = requests.Session() 21 | session.headers.update( 22 | {'Authorization': ' Token ' + self.key, 23 | 'Accept': 'application/json', 24 | 'Content-Type': 'application/json'}) 25 | return session 26 | 27 | def get_users(self): 28 | return self.__get('user') 29 | 30 | def get_persons(self): 31 | return self.__get('person') 32 | 33 | def get_organisations(self): 34 | return self.__get('organisation') 35 | 36 | def __get(self, model): 37 | session = self.__prepare_session() 38 | return session.get('{}/{}s/'.format(self.url, model)) 39 | 40 | def get_person_by_name(self, name): 41 | return self.__get_by_name('person', name) 42 | 43 | def get_org_by_name(self, name): 44 | return self.__get_by_name('organisation', name) 45 | 46 | def __get_by_name(self, model, name): 47 | session = self.__prepare_session() 48 | return session.get('{}/{}s/?name={}'.format(self.url, model, name)) 49 | 50 | def get_PGP_Key(self, fingerprint): 51 | session = self.__prepare_session() 52 | return session.get('{}/pgpkeys/{}/'.format(self.url, fingerprint)) 53 | 54 | def get_asn(self, asn): 55 | session = self.__prepare_session() 56 | return session.get('{}/asns/{}/'.format(self.url, asn)) 57 | 58 | def post_organisation(self, organisation): 59 | return self.__post('organisation', organisation) 60 | 61 | def post_person(self, person): 62 | return self.__post('person', person) 63 | 64 | def post_source(self, source): 65 | return self.__post('source', source) 66 | 67 | def post_cc(self, cc): 68 | return self.__post('countrycode', cc) 69 | 70 | def post_asn(self, asn): 71 | return self.__post('asn', asn) 72 | 73 | def __post(self, model, obj): 74 | session = self.__prepare_session() 75 | return session.post('{}/{}s/'.format(self.url, model), obj) 76 | 77 | def update_asn(self, asn): 78 | return self.__update('asn', asn) 79 | 80 | def update_person(self, person): 81 | return self.__update('person', person) 82 | 83 | def __update(self, model, obj): 84 | session = self.__prepare_session() 85 | if model == 'asn': 86 | o_id = obj['asn'] 87 | else: 88 | o_id = obj['id'] 89 | return session.put('{}/{}s/{}/'.format(self.url, model, o_id), 90 | json.dumps(obj)) 91 | -------------------------------------------------------------------------------- /old2/client/cc_import.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import urllib2 4 | import json 5 | 6 | from api import PyContactBD 7 | from keys import key 8 | 9 | 10 | def dump_import(contactdb, dump): 11 | for entry in dump: 12 | cc = { 13 | 'cc': entry['alpha-2'], 14 | #'cc3': entry['alpha-3'], 15 | 'country_name': entry['name'] 16 | } 17 | contactdb.post_cc(json.dumps(cc)) 18 | 19 | 20 | def add_custom_TI(contactdb): 21 | worldwide = {'cc': 'WW', 'cc3': 'WWD', 'country_name': 'World Wide'} 22 | europe = {'cc': 'EU', 'cc3': 'EUR', 'country_name': 'Europe'} 23 | uk = {'cc': 'UK', 'cc3': 'GBR', 'country_name': 'United Kingdom'} 24 | contactdb.post_cc(json.dumps(worldwide)) 25 | contactdb.post_cc(json.dumps(europe)) 26 | contactdb.post_cc(json.dumps(uk)) 27 | 28 | 29 | if __name__ == '__main__': 30 | source = 'https://raw.githubusercontent.com/lukes/ISO-3166-Countries-with-Regional-Codes/master/all/all.json' 31 | 32 | response = urllib2.urlopen(source) 33 | json_iso3166 = json.loads(response.read()) 34 | 35 | url = 'http://127.0.0.1:8000' 36 | 37 | contactdb = PyContactBD(url, key) 38 | dump_import(contactdb, json_iso3166) 39 | add_custom_TI(contactdb) 40 | -------------------------------------------------------------------------------- /old2/client/get_API_key.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import requests 4 | import json 5 | 6 | url = 'http://127.0.0.1:8000' 7 | username = 'raphael' 8 | password = 'testest' 9 | 10 | 11 | def get_auth_token(url, username, password): 12 | auth_obj = json.dumps({'username': username, 'password': password}) 13 | session = requests.Session() 14 | session.headers.update( 15 | {'Accept': 'application/json', 'Content-Type': 'application/json'}) 16 | 17 | return session.post(url + '/api-token-auth/', auth_obj) 18 | 19 | if __name__ == '__main__': 20 | response = get_auth_token(url, username, password) 21 | try: 22 | token = response.json()['token'] 23 | with open('keys.py', 'w') as f: 24 | to_write = 'key="{}"'.format(token) 25 | f.write(to_write) 26 | except: 27 | print(response.json()) 28 | -------------------------------------------------------------------------------- /old2/client/get_PGP_Key.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | from api import PyContactBD 4 | from keys import key 5 | 6 | fp = 'CA572205C0024E06BA70BE89EAADCFFC22BD4CD5' 7 | 8 | 9 | if __name__ == '__main__': 10 | url = 'http://127.0.0.1:8000' 11 | contactdb = PyContactBD(url, key) 12 | 13 | r = contactdb.get_PGP_Key(fp) 14 | print r.json()[fp] 15 | -------------------------------------------------------------------------------- /old2/client/sources_import.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | from api import PyContactBD 3 | import json 4 | from keys import key 5 | 6 | sources = {('TI', 0.9), ('FIRST', 0.9)} 7 | 8 | 9 | def sources_import(contactdb): 10 | for source, reliability in sources: 11 | s = {'name': source, 'reliability': reliability} 12 | response = contactdb.post_source(json.dumps(s)) 13 | if response.status_code >= 300: 14 | print response.text 15 | 16 | if __name__ == '__main__': 17 | url = 'http://127.0.0.1:8000' 18 | 19 | contactdb = PyContactBD(url, key) 20 | 21 | sources_import(contactdb) 22 | -------------------------------------------------------------------------------- /old2/client/ti_import.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | from csv import DictReader 3 | from api import PyContactBD 4 | import json 5 | import os 6 | import re 7 | import gnupg 8 | from keys import key 9 | 10 | debug = False 11 | 12 | 13 | def import_gpg(filename): 14 | gpg = gnupg.GPG(homedir=os.environ['GNUPGHOME']) 15 | gpg.dirty_encoding_ignore() 16 | import_result = gpg.import_keys(open(filename, 'r').read()) 17 | return import_result 18 | 19 | 20 | def make_short_id(): 21 | gpg = gnupg.GPG(homedir=os.environ['GNUPGHOME']) 22 | gpg.dirty_encoding_ignore() 23 | keys = gpg.list_keys() 24 | to_return = {} 25 | for k in keys: 26 | short_id = k['keyid'][-8:] 27 | to_return[short_id] = k['fingerprint'] 28 | return to_return 29 | 30 | 31 | def get_fingerprint(dict_shortid, shortid): 32 | if shortid.startswith('0x'): 33 | shortid = shortid[2:] 34 | return dict_shortid.get(shortid, '') 35 | 36 | 37 | def team_import(l, country, status, pgp_keys_shortids): 38 | phone_number = re.split(' - ', l['Telephone'])[0] 39 | url = re.split(' - ', l['WWW'])[0] 40 | if len(url) > 0 and not url.startswith('http'): 41 | url = 'http://' + url 42 | pattern = re.compile(' - ') 43 | address = pattern.sub('\n', l['Address']) 44 | if debug: 45 | print "XXXXX address: '" + address + "'" 46 | fingerprint = '' 47 | if status in ['Accredited', 'Certified']: 48 | if len(l['PGP Key (Team)']) == 0: 49 | print l['Team Name'], 'has no PGP key.' 50 | else: 51 | fingerprint = get_fingerprint(pgp_keys_shortids, l['PGP Key (Team)']) 52 | if len(fingerprint) == 0: 53 | print 'Key not found', l['PGP Key (Team)'], l['Official Team Name'] 54 | org = { 55 | 'name': l['Team Name'].strip(), 56 | 'long_name': l['Official Team Name'], 57 | 'countrycodes': [country], 58 | 'address': address, 59 | 'phone_number': phone_number, 60 | 'emergency_phone': l['Emergency Phone'], 61 | 'fax': l['Telefax'], 62 | 'email': l['Email'], 63 | # 'business_hh_start': l['-Business Hours'], 64 | # 'business_hh_end': l['-Business Hours'], 65 | 'pgp_fingerprint': fingerprint, 66 | 'url': url, 67 | 'ti_url': l['TI URL'], 68 | 'created': l['Date of Establishment'], 69 | 'date_established': l['Date of Establishment'], 70 | 'last_updated': l['Last changed'], 71 | 'source': ['TI'], 72 | } 73 | if debug: 74 | print org 75 | response = contactdb.post_organisation(json.dumps(org)) 76 | if response.status_code >= 300: 77 | print response.text 78 | print org 79 | 80 | 81 | def asn_import(l): 82 | asns_dirty = re.split('. - ', l['Constituency ASNs']) 83 | asns = [] 84 | for asn in asns_dirty: 85 | if asn.upper().startswith('AS'): 86 | asns.append(asn[2:]) 87 | else: 88 | asns.append(asn) 89 | org = contactdb.get_org_by_name(l['Team Name']).json() 90 | for a in asns: 91 | asn = { 92 | 'active': True, 93 | 'source': 'TI', 94 | 'owners': [org[0]['id']], 95 | 'asn': a 96 | } 97 | j_asn = json.dumps(asn) 98 | response = contactdb.post_asn(j_asn) 99 | if response.status_code == 400: 100 | asn = contactdb.get_asn(a).json() 101 | if org[0]['id'] not in asn['owners']: 102 | asn['owners'].append(org[0]['id']) 103 | response = contactdb.update_asn(asn) 104 | if response.status_code >= 300: 105 | print response.text 106 | print asn 107 | 108 | 109 | def rep_import(l, country, status, pgp_keys_shortids): 110 | fingerprint = '' 111 | if status in ['Accredited', 'Certified']: 112 | if len(l['PGP Key (Rep)']) == 0: 113 | print l['Team Representative'], 'has no PGP key.' 114 | else: 115 | fingerprint = get_fingerprint(pgp_keys_shortids, l['PGP Key (Rep)']) 116 | if len(fingerprint) == 0: 117 | print 'Key not found', l['PGP Key (Rep)'], l['Team Representative'] 118 | org = contactdb.get_org_by_name(l['Team Name']).json() 119 | existing_persons = contactdb.get_person_by_name(l['Team Representative']).json() 120 | is_new = True 121 | has_changed = False 122 | person = None 123 | if len(existing_persons) >= 0: 124 | for p in existing_persons: 125 | if p['pgp_fingerprint'] == fingerprint: 126 | # same person 127 | is_new = False 128 | if country not in p['countrycodes']: 129 | p['countrycodes'].append(country) 130 | has_changed = True 131 | if org[0]['id'] not in p['organisations']: 132 | p['organisations'].append(org[0]['id']) 133 | has_changed = True 134 | person = p 135 | break 136 | if is_new: 137 | person = { 138 | # name as primary key for person: issue with duplicated name 139 | 'name': l['Team Representative'], 140 | 'source': ['TI'], 141 | 'email': l['Email (Rep)'], 142 | 'pgp_fingerprint': fingerprint, 143 | 'countrycodes': [country], 144 | 'organisations': [org[0]['id']] 145 | } 146 | response = contactdb.post_person(json.dumps(person)) 147 | elif has_changed: 148 | response = contactdb.update_person(json.dumps(person)) 149 | if response.status_code >= 300: 150 | print response.text 151 | print person 152 | 153 | 154 | def dump_import(contactdb, filename): 155 | reader = DictReader(open(filename), delimiter=';') 156 | pgp_keys_shortids = make_short_id() 157 | for l in reader: 158 | # check if we should import it 159 | status = l['TI Level'] 160 | if status not in ['Listed', 'Accredited', 'Certified']: 161 | continue 162 | # FIXME: we lose the special meaning of the * in the country name 163 | country = l['Country'].strip('*') 164 | if country == 'World Wide': 165 | country = 'WW' 166 | elif country == 'Europe': 167 | country = 'EU' 168 | team_import(l, country, status, pgp_keys_shortids) 169 | if len(l['Constituency ASNs']) > 0: 170 | asn_import(l) 171 | if len(l['Team Representative']) > 0: 172 | rep_import(l, country, status, pgp_keys_shortids) 173 | 174 | if __name__ == '__main__': 175 | url = 'http://127.0.0.1:8000' 176 | # url = 'http://193.191.172.240:80' 177 | 178 | contactdb = PyContactBD(url, key) 179 | import_gpg('../ti/ti-l2-pgpkeys.asc') 180 | 181 | dump_import(contactdb, '../ti/ti-l2-l1-l0-info.v2.csv') 182 | -------------------------------------------------------------------------------- /old2/contactdb/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/certtools/contactdb/51c19fc235a73a447e388c9778b34255eb420221/old2/contactdb/__init__.py -------------------------------------------------------------------------------- /old2/contactdb/admin.py: -------------------------------------------------------------------------------- 1 | from contactdb.models import Organisation, Person 2 | from contactdb.models import Source, Countrycode, Tag 3 | from contactdb.models import ASN, Inetnum 4 | from contactdb.models import OtherCommunicationChannel, OTRFingerprint 5 | from django.contrib import admin 6 | 7 | from contactdb.inetnumadmin import InetnumAdminPage 8 | 9 | 10 | def createInlineAdmin(model_class, number_of_lines=0, key_name=None): 11 | class InlineAdmin(admin.TabularInline): 12 | model = model_class 13 | extra = number_of_lines 14 | fk_name = key_name 15 | 16 | return InlineAdmin 17 | 18 | 19 | class OrganisationAdminPage(admin.ModelAdmin): 20 | filter_horizontal = ['countrycodes', 'tags'] 21 | fields = ('name', 'long_name', 'countrycodes', 'address', 22 | ('email', 'pgp_fingerprint'), 'phone_number', 'url', 23 | 'business_hh_start', 'business_hh_end', 'comment', 'tags') 24 | search_fields = ['name', 'email'] 25 | inlines = [createInlineAdmin(OtherCommunicationChannel), ] 26 | 27 | 28 | class PersonAdminPage(admin.ModelAdmin): 29 | filter_horizontal = ['countrycodes', 'tags'] 30 | fields = ('name', 'long_name', 'user', 'countrycodes', 31 | ('email', 'pgp_fingerprint'), 'phone_number', 32 | 'jabber_handle', 'organisation', 'picture', 'comment', 'tags') 33 | search_fields = ['name', 'user', 'email'] 34 | inlines = [createInlineAdmin(OTRFingerprint), 35 | createInlineAdmin(OtherCommunicationChannel), ] 36 | 37 | exclude = ('last_logged_in', ) 38 | 39 | 40 | class ASNAdminPage(admin.ModelAdmin): 41 | fields = ('asn', 'asname', 'owner', 'source') 42 | search_fields = ['asn', 'asname'] 43 | 44 | admin.site.register(Organisation, OrganisationAdminPage) 45 | admin.site.register(Inetnum, InetnumAdminPage) 46 | admin.site.register(Person, PersonAdminPage) 47 | admin.site.register(ASN, ASNAdminPage) 48 | 49 | admin.site.register(Countrycode) 50 | admin.site.register(Source) 51 | admin.site.register(Tag) 52 | -------------------------------------------------------------------------------- /old2/contactdb/api/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/certtools/contactdb/51c19fc235a73a447e388c9778b34255eb420221/old2/contactdb/api/__init__.py -------------------------------------------------------------------------------- /old2/contactdb/api/resources.py: -------------------------------------------------------------------------------- 1 | from tastypie.resources import ModelResource 2 | from tastypie.constants import ALL 3 | from contactdb.models import * 4 | 5 | 6 | # NOTE: these classes are semi-automatically generated via scripts/gen-api.sh 7 | 8 | 9 | class Country(ModelResource): 10 | class Meta: 11 | filtering = { 12 | "country_name" : ("exact", "startswith", "contains") 13 | } 14 | 15 | queryset = Countrycode.objects.all() 16 | allowed_methods = ['get'] 17 | 18 | 19 | class PGPKey(ModelResource): 20 | class Meta: 21 | queryset = PGPKey.objects.all() 22 | allowed_methods = ['get'] 23 | 24 | 25 | class PGPUid(ModelResource): 26 | class Meta: 27 | queryset = PGPUid.objects.all() 28 | allowed_methods = ['get'] 29 | 30 | 31 | class Source(ModelResource): 32 | class Meta: 33 | queryset = Source.objects.all() 34 | allowed_methods = ['get'] 35 | 36 | 37 | class Organisation(ModelResource): 38 | class Meta: 39 | queryset = Organisation.objects.all() 40 | allowed_methods = ['get'] 41 | 42 | 43 | class OrganisationTel(ModelResource): 44 | class Meta: 45 | queryset = OrganisationTel.objects.all() 46 | allowed_methods = ['get'] 47 | 48 | 49 | class OrganisationEmail(ModelResource): 50 | class Meta: 51 | queryset = OrganisationEmail.objects.all() 52 | allowed_methods = ['get'] 53 | 54 | 55 | class Person(ModelResource): 56 | class Meta: 57 | queryset = Person.objects.all() 58 | allowed_methods = ['get'] 59 | 60 | 61 | 62 | class NetObject(ModelResource): 63 | class Meta: 64 | queryset = NetObject.objects.all() 65 | allowed_methods = ['get'] 66 | 67 | class NetObjContactLink(ModelResource): 68 | class Meta: 69 | #queryset = NetObjContactLink.objects.all() 70 | allowed_methods = [] 71 | 72 | class ASN(ModelResource): 73 | class Meta: 74 | queryset = ASN.objects.all() 75 | allowed_methods = ['get'] 76 | 77 | 78 | class Inetnum(ModelResource): 79 | class Meta: 80 | queryset = Inetnum.objects.all() 81 | allowed_methods = ['get'] 82 | 83 | 84 | class IPAddress(ModelResource): 85 | class Meta: 86 | queryset = IPAddress.objects.all() 87 | allowed_methods = ['get'] 88 | 89 | 90 | class Hostname(ModelResource): 91 | class Meta: 92 | queryset = Hostname.objects.all() 93 | allowed_methods = ['get'] 94 | 95 | 96 | class Domainname(ModelResource): 97 | class Meta: 98 | queryset = Domainname.objects.all() 99 | allowed_methods = ['get'] 100 | 101 | 102 | 103 | -------------------------------------------------------------------------------- /old2/contactdb/fields/__init__.py: -------------------------------------------------------------------------------- 1 | from jsonfield import * 2 | -------------------------------------------------------------------------------- /old2/contactdb/fields/jsonfield.py: -------------------------------------------------------------------------------- 1 | from django.db import models 2 | from contactdb.forms.fields import JSONFormField 3 | 4 | import json 5 | 6 | # JSONWidget, JSONFormField and JSONField adapted from http://justcramer.com/2009/04/14/cleaning-up-with-json-and-sql/ 7 | # Added support for custom JSON encoders/decoders 8 | class JSONField(models.TextField): 9 | __metaclass__ = models.SubfieldBase 10 | 11 | def formfield(self, **kwargs): 12 | return super(JSONField, self).formfield(form_class=self.form_class, validator=self.validator, **kwargs) 13 | 14 | def __init__(self, *args, **kwargs): 15 | self.validator = kwargs.pop('validator', None) 16 | self.encoder = kwargs.pop('encoder', None) 17 | self.decoder = kwargs.pop('decoder', None) 18 | self.form_class = kwargs.pop('form_class', JSONFormField) 19 | super(JSONField, self).__init__(*args, **kwargs) 20 | 21 | def to_python(self, value): 22 | if not value: 23 | return 24 | elif isinstance(value, basestring): 25 | if self.decoder: 26 | return json.loads(value, cls=self.decoder) 27 | else: 28 | return json.loads(value) 29 | else: 30 | return value 31 | 32 | def get_db_prep_value(self, value, *args, **kwargs): 33 | if not value: 34 | return 35 | elif self.encoder: 36 | return json.dumps(value, cls=self.encoder) 37 | else: 38 | return json.dumps(value) 39 | 40 | def value_to_string(self, obj): 41 | value = self._get_val_from_obj(obj) 42 | return self.get_db_prep_value(value) 43 | 44 | -------------------------------------------------------------------------------- /old2/contactdb/filters.py: -------------------------------------------------------------------------------- 1 | import django_filters 2 | 3 | from contactdb.models import Organisation 4 | from contactdb.models import Person 5 | 6 | 7 | class OrganisationFilter(django_filters.FilterSet): 8 | 9 | class Meta: 10 | model = Organisation 11 | fields = ['name'] 12 | 13 | 14 | class PersonFilter(django_filters.FilterSet): 15 | 16 | class Meta: 17 | model = Person 18 | fields = ['name'] 19 | -------------------------------------------------------------------------------- /old2/contactdb/forms/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/certtools/contactdb/51c19fc235a73a447e388c9778b34255eb420221/old2/contactdb/forms/__init__.py -------------------------------------------------------------------------------- /old2/contactdb/forms/fields.py: -------------------------------------------------------------------------------- 1 | from django import forms 2 | from django.utils.translation import ugettext_lazy 3 | from django.core.exceptions import ValidationError 4 | 5 | import json 6 | 7 | from contactdb.forms.widgets import JSONListToNewlineWidget, JSONWidget 8 | 9 | 10 | class JSONListToNewlineField(forms.CharField): 11 | def __init__(self, *args, **kwargs): 12 | self.validator = kwargs.pop('validator', None) 13 | 14 | messages = getattr(self, 'error_messages', {}) 15 | custom_error_messages = kwargs.pop('error_messages', {}) 16 | 17 | for (key, value) in custom_error_messages: 18 | messages[key] = value 19 | 20 | self.error_messages = messages 21 | 22 | kwargs['widget'] = JSONListToNewlineWidget 23 | super(JSONListToNewlineField, self).__init__(*args, **kwargs) 24 | 25 | print 'Got here' 26 | 27 | if self.validator: 28 | if self.validators: 29 | self.validators.append(self.validate_elements) 30 | else: 31 | self.validators = [self.validate_elements] 32 | 33 | print 'Got here too: %r' % self.validators 34 | 35 | def validate_elements(self, value): 36 | errors = [] 37 | values = [v.strip() for v in value.split('\n')] 38 | i = 0 39 | 40 | if not self.validator: 41 | return values 42 | 43 | print 'Validate elements' 44 | 45 | for v in values: 46 | i += 1 47 | try: 48 | self.validator(v) 49 | except ValidationError as e: 50 | message = 'Error in element %s: %s' % (i, str(e)) 51 | errors.append(message) 52 | if errors: 53 | raise ValidationError(errors) 54 | else: 55 | return values 56 | 57 | def clean(self, value): 58 | print 'Cleaning elements' 59 | if not value: 60 | return 61 | try: 62 | return self.validate_elements(value) 63 | except Exception, exc: 64 | raise exc 65 | 66 | #class JSONListToNewlineField(forms.CharField): 67 | # def __init__(self, *args, **kwargs): 68 | # kwargs['widget'] = JSONListToNewlineWidget 69 | # super(JSONListToNewlineField, self).__init__(*args, **kwargs) 70 | # 71 | # def clean(self, value): 72 | # if not value: 73 | # return 74 | # try: 75 | # return [v.strip() for v in value.split('\n')] 76 | # except Exception, exc: 77 | # raise forms.ValidationError(u'Split error: %s' % unicode(exc)) 78 | 79 | class JSONFormField(forms.CharField): 80 | def __init__(self, *args, **kwargs): 81 | kwargs['widget'] = JSONWidget 82 | super(JSONFormField, self).__init__(*args, **kwargs) 83 | 84 | def clean(self, value): 85 | if not value: 86 | return 87 | try: 88 | if self.decoder: 89 | return json.loads(value, cls=self.decoder) 90 | else: 91 | return json.loads(value) 92 | except Exception, exc: 93 | raise forms.ValidationError(u'JSON decode error: %s' % unicode(exc)) 94 | -------------------------------------------------------------------------------- /old2/contactdb/forms/widgets.py: -------------------------------------------------------------------------------- 1 | from django import forms 2 | 3 | import json 4 | 5 | class JSONListToNewlineWidget(forms.Textarea): 6 | def render(self, name, value, attrs=None): 7 | if not isinstance(value, basestring): 8 | display_string = '' 9 | if value is not None and len(value) > 0: 10 | for v in value: 11 | display_string += v + '\n' 12 | 13 | value = display_string 14 | return super(JSONListToNewlineWidget, self).render(name, value, attrs) 15 | 16 | class JSONWidget(forms.Textarea): 17 | def render(self, name, value, attrs=None): 18 | if not isinstance(value, basestring): 19 | value = json.dumps(value, indent=2) 20 | return super(JSONWidget, self).render(name, value, attrs) 21 | -------------------------------------------------------------------------------- /old2/contactdb/inetnum.py: -------------------------------------------------------------------------------- 1 | from django.db import models 2 | 3 | class InetnumModel(models.Model): 4 | inet = models.GenericIPAddressField() 5 | init_ip = models.GenericIPAddressField(editable=False, blank=True, null=True) 6 | end_ip = models.GenericIPAddressField(editable=False, blank=True, null=True) 7 | prefix_length = models.PositiveSmallIntegerField(editable=False, blank=True, null=True) 8 | 9 | @staticmethod 10 | def int_to_ipstr(ip): 11 | import socket 12 | import binascii 13 | 14 | hex_ip = hex(ip)[2:] 15 | 16 | if len(hex_ip) == 8: 17 | return socket.inet_ntop(socket.AF_INET, hex_ip) 18 | elif len(hex_ip) == 33: 19 | hex_ip = binascii.unhexlify(hex_ip[:-1]) 20 | return socket.inet_ntop(socket.AF_INET6, hex_ip) 21 | else: 22 | return '' 23 | 24 | @staticmethod 25 | def ipstr_to_int(ip): 26 | import socket 27 | import binascii 28 | 29 | int_ip = 0 30 | ip_size = 0 31 | try: 32 | int_ip = socket.inet_pton(socket.AF_INET, ip) 33 | ip_size = 32 34 | except: 35 | try: 36 | int_ip = socket.inet_pton(socket.AF_INET6, ip) 37 | ip_size = 128 38 | except: 39 | int_ip = -1 40 | ip_size = -1 41 | 42 | if int_ip == -1: 43 | return None 44 | 45 | int_ip = int(binascii.hexlify(int_ip), 16) 46 | 47 | return (int_ip, ip_size) 48 | 49 | @staticmethod 50 | def split_inet(inet): 51 | splitted_inet = inet.split('/') 52 | 53 | prefix = None 54 | if len(splitted_inet) == 2: 55 | prefix = int(splitted_inet[1]) 56 | 57 | (ip_int, ip_size) = Inetnum.ipstr_to_int(splitted_inet[0]) 58 | 59 | return (ip_int, ip_size, prefix) 60 | 61 | @staticmethod 62 | def inet_borders(inet): 63 | (ip_int, ip_size, prefix) = Inetnum.split_inet(inet) 64 | imask = ('1' * prefix) + ('0' * (ip_size - prefix)) 65 | init_ip = Inetnum.int_to_ipstr(int(imask, 2) & ip_int) 66 | emask = ('0' * prefix) + ('1' * (ip_size - prefix)) 67 | end_ip = Inetnum.int_to_ipstr(int(emask, 2) | ip_int) 68 | 69 | return (init_ip, end_ip) 70 | 71 | def _update_inet_borders(self): 72 | (init_ip, end_ip) = Inetnum.inet_borders(self.inet) 73 | self.init_ip = init_ip 74 | self.end_ip = end_ip 75 | 76 | def _update_prefix(self): 77 | (ip_int, ip_size, prefix) = Inetnum.split_inet(self.inet) 78 | self.prefix_length = str(prefix) 79 | 80 | def save(self, *args, **kwargs): 81 | self._update_inet_borders() 82 | self._update_prefix() 83 | 84 | super(Inetnum, self).save(*args, **kwargs) 85 | 86 | def __unicode__(self): 87 | return unicode(self.inet) 88 | 89 | class Meta: 90 | abstract = True 91 | ordering = ['prefix_length', ] 92 | -------------------------------------------------------------------------------- /old2/contactdb/inetnumadmin.py: -------------------------------------------------------------------------------- 1 | from django import forms 2 | from django.contrib import admin 3 | from contactdb.models import Inetnum 4 | 5 | class InetnumFormField(forms.GenericIPAddressField): 6 | def validate_ipv46_address(self, value): 7 | from django.core.validators import validate_ipv4_address, validate_ipv6_address 8 | try: 9 | validate_ipv4_address(value) 10 | return 'ipv4' 11 | except ValidationError: 12 | try: 13 | validate_ipv6_address(value) 14 | return 'ipv6' 15 | except ValidationError: 16 | raise ValidationError(_('Enter a valid IPv4 or IPv6 address.'), code='invalid') 17 | 18 | def validate_ipv46_cidr(self, value): 19 | if '/' in value: 20 | splitted = value.split('/') 21 | prot = self.validate_ipv46_address(splitted[0]) 22 | 23 | if prot is 'ipv4': 24 | try: 25 | ipv4_prefix = int(splitted[1]) 26 | if ipv4_prefix < 0 or ipv4_prefix > 32: 27 | raise ValidationError(_('IPv4 prefix is not valid.'), code='invalid') 28 | except: 29 | raise ValidationError(_('IPv4 prefix is not valid.'), code='invalid') 30 | elif prot is 'ipv6': 31 | try: 32 | ipv6_prefix = int(splitted[1]) 33 | if ipv6_prefix < 0 or ipv6_prefix > 128: 34 | raise ValidationError(_('IPv6 prefix is not valid.'), code='invalid') 35 | except: 36 | raise ValidationError(_('IPv6 prefix is not valid.'), code='invalid') 37 | else: 38 | self.validate_ipv46_address(value) 39 | 40 | def run_validators(self, value): 41 | return self.validate_ipv46_cidr(value) 42 | 43 | def to_python(self, value): 44 | try: 45 | if '/' in value: 46 | splitted = value.split('/') 47 | toRet = super(InetnumFormField, self).to_python(splitted[0]) + '/' + splitted[1] 48 | else: 49 | toRet = super(InetnumFormField, self).to_python(splitted[0]) 50 | 51 | return toRet 52 | except Exception, e: 53 | print 'To_Python returned an exception: %r' % e 54 | 55 | class InetnumForm(forms.ModelForm): 56 | inet = InetnumFormField(required=True) 57 | save_inetnum = None 58 | 59 | class Meta: 60 | model = Inetnum 61 | exclude = [] 62 | 63 | def clean(self): 64 | toRet = super(InetnumForm, self).clean() 65 | self.data['inet'] = self.cleaned_data['inet'] 66 | return toRet 67 | 68 | def full_clean(self): 69 | toRet = super(InetnumForm, self).full_clean() 70 | if 'inet' in self._errors: 71 | del self._errors['inet'] 72 | 73 | self.cleaned_data['inet'] = self.data['inet'] 74 | 75 | return toRet 76 | 77 | class InetnumAdminPage(admin.ModelAdmin): 78 | form = InetnumForm 79 | 80 | def save_model(self, request, obj, form, change): 81 | #print 'Inetnum: ' + str(form.cleaned_data['inet']) 82 | obj.save() 83 | -------------------------------------------------------------------------------- /old2/contactdb/migrations/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/certtools/contactdb/51c19fc235a73a447e388c9778b34255eb420221/old2/contactdb/migrations/__init__.py -------------------------------------------------------------------------------- /old2/contactdb/models.py: -------------------------------------------------------------------------------- 1 | from django.db.models import Model, CharField, FloatField, ForeignKey, \ 2 | EmailField, TextField, DateTimeField, TimeField, BooleanField, \ 3 | DateField, ImageField, URLField, IntegerField, ManyToManyField, \ 4 | OneToOneField 5 | from django.contrib.auth.models import User 6 | 7 | from contactdb.inetnum import InetnumModel 8 | 9 | from datetime import datetime 10 | 11 | MEDIA_ROOT = '/var/www/upload/' 12 | MEDIA_URL = '/upload/' 13 | 14 | # Auto generate an auth token for all the users 15 | 16 | from django.conf import settings 17 | from django.contrib.auth import get_user_model 18 | from django.db.models.signals import post_save 19 | from django.dispatch import receiver 20 | from rest_framework.authtoken.models import Token 21 | 22 | 23 | @receiver(post_save, sender=settings.AUTH_USER_MODEL) 24 | def create_auth_token(sender, instance=None, created=False, **kwargs): 25 | if created: 26 | Token.objects.create(user=instance) 27 | 28 | # --------------------------------------------- 29 | 30 | 31 | class Countrycode(Model): 32 | # http://en.wikipedia.org/wiki/ISO_3166-1_alpha-2 33 | # we can load this automatically from ripe.NET 34 | cc = CharField(max_length=2, primary_key=True) 35 | country_name = CharField(max_length=100) 36 | 37 | def __unicode__(self): 38 | return self.country_name 39 | 40 | class Meta: 41 | verbose_name = "country" 42 | verbose_name_plural = "countries" 43 | 44 | 45 | class Source(Model): 46 | name = CharField(max_length=50, primary_key=True) 47 | reliability = FloatField(default=0.0) # between 0 and 1, with 1 being super reliable 48 | 49 | def __unicode__(self): 50 | return self.name 51 | 52 | class Meta: 53 | verbose_name = "data source" 54 | 55 | 56 | class Tag(Model): 57 | name = CharField(max_length=30, primary_key=True) 58 | 59 | def __unicode__(self): 60 | return self.name 61 | 62 | 63 | class Entity(Model): 64 | name = CharField(max_length=50) 65 | long_name = CharField(max_length=1000, null=True, blank=True) 66 | 67 | #################### 68 | countrycodes = ManyToManyField(Countrycode, related_name="%(app_label)s_%(class)s", blank=True, null=True) 69 | tags = ManyToManyField(Tag, related_name="%(app_label)s_%(class)s", blank=True, null=True) 70 | #################### 71 | 72 | source = ForeignKey(Source, null=True, blank=True) 73 | 74 | email = EmailField(null=False) 75 | pgp_fingerprint = CharField(max_length=50, null=True, blank=True) 76 | 77 | phone_number = CharField(max_length=30, null=True, blank=True) 78 | url = URLField("URL", null=True, blank=True) 79 | comment = TextField(max_length=1000, null=True, blank=True) 80 | 81 | created = DateTimeField(auto_now_add=True) 82 | last_updated = DateTimeField(auto_now=True) 83 | 84 | def __unicode__(self): 85 | return self.name 86 | 87 | 88 | class Organisation(Entity): 89 | address = TextField(max_length=1000, null=True, blank=True) 90 | 91 | business_hh_start = TimeField(verbose_name="business hours start", 92 | null=True, blank=True) 93 | business_hh_end = TimeField(verbose_name="business hours end", 94 | null=True, blank=True) 95 | date_established = DateField(verbose_name="date established", 96 | null=True, blank=True) 97 | 98 | confirmed = BooleanField(verbose_name="confirmed to exist", default=False) 99 | active = BooleanField(verbose_name="still active", default=False) 100 | 101 | ti_url = CharField(max_length=500, null=True, blank=True) 102 | first_url = CharField(max_length=500, null=True, blank=True) 103 | 104 | 105 | class Person(Entity): 106 | user = OneToOneField(User, related_name='persons', null=True, blank=True) 107 | organisations = ManyToManyField(Organisation, related_name="%(app_label)s_%(class)s", blank=True, null=True) 108 | picture = ImageField(upload_to='/static/person/pics/', null=True, blank=True) 109 | last_logged_in = TimeField(null=False, default=datetime.now) 110 | 111 | jabber_handle = EmailField(max_length=100, null=True, blank=True) 112 | 113 | 114 | class CommunicationChannel(Model): 115 | description = CharField(max_length=200, null=True, blank=True) 116 | created = DateTimeField(auto_now_add=True) 117 | last_updated = DateTimeField(auto_now=True) 118 | 119 | class Meta: 120 | abstract = True 121 | 122 | 123 | class OTRFingerprint(Model): 124 | otr_fingerprint = CharField(max_length=50, null=False) 125 | handle = ForeignKey(Person, related_name='otr_fingerprints') 126 | 127 | 128 | class OtherCommunicationChannel(CommunicationChannel): 129 | entity = ForeignKey(Entity) 130 | value = CharField("communication channel", max_length=1000, null=False, 131 | blank=False) 132 | channel_type = CharField(max_length=100, null=False) 133 | 134 | class Meta: 135 | verbose_name = "communication channel" 136 | 137 | 138 | class NetObject(Model): 139 | active = BooleanField(default=False) 140 | 141 | source = ForeignKey(Source, null=True) 142 | owners = ManyToManyField(Entity, related_name="%(app_label)s_%(class)s") 143 | created = DateTimeField(auto_now_add=True) 144 | last_updated = DateTimeField(auto_now=True) 145 | 146 | class Meta: 147 | abstract = True 148 | 149 | 150 | class ASN(NetObject): 151 | asn = IntegerField(primary_key=True) 152 | asname = CharField(max_length=500) 153 | 154 | def __unicode__(self): 155 | return str(self.asn) 156 | 157 | class Meta: 158 | verbose_name = "Autonomous System Number" 159 | verbose_name_plural = "Autonomous System Numbers" 160 | 161 | 162 | class Inetnum(NetObject, InetnumModel): 163 | pass 164 | 165 | 166 | class DomainName(NetObject): 167 | domain = CharField(max_length=1000) 168 | 169 | def __unicode__(self): 170 | return self.domain 171 | 172 | 173 | class TLD(NetObject): 174 | tld = CharField(max_length=100, primary_key=True) 175 | 176 | def __unicode__(self): 177 | return self.tld 178 | -------------------------------------------------------------------------------- /old2/contactdb/netobjects.py: -------------------------------------------------------------------------------- 1 | import models 2 | 3 | def addCountry(cc, cc3, cname): 4 | """ 5 | @TODO ADD CHECK ON VALUES GIVEN 6 | len(cc) == 2 7 | len(cc3) == 3 8 | ... 9 | TO BE MOVED TO ANOTHER CLASS!! - HERE NET OBJECTS ONLY - USE FOR DEBUG ;) 10 | """ 11 | cobj = models.Countrycode(cc=cc, cc3=cc3, country_name=cname) 12 | cobj.save() 13 | return 14 | 15 | 16 | 17 | def addASN(asn, asname): 18 | print("DEBUG - in addASN =%d= =%s=" % (int(asn), asname)) 19 | asnobj = models.ASN(asn=3, asname="test") 20 | print("DEBUG - asnobj created") 21 | asnobj.save() 22 | print("DEBUG - saved to DB") 23 | return 24 | 25 | 26 | def addIP(range, asn): 27 | return -------------------------------------------------------------------------------- /old2/contactdb/permissions.py: -------------------------------------------------------------------------------- 1 | from rest_framework import permissions 2 | from django.contrib.auth.models import Group 3 | 4 | 5 | class IsUserOrReadOnly(permissions.BasePermission): 6 | """ 7 | Custom permission to only allow User of an Person to edit it. 8 | """ 9 | 10 | def has_object_permission(self, request, view, obj): 11 | # Read permissions are allowed to any request 12 | if request.method in permissions.SAFE_METHODS: 13 | return True 14 | 15 | # Write permissions are only allowed to the User of a Person 16 | return request.user.is_staff or obj.user == request.user 17 | 18 | 19 | class IsInOrgOrReadOnly(permissions.BasePermission): 20 | 21 | def has_object_permission(self, request, view, obj): 22 | # Read permissions are allowed to any request 23 | if request.method in permissions.SAFE_METHODS: 24 | return True 25 | return request.user.is_staff or \ 26 | Group.objects.get(name=obj.name) in request.user.groups.all() 27 | -------------------------------------------------------------------------------- /old2/contactdb/serializers.py: -------------------------------------------------------------------------------- 1 | from rest_framework import serializers 2 | from django.contrib.auth.models import User 3 | from django.contrib.auth.models import Group 4 | from contactdb.models import Person 5 | from contactdb.models import Organisation 6 | from contactdb.models import Countrycode 7 | from contactdb.models import Source 8 | from contactdb.models import Tag 9 | from contactdb.models import ASN 10 | 11 | 12 | class CountrycodeSerializer(serializers.ModelSerializer): 13 | 14 | class Meta: 15 | model = Countrycode 16 | fields = ('cc', 'country_name', ) 17 | 18 | 19 | class SourceSerializer(serializers.HyperlinkedModelSerializer): 20 | 21 | class Meta: 22 | model = Source 23 | fields = ('name', 'reliability') 24 | 25 | 26 | class OrganisationSerializer(serializers.ModelSerializer): 27 | 28 | class Meta: 29 | model = Organisation 30 | fields = ('id', 'name', 'long_name', 'countrycodes', 'address', 'email', 'pgp_fingerprint', 'phone_number', 'url', 'business_hh_start', 'business_hh_end', 'comment', 'tags', 'date_established', 'confirmed', 'active', 'ti_url', 'first_url') 31 | 32 | 33 | class PersonSerializer(serializers.ModelSerializer): 34 | 35 | class Meta: 36 | model = Person 37 | fields = ('id', 'name', 'long_name', 'user', 'countrycodes', 'email', 'pgp_fingerprint', 'phone_number', 'jabber_handle', 'organisations', 'picture', 'comment', 'tags') 38 | 39 | 40 | class UserSerializer(serializers.HyperlinkedModelSerializer): 41 | 42 | class Meta: 43 | model = User 44 | fields = ('url', 'groups', 'username') 45 | 46 | 47 | class GroupSerializer(serializers.HyperlinkedModelSerializer): 48 | 49 | class Meta: 50 | model = Group 51 | fields = ('url', 'name', 'permissions') 52 | 53 | 54 | class TagSerializer(serializers.ModelSerializer): 55 | 56 | class Meta: 57 | model = Tag 58 | fields = ('name', ) 59 | 60 | 61 | class ASNSerializer(serializers.ModelSerializer): 62 | 63 | class Meta: 64 | model = ASN 65 | fields = ('asn', 'owners', 'source', 'active', ) 66 | -------------------------------------------------------------------------------- /old2/contactdb/templates/add_asn.html: -------------------------------------------------------------------------------- 1 |

Add ASN

2 | 3 | {% if error_message %}

{{ error_message }}

{% endif %} 4 | 5 |
6 | {% csrf_token %} 7 | 8 | 9 |
10 | 11 | 12 |
13 | 14 |
-------------------------------------------------------------------------------- /old2/contactdb/templates/add_source.html: -------------------------------------------------------------------------------- 1 |

Add Source

2 | 3 | {% if error_message %}

{{ error_message }}

{% endif %} 4 | 5 |
6 | {% csrf_token %} 7 | 8 | 9 |
10 | 11 | 12 |
13 | 14 |
-------------------------------------------------------------------------------- /old2/contactdb/templates/index.html: -------------------------------------------------------------------------------- 1 | {% if country_list %} 2 |
    3 | {% for country in country_list %} 4 |
  • {{ country.cc }} = {{ country.country_name }}
    • 5 | {% endfor %} 6 |
7 | {% else %} 8 |

No countries are available.

9 | {% endif %} 10 | -------------------------------------------------------------------------------- /old2/contactdb/templates/search.html: -------------------------------------------------------------------------------- 1 |

Search

2 | 3 | {% if error_message %}

{{ error_message }}

{% endif %} 4 | 5 |
6 | {% csrf_token %} 7 | 8 | 9 | 10 |
11 | -------------------------------------------------------------------------------- /old2/contactdb/views.py: -------------------------------------------------------------------------------- 1 | from django.contrib.auth.models import User 2 | from django.contrib.auth.models import Group 3 | from rest_framework import permissions 4 | from rest_framework.authentication import SessionAuthentication, TokenAuthentication 5 | from rest_framework import viewsets 6 | from rest_framework.exceptions import PermissionDenied 7 | from rest_framework.decorators import api_view 8 | from rest_framework.decorators import permission_classes 9 | from rest_framework.permissions import IsAuthenticated 10 | from rest_framework.response import Response 11 | from contactdb.permissions import IsUserOrReadOnly 12 | from contactdb.permissions import IsInOrgOrReadOnly 13 | from contactdb.serializers import UserSerializer 14 | from contactdb.serializers import GroupSerializer 15 | 16 | from contactdb.models import Person 17 | from contactdb.serializers import PersonSerializer 18 | from contactdb.filters import PersonFilter 19 | 20 | from contactdb.models import Organisation 21 | from contactdb.serializers import OrganisationSerializer 22 | from contactdb.filters import OrganisationFilter 23 | 24 | from contactdb.models import Countrycode 25 | from contactdb.serializers import CountrycodeSerializer 26 | 27 | from contactdb.models import Source 28 | from contactdb.serializers import SourceSerializer 29 | 30 | from contactdb.models import Tag 31 | from contactdb.serializers import TagSerializer 32 | 33 | from contactdb.models import ASN 34 | from contactdb.serializers import ASNSerializer 35 | 36 | import gnupg 37 | import os 38 | 39 | 40 | @api_view(['GET']) 41 | @permission_classes((IsAuthenticated, )) 42 | def PGPKey(request, fingerprint): 43 | if request.method == 'GET': 44 | gpg = gnupg.GPG(homedir=os.environ['GNUPGHOME']) 45 | key = gpg.export_keys(fingerprint) 46 | return Response({fingerprint: key}) 47 | 48 | 49 | class CountrycodeViewSet(viewsets.ModelViewSet): 50 | queryset = Countrycode.objects.all() 51 | serializer_class = CountrycodeSerializer 52 | 53 | 54 | class SourceViewSet(viewsets.ModelViewSet): 55 | queryset = Source.objects.all() 56 | serializer_class = SourceSerializer 57 | 58 | 59 | class OrganisationViewSet(viewsets.ModelViewSet): 60 | authentication_classes = (SessionAuthentication, 61 | TokenAuthentication) 62 | queryset = Organisation.objects.all() 63 | serializer_class = OrganisationSerializer 64 | filter_class = OrganisationFilter 65 | permission_classes = (permissions.IsAuthenticatedOrReadOnly, 66 | IsInOrgOrReadOnly,) 67 | 68 | def pre_save(self, obj): 69 | g, created = Group.objects.get_or_create(name=obj.name) 70 | if not self.request.user.is_staff: 71 | g.user_set.add(self.request.user) 72 | 73 | 74 | class PersonViewSet(viewsets.ModelViewSet): 75 | queryset = Person.objects.all() 76 | serializer_class = PersonSerializer 77 | filter_class = PersonFilter 78 | permission_classes = (permissions.IsAuthenticatedOrReadOnly, 79 | IsUserOrReadOnly,) 80 | 81 | def post_save(self, obj, **kwargs): 82 | if self.request.user.is_staff or obj.user == self.request.user: 83 | if obj.organisations is not None: 84 | for o in obj.organisations.all(): 85 | g, created = Group.objects.get_or_create(name=o.name) 86 | if created or o.name in self.request.user.groups.all(): 87 | # only allow to add an organisation to an user if the user 88 | # doing so is in the organisation 89 | g.user_set.add(self.request.user) 90 | else: 91 | raise PermissionDenied(detail='User of Person has to be you.') 92 | 93 | 94 | class UserViewSet(viewsets.ReadOnlyModelViewSet): 95 | """ 96 | This endpoint presents the users in the system. 97 | 98 | As you can see, the collection of snippet instances owned by a user are 99 | serialized using a hyperlinked representation. 100 | """ 101 | queryset = User.objects.all() 102 | serializer_class = UserSerializer 103 | 104 | 105 | class GroupViewSet(viewsets.ReadOnlyModelViewSet): 106 | """ 107 | API endpoint that allows groups to be viewed or edited. 108 | """ 109 | queryset = Group.objects.all() 110 | serializer_class = GroupSerializer 111 | 112 | 113 | class TagViewSet(viewsets.ReadOnlyModelViewSet): 114 | """ 115 | API endpoint that allows tags to be viewed or edited. 116 | """ 117 | queryset = Tag.objects.all() 118 | serializer_class = TagSerializer 119 | 120 | 121 | class ASNViewSet(viewsets.ModelViewSet): 122 | """ 123 | API endpoint that allows tags to be viewed or edited. 124 | """ 125 | queryset = ASN.objects.all() 126 | serializer_class = ASNSerializer 127 | -------------------------------------------------------------------------------- /old2/doc/.gitignore: -------------------------------------------------------------------------------- 1 | contact-databases-for-abuse-handling.aux 2 | contact-databases-for-abuse-handling.log 3 | contact-databases-for-abuse-handling.pdf 4 | contact-databases-for-abuse-handling.tex 5 | contact-databases-for-abuse-handling.toc 6 | -------------------------------------------------------------------------------- /old2/doc/20140603-meeting-notes.mkd: -------------------------------------------------------------------------------- 1 | 2 | 3 | # PGP key maintenance 4 | 5 | * the system needs to check the PGP keyring once per day. Find expired / revoked keys and inform the users that their key expires soon and send a link to upload a new key or edit user details 6 | * sync the keyring vs. the global key servers (pull to local only!) 7 | 8 | * invariant: if a fingerprint changes, then all the fingerprints in the corresponding Email Address object need to change as well 9 | (do that via script) 10 | 11 | * The model allows for multiple email address <-> to one fingerprint and one key (==fingerprint) per email address. A person can have only one (!) email 12 | * we decided to abandon the approach of having multiple email addresses per person/org in order to reduce complexity 13 | * remove cc3 from Country 14 | * remove TZ 15 | 16 | 17 | ``` 18 | 19 | is_a (entity) 20 | +----------+ 1 1 +---------+ m 1 +-------------+ 21 | | Person | ----- | Email | ------ | fingerprint | 22 | +----------+ +---------+ +-------------+ 23 | / 24 | 1 /---- 1 25 | +---------+ 26 | | Org | 27 | +---------+ 28 | is_a(entity) 29 | 30 | ``` 31 | 32 | 33 | # OTR fingerprints 34 | 35 | ``` 36 | 37 | +---------+ m 1 +-----------+ 1 1 +--------+ 38 | | OTRfpr | --------- | JabberID | ----- | Person | 39 | +---------+ +-----------+ +--------+ 40 | 41 | ``` 42 | -------------------------------------------------------------------------------- /old2/doc/Makefile: -------------------------------------------------------------------------------- 1 | #pdf1: 2 | # ./update-metadata-for-gitinfo 3 | # #pandoc -f markdown_github -o notes.pdf notes.md 4 | 5 | all: datasets.pdf 6 | 7 | latex: datasets.tex 8 | 9 | datasets.tex: datasets.mkd 10 | pandoc -N --listings -T "Abuse Contact datasets for IT security incident handling" -s -f markdown_github+implicit_figures+pandoc_title_block+table_captions+multiline_tables --toc -t latex -o datasets.tex datasets.mkd 11 | #pandoc -N --listings -T "Contact database for IT security incident handling" -s -f markdown_github+pandoc_title_block+pipe_tables --toc -o datasets.pdf datasets.mkd 12 | 13 | datasets.pdf: datasets.tex 14 | # add the proper stylesheets 15 | cat styleheader.tex > r 16 | tail -n +2 datasets.tex >> r 17 | mv r datasets.tex 18 | # compute version info 19 | ./update-metadata-for-gitinfo 20 | # update the toc 21 | pdflatex datasets.tex 22 | pdflatex datasets.tex 23 | 24 | clean: 25 | rm -f datasets.pdf datasets.tex datasets.log datasets.out datasets.aux datasets.toc r 26 | -------------------------------------------------------------------------------- /old2/doc/README.md: -------------------------------------------------------------------------------- 1 | 2 | What's here? 3 | ============ 4 | 5 | 6 | |File | Explanation | 7 | |:--------------|----------------------------------------------------------------------| 8 | |Makefile |Makefile for the markdown documents | 9 | |datasets.mkd |Part I: an overview of all datasets which contain abuse contat lookup data | 10 | |abuse-lookups.mkd|Part II: a proposed mechanism for generic abuse contact lookups | 11 | |common/ |supporting .tex templates for generating a nice PDF out of the markdown document | 12 | |img/ |supporting images for generating a nice PDF out of the markdown document | 13 | |contactDB-pgp support.pptx|slides describing the contact lookup for CERTs project. | 14 | 15 | 16 | How to build the document 17 | ========================= 18 | 19 | ``` 20 | make clean; make 21 | ``` 22 | 23 | Pre-requisites 24 | ============== 25 | 26 | * pandoc 27 | * macTeX (tested with MacTex as well as TexLive) 28 | or 29 | * TexLive (Linux) 30 | or a similar Tex distribution under Windows 31 | -------------------------------------------------------------------------------- /old2/doc/common/cipherStringB.tex: -------------------------------------------------------------------------------- 1 | \seqsplit{EDH+CAMELLIA:EDH+aRSA:EECDH+aRSA+AESGCM:EECDH+aRSA+SHA384:EECDH+aRSA+SHA256:EECDH:+CAMELLIA256:+AES256:+CAMELLIA128:+AES128:+SSLv3:!aNULL:!eNULL:!LOW:!3DES:!MD5:!EXP:!PSK:!DSS:!RC4:!SEED:!ECDSA:CAMELLIA256-SHA:AES256-SHA:CAMELLIA128-SHA:AES128-SHA} 2 | -------------------------------------------------------------------------------- /old2/doc/common/commands.tex: -------------------------------------------------------------------------------- 1 | %%% 2 | %%% commands.tex 3 | %%% Document-specific commands 4 | %%% 5 | 6 | % Outputs red TODOs in the document. Requires \usepackage{color}. 7 | % 8 | % Usage: \todo{Document the TODO command.} 9 | % 10 | % Comment out second line to disable. 11 | \AtBeginDocument{\providecommand{\todo}[1]{}} 12 | \newcommand*{\todo}[1]{{\color{Red} TODO: {#1}}} 13 | 14 | % Creating a horizontal rule 15 | \newcommand*{\HorRule}{% 16 | \color{darkblue}% 17 | \rule{\linewidth}{1pt}% 18 | } 19 | 20 | %%% CIPHERSTRING 21 | \usepackage{seqsplit} % Use Sequence split. Basically it inserts between every character pair a box with zero width to allow linebreaks everywhere. Better solution wanted, but is there any better? 22 | \CatchFileDef{\cipherStringB}{common/cipherStringB.tex}{\endlinechar=-1 }% 23 | 24 | %%% Local Variables: 25 | %%% mode: latex 26 | %%% TeX-master: "../applied-crypto-hardening" 27 | %%% End: 28 | -------------------------------------------------------------------------------- /old2/doc/common/style.tex: -------------------------------------------------------------------------------- 1 | %%% 2 | %%% style.tex 3 | %%% Stylistic configuration 4 | %%% 5 | 6 | % Colors 7 | \definecolor{green}{RGB}{32,113,10} 8 | \definecolor{orange}{RGB}{251,111,16} 9 | \definecolor{red}{RGB}{247,56,0} 10 | \definecolor{blue}{RGB}{0,28,128} 11 | \definecolor{lightgreen}{RGB}{187,218,216} 12 | \definecolor{intersectgreen}{RGB}{103,133,155} 13 | \definecolor{darkblue}{RGB}{76,87,117} 14 | \definecolor{Brown}{cmyk}{0,0.81,1,0.60} 15 | \definecolor{OliveGreen}{cmyk}{0.64,0,0.95,0.40} 16 | \definecolor{CadetBlue}{cmyk}{0.62,0.57,0.23,0} 17 | \definecolor{lightlightgray}{gray}{0.9} 18 | 19 | % Draft stuff 20 | \ifdraft{ 21 | % Ensure that everything is neatly set up for us 22 | \AtBeginDocument{ 23 | \def\draftInfo{% 24 | Draft revision\gitVtags: \gitAbbrevHash{} % 25 | (\gitCommitterIsoDate) \gitCommitterName} 26 | \sbox{\draftWatermark}{% 27 | \includegraphics[width=\paperwidth]{img/draft}} 28 | \sbox{\draftPageLine}{% 29 | \colorbox{black!10}{% 30 | % enlarge box vertically by 2/3 lines 31 | \raisebox{0pt}% 32 | [\dimexpr .33\baselineskip + \height]% 33 | [\dimexpr .33\baselineskip + \depth]{% 34 | \makebox[\paperwidth]{\color{black!50}\draftInfo}}}} 35 | } 36 | }{} 37 | 38 | 39 | %%% Fonts 40 | % Fonts are loaded, now is the right time for mictorype. 41 | \microtypesetup{stretch=9,shrink=15,step=3,tracking=smallcaps,letterspace=75} 42 | % makes default font sans-serif 43 | \renewcommand{\familydefault}{\sfdefault} 44 | 45 | % Captions 46 | \setcapindent{1em} 47 | \addtokomafont{caption}{\small\itshape} 48 | \addtokomafont{captionlabel}{\bfseries} 49 | 50 | % Section headers 51 | \addtokomafont{disposition}{\color{darkblue}\bfseries} 52 | 53 | % Page foot 54 | \setkomafont{pagefoot}{\normalfont\sffamily\footnotesize} 55 | \addtolength{\headheight}{2em} 56 | \ohead{% 57 | %\includegraphics[height=3em]{img/holistic.pdf} 58 | %\hspace{2em} 59 | %\includegraphics[height=3em]{img/cert.png} 60 | } 61 | \ifoot% 62 | [Contact databases for abuse handling\ifdraft{ % Space! 63 | \textbullet{} \draftInfo}{}]% 64 | {Contact databases for abuse handling \ifdraft{ % Space! 65 | \textbullet{} \draftInfo}{}} 66 | \cfoot[]{} 67 | \ofoot% 68 | [Seite \thepage\ ]% 69 | {Seite \thepage\ } 70 | %[page \thepage\ of \pageref{LastPage}]% 71 | %{page \thepage\ of \pageref{LastPage}} 72 | \pagestyle{scrheadings} 73 | % Epigraph / dictum 74 | \newcommand*{\epigraph}[3][]{\dictum[#3]{#2}\bigskip} 75 | \renewcommand*{\dictumrule}{} 76 | \renewcommand*{\dictumauthorformat}[1]{--- #1} 77 | \addtokomafont{dictumtext}{\itshape} 78 | \setkomafont{dictumauthor}{\normalfont} 79 | \renewcommand{\dictumwidth}{8cm} 80 | 81 | % Graphics 82 | %tell TeX where to look for graphics/logos 83 | \graphicspath{ {/img/} } 84 | 85 | % This block is for listings 86 | \usepackage[framemethod=TikZ]{mdframed} % mdframed is used to draw a grey box 87 | \mdfdefinestyle{listingstyle}{ 88 | backgroundcolor=black!10,outerlinewidth=0,outerlinecolor=black, 89 | innerleftmargin=9pt,innerrightmargin=0,innertopmargin=9pt,innerbottommargin=2pt 90 | } 91 | %\usepackage{amssymb}% for \curvearrowright 92 | % Insert a grey box behind the listing for uniform background color (The \cipherstring would the listing and the background would turn white) 93 | \BeforeBeginEnvironment{lstlisting}{\vspace{0.2cm}\begin{mdframed}[style=listingstyle]} 94 | \AfterEndEnvironment{lstlisting}{\end{mdframed}} 95 | % Listings 96 | \lstset{ 97 | basicstyle=\ttfamily, 98 | keywordstyle=\color{OliveGreen}, 99 | commentstyle=\color{gray}, 100 | backgroundcolor=\color{lightlightgray}, 101 | upquote=true, 102 | showstringspaces=false, 103 | tabsize=2, 104 | captionpos=b, 105 | breaklines=true, 106 | breakatwhitespace=false, 107 | inputencoding=utf8, 108 | breakatwhitespace=false, 109 | showspaces=false, 110 | columns=fullflexible, % Column format: no spaces are inserted for monospaced appearance 111 | breakindent=10pt, 112 | morekeywords={__global__, __device__},% 113 | escapechar=\`, 114 | escapeinside={\%*}{*)}, % Escape TeX commands inside %* and *) 115 | % prebreak=\mbox{$\curvearrowright$}, % Disply curved arrow before linebreak 116 | prebreak=\small\symbol{'134}, 117 | } 118 | 119 | % red warning box (for SSH section) 120 | \mdfdefinestyle{warningboxstyle}{ 121 | backgroundcolor=pink!20, 122 | innerleftmargin=9pt,innerrightmargin=9pt,innertopmargin=9pt,innerbottommargin=9pt 123 | } 124 | 125 | % Hyperref styles 126 | %\hypersetup{% 127 | % breaklinks,% 128 | % colorlinks,% 129 | % linkcolor=darkblue,citecolor=blue,urlcolor=blue,% 130 | % breaklinks=true,% 131 | % unicode,% 132 | % pdfnewwindow=true,% 133 | % final 134 | %} 135 | %\urlstyle{same} 136 | 137 | % Bibliography 138 | \bibliographystyle{alphalink} 139 | 140 | % Disable single lines at the start of a paragraph (Schusterjungen) 141 | \clubpenalty = 10000 142 | % Disable single lines at the end of a paragraph (Hurenkinder) 143 | \widowpenalty = 10000 144 | \displaywidowpenalty = 10000 % formulas 145 | \setlength{\textfloatsep}{\baselineskip} 146 | \setlength{\floatsep}{\baselineskip} 147 | 148 | \frenchspacing 149 | \raggedbottom 150 | 151 | % customized spaces between text and footnotes 152 | \setlength{\skip\footins}{2\baselineskip} 153 | 154 | % Better float parameters: (from the TeX FAQ) 155 | \renewcommand{\topfraction}{.85} 156 | \renewcommand{\bottomfraction}{.7} 157 | \renewcommand{\textfraction}{.15} 158 | \renewcommand{\floatpagefraction}{.66} 159 | \renewcommand{\dbltopfraction}{.66} 160 | \renewcommand{\dblfloatpagefraction}{.66} 161 | \setcounter{topnumber}{9} 162 | \setcounter{bottomnumber}{9} 163 | \setcounter{totalnumber}{20} 164 | \setcounter{dbltopnumber}{9} 165 | 166 | % Starred lists (\begin{itemize*}) for less space between items 167 | \usepackage{mdwlist} 168 | 169 | %%% Local Variables: 170 | %%% mode: latex 171 | %%% TeX-master: "../applied-crypto-hardening" 172 | %%% End: 173 | -------------------------------------------------------------------------------- /old2/doc/common/system.tex: -------------------------------------------------------------------------------- 1 | %%% 2 | %%% system.tex 3 | %%% Necessary packages and sytem changes 4 | %%% 5 | % Document Encoding. important. 6 | \usepackage[utf8]{inputenc} 7 | 8 | % changes font encoding to T1 9 | \usepackage[T1]{fontenc} 10 | \usepackage{textcomp} 11 | 12 | % For searchable pdfs 13 | \input glyphtounicode 14 | \pdfgentounicode=1 15 | 16 | \usepackage{fixltx2e} 17 | 18 | % Setup KOMA script 19 | \usepackage{scrhack} 20 | \KOMAoptions{paper=a4% 21 | ,fontsize=10pt% 22 | ,DIV=12% 23 | ,parskip=true% 24 | } 25 | 26 | % Language 27 | \usepackage[english]{babel} 28 | 29 | % Color 30 | \usepackage[usenames,dvipsnames,svgnames,table]{xcolor} 31 | \usepackage{color} 32 | 33 | % Packages for fonts 34 | \usepackage{lmodern} 35 | \usepackage[defaultsans]{opensans} 36 | \usepackage[final,babel=true]{microtype}[2011/08/18] 37 | 38 | \usepackage{pifont} 39 | \newcommand{\yes}{\textcolor{green}{\ding{51}}} 40 | \newcommand{\no}{\textcolor{red}{\ding{55}}} 41 | 42 | % Figures and graphics 43 | \usepackage[final]{graphicx} 44 | \usepackage{epstopdf} 45 | \usepackage{float} 46 | \usepackage{subfig} 47 | \usepackage{placeins} 48 | \usepackage{wrapfig} 49 | \usepackage{tikz} 50 | \usetikzlibrary{shapes,arrows} 51 | 52 | % Tables 53 | \usepackage{longtable} 54 | \usepackage{booktabs} 55 | \renewcommand{\arraystretch}{1.25} 56 | \usepackage{multicol} 57 | 58 | % Verbatims and listings 59 | \usepackage{fancyvrb} 60 | \usepackage[final]{listings} 61 | 62 | % The page 63 | \usepackage[footsepline]{scrpage2} 64 | \usepackage{lastpage} 65 | 66 | % Misc 67 | \usepackage{gitinfo} 68 | \usepackage{catchfile} 69 | %\usepackage{hyperref} 70 | 71 | % for development 72 | \usepackage{ifdraft} 73 | \ifdraft{% 74 | %% Heavy debugging 75 | %\usepackage{showframe} 76 | \usepackage{blindtext} 77 | \usepackage{eso-pic} 78 | \newsavebox{\draftPageLine} 79 | \newsavebox{\draftWatermark} 80 | \AddToShipoutPicture{% 81 | \AtPageLowerLeft{\usebox{\draftWatermark}} 82 | \AtPageUpperLeft{% 83 | \raisebox{-\height}[\height][0pt]{\usebox{\draftPageLine}}}% 84 | \AtPageLowerLeft{% 85 | \raisebox{\depth}[\height][0pt]{\usebox{\draftPageLine}}}% 86 | } 87 | % \AtEndDocument{\listoftodos} 88 | }{ 89 | \let\blindtext\relax 90 | \let\Blindtext\relax 91 | \let\blinddocument\relax 92 | \let\Blinddocument\relax 93 | } 94 | 95 | %%% Local Variables: 96 | %%% mode: latex 97 | %%% TeX-master: "../applied-crypto-hardening" 98 | %%% End: 99 | -------------------------------------------------------------------------------- /old2/doc/contactDB-pgp support.pptx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/certtools/contactdb/51c19fc235a73a447e388c9778b34255eb420221/old2/doc/contactDB-pgp support.pptx -------------------------------------------------------------------------------- /old2/doc/gitHeadInfo.gin: -------------------------------------------------------------------------------- 1 | \usepackage[% 2 | shash={5ba2de6}, 3 | lhash={5ba2de6bd3eb9ec0b8c38d559eaa1e37408ba60a}, 4 | authname={Aaron Kaplan}, 5 | authemail={aaron@lo-res.org}, 6 | authsdate={2014-09-26}, 7 | authidate={2014-09-26 17:08:15 +0200}, 8 | authudate={1411744095}, 9 | commname={Aaron Kaplan}, 10 | commemail={aaron@lo-res.org}, 11 | commsdate={2014-09-26}, 12 | commidate={2014-09-26 17:08:15 +0200}, 13 | commudate={1411744095}, 14 | refnames={ (HEAD, rest)} 15 | ]{gitsetinfo} -------------------------------------------------------------------------------- /old2/doc/img/cert.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/certtools/contactdb/51c19fc235a73a447e388c9778b34255eb420221/old2/doc/img/cert.png -------------------------------------------------------------------------------- /old2/doc/img/contact-lookup-flow.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/certtools/contactdb/51c19fc235a73a447e388c9778b34255eb420221/old2/doc/img/contact-lookup-flow.png -------------------------------------------------------------------------------- /old2/doc/img/contact-lookup-flow.pptx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/certtools/contactdb/51c19fc235a73a447e388c9778b34255eb420221/old2/doc/img/contact-lookup-flow.pptx -------------------------------------------------------------------------------- /old2/doc/img/draft.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/certtools/contactdb/51c19fc235a73a447e388c9778b34255eb420221/old2/doc/img/draft.png -------------------------------------------------------------------------------- /old2/doc/structure.mkd: -------------------------------------------------------------------------------- 1 | ## $Name of the dataset 2 | 3 | ### Overview 4 | 5 | ### Data in the dataset 6 | 7 | ### Original or synthetic data? 8 | 9 | ### Where to find it? 10 | 11 | ### Who has access? 12 | 13 | ### APIs / automatic querying 14 | 15 | ### Maintainers 16 | 17 | ### Liveliness 18 | 19 | ### Known issues 20 | -------------------------------------------------------------------------------- /old2/doc/styleheader.tex: -------------------------------------------------------------------------------- 1 | %%% 2 | %%% Salzburg-AG hack Dokument 3 | %%% 4 | %%% 5 | \RequirePackage{fix-cm} 6 | \documentclass[draft]{scrartcl} 7 | %\documentclass{scrreprt} % uncomment this if you want to make a final version (and remove the DRAFT watermark) 8 | \input{common/system} 9 | \input{common/style} 10 | \input{common/commands} 11 | -------------------------------------------------------------------------------- /old2/doc/update-metadata-for-gitinfo: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | GITINFO_INDEX_FILENAME="gitHeadInfo.gin" 4 | GITINFO_GIT_LOG_PRETTY_FORMAT=$(cat <<'__EOS__' 5 | \usepackage[% 6 | shash={%h}, 7 | lhash={%H}, 8 | authname={%an}, 9 | authemail={%ae}, 10 | authsdate={%ad}, 11 | authidate={%ai}, 12 | authudate={%at}, 13 | commname={%an}, 14 | commemail={%ae}, 15 | commsdate={%ad}, 16 | commidate={%ai}, 17 | commudate={%at}, 18 | refnames={%d} 19 | ]{gitsetinfo} 20 | __EOS__ 21 | ) 22 | 23 | git log -1 --date=short \ 24 | --pretty=format:"${GITINFO_GIT_LOG_PRETTY_FORMAT}" \ 25 | HEAD > ./${GITINFO_INDEX_FILENAME} 26 | 27 | 28 | -------------------------------------------------------------------------------- /old2/install_deps.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -e 4 | set -x 5 | 6 | # Debian Wheezy has 9.1 7 | #POSTGRES_VERSION="9.1" 8 | 9 | # Ubuntu 14.04 has 9.3 10 | POSTGRES_VERSION="9.3" 11 | 12 | sudo apt-get update 13 | sudo apt-get install postgresql-${POSTGRES_VERSION} 14 | sudo apt-get install postgresql-server-dev-${POSTGRES_VERSION} 15 | sudo apt-get install python-virtualenv 16 | sudo apt-get install python-dev 17 | sudo sed -i "s/^\(local[ ]*all[ ]*all.*\)peer/\1trust/" /etc/postgresql/${POSTGRES_VERSION}/main/pg_hba.conf 18 | sudo service postgresql restart 19 | 20 | sudo su - postgres -c 'createuser -s contactdb' 21 | #sudo su - postgres -c 'dropdb contactdb' 22 | sudo su - postgres -c 'createdb contactdb' 23 | 24 | virtualenv virtenv 25 | 26 | echo export CONTACTDB_HOME=$(pwd) >> ./virtenv/bin/activate 27 | echo export GNUPGHOME=$(pwd)/.gnupg >> ./virtenv/bin/activate 28 | echo export PYTHONPATH=\$PYTHONPATH:$(pwd) >> ./virtenv/bin/activate 29 | echo export DJANGO_SETTINGS_MODULE='certdir.settings' >> ./virtenv/bin/activate 30 | 31 | . ./virtenv/bin/activate 32 | pip install -r requirements.txt --upgrade 33 | 34 | django-admin syncdb 35 | -------------------------------------------------------------------------------- /old2/manage.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import os 3 | import sys 4 | import django 5 | 6 | if __name__ == "__main__": 7 | os.environ.setdefault("DJANGO_SETTINGS_MODULE", "certdir.settings") 8 | django.setup() 9 | 10 | from django.core.management import execute_from_command_line 11 | 12 | 13 | execute_from_command_line(sys.argv) 14 | -------------------------------------------------------------------------------- /old2/old/API: -------------------------------------------------------------------------------- 1 | 2 | http://localhost:8000/api/v1/countries/?format=json&limit=10000&country_name__contains=FRA 3 | http://localhost:8000/api/v1/countries/?format=json&limit=10000&country_name__startswith=BE 4 | http://localhost:8000/api/v1/countries/?format=json&limit=10000 5 | http://localhost:8000/api/v1/airports/?format=json&limit=10000 6 | 7 | -------------------------------------------------------------------------------- /old2/old/contrib/README.rst: -------------------------------------------------------------------------------- 1 | Demos for importing the TI data, handling pgp keys etc 2 | Needs more work 3 | -------------------------------------------------------------------------------- /old2/old/contrib/TI-import.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python2.7 2 | 3 | import sys 4 | import csv 5 | import pprint 6 | import re 7 | import gnupg 8 | import os 9 | 10 | gpg = gnupg.GPG(gnupghome=os.environ['CONTACTDB_HOME'] + '/.gnupg/') 11 | 12 | 13 | #def get_pgpkey(key_id): 14 | # 15 | 16 | def extract_workinghours(field): 17 | # input format: 18 | # 09:00 to 17:00 Monday to Friday except public holidays. 19 | # Timezone: GMT+01. 20 | # Timezone with DST: GMT+02 21 | match = re.search('.*([0-9]{2}:[0-9]{2}).*(to|\-).*([0-9]{2}:[0-9]{2}).*Timezone: ([^\.]+)', field, re.MULTILINE | re.IGNORECASE | re.DOTALL) 22 | if (match == None): 23 | begin_hh = '' 24 | end_hh = '' 25 | tz = '' 26 | else: 27 | begin_hh = match.group(1) 28 | end_hh= match.group(3) 29 | tz = match.group(4) 30 | return (begin_hh, end_hh, tz) 31 | 32 | 33 | # new TI format: (number indicates field number) 34 | # 0 1 2 3 4 5 6 7 8 9 35 | #Team Name,TI Level,First entered,Last changed,FIRST Membership,TI URL,Official Team Name,Former Team Names,Country, 36 | # 9 10 11 12 13 14 37 | #Date of Establishment,-Type of Constituency,Constituency ASNs,Constituency Domains,Costituency Nets,Country of Constituents, 38 | # 15 16 17 18 19 20 21 39 | #Email,PGP Key (Team),Telephone,Emergency Phone,Telefax,Other communication,Address, 40 | # 22 23 24 25 41 | #-Business Hours,Contacting outside Business Hours,Team Representative,Email (Rep), 42 | # 26 27 28 29 30 43 | #PGP Key (Rep),WWW,FTP,*RFC2350,Operating Status 44 | # 45 | 46 | 47 | #ACOnet-CERT,Accredited,8/31/00,3/28/03,Full Member,https://tiw.trusted-introducer.org/directory/teams/aconet-cert.html,ACOnet-CERT,,AT,1/1/03,Research & Education,AS1853. - AS679. - AS760. - AS1109-AS1123. - AS1205. - AS1776. - AS1921. - AS2036. - AS2494. - AS2604. - AS6720. - AS8692. - AS12991. - AS16314. - AS30971. - AS39837. - AS41915. - AS42685. - AS47515,ac.at,,AT,cert@aco.net,0x86EDDB8A,+43 1 427714045,+43 1 427714045,+43 1 42779140,N/A,Zentraler Informatikdienst. - Universitaet Wien. - Universitaetsstrasse 7. - A-1010 Wien. - Austria,09:00 to 17:00 Monday to Friday except public holidays. - Timezone: GMT+01. - Timezone with DST: GMT+02,eMail or leave Message on the Voicebox,Alexander Talos-Zens,alexander.talos-zens@univie.ac.at,0x9D9731C5,http://cert.aco.net/,,, 48 | 49 | reader = csv.reader(sys.stdin, delimiter=',') 50 | headers = reader.next() 51 | for r in reader: 52 | r = [ x.replace( " - ", "\n") for x in r ] 53 | (begin_hh, end_hh, tz) = extract_workinghours(r[22]) 54 | mapping = { "name": r[0], "fullname": r[6], 55 | "address":r[21], 56 | "country_id":r[8], "phone":r[17], "emergency_phone":r[18], 57 | "fax":r[19], "email":r[15], "website":r[27], "timezone":tz, ## XXX FIXME: timezone parsing from field r[22] 58 | "business_hh_start": begin_hh, "business_hh_end": end_hh, ## XXX FIXME: parse and split this field r[22] 59 | "date_established": r[9] , 60 | "isCERT": "t", 61 | "ti_url": r[5], 62 | "pgp_key_id": r[16], 63 | "confirmed": "t", 64 | "active": "t", 65 | "source_id": "TI", 66 | } 67 | #pprint.pprint(mapping) 68 | # do the mapping 69 | keystr="" 70 | valstr="" 71 | for key in mapping.keys(): 72 | keystr += key + ", " 73 | valstr += "" + ( 'E' + repr(mapping[key]) if (mapping[key] != '') else 'NULL' ) + ", " 74 | keystr += 'parent_id' 75 | valstr += 'NULL' 76 | 77 | # order: 78 | # first insert the pgp key and uids if it does not exist yet 79 | # then the person 80 | # then the organisation 81 | print "INSERT INTO contactdb_pgpkey (pgp_key_id) values ( " + repr(r[16]) + " );" 82 | print "INSERT INTO contactdb_organisation( " + keystr + ") values (" + valstr + ");" 83 | 84 | # now insert all countries (if they don't exist yet) into contactdb_organisation_country 85 | 86 | # database format: 87 | # contactdb=# \d contactdb_organisation 88 | #Column | Type | Modifiers 89 | #-------------------------+--------------------------+--------------------------------------------------------------------- 90 | #id | integer | not null default nextval('contactdb_organisation_id_seq'::regclass) 91 | #parent_id | integer | 92 | #name | character varying(1000) | not null 93 | #fullname | character varying(1000) | 94 | #nesting | character varying(5000) | 95 | #protection_profile | character varying(30) | 96 | #isCERT | boolean 97 | #address | character varying(1000) | 98 | #housenr | character varying(50) | 99 | #pobox | character varying(50) | 100 | #city | character varying(200) | 101 | #zipcode | character varying(20) | 102 | #country_id | character varying(2) | not null 103 | #phone | character varying(64) | not null 104 | #emergency_phone | character varying(64) | 105 | #fax | character varying(64) | 106 | #email | character varying(256) | not null 107 | #website | character varying(1000) | 108 | #timezone | character varying(10) | 109 | #business_hh_start | time without time zone | not null 110 | #business_hh_end | time without time zone | not null 111 | #date_established | date | 112 | #pgp_key_id | character varying(1000) | 113 | #confirmed | boolean | not null 114 | #active | boolean | not null 115 | #source_id | character varying(1000) | 116 | #vouching_proposed_by_id | integer | not null 117 | #ti_url | character varying(1000) | 118 | #first_url | character varying(1000) | 119 | #created | timestamp with time zone | not null 120 | #last_updated | timestamp with time zone | not null 121 | 122 | -------------------------------------------------------------------------------- /old2/old/contrib/geolocation.py: -------------------------------------------------------------------------------- 1 | # geolocation module 2 | # Thanks to romain.bourgue@gmail.com 3 | 4 | 5 | import json 6 | import sys 7 | from geopy import geocoders 8 | 9 | # returning latitude and longitude coordinates to a given address using Google API 10 | # @version 1.0 20131209 11 | # @param location - text, specifies the address 12 | # @param geoQuality - specifies the quality of the returning lat/long coordinats. starting at 100% as default 13 | # @param coordinatesOnly - true by default, returns only coordinates, if 0: returning all information (place, latitude + longitude, quality) 14 | # @return latitude and longitude coordinates, optional quality of geolocation and place if specified 15 | def geocode(location,coordinatesOnly=1,geoQuality=100): 16 | try: 17 | place, (lat, lng)= geocoders.GoogleV3().geocode(location.encode("utf-8")) 18 | 19 | except geocoders.google.GQueryError: 20 | print("Error couldn't geocode %s" % location) 21 | if location.count('-')==0: 22 | print "ERROR Returning Nowhere" 23 | return nowhere 24 | print("Removing first info") 25 | geoQuality=geoQuality-10 26 | return geocode('-'.join(location.split('-')[1::]),geoQuality) 27 | if coordinatesOnly == 1: 28 | return (lat,lng) 29 | else: 30 | return place,[(lat,lng)],geoQuality 31 | 32 | # usage example 33 | # geocode("1010 Vienna", 100) 34 | print(geocode("vienna")) 35 | 36 | -------------------------------------------------------------------------------- /old2/old/contrib/kmlcertmap.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | 3 | """ 4 | This script parses a CSC file as input and geocode the 'Address' field. 5 | Each entry is then place on a KML file or output as a JSON object. 6 | 7 | In json mode, callback function is setCerts() 8 | 9 | Usage: kmlcertmap.py CSVFILE OUTPUT [json] 10 | 11 | 12 | Author: romain.bourgue@gmail.com 13 | """ 14 | 15 | import re 16 | import json 17 | import simplekml 18 | import time 19 | import csv 20 | import sys 21 | from geopy import geocoders 22 | 23 | nowhere=[(-35.537113,46.316584)] 24 | kml=simplekml.Kml() 25 | def geocode(location,geoQuality=100): 26 | print("Geocoding"+location+"...") 27 | try: 28 | place, (lat, lng)= geocoders.GoogleV3().geocode(location.encode("utf-8"),exactly_one=False)[0] 29 | except geocoders.googlev3.GTooManyQueriesError: 30 | print('Too many requests, sleeping a bit') 31 | time.sleep(0.3) 32 | return geocode(location,geoQuality) 33 | 34 | except geocoders.googlev3.GQueryError: 35 | 36 | print("Error couldn't geocode %s" % location) 37 | if location.count('-')==0: 38 | print "ERROR Returning Nowhere" 39 | return nowhere 40 | print("Removing first info") 41 | geoQuality=geoQuality-10 42 | return geocode('-'.join(location.split('-')[1::]),geoQuality) 43 | 44 | 45 | return [(lng,lat)],geoQuality 46 | 47 | class Cert: 48 | def jsonable(self): 49 | return self.__dict__ 50 | pass 51 | 52 | def addToKML(cert): 53 | 54 | pnt=kml.newpoint(name=cert.title, description=cert.description, coords=cert.coords) 55 | pnt.timespan.begin = cert.date 56 | #pnt.style.iconstyle.icon.href=job.icon 57 | 58 | 59 | def parseRow(row): 60 | cert=Cert() 61 | cert.geoQuality=100 62 | if row["Address"]=='. - ': 63 | print "no address for %s, using country %s" % (row["Official Team Name"],row["Country"]) 64 | row["Address"]=row["Country"] 65 | cert.geoQuality=10 66 | 67 | cert.coords,cert.geoQuality=geocode(row["Address"],cert.geoQuality) 68 | print "Geoquality is "+str(cert.geoQuality) 69 | cert.x=cert.coords[0][0] 70 | cert.y=cert.coords[0][1] 71 | 72 | cert.date=row.pop("First entered") 73 | 74 | 75 | cert.title=row.pop("Team Name") 76 | cert.description="" 77 | for key in row: 78 | if row[key]!="": 79 | 80 | if re.match("https?://",row[key]): 81 | cert.description+="- %s: %s
" % (key,row[key],row[key]) 82 | elif re.match("@",row[key]): 83 | cert.description+="- %s: %s
" % (key,row[key],row[key]) 84 | else: 85 | cert.description+="- %s: %s
" % (key,row[key]) 86 | setattr(cert,re.sub(' ','_',key),row[key]) 87 | 88 | return cert 89 | 90 | certs = [] 91 | 92 | #Internet Addresses;Email (Rep);Official Team Name;Telefax;Former Team Names;Email;Operating Status;Emergency Phone;TI Level;First entered;Address;PGP Key (T 93 | #eam);Date of Establishment;Last changed;FTP;PGP Key (Rep);Team Representative;Country;Contacting outside Business Hours;Other communication;-Type of Constituency;Country o 94 | # Constituents;Team Name;-Business Hours;WWW;Telephone;*RFC2350;TI URL;FIRST Membership 95 | 96 | if len(sys.argv) == 1: 97 | print "Usage: %s CSVinputfile output [json]" % sys.argv[0] 98 | exit(0) 99 | 100 | csvDictReader = csv.DictReader(open(sys.argv[1], 'rb'), restkey='rest', dialect='excel', delimiter=';', quotechar='"') 101 | 102 | for row in csvDictReader: 103 | 104 | if ((row["TI Level"]=="Outdated") or (row["TI Level"]=="Not Operational")): 105 | print "Skiping outdated info...." 106 | continue 107 | cert=parseRow(row) 108 | 109 | addToKML(cert); 110 | certs.append(cert.__dict__) 111 | 112 | if (len(sys.argv)>3) and (sys.argv[3]=="json"): 113 | text_file = open(sys.argv[2], "w") 114 | text_file.write('setCerts('+ json.dumps(certs) +");") 115 | text_file.close() 116 | else: 117 | kml.save("/var/www/jobmap/jobmap.kml") 118 | print("KML document saved") 119 | -------------------------------------------------------------------------------- /old2/old/contrib/pgpwrapper.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import os 4 | import gnupg 5 | from __future__ import print_function 6 | 7 | pgphome=gnupghome=os.environ['CONTACTDB_HOME'] + '/.gnupg/' 8 | gpg = gnupg.GPG(gnupghome=pgphome) 9 | keyserver = 'subkeys.pgp.net' 10 | 11 | cipherprefs="""Cipher: AES256, AES192, AES, CAST5, 3DES 12 | Digest: SHA512, SHA384, SHA256, SHA224, SHA1 13 | Compression: ZLIB, BZIP2, ZIP, Uncompressed 14 | Features: MDC, Keyserver no-modify""" 15 | 16 | def genkey(): 17 | #keyparams = dict({ 'name_real': 'ContactDB', 18 | # 'name_email': 'contactdb@contactd.be', 19 | # 'nname_comment': 'automatically generated, signing only key', 20 | # 'key_type': 'RSA', 21 | # 'key_length': 4096, 22 | # 'key_usage': '', 23 | # 'subkey_type': 'RSA', 24 | # 'subkey_length': 4096, 25 | # 'subkey_usage': 'encrypt,sign,auth', 26 | # 'passphrase': 'secretSauce' }) 27 | # #'preferences': cipherprefs, } 28 | alice = { 'name_real': 'Alice', 29 | 'name_email': 'alice@inter.net', 30 | 'expire_date': '2014-04-01', 31 | 'key_type': 'RSA', 32 | 'key_length': 4096, 33 | 'key_usage': '', 34 | 'subkey_type': 'RSA', 35 | 'subkey_length': 4096, 36 | 'subkey_usage': 'encrypt,sign,auth', 37 | 'passphrase': 'sekrit'} 38 | 39 | print alice 40 | 41 | #keyinput = gpg.gen_key_input(**keyparams) 42 | #print keyinput 43 | key = gpg.gen_key(keyinput) 44 | assert key is not None 45 | assert key.fingerprint is not None 46 | print key 47 | assert key.fingerprint 48 | return key 49 | 50 | def get_pgpkey(key_id): 51 | #gpg.search_keys(key_id, keyserver) 52 | import_result = gpg.recv_keys(keyserver, key_id) 53 | print import_result 54 | 55 | 56 | 57 | if __name__ == "__main__": 58 | genkey() 59 | get_pgpkey('AAAAAAAA') 60 | 61 | 62 | -------------------------------------------------------------------------------- /old2/old/db/initialize/README.rst: -------------------------------------------------------------------------------- 1 | ========== 2 | README 3 | ========== 4 | 5 | The files in this directory will help you get initialize the DB with basic data such as country codes etc. 6 | 7 | Here is how you get the data into the DB: 8 | 9 | $ ./initialize-db.sh 10 | 11 | Note that the DB structure must exist prior to this step 12 | 13 | 14 | ============ 15 | TI Database 16 | ============ 17 | For getting the TI database you must be a member of Trusted Introducer (trusted-introducer.org). 18 | 19 | -------------------------------------------------------------------------------- /old2/old/db/initialize/import-TI-pgp.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python2.7 2 | 3 | """ NOTE: this is outdated... """ 4 | 5 | import csv 6 | import pprint 7 | import sys 8 | import gnupg 9 | import os 10 | import re 11 | import logging 12 | 13 | #gpg = gnupg.GPG(gnupghome='/Users/aaron/.gnupg', verbose=True) # os.environ['HOME']) 14 | gpg = gnupg.GPG(verbose=False, use_agent=True) 15 | #gpg.encoding = 'UTF-8' 16 | #gpg.encoding = 'latin-1' 17 | gpg.encoding = 'latin-1' 18 | public_keys = gpg.list_keys() 19 | 20 | 21 | def extract_email(str): 22 | res = re.search('<([^>]+)>', str, re.I) 23 | if (None!= res and res.groups): 24 | return (res.group(1)) 25 | 26 | 27 | def print_insert_stmt(email, key_id): 28 | try: 29 | if (email and key_id): 30 | print "INSERT into contactdb_pgpuid (pgp_email, pgp_key_id) VALUES ('%s', '%s'); " %(email, key_id) 31 | except: 32 | logging.warning('could not convert email: ' + email) 33 | 34 | 35 | for k in public_keys: 36 | k_id = "0x" + k['keyid'][8:] 37 | print "-- %s" %k_id # just for info 38 | emails = list(set(map(extract_email, k['uids']) )) 39 | #print emails 40 | for e in emails: 41 | print_insert_stmt(e, k_id) 42 | 43 | #for e in emails: 44 | #e = unicode(e, 'utf-16') 45 | # print "%s" %e 46 | # print "INSERT into contactdb_pgpuid (email, pgp_key_id) VALUES ('%s', '%s')" %(e, k_id) 47 | 48 | 49 | 50 | exit (0) 51 | 52 | 53 | csvDictReader = csv.DictReader(open(sys.argv[1], 'rb'), restkey='rest', dialect='excel') #, delimiter=',', quotechar='"') 54 | 55 | for row in csvDictReader: 56 | k_id = row['PGP Key (Team)'] 57 | print """INSERT INTO contactdb_pgpkey ( pgp_key_id, pgp_key, pgp_key_email, pgp_key_created, pgp_key_expires ) 58 | VALUES ( '%s', '%s', '%s', '%s', '%s' ); 59 | """ %( k_id, key(k_id), email(k_id), created(k_id), expires(k_id) ) 60 | 61 | -------------------------------------------------------------------------------- /old2/old/db/initialize/pgpImporter.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python2.7 2 | 3 | import pprint 4 | import gnupg 5 | import re 6 | import logging 7 | import psycopg2 8 | from psycopg2 import errorcodes 9 | 10 | 11 | # need to call init() to initialize this 12 | gpgdict = dict() 13 | 14 | # ---------------------------------------------------- 15 | # functions 16 | 17 | def insert_pgp_key(k, dbcursor): 18 | if (k == None or k == ""): 19 | print "skipping empty key '%s'" %k 20 | return 21 | 22 | # first try inserting the key 23 | try: 24 | print "inserting pgp key %s" %k 25 | #print row 26 | #print dbcursor.mogrify("INSERT INTO contactdb_pgpkey (pgp_key_id) VALUES (%s)", (k,)) 27 | dbcursor.execute("INSERT INTO contactdb_pgpkey (pgp_key_id) VALUES (%s)", (k,)) 28 | #print "inserted id %d" %(dbcursor.lastrowid) 29 | except Exception, e: 30 | print "could not insert pgp key %s" %k 31 | print errorcodes.lookup(e) 32 | # next try to insert email addresses for that key into the contactdb_pgpuid 33 | # table 34 | # fetch emails for this key id 35 | print "searching for key %s in gpgdict" %k 36 | if (k in gpgdict): 37 | #print "k in gpgdict" 38 | # found it, insert into contactdb_pgpuid 39 | for e in gpgdict[k]: 40 | if (e != None and e != ""): 41 | #print dbcursor.mogrify("INSERT INTO contactdb_pgpuid (pgp_key_id, pgp_email) VALUES (%s, %s)", (k, e)) 42 | dbcursor.execute("INSERT INTO contactdb_pgpuid (pgp_key_id, pgp_email) VALUES (%s, %s)", (k, e)) 43 | else: 44 | print "could not find pgp key %s in keyring" %(k) 45 | 46 | return 47 | 48 | 49 | def extract_email(str): 50 | res = re.search('<([^>]+)>', str, re.I) 51 | if (None!= res and res.groups): 52 | return (res.group(1)) 53 | 54 | 55 | 56 | def init(): 57 | gpg = gnupg.GPG(verbose=False, use_agent=True) 58 | #gpg.encoding = 'UTF-8' 59 | gpg.encoding = 'latin-1' 60 | public_keys = gpg.list_keys() 61 | # read in all the keys, now make a dict (key_id -> [ emails,...] 62 | for k in public_keys: 63 | k_id = "0x" + k['keyid'][8:] 64 | print "-- %s" %k_id # just for info 65 | emails = list(set(map(extract_email, k['uids']) )) 66 | gpgdict[k_id] = emails 67 | #print gpgdict 68 | 69 | 70 | 71 | def print_insert_stmt(email, key_id): 72 | try: 73 | if (email and key_id): 74 | print "INSERT into contactdb_pgpuid (pgp_email, pgp_key_id) VALUES ('%s', '%s'); " %(email, key_id) 75 | except: 76 | logging.warning('could not convert email: ' + email) 77 | 78 | 79 | 80 | -------------------------------------------------------------------------------- /old2/old/html/README: -------------------------------------------------------------------------------- 1 | jobmap.kml 2 | out.json - json file output from kmlcertmap.py 3 | page.html - sample page using out.json 4 | pagekml.html - sample page displaying a KML file 5 | -------------------------------------------------------------------------------- /old2/old/html/lib/oms.min.js: -------------------------------------------------------------------------------- 1 | (function(){/* 2 | OverlappingMarkerSpiderfier 3 | https://github.com/jawj/OverlappingMarkerSpiderfier 4 | Copyright (c) 2011 - 2012 George MacKerron 5 | Released under the MIT licence: http://opensource.org/licenses/mit-license 6 | Note: The Google Maps API v3 must be included *before* this code 7 | */ 8 | var h=!0,i=null,n=!1,p,q={}.hasOwnProperty,s=[].slice; 9 | if(((p=this.google)!=i?p.maps:void 0)!=i){var u=function(b,c){var a,e,d,f,g=this;this.map=b;c==i&&(c={});for(a in c)q.call(c,a)&&(e=c[a],this[a]=e);this.e=new this.constructor.g(this.map);this.n();this.b={};f=["click","zoom_changed","maptypeid_changed"];e=0;for(d=f.length;ec)return this;e=this.j.splice(c,1)[0];d=0;for(f=e.length;da||this.b[b].splice(a,1);return this};z.clearListeners=function(b){this.b[b]=[];return this};z.trigger=function(){var b,c,a,e,d,f;c=arguments[0];b=2<=arguments.length?s.call(arguments,1):[];c=(a=this.b[c])!=i?a:[];f=[];e=0;for(d=c.length;eb;a=0<=b?++f:--f)a=this.circleStartAngle+ 14 | a*e,g.push(new v.Point(c.x+d*Math.cos(a),c.y+d*Math.sin(a)));return g};z.v=function(b,c){var a,e,d,f,g;d=this.spiralLengthStart;a=0;g=[];for(e=f=0;0<=b?fb;e=0<=b?++f:--f)a+=this.spiralFootSeparation/d+5E-4*e,e=new v.Point(c.x+d*Math.cos(a),c.y+d*Math.sin(a)),d+=A*this.spiralLengthFactor/a,g.push(e);return g};z.F=function(b){var c,a,e,d,f,g,j,k,m;d=b._omsData!=i;(!d||!this.keepSpiderfied)&&this.unspiderfy();if(d||this.map.getStreetView().getVisible())return this.trigger("click",b);d=[];f=[];c= 15 | this.nearbyDistance;g=c*c;e=this.c(b.position);m=this.a;j=0;for(k=m.length;j=this.circleSpiralSwitchover?this.v(d,a).reverse():this.u(d,a);var k,m,l,r=this;l=[];k=0;for(m=d.length;k 2 | 3 | 4 | 5 | Cert Map - json 6 | 7 | 8 | 9 | 100 | 101 | 102 | 103 |
Country:
TI-Level:
Constituency:
104 |
105 | 106 | 107 | 108 | -------------------------------------------------------------------------------- /old2/old/html/pagekml.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | Google Maps V3 API Sample 6 | 7 | 8 | 25 | 26 | 27 |
Legende:
28 |
Jobs without geocode:
29 | 30 | 31 | 32 | -------------------------------------------------------------------------------- /old2/old/installation/contactdb.macosx.install.txt: -------------------------------------------------------------------------------- 1 | 2 | Author: L. Aaron Kaplan 3 | Last modified: 2013/01/29 4 | 5 | 6 | 7 | HOW TO INSTALL the Django AH contactdb 8 | ====================================== 9 | 10 | 11 | PREREQUISITES 12 | ============= 13 | 14 | Python2.7 15 | py27-psycopg2 16 | py27-django 17 | south (http://south.readthedocs.org/en/latest/about.html) 18 | tastypie (http://django-tastypie.readthedocs.org/) 19 | py27-gnupg 20 | postgresql91 or higher 21 | 22 | OPTIONAL pyyaml (http://pyyaml.org/) - used for YAML RESTful output 23 | OPTIONAL lxml (http://lxml.de/) - used for XML RESTful output 24 | 25 | 26 | On OS X: 27 | sudo port install postgresql91-server 28 | sudo port install postgresql91 29 | sudo port install py27-django 30 | sudo port install py27-gnupg 31 | sudo port install py27-psycopg2 32 | sudo port install py27-pil 33 | sudo easy_install geopy 34 | sudo port install py27-south 35 | sudo port install py27-tastypie 36 | sudo port install py27-lxml 37 | sudo port install py27-yaml 38 | 39 | Note: in case you had some issues with installing postgresql on OS X , you can can also try the Mac OS X package provided on postgresql.org 40 | 41 | 42 | 43 | Setting up the Database 44 | ======================= 45 | 46 | $ sudo su - 47 | # su - postgres 48 | 49 | $ createuser -s contactdb 50 | $ createdb contactdb 51 | 52 | 53 | 54 | 55 | 56 | Creating the DB structure 57 | ========================= 58 | $ ./manage.py syncdb 59 | 60 | 61 | 62 | Edit the settings.py file to suite your needs 63 | ============================================= 64 | 65 | $ vi AHDjango/settings.py 66 | 67 | DATABASES = { 68 | 'default': { 69 | 'ENGINE': 'django.db.backends.postgresql_psycopg2', # Add 'postgresql_psycopg2', 'mysql', 'sqlite3' or 'oracle'. 70 | 'NAME': 'contactdb', # Or path to database file if using sqlite3. 71 | 'USER': 'aaron', # Not used with sqlite3. 72 | 'PASSWORD': '', # Not used with sqlite3. 73 | 'HOST': '', # Set to empty string for localhost. Not used with sqlite3. 74 | ^^^^^^^^^^^^^^^ change these values 75 | 'PORT': '', # Set to empty string for default. Not used with sqlite3. 76 | } 77 | } 78 | 79 | 80 | + pyc cleanup 81 | 82 | 83 | 84 | Install Tastypie (REST-API) 85 | ========================== 86 | http://tastypieapi.org/ 87 | 88 | Install using Pip: pip install django-tastypie 89 | Add to installed apps: INSTALLED_APPS += ['tastypie'] 90 | Syncdb: ./manage.py syncdb 91 | Create your resource(s) 92 | Hook them up in the URLconf 93 | 94 | 95 | 96 | Set your environment variables to use UTF-8 97 | =========================================== 98 | export LANG="en_EN.UTF-8" 99 | export LC_COLLATE="en_EN.UTF-8" 100 | export LC_CTYPE="en_EN.UTF-8" 101 | export LC_MESSAGES="en_EN.UTF-8" 102 | export LC_MONETARY="en_EN.UTF-8" 103 | export LC_NUMERIC="en_EN.UTF-8" 104 | export LC_TIME="en_EN.UTF-8" 105 | export LC_ALL= 106 | 107 | export CONTACTDB_HOME=$(pwd) 108 | 109 | Fill in the DB values 110 | ===================== 111 | 112 | $ ./initialize-db.sh 113 | 114 | 115 | Fetch the TI database: 116 | 117 | you need to have access to the internal TIW website for this. Ask Trusted Introducer for access in case you are member but don't have access yet. 118 | It requires a client certificate. 119 | 120 | Please use the "new" format: https://tiw.trusted-introducer.org/directory/ti-l2-info.v2.csv 121 | 122 | Import the TI database into the contactdb: 123 | 124 | % cd contrib 125 | % ./TI-import.py < ti-l2-info.v2.csv > foo.sql 126 | % psql contactdb < foo.sql 127 | 128 | 129 | 130 | Start the local Django server 131 | ============================ 132 | $ ./manage.py runserver 133 | 134 | connect your browser to http://localhost:8000/admin 135 | 136 | You should see lots of data in the Organisations table 137 | 138 | Test the RESTful API 139 | =================== 140 | 141 | http://localhost:8000/api/v1/?format=json 142 | http://localhost:8000/api/v1/country/?format=json&limit=1000 143 | 144 | See the API document for details. 145 | 146 | 147 | 148 | 149 | 150 | 151 | -------------------------------------------------------------------------------- /old2/old/installation/contactdb.ubuntu.install.sh: -------------------------------------------------------------------------------- 1 | # Contributors: 2 | # "Tomas Lima (CERT.PT)" , 3 | # "Mauro Silva (CERT.PT)" 4 | 5 | 6 | 7 | echo 8 | echo "=== Warning ===" 9 | echo 10 | echo "Please, execute this script as root" 11 | echo " # sudo su" 12 | echo " # sh INSTALL.ubuntu.sh" 13 | echo 14 | echo "If you are executing as root, press ENTER" 15 | echo 16 | read choice 17 | 18 | 19 | clear 20 | echo "Installing Dependencies [PRESS ENTER]" 21 | echo "=====================================" 22 | read choice 23 | apt-get update 24 | apt-get upgrade -y 25 | apt-get install postgresql-9.1 -y 26 | apt-get install python2.7 -y 27 | apt-get install python-setuptools -y 28 | apt-get install python-pip -y 29 | #apt-get install python-django -y 30 | wget https://launchpad.net/ubuntu/+archive/test-rebuild-20130917/+build/5014357/+files/python-django_1.5.3-1_all.deb 31 | dpkg -i python-django_1.5.3-1_all.deb 32 | rm python-django_1.5.3-1_all.deb 33 | apt-get install python-psycopg2 -y 34 | apt-get install python-imaging -y 35 | apt-get install python-ipy -y 36 | apt-get install python-mimeparse -y 37 | pip install django-ipyfield 38 | pip install django-tastypie 39 | pip install python-gnupg 40 | easy_install geopy 41 | 42 | 43 | sleep 3 44 | clear 45 | echo 46 | echo "Edit the certdir/settings.py file to suite your needs [PRESS ENTER]" 47 | echo "=====================================" 48 | echo -n " 49 | 50 | --- Change this parameters --- 51 | 52 | DATABASES = { 53 | 'default': { 54 | 'ENGINE': 'django.db.backends.postgresql_psycopg2', # Add 'postgresql_psycopg2', 'mysql', 'sqlite3' or 'oracle'. 55 | 'NAME': 'contactdb', # Or path to database file if using sqlite3. 56 | 'USER': 'contactdb', # Not used with sqlite3. 57 | 'PASSWORD': '', # Not used with sqlite3. 58 | 'HOST': '', # Set to empty string for localhost. Not used with sqlite3. 59 | 'PORT': '', # Set to empty string for default. Not used with sqlite3. 60 | } 61 | } 62 | 63 | 64 | --- Add to variable INSTALLED_APPS the following 'tastypie' parameter --- 65 | INSTALLED_APPS = ( 66 | 'tastypie', 67 | ... 68 | ) 69 | " 70 | echo 71 | read choice 72 | 73 | 74 | clear 75 | echo 76 | echo "Change PostreSQL configuration [PRESS ENTER]" 77 | echo "=====================================" 78 | echo "Edit the file: /etc/postgresql/9.1/main/pg_hba.conf 79 | Change this line: 80 | 'local all all peer' 81 | To: 82 | local all all trust" 83 | read choice 84 | 85 | 86 | clear 87 | echo 88 | echo "Setting up the Database [PRESS ENTER]" 89 | echo "=====================================" 90 | read choice 91 | echo "... creating user 'contactdb' ..." 92 | su - postgres -c 'createuser -s contactdb' 93 | echo "... creating database 'contactdb' ..." 94 | su - postgres -c 'createdb contactdb' 95 | echo 96 | echo "... creating database structure ..." 97 | python ./manage.py syncdb 98 | 99 | 100 | sleep 3 101 | clear 102 | echo 103 | echo "Set your environment variables to use UTF-8 [PRESS ENTER]" 104 | echo "=====================================" 105 | read choice 106 | export LANG="en_EN.UTF-8" 107 | export LC_COLLATE="en_EN.UTF-8" 108 | export LC_CTYPE="en_EN.UTF-8" 109 | export LC_MESSAGES="en_EN.UTF-8" 110 | export LC_MONETARY="en_EN.UTF-8" 111 | export LC_NUMERIC="en_EN.UTF-8" 112 | export LC_TIME="en_EN.UTF-8" 113 | export LC_ALL= 114 | export CONTACTDB_HOME=$(pwd) 115 | 116 | 117 | clear 118 | echo 119 | echo "Fill in the DB values [PRESS ENTER]" 120 | echo "=====================================" 121 | read choice 122 | cd ./db/initialize/ 123 | sh ./initialize-db.sh 124 | cd ../.. 125 | 126 | 127 | sleep 3 128 | clear 129 | echo 130 | echo "Import the TI csv into the database [PRESS ENTER]" 131 | echo "=====================================" 132 | echo -n "Execute: psql -U postgres contactdb < tidb.csv" 133 | read choice 134 | 135 | 136 | clear 137 | echo 138 | echo "Start the local Django server [PRESS ENTER]" 139 | echo "=====================================" 140 | echo -n "./manage.py runserver" 141 | read choice 142 | 143 | 144 | clear 145 | echo 146 | echo "Ready! [PRESS ENTER]" 147 | echo "=====================================" 148 | echo -n "Connect your browser to http://localhost:8000/admin 149 | You should see lots of data in the Organisations table" 150 | read choice 151 | -------------------------------------------------------------------------------- /old2/old/scripts/drop_and_load.sh: -------------------------------------------------------------------------------- 1 | BASEDIR=$(dirname $0) 2 | 3 | echo "Droping database.... [PRESS ENTER]" 4 | read 5 | echo "DROP DATABASE contactdb" | psql -U contactdb -d postgres 6 | 7 | echo "Recreating database and tables.... [PRESS ENTER]" 8 | read 9 | echo "CREATE DATABASE contactdb" | psql -U contactdb -d postgres 10 | python $BASEDIR/../manage.py syncdb 11 | 12 | echo "Importing country codes.... [PRESS ENTER]" 13 | read 14 | psql -U contactdb contactdb < $BASEDIR/../db/initialize/countries.sql 15 | 16 | echo "Adding TI as a source.... [PRESS ENTER]" 17 | read 18 | psql -U contactdb contactdb < $BASEDIR/../db/initialize/sources.sql 19 | 20 | echo "Import TI data.... [PRESS ENTER]" 21 | read 22 | $BASEDIR/../contrib/TI-import.py <$BASEDIR/../contrib/TI-dump.20130918.csv >foo.sql 23 | psql -U contactdb contactdb < foo.sql 24 | rm foo.sql 25 | -------------------------------------------------------------------------------- /old2/old/scripts/gen-api.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | CLASSES=$(scripts/get-all-classes.sh ) 4 | 5 | for c in $CLASSES; do 6 | cat < ti-l2-pgpkeys.asc 5 | echo "cURL is going to ask you for the passphrase you used while converting your private key from PKCS#12 to PEM format." 6 | curl -k https://tiw.trusted-introducer.org/directory/ti-l2-l1-l0-info.v2.csv --key key.pem --cacert ca.pem --cert client.pem: > ti-l2-l1-l0-info.v2.csv 7 | 8 | -------------------------------------------------------------------------------- /old2/ti/prepare_certificate.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | echo "OpenSSL is going to ask you for the passphrase you used while exporting your cert from the browser." 4 | openssl pkcs12 -in cert.p12 -out ca.pem -cacerts -nokeys 5 | echo "OpenSSL is going to ask you for the passphrase you used while exporting your cert from the browser." 6 | openssl pkcs12 -in cert.p12 -out client.pem -clcerts -nokeys 7 | echo "OpenSSL is going to ask you for the passphrase you used while exporting your cert from the browser." 8 | echo "Then, you will also have to enter, and confirm, a passphrase for the private key in PEM format." 9 | openssl pkcs12 -in cert.p12 -out key.pem -nocerts 10 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | django 2 | djangorestframework 3 | psycopg2 4 | ldif3 5 | --------------------------------------------------------------------------------