├── 978-1-4842-0218-0_Source_Code_Ch01 ├── details.tpl ├── index.tpl ├── snmp-manager.cfg ├── snmp-manager.py └── snmp-pages.py ├── 978-1-4842-0218-0_Source_Code_Ch03 └── www_example_com │ ├── ip_addresses │ ├── __init__.py │ ├── admin.py │ ├── models.py │ ├── templates │ │ ├── add.html │ │ └── display.html │ ├── tests.py │ ├── urls.py │ └── views.py │ ├── manage.py │ ├── sample_data.json │ └── www_example_com │ ├── __init__.py │ ├── settings.py │ ├── urls.py │ └── wsgi.py ├── 978-1-4842-0218-0_Source_Code_Ch04 └── www_example_com │ ├── ip_addresses │ ├── __init__.py │ ├── admin.py │ ├── models.py │ ├── templates │ │ ├── add.html │ │ ├── base.html │ │ ├── delete_confirm_classrule.html │ │ ├── dhcpd.conf.txt │ │ ├── display.html │ │ ├── display.html.bak │ │ ├── display_classrule.html │ │ └── display_dhcp.html │ ├── tests.py │ ├── urls.py │ └── views.py │ ├── manage.py │ ├── sample_data.json │ └── www_example_com │ ├── __init__.py │ ├── settings.py │ ├── urls.py │ └── wsgi.py ├── 978-1-4842-0218-0_Source_Code_Ch05 └── www_example_com │ ├── httpconfig │ ├── __init__.py │ ├── admin.py │ ├── fixtures │ │ └── initial_data.json │ ├── models.py │ ├── templates │ │ └── full_config.txt │ ├── tests.py │ ├── urls.py │ └── views.py │ ├── manage.py │ └── www_example_com │ ├── __init__.py │ ├── settings.py │ ├── urls.py │ └── wsgi.py ├── 978-1-4842-0218-0_Source_Code_Ch06 ├── http_log_parser.py ├── manager.py └── plugins │ ├── plugin_count_200.py │ ├── plugin_geoip_stats.py │ └── world_borders │ ├── Readme.txt │ ├── TM_WORLD_BORDERS-0.3.dbf │ ├── TM_WORLD_BORDERS-0.3.prj │ ├── TM_WORLD_BORDERS-0.3.shp │ └── TM_WORLD_BORDERS-0.3.shx ├── 978-1-4842-0218-0_Source_Code_Ch07 ├── FileServer.java ├── catalina.log ├── exctractor.py └── exctractor.xml ├── 978-1-4842-0218-0_Source_Code_Ch08 ├── check_website_login.py ├── check_website_login_requests.py └── check_website_navigation.py ├── 978-1-4842-0218-0_Source_Code_Ch09 ├── client-server │ ├── client │ │ ├── client.cfg │ │ ├── client_daemon.py │ │ ├── sensors │ │ │ ├── cpu_load │ │ │ │ └── check │ │ │ ├── disk │ │ │ │ └── check │ │ │ ├── memory │ │ │ │ └── check │ │ │ └── processes │ │ │ │ └── check │ │ └── sensors_backup │ │ │ ├── disk_2010-03-07T21_24_33 │ │ │ └── check │ │ │ ├── disk_2010-03-07T21_25_16 │ │ │ └── check │ │ │ ├── disk_2010-03-07T21_27_07 │ │ │ └── check │ │ │ └── disk_2010-03-07T21_32_32 │ │ │ └── check │ └── server │ │ ├── MonitorLib.py │ │ ├── monitor.db │ │ ├── monitor_cli.py │ │ ├── monitor_daemon.py │ │ ├── monitor_db_init.sql │ │ ├── monitor_scheduler.py │ │ ├── sensors │ │ ├── cpu_load.tar.bz2 │ │ ├── disk.tar.bz2 │ │ ├── memory.tar.bz2 │ │ └── processes.tar.bz2 │ │ └── server.cfg ├── example_cherrypy.py ├── example_oscillator.py └── example_processes.py ├── 978-1-4842-0218-0_Source_Code_Ch10 ├── ctrl_example.py ├── example.cfg ├── example2.cfg ├── example3.cfg ├── fd_example.py └── setsid_example.py ├── 978-1-4842-0218-0_Source_Code_Ch11 └── sensor-db │ ├── generate-data.py │ ├── monitor.db │ ├── regen_data.sh │ ├── sample_data.sql │ ├── stats_generator.py │ └── templates │ ├── host.template │ ├── host_probe_details.template │ ├── host_scale_details.template │ └── index.template ├── 978-1-4842-0218-0_Source_Code_Ch13 ├── mysql_db.cfg ├── mysql_inspector.py ├── plugin_manager.py └── plugins │ ├── plugin_advisor.py │ └── plugin_system_query.py ├── 978-1-4842-0218-0_Source_Code_Ch14 ├── backup.cfg └── db_backup.py ├── 9781484202180.jpg ├── LICENSE.txt ├── README.md └── contributing.md /978-1-4842-0218-0_Source_Code_Ch01/details.tpl: -------------------------------------------------------------------------------- 1 |

{{ check.description }}

2 |

OID: {{ check.oid }}

3 | 4 | -------------------------------------------------------------------------------- /978-1-4842-0218-0_Source_Code_Ch01/index.tpl: -------------------------------------------------------------------------------- 1 |

System checks

2 | {% if systems %} 3 | {% for system in systems %} 4 |

{{ systems[system].description }}

5 |

{{ systems[system].address }}:{{ systems[system].port }}

6 | {% if systems[system].checks %} 7 | The following checks are available: 8 | 13 | {% else %} 14 | There are no checks defined for this system 15 | {% endif %} 16 | {% endfor %} 17 | {% else %} 18 | No system configuration available 19 | {% endif %} 20 | -------------------------------------------------------------------------------- /978-1-4842-0218-0_Source_Code_Ch01/snmp-manager.cfg: -------------------------------------------------------------------------------- 1 | [system_1] 2 | description=My Laptop 3 | address=192.168.1.68 4 | port=161 5 | communityro=public 6 | 7 | #[system_2] 8 | #description=My Server 9 | #address=192.168.1.68 10 | #port=161 11 | #communityro=public 12 | 13 | [check_1] 14 | description=WLAN incoming traffic 15 | oid=1.3.6.1.2.1.2.2.1.10.3 16 | system=system_1 17 | sampling_rate=600 18 | 19 | [check_2] 20 | description=WLAN outgoing traffic 21 | oid=1.3.6.1.2.1.2.2.1.16.3 22 | system=system_1 23 | sampling_rate=600 24 | 25 | 26 | -------------------------------------------------------------------------------- /978-1-4842-0218-0_Source_Code_Ch01/snmp-manager.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import sys, os.path, time 4 | from ConfigParser import SafeConfigParser 5 | from pysnmp.entity.rfc3413.oneliner import cmdgen 6 | import rrdtool 7 | 8 | 9 | class SnmpManager: 10 | def __init__(self): 11 | self.systems = {} 12 | self.databases_initialised = False 13 | 14 | def add_system(self, id, descr, addr, port, comm_ro): 15 | self.systems[id] = {'description' : descr, 16 | 'address' : addr, 17 | 'port' : int(port), 18 | 'communityro' : comm_ro, 19 | 'checks' : {} 20 | } 21 | 22 | def add_check(self, id, oid, descr, system, sampling_rate): 23 | oid_tuple = tuple([int(i) for i in oid.split('.')]) 24 | self.systems[system]['checks'][id] = {'description': descr, 25 | 'oid' : oid_tuple, 26 | 'result' : None, 27 | 'sampling_rate' : sampling_rate 28 | } 29 | 30 | def query_all_systems(self): 31 | if not self.databases_initialised: 32 | self.initialise_databases() 33 | self.databases_initialised = True 34 | cg = cmdgen.CommandGenerator() 35 | for system in self.systems.values(): 36 | comm_data = cmdgen.CommunityData('my-manager', system['communityro']) 37 | transport = cmdgen.UdpTransportTarget((system['address'], system['port'])) 38 | for key, check in system['checks'].iteritems(): 39 | oid = check['oid'] 40 | errInd, errStatus, errIdx, result = cg.getCmd(comm_data, transport, oid) 41 | if not errInd and not errStatus: 42 | file_name = "%s.rrd" % key 43 | rrdtool.update(file_name, 44 | "%d:%d" % (int(time.time(),), 45 | float(result[0][1]),) 46 | ) 47 | 48 | def initialise_databases(self): 49 | for system in self.systems.values(): 50 | for check in system['checks']: 51 | data_file = "%s.rrd" % check 52 | if not os.path.isfile(data_file): 53 | print data_file, 'does not exist' 54 | rrdtool.create(data_file, 55 | "DS:%s:COUNTER:%s:U:U" % (check, 56 | system['checks'][check]['sampling_rate']), 57 | "RRA:AVERAGE:0.5:1:288",) 58 | 59 | 60 | def main(conf_file=""): 61 | if not conf_file: 62 | sys.exit(-1) 63 | config = SafeConfigParser() 64 | config.read(conf_file) 65 | snmp_manager = SnmpManager() 66 | for system in [s for s in config.sections() if s.startswith('system')]: 67 | snmp_manager.add_system(system, 68 | config.get(system, 'description'), 69 | config.get(system, 'address'), 70 | config.get(system, 'port'), 71 | config.get(system, 'communityro')) 72 | for check in [c for c in config.sections() if c.startswith('check')]: 73 | snmp_manager.add_check(check, 74 | config.get(check, 'oid'), 75 | config.get(check, 'description'), 76 | config.get(check, 'system'), 77 | config.get(check, 'sampling_rate')) 78 | snmp_manager.query_all_systems() 79 | 80 | if __name__ == '__main__': 81 | main(conf_file='snmp-manager.cfg') 82 | -------------------------------------------------------------------------------- /978-1-4842-0218-0_Source_Code_Ch01/snmp-pages.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | from jinja2 import Environment, FileSystemLoader 4 | from ConfigParser import SafeConfigParser 5 | import rrdtool 6 | import sys 7 | 8 | WEBSITE_ROOT = '/home/rytis/public_html/snmp-monitor/' 9 | 10 | def generate_index(systems, env, website_root): 11 | template = env.get_template('index.tpl') 12 | f = open("%s/index.html" % website_root, 'w') 13 | f.write(template.render({'systems': systems})) 14 | f.close() 15 | 16 | def generate_details(system, env, website_root): 17 | template = env.get_template('details.tpl') 18 | for check_name, check_obj in system['checks'].iteritems(): 19 | rrdtool.graph ("%s/%s.png" % (website_root, check_name), 20 | '--title', "%s" % check_obj['description'], 21 | "DEF:data=%(name)s.rrd:%(name)s:AVERAGE" % {'name': check_name}, 22 | 'AREA:data#0c0c0c') 23 | f = open("%s/%s.html" % (website_root, str(check_name)), 'w') 24 | f.write(template.render({'check': check_obj, 'name': check_name})) 25 | f.close() 26 | 27 | def generate_website(conf_file="", website_root=WEBSITE_ROOT): 28 | if not conf_file: 29 | sys.exit(-1) 30 | config = SafeConfigParser() 31 | config.read(conf_file) 32 | loader = FileSystemLoader('.') 33 | env = Environment(loader=loader) 34 | systems = {} 35 | for system in [s for s in config.sections() if s.startswith('system')]: 36 | systems[system] = {'description': config.get(system, 'description'), 37 | 'address' : config.get(system, 'address'), 38 | 'port' : config.get(system, 'port'), 39 | 'checks' : {} 40 | } 41 | for check in [c for c in config.sections() if c.startswith('check')]: 42 | systems[config.get(check, 'system')]['checks'][check] = { 43 | 'oid' : config.get(check, 'oid'), 44 | 'description': config.get(check, 'description'), 45 | } 46 | 47 | generate_index(systems, env, website_root) 48 | for system in systems.values(): 49 | generate_details(system, env, website_root) 50 | 51 | 52 | if __name__ == '__main__': 53 | generate_website(conf_file='snmp-manager.cfg') 54 | -------------------------------------------------------------------------------- /978-1-4842-0218-0_Source_Code_Ch03/www_example_com/ip_addresses/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Apress/pro-python-system-admin-14/2b70cbe957969ce2f72e59858ffc4c0bbfe77a05/978-1-4842-0218-0_Source_Code_Ch03/www_example_com/ip_addresses/__init__.py -------------------------------------------------------------------------------- /978-1-4842-0218-0_Source_Code_Ch03/www_example_com/ip_addresses/admin.py: -------------------------------------------------------------------------------- 1 | from ip_addresses.models import NetworkAddress 2 | from django.contrib import admin 3 | 4 | class NetworkAddressAdmin(admin.ModelAdmin): 5 | pass 6 | 7 | admin.site.register(NetworkAddress, NetworkAddressAdmin) 8 | 9 | -------------------------------------------------------------------------------- /978-1-4842-0218-0_Source_Code_Ch03/www_example_com/ip_addresses/models.py: -------------------------------------------------------------------------------- 1 | from django.db import models 2 | from django.forms import ModelForm 3 | 4 | class NetworkAddress(models.Model): 5 | address = models.IPAddressField() 6 | network_size = models.PositiveIntegerField() 7 | description = models.CharField(max_length=400) 8 | parent = models.ForeignKey('self', blank=True, null=True) 9 | 10 | class NetworkAddressAddForm(ModelForm): 11 | class Meta: 12 | model = NetworkAddress 13 | exclude = ('parent', ) 14 | 15 | class NetworkAddressModifyForm(ModelForm): 16 | class Meta: 17 | model = NetworkAddress 18 | fields = ('description',) 19 | 20 | -------------------------------------------------------------------------------- /978-1-4842-0218-0_Source_Code_Ch03/www_example_com/ip_addresses/templates/add.html: -------------------------------------------------------------------------------- 1 |
2 | {% csrf_token %} 3 | {{ form.as_p }} 4 | 5 |
6 | 7 | -------------------------------------------------------------------------------- /978-1-4842-0218-0_Source_Code_Ch03/www_example_com/ip_addresses/templates/display.html: -------------------------------------------------------------------------------- 1 | {% if parent %} 2 |

Current address: {{ parent.address }}/{{ parent.network_size }}

3 |

Go back

4 | {% else %} 5 |

At the top of the networks tree

6 | {% endif %} 7 | 8 | {% if addresses_list %} 9 | 20 | {% else %} 21 | {% ifequal parent.network_size 32 %} 22 | This is a node IP 23 | 24 | {% else %} 25 | No addresses or subnets in this range 26 | {% endifequal %} 27 | {% endif %} 28 |

Add new subnet or node IP

29 | 30 | -------------------------------------------------------------------------------- /978-1-4842-0218-0_Source_Code_Ch03/www_example_com/ip_addresses/tests.py: -------------------------------------------------------------------------------- 1 | from django.test import TestCase 2 | 3 | # Create your tests here. 4 | -------------------------------------------------------------------------------- /978-1-4842-0218-0_Source_Code_Ch03/www_example_com/ip_addresses/urls.py: -------------------------------------------------------------------------------- 1 | from django.conf.urls import patterns, include, url 2 | 3 | urlpatterns = patterns('ip_addresses.views', 4 | url(r'^networkaddress/$', 'display'), 5 | url(r'^networkaddress/(?P
\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\/\d{1,2})/$', 'display'), 6 | url(r'^networkaddress/(?P
\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\/\d{1,2})/delete/$', 'delete'), 7 | url(r'^networkaddress/(?P
\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\/\d{1,2})/add/$', 'add'), 8 | url(r'^networkaddress/add/$', 'add'), 9 | url(r'^networkaddress/(?P
\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\/\d{1,2})/modify/$', 'modify'), 10 | url(r'^networkaddress/modify/$', 'modify'), 11 | ) 12 | 13 | -------------------------------------------------------------------------------- /978-1-4842-0218-0_Source_Code_Ch03/www_example_com/ip_addresses/views.py: -------------------------------------------------------------------------------- 1 | from django.shortcuts import render, render_to_response 2 | from ip_addresses.models import * 3 | from django.http import HttpResponse, HttpResponseRedirect 4 | from django.template import RequestContext 5 | 6 | def display(request, address=None): 7 | if not address: 8 | parent = None 9 | else: 10 | ip, net_size = address.split('/') 11 | parent = NetworkAddress.objects.get(address=ip, network_size=int(net_size)) 12 | addr_list = NetworkAddress.objects.filter(parent=parent) 13 | return render_to_response('display.html', 14 | {'parent': parent, 'addresses_list': addr_list}) 15 | 16 | def delete(request, address=None): 17 | ip, net_size = address.split('/') 18 | parent = NetworkAddress.objects.get(address=ip, 19 | network_size=int(net_size)).parent 20 | NetworkAddress.objects.get(address=ip, network_size=int(net_size)).delete() 21 | redirect_to = '../../../' 22 | if parent: 23 | redirect_to += '%s/%s/' % (parent.address, int(parent.network_size)) 24 | return HttpResponseRedirect(redirect_to) 25 | 26 | def add(request, address=None): 27 | if request.method == 'POST': 28 | parent = None 29 | if address: 30 | ip, net_size = address.split('/') 31 | parent = NetworkAddress.objects.get(address=ip, 32 | network_size=int(net_size)) 33 | new_address = NetworkAddress(parent=parent) 34 | form = NetworkAddressAddForm(request.POST, instance=new_address) 35 | if form.is_valid(): 36 | form.save() 37 | return HttpResponseRedirect("..") 38 | else: 39 | form = NetworkAddressAddForm() 40 | return render_to_response('add.html', {'form': form,}, context_instance=RequestContext(request)) 41 | 42 | def modify(request, address=None): 43 | if request.method == 'POST': 44 | # submitting changes 45 | ip, net_size = address.split('/') 46 | address_obj = NetworkAddress.objects.get(address=ip, 47 | network_size=int(net_size)) 48 | form = NetworkAddressModifyForm(request.POST, instance=address_obj) 49 | if form.is_valid(): 50 | form.save() 51 | return HttpResponseRedirect("..") 52 | else: 53 | # first time display 54 | ip, net_size = address.split('/') 55 | address_obj = NetworkAddress.objects.get(address=ip, 56 | network_size=int(net_size)) 57 | form = NetworkAddressModifyForm(initial={ 'description': 58 | address_obj.description, }) 59 | return render_to_response('add.html', {'form': form,}, context_instance=RequestContext(request)) 60 | 61 | -------------------------------------------------------------------------------- /978-1-4842-0218-0_Source_Code_Ch03/www_example_com/manage.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import os 3 | import sys 4 | 5 | if __name__ == "__main__": 6 | os.environ.setdefault("DJANGO_SETTINGS_MODULE", "www_example_com.settings") 7 | 8 | from django.core.management import execute_from_command_line 9 | 10 | execute_from_command_line(sys.argv) 11 | -------------------------------------------------------------------------------- /978-1-4842-0218-0_Source_Code_Ch03/www_example_com/sample_data.json: -------------------------------------------------------------------------------- 1 | 2 | [ 3 | { 4 | "model": "ip_addresses.networkaddress", 5 | "pk": 1, 6 | "fields": { 7 | "address": "192.168.0.0", 8 | "network_size": 24, 9 | "description": "First top level network" 10 | } 11 | }, 12 | { 13 | "model": "ip_addresses.networkaddress", 14 | "pk": 2, 15 | "fields": { 16 | "address": "192.168.1.0", 17 | "network_size": 24, 18 | "description": "Second top level network" 19 | } 20 | }, 21 | { 22 | "model": "ip_addresses.networkaddress", 23 | "pk": 3, 24 | "fields": { 25 | "address": "192.168.0.0", 26 | "network_size": 25, 27 | "description": "Subnet 1-1", 28 | "parent": 1 29 | } 30 | }, 31 | { 32 | "model": "ip_addresses.networkaddress", 33 | "pk": 4, 34 | "fields": { 35 | "address": "192.168.0.128", 36 | "network_size": 25, 37 | "description": "Subnet 1-2", 38 | "parent": 1 39 | } 40 | }, 41 | { 42 | "model": "ip_addresses.networkaddress", 43 | "pk": 5, 44 | "fields": { 45 | "address": "192.168.1.0", 46 | "network_size": 26, 47 | "description": "Subnet 2-1", 48 | "parent": 2 49 | } 50 | }, 51 | { 52 | "model": "ip_addresses.networkaddress", 53 | "pk": 6, 54 | "fields": { 55 | "address": "192.168.1.64", 56 | "network_size": 26, 57 | "description": "Subnet 2-2", 58 | "parent": 2 59 | } 60 | }, 61 | { 62 | "model": "ip_addresses.networkaddress", 63 | "pk": 7, 64 | "fields": { 65 | "address": "192.168.1.128", 66 | "network_size": 26, 67 | "description": "Subnet 2-3", 68 | "parent": 2 69 | } 70 | }, 71 | { 72 | "model": "ip_addresses.networkaddress", 73 | "pk": 8, 74 | "fields": { 75 | "address": "192.168.1.192", 76 | "network_size": 26, 77 | "description": "Subnet 2-4", 78 | "parent": 2 79 | } 80 | }, 81 | { 82 | "model": "ip_addresses.networkaddress", 83 | "pk": 9, 84 | "fields": { 85 | "address": "192.168.0.1", 86 | "network_size": 32, 87 | "description": "IP 1 in Subnet 1-1", 88 | "parent": 3 89 | } 90 | }, 91 | { 92 | "model": "ip_addresses.networkaddress", 93 | "pk": 10, 94 | "fields": { 95 | "address": "192.168.0.2", 96 | "network_size": 32, 97 | "description": "IP 2 in Subnet 1-1", 98 | "parent": 3 99 | } 100 | }, 101 | { 102 | "model": "ip_addresses.networkaddress", 103 | "pk": 11, 104 | "fields": { 105 | "address": "192.168.0.129", 106 | "network_size": 32, 107 | "description": "IP 1 in Subnet 1-2", 108 | "parent": 4 109 | } 110 | }, 111 | { 112 | "model": "ip_addresses.networkaddress", 113 | "pk": 12, 114 | "fields": { 115 | "address": "192.168.0.130", 116 | "network_size": 32, 117 | "description": "IP 2 in Subnet 1-2", 118 | "parent": 4 119 | } 120 | }, 121 | { 122 | "model": "ip_addresses.networkaddress", 123 | "pk": 13, 124 | "fields": { 125 | "address": "192.168.1.1", 126 | "network_size": 32, 127 | "description": "IP 1 in Subnet 2-1", 128 | "parent": 5 129 | } 130 | }, 131 | { 132 | "model": "ip_addresses.networkaddress", 133 | "pk": 14, 134 | "fields": { 135 | "address": "192.168.1.2", 136 | "network_size": 32, 137 | "description": "IP 2 in Subnet 2-1", 138 | "parent": 5 139 | } 140 | }, 141 | { 142 | "model": "ip_addresses.networkaddress", 143 | "pk": 15, 144 | "fields": { 145 | "address": "192.168.1.65", 146 | "network_size": 32, 147 | "description": "IP 1 in Subnet 2-2", 148 | "parent": 6 149 | } 150 | }, 151 | { 152 | "model": "ip_addresses.networkaddress", 153 | "pk": 16, 154 | "fields": { 155 | "address": "192.168.1.66", 156 | "network_size": 32, 157 | "description": "IP 2 in Subnet 2-2", 158 | "parent": 6 159 | } 160 | }, 161 | { 162 | "model": "ip_addresses.networkaddress", 163 | "pk": 17, 164 | "fields": { 165 | "address": "192.168.1.129", 166 | "network_size": 32, 167 | "description": "IP 1 in Subnet 2-3", 168 | "parent": 7 169 | } 170 | }, 171 | { 172 | "model": "ip_addresses.networkaddress", 173 | "pk": 18, 174 | "fields": { 175 | "address": "192.168.1.130", 176 | "network_size": 32, 177 | "description": "IP 2 in Subnet 2-3", 178 | "parent": 7 179 | } 180 | }, 181 | { 182 | "model": "ip_addresses.networkaddress", 183 | "pk": 19, 184 | "fields": { 185 | "address": "192.168.1.193", 186 | "network_size": 32, 187 | "description": "IP 1 in Subnet 2-4", 188 | "parent": 8 189 | } 190 | }, 191 | { 192 | "model": "ip_addresses.networkaddress", 193 | "pk": 20, 194 | "fields": { 195 | "address": "192.168.1.194", 196 | "network_size": 32, 197 | "description": "IP 2 in Subnet 2-4", 198 | "parent": 8 199 | } 200 | } 201 | 202 | 203 | 204 | 205 | ] 206 | -------------------------------------------------------------------------------- /978-1-4842-0218-0_Source_Code_Ch03/www_example_com/www_example_com/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Apress/pro-python-system-admin-14/2b70cbe957969ce2f72e59858ffc4c0bbfe77a05/978-1-4842-0218-0_Source_Code_Ch03/www_example_com/www_example_com/__init__.py -------------------------------------------------------------------------------- /978-1-4842-0218-0_Source_Code_Ch03/www_example_com/www_example_com/settings.py: -------------------------------------------------------------------------------- 1 | """ 2 | Django settings for www_example_com project. 3 | 4 | For more information on this file, see 5 | https://docs.djangoproject.com/en/1.6/topics/settings/ 6 | 7 | For the full list of settings and their values, see 8 | https://docs.djangoproject.com/en/1.6/ref/settings/ 9 | """ 10 | 11 | # Build paths inside the project like this: os.path.join(BASE_DIR, ...) 12 | import os 13 | BASE_DIR = os.path.dirname(os.path.dirname(__file__)) 14 | 15 | 16 | # Quick-start development settings - unsuitable for production 17 | # See https://docs.djangoproject.com/en/1.6/howto/deployment/checklist/ 18 | 19 | # SECURITY WARNING: keep the secret key used in production secret! 20 | SECRET_KEY = '_pfz(p3b$e8$_h%n920i@#o1fzrgg3ishgv%n$rl-5!%f#6xm-' 21 | 22 | # SECURITY WARNING: don't run with debug turned on in production! 23 | DEBUG = True 24 | 25 | TEMPLATE_DEBUG = True 26 | 27 | ALLOWED_HOSTS = [] 28 | 29 | 30 | # Application definition 31 | 32 | INSTALLED_APPS = ( 33 | 'django.contrib.admin', 34 | 'django.contrib.auth', 35 | 'django.contrib.contenttypes', 36 | 'django.contrib.sessions', 37 | 'django.contrib.messages', 38 | 'django.contrib.staticfiles', 39 | 'ip_addresses', 40 | ) 41 | 42 | MIDDLEWARE_CLASSES = ( 43 | 'django.contrib.sessions.middleware.SessionMiddleware', 44 | 'django.middleware.common.CommonMiddleware', 45 | 'django.middleware.csrf.CsrfViewMiddleware', 46 | 'django.contrib.auth.middleware.AuthenticationMiddleware', 47 | 'django.contrib.messages.middleware.MessageMiddleware', 48 | 'django.middleware.clickjacking.XFrameOptionsMiddleware', 49 | ) 50 | 51 | ROOT_URLCONF = 'www_example_com.urls' 52 | 53 | WSGI_APPLICATION = 'www_example_com.wsgi.application' 54 | 55 | 56 | # Database 57 | # https://docs.djangoproject.com/en/1.6/ref/settings/#databases 58 | 59 | DATABASES = { 60 | 'default': { 61 | 'ENGINE': 'django.db.backends.sqlite3', 62 | 'NAME': os.path.join(BASE_DIR, 'db.sqlite3'), 63 | } 64 | } 65 | 66 | # Internationalization 67 | # https://docs.djangoproject.com/en/1.6/topics/i18n/ 68 | 69 | LANGUAGE_CODE = 'en-us' 70 | 71 | TIME_ZONE = 'UTC' 72 | 73 | USE_I18N = True 74 | 75 | USE_L10N = True 76 | 77 | USE_TZ = True 78 | 79 | 80 | # Static files (CSS, JavaScript, Images) 81 | # https://docs.djangoproject.com/en/1.6/howto/static-files/ 82 | 83 | STATIC_URL = '/static/' 84 | -------------------------------------------------------------------------------- /978-1-4842-0218-0_Source_Code_Ch03/www_example_com/www_example_com/urls.py: -------------------------------------------------------------------------------- 1 | from django.conf.urls import patterns, include, url 2 | 3 | from django.contrib import admin 4 | admin.autodiscover() 5 | 6 | urlpatterns = patterns('', 7 | # Examples: 8 | # url(r'^$', 'www_example_com.views.home', name='home'), 9 | # url(r'^blog/', include('blog.urls')), 10 | 11 | url(r'^admin/', include(admin.site.urls)), 12 | # ip_addresses application 13 | url(r'^ip_addresses/', include('ip_addresses.urls')), 14 | 15 | ) 16 | -------------------------------------------------------------------------------- /978-1-4842-0218-0_Source_Code_Ch03/www_example_com/www_example_com/wsgi.py: -------------------------------------------------------------------------------- 1 | """ 2 | WSGI config for www_example_com project. 3 | 4 | It exposes the WSGI callable as a module-level variable named ``application``. 5 | 6 | For more information on this file, see 7 | https://docs.djangoproject.com/en/1.6/howto/deployment/wsgi/ 8 | """ 9 | 10 | import os 11 | os.environ.setdefault("DJANGO_SETTINGS_MODULE", "www_example_com.settings") 12 | 13 | from django.core.wsgi import get_wsgi_application 14 | application = get_wsgi_application() 15 | -------------------------------------------------------------------------------- /978-1-4842-0218-0_Source_Code_Ch04/www_example_com/ip_addresses/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Apress/pro-python-system-admin-14/2b70cbe957969ce2f72e59858ffc4c0bbfe77a05/978-1-4842-0218-0_Source_Code_Ch04/www_example_com/ip_addresses/__init__.py -------------------------------------------------------------------------------- /978-1-4842-0218-0_Source_Code_Ch04/www_example_com/ip_addresses/admin.py: -------------------------------------------------------------------------------- 1 | from ip_addresses.models import * 2 | from django.contrib import admin 3 | 4 | class NetworkAddressAdmin(admin.ModelAdmin): 5 | pass 6 | 7 | class DHCPNetworkAdmin(admin.ModelAdmin): 8 | pass 9 | 10 | class DNSServerAdmin(admin.ModelAdmin): 11 | pass 12 | 13 | class DomainNameAdmin(admin.ModelAdmin): 14 | pass 15 | 16 | class DHCPAddressPoolAdmin(admin.ModelAdmin): 17 | pass 18 | 19 | class ClassRuleAdmin(admin.ModelAdmin): 20 | pass 21 | 22 | admin.site.register(NetworkAddress, NetworkAddressAdmin) 23 | admin.site.register(DHCPNetwork, DHCPNetworkAdmin) 24 | admin.site.register(DHCPAddressPool, DHCPAddressPoolAdmin) 25 | admin.site.register(DNSServer, DNSServerAdmin) 26 | admin.site.register(DomainName, DomainNameAdmin) 27 | admin.site.register(ClassRule, ClassRuleAdmin) 28 | -------------------------------------------------------------------------------- /978-1-4842-0218-0_Source_Code_Ch04/www_example_com/ip_addresses/models.py: -------------------------------------------------------------------------------- 1 | from django.db import models 2 | from django.forms import ModelForm 3 | import socket 4 | 5 | # Create your models here. 6 | 7 | class NetworkAddress(models.Model): 8 | address = models.IPAddressField() 9 | network_size = models.PositiveIntegerField() 10 | description = models.CharField(max_length=400) 11 | parent = models.ForeignKey('self', null=True, blank=True) 12 | 13 | def __unicode__(self): 14 | return "%s/%d" % (self.address, self.network_size) 15 | 16 | @models.permalink 17 | def get_absolute_url(self): 18 | return ('networkaddress_display', (), {'address': '%s/%s' % (self.address, self.network_size)}) 19 | 20 | def get_hostname(self): 21 | try: 22 | fqdn = socket.gethostbyaddr(str(self.address))[0] 23 | except: 24 | fqdn = '' 25 | return fqdn 26 | 27 | def get_formated_address(self): 28 | return str(self.address).replace('.', '_') 29 | 30 | def get_netmask(self): 31 | bit_netmask = 0; 32 | bit_netmask = pow(2, self.network_size) - 1 33 | bit_netmask = bit_netmask << (32 - self.network_size) 34 | nmask_array = [] 35 | for c in range(4): 36 | dec = bit_netmask & 255 37 | bit_netmask = bit_netmask >> 8 38 | nmask_array.insert(0, str(dec)) 39 | return ".".join(nmask_array) 40 | 41 | 42 | 43 | class NetworkAddressAddForm(ModelForm): 44 | class Meta: 45 | model = NetworkAddress 46 | exclude = ('parent', ) 47 | 48 | class NetworkAddressModifyForm(ModelForm): 49 | class Meta: 50 | model = NetworkAddress 51 | fields = ('description',) 52 | 53 | 54 | class DNSServer(models.Model): 55 | address = models.IPAddressField() 56 | comment = models.CharField(max_length=400) 57 | 58 | def __unicode__(self): 59 | return "%s (%s)" % (self.address, self.comment[:20]) 60 | 61 | class DomainName(models.Model): 62 | name = models.CharField(max_length=100) 63 | comment = models.CharField(max_length=400) 64 | 65 | def __unicode__(self): 66 | return "%s (%s)" % (self.name, self.comment[:20]) 67 | 68 | class DHCPNetwork(models.Model): 69 | physical_net = models.OneToOneField(NetworkAddress) 70 | router = models.IPAddressField() 71 | dns_server = models.ForeignKey(DNSServer) 72 | domain_name = models.ForeignKey(DomainName) 73 | 74 | def __unicode__(self): 75 | return "%s/%s" % (self.physical_net.address, self.physical_net.network_size) 76 | 77 | @models.permalink 78 | def get_absolute_url(self): 79 | return ('dhcpnetwork-display', (), {'address': '%s/%s' % (self.physical_net.address, self.physical_net.network_size)}) 80 | 81 | class DHCPNetworkAddForm(ModelForm): 82 | class Meta: 83 | model = DHCPNetwork 84 | exclude = ('physical_net',) 85 | 86 | class ClassRule(models.Model): 87 | rule = models.TextField() 88 | description = models.CharField(max_length=400) 89 | 90 | def __unicode__(self): 91 | return self.description[:20] 92 | 93 | @models.permalink 94 | def get_absolute_url(self): 95 | return ('classrule_display', (), {'pk': self.id}) 96 | 97 | class ClassRuleForm(ModelForm): 98 | class Meta: 99 | model = ClassRule 100 | 101 | class DHCPAddressPool(models.Model): 102 | dhcp_network = models.ForeignKey(DHCPNetwork) 103 | class_rule = models.ForeignKey(ClassRule, null=True, blank=True) 104 | range_start = models.IPAddressField() 105 | range_finish = models.IPAddressField() 106 | 107 | def __unicode__(self): 108 | return "%s/%s" % (self.range_start, self.range_finish) 109 | 110 | def __str__(self): 111 | return "%s/%s" % (self.range_start, self.range_finish) 112 | 113 | @models.permalink 114 | def get_absolute_url(self): 115 | return ('dhcpaddresspool-display', (), {'range_start': self.range_start, 'range_finish': self.range_finish}) 116 | 117 | class DHCPAddressPoolForm(ModelForm): 118 | class Meta: 119 | model = DHCPAddressPool 120 | exclude = ('dhcp_network',) 121 | 122 | -------------------------------------------------------------------------------- /978-1-4842-0218-0_Source_Code_Ch04/www_example_com/ip_addresses/templates/add.html: -------------------------------------------------------------------------------- 1 |
2 | {% csrf_token %} 3 | {{ form.as_p }} 4 | 5 |
6 | -------------------------------------------------------------------------------- /978-1-4842-0218-0_Source_Code_Ch04/www_example_com/ip_addresses/templates/base.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 24 | 25 | 26 | {% block menu %} 27 | 32 | {% endblock %} 33 |
34 | {% block contents %} 35 | {% endblock %} 36 | 37 | -------------------------------------------------------------------------------- /978-1-4842-0218-0_Source_Code_Ch04/www_example_com/ip_addresses/templates/delete_confirm_classrule.html: -------------------------------------------------------------------------------- 1 |
2 | {% csrf_token %} 3 |

Are you sure?

4 | 5 |
6 | -------------------------------------------------------------------------------- /978-1-4842-0218-0_Source_Code_Ch04/www_example_com/ip_addresses/templates/dhcpd.conf.txt: -------------------------------------------------------------------------------- 1 | {% autoescape off %} 2 | ignore client-updates; 3 | ddns-update-style interim; 4 | 5 | {% if class_rules %} 6 | {% for cr in class_rules %} 7 | # {{cr.description }} 8 | class "class_rule_{{ cr.id }}" { 9 | {{ cr.rule }}; 10 | } 11 | {% endfor %} 12 | {% endif %} 13 | 14 | {% if networks %} 15 | {% for net in networks %} 16 | shared-network network_{{ net.dhcp_net.id }} { 17 | subnet {{ net.dhcp_net.physical_net.address }} netmask {{ net.dhcp_net.physical_net.get_netmask }} { 18 | option routers {{ net.dhcp_net.router }}; 19 | option domain-name-servers {{ net.dhcp_net.dns_server.address }}; 20 | option domain-name {{ net.dhcp_net.domain_name.name }}; 21 | 22 | {% if net.pools %} 23 | {% for pool in net.pools %} 24 | pool { 25 | allow members of "class_rule_{{ pool.class_rule.id }}"; 26 | range {{ pool.range_start }} {{ pool.range_finish }}; 27 | } 28 | {% endfor %} 29 | {% endif %} 30 | } 31 | } 32 | {% endfor %} 33 | {% endif %} 34 | 35 | {% endautoescape %} 36 | -------------------------------------------------------------------------------- /978-1-4842-0218-0_Source_Code_Ch04/www_example_com/ip_addresses/templates/display.html: -------------------------------------------------------------------------------- 1 | {% extends "base.html" %} 2 | {% block contents %} 3 | {% if parent %} 4 |

Current address: {{ parent.address }}/{{ parent.network_size }}

5 |

Go back

6 | {% else %} 7 |

At the top of the networks tree

8 | {% endif %} 9 | 10 | {% if addresses_list %} 11 |
    12 | {% for address in addresses_list %} 13 |
  • {{ address.address }}/{{ address.network_size }} 14 | {% ifequal address.network_size 32 %}(host){% else %}(network){% endifequal %} 15 | {{ address.description }} {% if address.get_hostname %} ({{ address.get_hostname }}) {% endif %} 16 | (delete | 17 | modify) 18 | {% ifequal address.network_size 32 %} 19 | [Status: Unknown ] 20 | {% endifequal %} 21 |
    • 22 | {% endfor %} 23 |
24 | {% else %} 25 | {% ifequal parent.network_size 32 %} 26 | This is a node IP 27 |
    28 |
  • Description: {{ parent.description }} ( modify )
    • 29 |
30 | {% else %} 31 | No addresses or subnets in this range 32 | {% endifequal %} 33 | {% endif %} 34 | 35 | {% ifnotequal parent.network_size 32 %} 36 |

Add new subnet or node IP

37 | {% endifnotequal %} 38 | 39 | {% if has_subnets %} 40 |

DHCP support cannot be enabled for networks with subnets

41 | {% else %} 42 | {% if dhcp_net %} 43 |

DHCP support is enabled for this subnet

44 |
    45 |
  • Router: {{ dhcp_net.router }}
    • 46 |
  • DNS: {{ dhcp_net.dns_server }}
    • 47 |
  • Domain: {{ dhcp_net.domain_name }}
    • 48 |
49 |

50 | ( modify | delete | details ) 51 |

52 | {% else %} 53 | {% ifnotequal parent.network_size 32 %} 54 |

Enable DHCP support

55 | {% endifnotequal %} 56 | {% endif %} 57 | {% endif %} 58 | {% endblock %} 59 | -------------------------------------------------------------------------------- /978-1-4842-0218-0_Source_Code_Ch04/www_example_com/ip_addresses/templates/display.html.bak: -------------------------------------------------------------------------------- 1 | {% if parent %} 2 |

Current address: {{ parent.address }}/{{ parent.network_size }}

3 |

Go back

4 | {% else %} 5 |

At the top of the networks tree

6 | {% endif %} 7 | 8 | {% if addresses_list %} 9 | 20 | {% else %} 21 | {% ifequal parent.network_size 32 %} 22 | This is a node IP 23 | 24 | {% else %} 25 | No addresses or subnets in this range 26 | {% endifequal %} 27 | {% endif %} 28 |

Add new subnet or node IP

29 | 30 | -------------------------------------------------------------------------------- /978-1-4842-0218-0_Source_Code_Ch04/www_example_com/ip_addresses/templates/display_classrule.html: -------------------------------------------------------------------------------- 1 | {% extends "base.html" %} 2 | {% block contents %} 3 | {% if object %} 4 |

Class Rules details

5 |
    6 |
  • ID: {{ object.id }}
    • 7 |
  • Description: {{ object.description }}
    • 8 |
  • Rule text: 9 | 
    10 |                 {{ object.rule }}
11 |
    12 |
    • 13 |
14 | ( modify | 15 | delete ) 16 | {% else %} 17 |

List of all Class Rules

18 | {% if object_list %} 19 |
    20 | {% for rule in object_list %} 21 |
  • {{ rule.description }} ( details | 22 | modify | 23 | delete )
    • 24 | {% endfor %} 25 |
26 | {% else %} 27 | No class rules defined yet. 28 | {% endif %} 29 |

Add new rule

30 | {% endif %} 31 | {% endblock %} 32 | -------------------------------------------------------------------------------- /978-1-4842-0218-0_Source_Code_Ch04/www_example_com/ip_addresses/templates/display_dhcp.html: -------------------------------------------------------------------------------- 1 | {% extends "base.html" %} 2 | {% block contents %} 3 |

DHCP details for {{ dhcp_net.physical_net.address }}/{{ dhcp_net.physical_net.network_size }} network

4 |

Go back to network details

5 |
    6 |
  • Router: {{ dhcp_net.router }} 7 |
  • DNS: {{ dhcp_net.dns_server }} 8 |
  • Domain: {{ dhcp_net.domain_name }} 9 |
10 |

( modify | delete )

11 | {% if dhcp_pools %} 12 |

Following DHCP pools are available:

13 |
    14 | {% for pool in dhcp_pools %} 15 |
  • {{ pool.range_start }} - {{ pool.range_finish }} ( delete )
    • 16 | {% endfor %} 17 |
18 | {% else %} 19 |

There are no DHCP pools defined

20 | {% endif %} 21 |

22 | ( add new pool ) 23 |

24 | {% if class_rules %} 25 |

Following Classification Rules are defined:

26 |
    27 | {% for rule in class_rules %} 28 |
  • {{ rule }} ( details | modify | delete )
    • 29 | {% endfor %} 30 |
31 | {% else %} 32 |

There are no Classification Rules defined

33 | {% endif %} 34 | ( add new rule ) 35 | {% endblock %} 36 | -------------------------------------------------------------------------------- /978-1-4842-0218-0_Source_Code_Ch04/www_example_com/ip_addresses/tests.py: -------------------------------------------------------------------------------- 1 | from django.test import TestCase 2 | 3 | # Create your tests here. 4 | -------------------------------------------------------------------------------- /978-1-4842-0218-0_Source_Code_Ch04/www_example_com/ip_addresses/urls.py: -------------------------------------------------------------------------------- 1 | from django.conf.urls import patterns, url, include 2 | #from django.views.generic import list_detail, create_update 3 | from models import * 4 | import views 5 | from django.core.urlresolvers import reverse 6 | 7 | classrule_info = { 8 | 'queryset': ClassRule.objects.all(), 9 | 'template_name': 'display_classrule.html', 10 | } 11 | 12 | classrule_form = { 13 | 'form_class': ClassRuleForm, 14 | 'template_name': 'add.html', 15 | } 16 | 17 | classrule_delete = { 18 | 'model': ClassRule, 19 | 'post_delete_redirect': '../..', 20 | 'template_name': 'delete_confirm_classrule.html', 21 | } 22 | 23 | urlpatterns = patterns('', 24 | url(r'^networkaddress/$', views.networkaddress_display, name='networkaddress_displaytop'), 25 | url(r'^networkaddress/add/$', views.networkaddress_add, name='networkaddress_addtop'), 26 | url(r'^networkaddress/(?P
\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\/\d{1,2})/$', views.networkaddress_display, name='networkaddress_display'), 27 | url(r'^networkaddress/(?P
\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\/\d{1,2})/delete/$', views.networkaddress_delete, name='networkaddress_delete'), 28 | url(r'^networkaddress/(?P
\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\/\d{1,2})/add/$', views.networkaddress_add, name='networkaddress_add'), 29 | url(r'^networkaddress/(?P
\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\/\d{1,2})/modify/$', views.networkaddress_modify, name='networkaddress_modify'), 30 | url(r'^networkaddress/(?P
\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\/\d{1,2})/add_dhcp/$', views.dhcpnetwork_add, name='networkaddress_adddhcp'), 31 | url(r'^networkaddress/(?P
\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})/ping/$', views.networkaddress_ping, name='networkaddress_ping'), 32 | url(r'^networkaddress/$', views.networkaddress_ping, name='networkaddress_ping_url'), 33 | 34 | url(r'^dhcpnetwork/(?P
\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\/\d{1,2})/$', views.dhcpnetwork_display, name='dhcpnetwork-display'), 35 | url(r'^dhcpnetwork/(?P
\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\/\d{1,2})/delete/$', views.dhcpnetwork_delete, name='dhcpnetwork-delete'), 36 | url(r'^dhcpnetwork/(?P
\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\/\d{1,2})/modify/$', views.dhcpnetwork_modify, name='dhcpnetwork-modify'), 37 | url(r'^dhcpnetwork/(?P
\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\/\d{1,2})/add_dhcppool/$', views.dhcpaddresspool_add, name='dhcpnetwork-addpool'), 38 | 39 | url(r'^dhcpaddresspool/add/$', views.dhcpaddresspool_add), 40 | url(r'^dhcpaddresspool/(?P\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})/(?P\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})/$', views.dhcpaddresspool_display, name='dhcpaddresspool-display'), 41 | url(r'^dhcpaddresspool/(?P\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\/\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})/delete/$', views.dhcpaddresspool_delete, name='dhcpaddresspool-delete'), 42 | 43 | url(r'^classrule/$', views.ClassRuleDisplay.as_view(), name='classrule_displaytop'), 44 | url(r'^classrule/(?P\d+)/$', views.ClassRuleDetailDisplay.as_view(), name='classrule_display'), 45 | url(r'^classrule/(?P\d+)/modify/$', views.ClassRuleUpdate.as_view(), name='classrule_modify'), 46 | url(r'^classrule/(?P\d+)/delete/$', views.ClassRuleDelete.as_view(), name='classrule_delete'), 47 | url(r'^classrule/add/$', views.ClassRuleCreate.as_view(), name='classrule_add'), 48 | 49 | url(r'^dhcpd.conf/$', views.dhcpd_conf_generate, name='dhcp_conf_generate'), 50 | ) 51 | 52 | -------------------------------------------------------------------------------- /978-1-4842-0218-0_Source_Code_Ch04/www_example_com/ip_addresses/views.py: -------------------------------------------------------------------------------- 1 | from ip_addresses.models import * 2 | from django.http import HttpResponse, HttpResponseRedirect 3 | from django.shortcuts import render_to_response, get_object_or_404 4 | from django.core.urlresolvers import reverse 5 | from django.views.generic import ListView, DetailView, CreateView, UpdateView, DeleteView 6 | from django.core.urlresolvers import reverse_lazy 7 | from django.template import RequestContext 8 | import subprocess 9 | 10 | # NETWORKADDRESS view functions 11 | 12 | def networkaddress_display(request, address=None): 13 | parent = get_network_object_from_address(address) 14 | addr_list = NetworkAddress.objects.filter(parent=parent) 15 | has_subnets = False 16 | for address in addr_list: 17 | if address.network_size != 32: 18 | has_subnets = True 19 | try: 20 | dhcp_net = DHCPNetwork.objects.get(physical_net=parent) 21 | except: 22 | dhcp_net = None 23 | return render_to_response('display.html', {'parent': parent, 24 | 'addresses_list': addr_list, 25 | 'has_subnets': has_subnets, 26 | 'dhcp_net': dhcp_net,}) 27 | 28 | def networkaddress_delete(request, address=None): 29 | address_obj = get_network_object_from_address(address) 30 | parent = address_obj.parent 31 | address_obj.delete() 32 | redirect_to = '../../../' 33 | if parent: 34 | redirect_to = parent.get_absolute_url() 35 | return HttpResponseRedirect(redirect_to) 36 | 37 | def networkaddress_add(request, address=None): 38 | if request.method == 'POST': 39 | parent = get_network_object_from_address(address) 40 | new_address = NetworkAddress(parent=parent) 41 | form = NetworkAddressAddForm(request.POST, instance=new_address) 42 | if form.is_valid(): 43 | form.save() 44 | url = parent.get_absolute_url() if parent else reverse('networkaddress_displaytop') 45 | return HttpResponseRedirect(url) 46 | else: 47 | form = NetworkAddressAddForm() 48 | return render_to_response('add.html', {'form': form,}, context_instance=RequestContext(request)) 49 | 50 | def networkaddress_modify(request, address=None): 51 | address_obj = get_network_object_from_address(address) 52 | if request.method == 'POST': 53 | # submitting changes 54 | form = NetworkAddressModifyForm(request.POST, instance=address_obj) 55 | if form.is_valid(): 56 | form.save() 57 | return HttpResponseRedirect(address_obj.parent.get_absolute_url()) 58 | else: 59 | # first time display 60 | form = NetworkAddressModifyForm(initial={ 'description': address_obj.description, }) 61 | return render_to_response('add.html', {'form': form,}, context_instance=RequestContext(request)) 62 | 63 | # DHCPNETWORK view functions 64 | 65 | def dhcpnetwork_add(request, address=None): 66 | if request.method == 'POST': 67 | network_addr = get_network_object_from_address(address) 68 | dhcp_net = DHCPNetwork(physical_net=network_addr) 69 | form = DHCPNetworkAddForm(request.POST, instance=dhcp_net) 70 | if form.is_valid(): 71 | form.save() 72 | return HttpResponseRedirect(network_addr.get_absolute_url()) 73 | else: 74 | form = DHCPNetworkAddForm() 75 | return render_to_response('add.html', {'form': form,}, context_instance=RequestContext(request)) 76 | 77 | def dhcpnetwork_delete(request, address=None): 78 | network_addr = get_network_object_from_address(address) 79 | get_dhcp_object_from_address(address).delete() 80 | return HttpResponseRedirect(network_addr.get_absolute_url()) 81 | 82 | def dhcpnetwork_modify(request, address=None): 83 | ip, net_size = address.split('/') 84 | network_addr = NetworkAddress.objects.get(address=ip, network_size=int(net_size)) 85 | dhcp_net = DHCPNetwork.objects.get(physical_net=network_addr) 86 | if request.method == 'POST': 87 | # submiting changes 88 | form = DHCPNetworkAddForm(request.POST, instance=dhcp_net) 89 | if form.is_valid(): 90 | form.save() 91 | return HttpResponseRedirect(dhcp_net.get_absolute_url()) 92 | else: 93 | # first time display 94 | form = DHCPNetworkAddForm(instance=dhcp_net) 95 | return render_to_response('add.html', {'form': form,}, context_instance=RequestContext(request)) 96 | 97 | def dhcpnetwork_display(request, address=None): 98 | dhcp_net = get_dhcp_object_from_address(address) 99 | dhcp_pools = DHCPAddressPool.objects.filter(dhcp_network=dhcp_net) 100 | class_rules = ClassRule.objects.all() 101 | return render_to_response('display_dhcp.html', {'dhcp_net': dhcp_net, 102 | 'dhcp_pools': dhcp_pools, 103 | 'class_rules': class_rules,}) 104 | 105 | # DHCPADDRESSPOOL views 106 | 107 | def dhcpaddresspool_display(request, range_start=None, range_finish=None): 108 | return HttpResponse('not implemented') 109 | 110 | def dhcpaddresspool_add(request, address=None): 111 | if request.method == 'POST': 112 | dhcp_net = get_dhcp_object_from_address(address) 113 | pool = DHCPAddressPool(dhcp_network=dhcp_net) 114 | form = DHCPAddressPoolForm(request.POST, instance=pool) 115 | if form.is_valid(): 116 | form.save() 117 | return HttpResponseRedirect(dhcp_net.get_absolute_url()) 118 | else: 119 | form = DHCPAddressPoolForm() 120 | return render_to_response('add.html', {'form': form,}, context_instance=RequestContext(request)) 121 | 122 | def dhcpaddresspool_delete(request, range=None): 123 | range_start, range_finish = range.split('/') 124 | pool_obj = DHCPAddressPool.objects.get(range_start=range_start, range_finish=range_finish) 125 | dhcp_net = pool_obj.dhcp_network 126 | pool_obj.delete() 127 | return HttpResponseRedirect(dhcp_net.get_absolute_url()) 128 | 129 | # CLASSRULE generic views 130 | 131 | class ClassRuleDisplay(ListView): 132 | model = ClassRule 133 | template_name = 'display_classrule.html' 134 | 135 | class ClassRuleDetailDisplay(DetailView): 136 | queryset = ClassRule.objects.all() 137 | template_name = 'display_classrule.html' 138 | 139 | def get_object(self): 140 | object = super(ClassRuleDetailDisplay, self).get_object() 141 | return object 142 | 143 | class ClassRuleCreate(CreateView): 144 | form_class = ClassRuleForm 145 | template_name = 'add.html' 146 | 147 | class ClassRuleUpdate(UpdateView): 148 | model = ClassRule 149 | form_class = ClassRuleForm 150 | template_name = 'add.html' 151 | 152 | class ClassRuleDelete(DeleteView): 153 | model = ClassRule 154 | success_url = reverse_lazy('classrule_displaytop') 155 | template_name = 'delete_confirm_classrule.html' 156 | 157 | # CLASSRULE views 158 | 159 | def classrule_display(request, rule_id=None): 160 | class_rules = ClassRule.objects.all() 161 | return render_to_response('display_classrules.html') 162 | 163 | def classrule_add(request): 164 | if request.method == 'POST': 165 | form = ClassRuleForm(request.POST) 166 | if form.is_valid(): 167 | form.save() 168 | return HttpResponseRedirect(request.META['HTTP_REFERER']) 169 | else: 170 | form = ClassRuleForm() 171 | return render_to_response('add.html', {'form': form,}, context_instance=RequestContext(request)) 172 | 173 | 174 | def networkaddress_ping(request, address=None): 175 | if responding_to_ping(address): 176 | msg = "Ping OK" 177 | else: 178 | msg = "No response" 179 | return HttpResponse(msg) 180 | 181 | def dhcpd_conf_generate(request): 182 | class_rules = ClassRule.objects.all() 183 | networks = [] 184 | for net in DHCPNetwork.objects.all(): 185 | networks.append( { 'dhcp_net': net, 186 | 'pools': DHCPAddressPool.objects.filter(dhcp_network=net), 187 | } ) 188 | 189 | return render_to_response('dhcpd.conf.txt', 190 | {'class_rules': class_rules, 191 | 'networks': networks, 192 | }, 193 | mimetype='text/plain') 194 | 195 | 196 | ############################################################################ 197 | # helper functions 198 | 199 | def get_network_object_from_address(address): 200 | if address: 201 | ip, net_size = address.split('/') 202 | object = NetworkAddress.objects.get(address=ip, network_size=int(net_size)) 203 | else: 204 | object = None 205 | return object 206 | 207 | def get_dhcp_object_from_address(address): 208 | if address: 209 | object = DHCPNetwork.objects.get(physical_net=get_network_object_from_address(address)) 210 | else: 211 | object = None 212 | return object 213 | 214 | def responding_to_ping(address, timeout=1): 215 | rc = subprocess.call("ping -c 1 -W %d %s" % (timeout, address), 216 | shell=True, stdout=open('/dev/null', 'w'), stderr=subprocess.STDOUT) 217 | if rc == 0: 218 | return True 219 | else: 220 | return False 221 | 222 | 223 | -------------------------------------------------------------------------------- /978-1-4842-0218-0_Source_Code_Ch04/www_example_com/manage.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import os 3 | import sys 4 | 5 | if __name__ == "__main__": 6 | os.environ.setdefault("DJANGO_SETTINGS_MODULE", "www_example_com.settings") 7 | 8 | from django.core.management import execute_from_command_line 9 | 10 | execute_from_command_line(sys.argv) 11 | -------------------------------------------------------------------------------- /978-1-4842-0218-0_Source_Code_Ch04/www_example_com/sample_data.json: -------------------------------------------------------------------------------- 1 | 2 | [ 3 | { 4 | "model": "ip_addresses.networkaddress", 5 | "pk": 1, 6 | "fields": { 7 | "address": "192.168.0.0", 8 | "network_size": 24, 9 | "description": "First top level network" 10 | } 11 | }, 12 | { 13 | "model": "ip_addresses.networkaddress", 14 | "pk": 2, 15 | "fields": { 16 | "address": "192.168.1.0", 17 | "network_size": 24, 18 | "description": "Second top level network" 19 | } 20 | }, 21 | { 22 | "model": "ip_addresses.networkaddress", 23 | "pk": 3, 24 | "fields": { 25 | "address": "192.168.0.0", 26 | "network_size": 25, 27 | "description": "Subnet 1-1", 28 | "parent": 1 29 | } 30 | }, 31 | { 32 | "model": "ip_addresses.networkaddress", 33 | "pk": 4, 34 | "fields": { 35 | "address": "192.168.0.128", 36 | "network_size": 25, 37 | "description": "Subnet 1-2", 38 | "parent": 1 39 | } 40 | }, 41 | { 42 | "model": "ip_addresses.networkaddress", 43 | "pk": 5, 44 | "fields": { 45 | "address": "192.168.1.0", 46 | "network_size": 26, 47 | "description": "Subnet 2-1", 48 | "parent": 2 49 | } 50 | }, 51 | { 52 | "model": "ip_addresses.networkaddress", 53 | "pk": 6, 54 | "fields": { 55 | "address": "192.168.1.64", 56 | "network_size": 26, 57 | "description": "Subnet 2-2", 58 | "parent": 2 59 | } 60 | }, 61 | { 62 | "model": "ip_addresses.networkaddress", 63 | "pk": 7, 64 | "fields": { 65 | "address": "192.168.1.128", 66 | "network_size": 26, 67 | "description": "Subnet 2-3", 68 | "parent": 2 69 | } 70 | }, 71 | { 72 | "model": "ip_addresses.networkaddress", 73 | "pk": 8, 74 | "fields": { 75 | "address": "192.168.1.192", 76 | "network_size": 26, 77 | "description": "Subnet 2-4", 78 | "parent": 2 79 | } 80 | }, 81 | { 82 | "model": "ip_addresses.networkaddress", 83 | "pk": 9, 84 | "fields": { 85 | "address": "192.168.0.1", 86 | "network_size": 32, 87 | "description": "IP 1 in Subnet 1-1", 88 | "parent": 3 89 | } 90 | }, 91 | { 92 | "model": "ip_addresses.networkaddress", 93 | "pk": 10, 94 | "fields": { 95 | "address": "192.168.0.2", 96 | "network_size": 32, 97 | "description": "IP 2 in Subnet 1-1", 98 | "parent": 3 99 | } 100 | }, 101 | { 102 | "model": "ip_addresses.networkaddress", 103 | "pk": 11, 104 | "fields": { 105 | "address": "192.168.0.129", 106 | "network_size": 32, 107 | "description": "IP 1 in Subnet 1-2", 108 | "parent": 4 109 | } 110 | }, 111 | { 112 | "model": "ip_addresses.networkaddress", 113 | "pk": 12, 114 | "fields": { 115 | "address": "192.168.0.130", 116 | "network_size": 32, 117 | "description": "IP 2 in Subnet 1-2", 118 | "parent": 4 119 | } 120 | }, 121 | { 122 | "model": "ip_addresses.networkaddress", 123 | "pk": 13, 124 | "fields": { 125 | "address": "192.168.1.1", 126 | "network_size": 32, 127 | "description": "IP 1 in Subnet 2-1", 128 | "parent": 5 129 | } 130 | }, 131 | { 132 | "model": "ip_addresses.networkaddress", 133 | "pk": 14, 134 | "fields": { 135 | "address": "192.168.1.2", 136 | "network_size": 32, 137 | "description": "IP 2 in Subnet 2-1", 138 | "parent": 5 139 | } 140 | }, 141 | { 142 | "model": "ip_addresses.networkaddress", 143 | "pk": 15, 144 | "fields": { 145 | "address": "192.168.1.65", 146 | "network_size": 32, 147 | "description": "IP 1 in Subnet 2-2", 148 | "parent": 6 149 | } 150 | }, 151 | { 152 | "model": "ip_addresses.networkaddress", 153 | "pk": 16, 154 | "fields": { 155 | "address": "192.168.1.66", 156 | "network_size": 32, 157 | "description": "IP 2 in Subnet 2-2", 158 | "parent": 6 159 | } 160 | }, 161 | { 162 | "model": "ip_addresses.networkaddress", 163 | "pk": 17, 164 | "fields": { 165 | "address": "192.168.1.129", 166 | "network_size": 32, 167 | "description": "IP 1 in Subnet 2-3", 168 | "parent": 7 169 | } 170 | }, 171 | { 172 | "model": "ip_addresses.networkaddress", 173 | "pk": 18, 174 | "fields": { 175 | "address": "192.168.1.130", 176 | "network_size": 32, 177 | "description": "IP 2 in Subnet 2-3", 178 | "parent": 7 179 | } 180 | }, 181 | { 182 | "model": "ip_addresses.networkaddress", 183 | "pk": 19, 184 | "fields": { 185 | "address": "192.168.1.193", 186 | "network_size": 32, 187 | "description": "IP 1 in Subnet 2-4", 188 | "parent": 8 189 | } 190 | }, 191 | { 192 | "model": "ip_addresses.networkaddress", 193 | "pk": 20, 194 | "fields": { 195 | "address": "192.168.1.194", 196 | "network_size": 32, 197 | "description": "IP 2 in Subnet 2-4", 198 | "parent": 8 199 | } 200 | } 201 | 202 | 203 | 204 | 205 | ] 206 | -------------------------------------------------------------------------------- /978-1-4842-0218-0_Source_Code_Ch04/www_example_com/www_example_com/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Apress/pro-python-system-admin-14/2b70cbe957969ce2f72e59858ffc4c0bbfe77a05/978-1-4842-0218-0_Source_Code_Ch04/www_example_com/www_example_com/__init__.py -------------------------------------------------------------------------------- /978-1-4842-0218-0_Source_Code_Ch04/www_example_com/www_example_com/settings.py: -------------------------------------------------------------------------------- 1 | """ 2 | Django settings for www_example_com project. 3 | 4 | For more information on this file, see 5 | https://docs.djangoproject.com/en/1.6/topics/settings/ 6 | 7 | For the full list of settings and their values, see 8 | https://docs.djangoproject.com/en/1.6/ref/settings/ 9 | """ 10 | 11 | # Build paths inside the project like this: os.path.join(BASE_DIR, ...) 12 | import os 13 | BASE_DIR = os.path.dirname(os.path.dirname(__file__)) 14 | 15 | 16 | # Quick-start development settings - unsuitable for production 17 | # See https://docs.djangoproject.com/en/1.6/howto/deployment/checklist/ 18 | 19 | # SECURITY WARNING: keep the secret key used in production secret! 20 | SECRET_KEY = '_pfz(p3b$e8$_h%n920i@#o1fzrgg3ishgv%n$rl-5!%f#6xm-' 21 | 22 | # SECURITY WARNING: don't run with debug turned on in production! 23 | DEBUG = True 24 | 25 | TEMPLATE_DEBUG = True 26 | 27 | ALLOWED_HOSTS = [] 28 | 29 | 30 | # Application definition 31 | 32 | INSTALLED_APPS = ( 33 | 'django.contrib.admin', 34 | 'django.contrib.auth', 35 | 'django.contrib.contenttypes', 36 | 'django.contrib.sessions', 37 | 'django.contrib.messages', 38 | 'django.contrib.staticfiles', 39 | 'ip_addresses', 40 | ) 41 | 42 | MIDDLEWARE_CLASSES = ( 43 | 'django.contrib.sessions.middleware.SessionMiddleware', 44 | 'django.middleware.common.CommonMiddleware', 45 | 'django.middleware.csrf.CsrfViewMiddleware', 46 | 'django.contrib.auth.middleware.AuthenticationMiddleware', 47 | 'django.contrib.messages.middleware.MessageMiddleware', 48 | 'django.middleware.clickjacking.XFrameOptionsMiddleware', 49 | ) 50 | 51 | ROOT_URLCONF = 'www_example_com.urls' 52 | 53 | WSGI_APPLICATION = 'www_example_com.wsgi.application' 54 | 55 | 56 | # Database 57 | # https://docs.djangoproject.com/en/1.6/ref/settings/#databases 58 | 59 | DATABASES = { 60 | 'default': { 61 | 'ENGINE': 'django.db.backends.sqlite3', 62 | 'NAME': os.path.join(BASE_DIR, 'db.sqlite3'), 63 | } 64 | } 65 | 66 | # Internationalization 67 | # https://docs.djangoproject.com/en/1.6/topics/i18n/ 68 | 69 | LANGUAGE_CODE = 'en-us' 70 | 71 | TIME_ZONE = 'UTC' 72 | 73 | USE_I18N = True 74 | 75 | USE_L10N = True 76 | 77 | USE_TZ = True 78 | 79 | 80 | # Static files (CSS, JavaScript, Images) 81 | # https://docs.djangoproject.com/en/1.6/howto/static-files/ 82 | 83 | STATIC_URL = '/static/' 84 | -------------------------------------------------------------------------------- /978-1-4842-0218-0_Source_Code_Ch04/www_example_com/www_example_com/urls.py: -------------------------------------------------------------------------------- 1 | from django.conf.urls import patterns, include, url 2 | 3 | from django.contrib import admin 4 | admin.autodiscover() 5 | 6 | urlpatterns = patterns('', 7 | # Examples: 8 | # url(r'^$', 'www_example_com.views.home', name='home'), 9 | # url(r'^blog/', include('blog.urls')), 10 | 11 | url(r'^admin/', include(admin.site.urls)), 12 | # ip_addresses application 13 | url(r'^ip_addresses/', include('ip_addresses.urls')), 14 | 15 | ) 16 | -------------------------------------------------------------------------------- /978-1-4842-0218-0_Source_Code_Ch04/www_example_com/www_example_com/wsgi.py: -------------------------------------------------------------------------------- 1 | """ 2 | WSGI config for www_example_com project. 3 | 4 | It exposes the WSGI callable as a module-level variable named ``application``. 5 | 6 | For more information on this file, see 7 | https://docs.djangoproject.com/en/1.6/howto/deployment/wsgi/ 8 | """ 9 | 10 | import os 11 | os.environ.setdefault("DJANGO_SETTINGS_MODULE", "www_example_com.settings") 12 | 13 | from django.core.wsgi import get_wsgi_application 14 | application = get_wsgi_application() 15 | -------------------------------------------------------------------------------- /978-1-4842-0218-0_Source_Code_Ch05/www_example_com/httpconfig/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Apress/pro-python-system-admin-14/2b70cbe957969ce2f72e59858ffc4c0bbfe77a05/978-1-4842-0218-0_Source_Code_Ch05/www_example_com/httpconfig/__init__.py -------------------------------------------------------------------------------- /978-1-4842-0218-0_Source_Code_Ch05/www_example_com/httpconfig/admin.py: -------------------------------------------------------------------------------- 1 | from django.contrib import admin 2 | from httpconfig.models import * 3 | 4 | 5 | class VHostDirectiveInLine(admin.TabularInline): 6 | model = VHostDirective 7 | extra = 1 8 | 9 | class VirtualHostAdmin(admin.ModelAdmin): 10 | inlines = (VHostDirectiveInLine,) 11 | list_display = ('description', 'is_default', 'is_template', 12 | 'bind_address', 'domain_names', 'code_snippet') 13 | 14 | actions = ('make_default', 'duplicate',) 15 | 16 | def make_default(self, request, queryset): 17 | if len(queryset) == 1: 18 | VirtualHost.objects.all().update(is_default=False) 19 | queryset.update(is_default=True) 20 | self.message_user(request, 21 | "Virtual host '%s' has been made the default virtual host" % queryset[0]) 22 | else: 23 | self.message_user(request, 'ERROR: Only one host can be set as the default!') 24 | make_default.short_description = 'Make selected Virtual Host default' 25 | 26 | def duplicate(self, request, queryset): 27 | msg = '' 28 | for vhost in queryset: 29 | new_vhost = VirtualHost() 30 | new_vhost.description = "%s (Copy)" % vhost.description 31 | new_vhost.bind_address = vhost.bind_address 32 | new_vhost.is_template = False 33 | new_vhost.is_default = False 34 | new_vhost.save() 35 | # recreate all 'orphan' directives that aren't parents 36 | o=vhost.vhostdirective_set.filter(parent=None).filter(directive__is_container=False) 37 | for vhd in o: 38 | new_vhd = VHostDirective() 39 | new_vhd.directive = vhd.directive 40 | new_vhd.value = vhd.value 41 | new_vhd.vhost = new_vhost 42 | new_vhd.save() 43 | # recreate all parent directives 44 | for vhd in vhost.vhostdirective_set.filter(directive__is_container=True): 45 | new_vhd = VHostDirective() 46 | new_vhd.directive = vhd.directive 47 | new_vhd.value = vhd.value 48 | new_vhd.vhost = new_vhost 49 | new_vhd.save() 50 | # and all their children 51 | for child_vhd in vhost.vhostdirective_set.filter(parent=vhd): 52 | msg += str(child_vhd) 53 | new_child_vhd = VHostDirective() 54 | new_child_vhd.directive = child_vhd.directive 55 | new_child_vhd.value = child_vhd.value 56 | new_child_vhd.vhost = new_vhost 57 | new_child_vhd.parent = new_vhd 58 | new_child_vhd.save() 59 | self.message_user(request, msg) 60 | duplicate.short_description = 'Duplicate selected Virtual Hosts' 61 | 62 | 63 | 64 | class VHostDirectiveAdmin(admin.ModelAdmin): 65 | pass 66 | 67 | class ConfigDirectiveAdmin(admin.ModelAdmin): 68 | fieldsets = [ 69 | (None, {'fields': ['name']}), 70 | ('Details', {'fields': ['is_container', 'documentation'], 71 | 'classes': ['collapse'], 72 | 'description': 'Specify the config directive details'}) 73 | ] 74 | 75 | 76 | 77 | admin.site.register(VirtualHost, VirtualHostAdmin) 78 | admin.site.register(ConfigDirective, ConfigDirectiveAdmin) 79 | admin.site.register(VHostDirective, VHostDirectiveAdmin) 80 | 81 | -------------------------------------------------------------------------------- /978-1-4842-0218-0_Source_Code_Ch05/www_example_com/httpconfig/fixtures/initial_data.json: -------------------------------------------------------------------------------- 1 | [ 2 | 3 | { 4 | "model": "httpconfig.configdirective", 5 | "pk": 1, 6 | "fields": { 7 | "name": "AcceptPathInfo", 8 | "documentation": "http://httpd.apache.org/docs/2.0/mod/core.html#AcceptPathInfo", 9 | "is_container": "False" 10 | } 11 | }, 12 | 13 | { 14 | "model": "httpconfig.configdirective", 15 | "pk": 2, 16 | "fields": { 17 | "name": "AccessFileName", 18 | "documentation": "http://httpd.apache.org/docs/2.0/mod/core.html#AccessFileName", 19 | "is_container": "False" 20 | } 21 | }, 22 | 23 | 24 | { 25 | "model": "httpconfig.configdirective", 26 | "pk": 3, 27 | "fields": { 28 | "name": "AddDefaultCharset", 29 | "documentation": "http://httpd.apache.org/docs/2.0/mod/core.html#AddDefaultCharset", 30 | "is_container": "False" 31 | } 32 | }, 33 | 34 | 35 | { 36 | "model": "httpconfig.configdirective", 37 | "pk": 4, 38 | "fields": { 39 | "name": "AddOutputFilterByType", 40 | "documentation": "http://httpd.apache.org/docs/2.0/mod/core.html#AddOutputFilterByType", 41 | "is_container": "False" 42 | } 43 | }, 44 | 45 | 46 | { 47 | "model": "httpconfig.configdirective", 48 | "pk": 5, 49 | "fields": { 50 | "name": "AllowEncodedSlashes", 51 | "documentation": "http://httpd.apache.org/docs/2.0/mod/core.html#AllowEncodedSlashes", 52 | "is_container": "False" 53 | } 54 | }, 55 | 56 | 57 | { 58 | "model": "httpconfig.configdirective", 59 | "pk": 6, 60 | "fields": { 61 | "name": "AllowOverride", 62 | "documentation": "http://httpd.apache.org/docs/2.0/mod/core.html#AllowOverride", 63 | "is_container": "False" 64 | } 65 | }, 66 | 67 | 68 | { 69 | "model": "httpconfig.configdirective", 70 | "pk": 7, 71 | "fields": { 72 | "name": "AuthName", 73 | "documentation": "http://httpd.apache.org/docs/2.0/mod/core.html#AuthName", 74 | "is_container": "False" 75 | } 76 | }, 77 | 78 | 79 | { 80 | "model": "httpconfig.configdirective", 81 | "pk": 8, 82 | "fields": { 83 | "name": "AuthType", 84 | "documentation": "http://httpd.apache.org/docs/2.0/mod/core.html#AuthType", 85 | "is_container": "False" 86 | } 87 | }, 88 | 89 | 90 | { 91 | "model": "httpconfig.configdirective", 92 | "pk": 9, 93 | "fields": { 94 | "name": "CGIMapExtension", 95 | "documentation": "http://httpd.apache.org/docs/2.0/mod/core.html#CGIMapExtension", 96 | "is_container": "False" 97 | } 98 | }, 99 | 100 | 101 | { 102 | "model": "httpconfig.configdirective", 103 | "pk": 10, 104 | "fields": { 105 | "name": "ContentDigest", 106 | "documentation": "http://httpd.apache.org/docs/2.0/mod/core.html#ContentDigest", 107 | "is_container": "False" 108 | } 109 | }, 110 | 111 | 112 | { 113 | "model": "httpconfig.configdirective", 114 | "pk": 11, 115 | "fields": { 116 | "name": "DefaultType", 117 | "documentation": "http://httpd.apache.org/docs/2.0/mod/core.html#DefaultType", 118 | "is_container": "False" 119 | } 120 | }, 121 | 122 | 123 | { 124 | "model": "httpconfig.configdirective", 125 | "pk": 12, 126 | "fields": { 127 | "name": "", 128 | "documentation": "http://httpd.apache.org/docs/2.0/mod/core.html#Directory", 129 | "is_container": "True" 130 | } 131 | }, 132 | 133 | 134 | { 135 | "model": "httpconfig.configdirective", 136 | "pk": 13, 137 | "fields": { 138 | "name": "", 139 | "documentation": "http://httpd.apache.org/docs/2.0/mod/core.html#DirectoryMatch", 140 | "is_container": "True" 141 | } 142 | }, 143 | 144 | 145 | { 146 | "model": "httpconfig.configdirective", 147 | "pk": 14, 148 | "fields": { 149 | "name": "DocumentRoot", 150 | "documentation": "http://httpd.apache.org/docs/2.0/mod/core.html#DocumentRoot", 151 | "is_container": "False" 152 | } 153 | }, 154 | 155 | 156 | { 157 | "model": "httpconfig.configdirective", 158 | "pk": 15, 159 | "fields": { 160 | "name": "EnableMMAP", 161 | "documentation": "http://httpd.apache.org/docs/2.0/mod/core.html#EnableMMAP", 162 | "is_container": "False" 163 | } 164 | }, 165 | 166 | 167 | { 168 | "model": "httpconfig.configdirective", 169 | "pk": 16, 170 | "fields": { 171 | "name": "EnableSendfile", 172 | "documentation": "http://httpd.apache.org/docs/2.0/mod/core.html#EnableSendfile", 173 | "is_container": "False" 174 | } 175 | }, 176 | 177 | 178 | { 179 | "model": "httpconfig.configdirective", 180 | "pk": 17, 181 | "fields": { 182 | "name": "ErrorDocument", 183 | "documentation": "http://httpd.apache.org/docs/2.0/mod/core.html#ErrorDocument", 184 | "is_container": "False" 185 | } 186 | }, 187 | 188 | 189 | { 190 | "model": "httpconfig.configdirective", 191 | "pk": 18, 192 | "fields": { 193 | "name": "ErrorLog", 194 | "documentation": "http://httpd.apache.org/docs/2.0/mod/core.html#ErrorLog", 195 | "is_container": "False" 196 | } 197 | }, 198 | 199 | 200 | { 201 | "model": "httpconfig.configdirective", 202 | "pk": 19, 203 | "fields": { 204 | "name": "FileETag", 205 | "documentation": "http://httpd.apache.org/docs/2.0/mod/core.html#FileETag", 206 | "is_container": "False" 207 | } 208 | }, 209 | 210 | 211 | { 212 | "model": "httpconfig.configdirective", 213 | "pk": 20, 214 | "fields": { 215 | "name": "", 216 | "documentation": "http://httpd.apache.org/docs/2.0/mod/core.html#Files", 217 | "is_container": "True" 218 | } 219 | }, 220 | 221 | 222 | { 223 | "model": "httpconfig.configdirective", 224 | "pk": 21, 225 | "fields": { 226 | "name": "", 227 | "documentation": "http://httpd.apache.org/docs/2.0/mod/core.html#FilesMatch", 228 | "is_container": "True" 229 | } 230 | }, 231 | 232 | 233 | { 234 | "model": "httpconfig.configdirective", 235 | "pk": 22, 236 | "fields": { 237 | "name": "ForceType", 238 | "documentation": "http://httpd.apache.org/docs/2.0/mod/core.html#ForceType", 239 | "is_container": "False" 240 | } 241 | }, 242 | 243 | 244 | { 245 | "model": "httpconfig.configdirective", 246 | "pk": 23, 247 | "fields": { 248 | "name": "HostnameLookups", 249 | "documentation": "http://httpd.apache.org/docs/2.0/mod/core.html#HostnameLookups", 250 | "is_container": "False" 251 | } 252 | }, 253 | 254 | 255 | { 256 | "model": "httpconfig.configdirective", 257 | "pk": 24, 258 | "fields": { 259 | "name": "IdentityCheck", 260 | "documentation": "http://httpd.apache.org/docs/2.0/mod/core.html#IdentityCheck", 261 | "is_container": "False" 262 | } 263 | }, 264 | 265 | 266 | { 267 | "model": "httpconfig.configdirective", 268 | "pk": 25, 269 | "fields": { 270 | "name": "", 271 | "documentation": "http://httpd.apache.org/docs/2.0/mod/core.html#IfDefine", 272 | "is_container": "True" 273 | } 274 | }, 275 | 276 | 277 | { 278 | "model": "httpconfig.configdirective", 279 | "pk": 26, 280 | "fields": { 281 | "name": "", 282 | "documentation": "http://httpd.apache.org/docs/2.0/mod/core.html#IfModule", 283 | "is_container": "True" 284 | } 285 | }, 286 | 287 | 288 | { 289 | "model": "httpconfig.configdirective", 290 | "pk": 27, 291 | "fields": { 292 | "name": "Include", 293 | "documentation": "http://httpd.apache.org/docs/2.0/mod/core.html#Include", 294 | "is_container": "False" 295 | } 296 | }, 297 | 298 | 299 | { 300 | "model": "httpconfig.configdirective", 301 | "pk": 28, 302 | "fields": { 303 | "name": "KeepAlive", 304 | "documentation": "http://httpd.apache.org/docs/2.0/mod/core.html#KeepAlive", 305 | "is_container": "False" 306 | } 307 | }, 308 | 309 | 310 | { 311 | "model": "httpconfig.configdirective", 312 | "pk": 29, 313 | "fields": { 314 | "name": "KeepAliveTimeout", 315 | "documentation": "http://httpd.apache.org/docs/2.0/mod/core.html#KeepAliveTimeout", 316 | "is_container": "False" 317 | } 318 | }, 319 | 320 | 321 | { 322 | "model": "httpconfig.configdirective", 323 | "pk": 30, 324 | "fields": { 325 | "name": "", 326 | "documentation": "http://httpd.apache.org/docs/2.0/mod/core.html#Limit", 327 | "is_container": "True" 328 | } 329 | }, 330 | 331 | 332 | { 333 | "model": "httpconfig.configdirective", 334 | "pk": 31, 335 | "fields": { 336 | "name": "", 337 | "documentation": "http://httpd.apache.org/docs/2.0/mod/core.html#LimitExcept", 338 | "is_container": "True" 339 | } 340 | }, 341 | 342 | 343 | { 344 | "model": "httpconfig.configdirective", 345 | "pk": 32, 346 | "fields": { 347 | "name": "LimitInternalRecursion", 348 | "documentation": "http://httpd.apache.org/docs/2.0/mod/core.html#LimitInternalRecursion", 349 | "is_container": "False" 350 | } 351 | }, 352 | 353 | 354 | { 355 | "model": "httpconfig.configdirective", 356 | "pk": 33, 357 | "fields": { 358 | "name": "LimitRequestBody", 359 | "documentation": "http://httpd.apache.org/docs/2.0/mod/core.html#LimitRequestBody", 360 | "is_container": "False" 361 | } 362 | }, 363 | 364 | 365 | { 366 | "model": "httpconfig.configdirective", 367 | "pk": 34, 368 | "fields": { 369 | "name": "LimitRequestFields", 370 | "documentation": "http://httpd.apache.org/docs/2.0/mod/core.html#LimitRequestFields", 371 | "is_container": "False" 372 | } 373 | }, 374 | 375 | 376 | { 377 | "model": "httpconfig.configdirective", 378 | "pk": 35, 379 | "fields": { 380 | "name": "LimitRequestFieldSize", 381 | "documentation": "http://httpd.apache.org/docs/2.0/mod/core.html#LimitRequestFieldSize", 382 | "is_container": "False" 383 | } 384 | }, 385 | 386 | 387 | { 388 | "model": "httpconfig.configdirective", 389 | "pk": 36, 390 | "fields": { 391 | "name": "LimitRequestLine", 392 | "documentation": "http://httpd.apache.org/docs/2.0/mod/core.html#LimitRequestLine", 393 | "is_container": "False" 394 | } 395 | }, 396 | 397 | 398 | { 399 | "model": "httpconfig.configdirective", 400 | "pk": 37, 401 | "fields": { 402 | "name": "LimitXMLRequestBody", 403 | "documentation": "http://httpd.apache.org/docs/2.0/mod/core.html#LimitXMLRequestBody", 404 | "is_container": "False" 405 | } 406 | }, 407 | 408 | 409 | { 410 | "model": "httpconfig.configdirective", 411 | "pk": 38, 412 | "fields": { 413 | "name": "", 414 | "documentation": "http://httpd.apache.org/docs/2.0/mod/core.html#Location", 415 | "is_container": "True" 416 | } 417 | }, 418 | 419 | 420 | { 421 | "model": "httpconfig.configdirective", 422 | "pk": 39, 423 | "fields": { 424 | "name": "", 425 | "documentation": "http://httpd.apache.org/docs/2.0/mod/core.html#LocationMatch", 426 | "is_container": "True" 427 | } 428 | }, 429 | 430 | 431 | { 432 | "model": "httpconfig.configdirective", 433 | "pk": 40, 434 | "fields": { 435 | "name": "LogLevel", 436 | "documentation": "http://httpd.apache.org/docs/2.0/mod/core.html#LogLevel", 437 | "is_container": "False" 438 | } 439 | }, 440 | 441 | 442 | { 443 | "model": "httpconfig.configdirective", 444 | "pk": 41, 445 | "fields": { 446 | "name": "MaxKeepAliveRequests", 447 | "documentation": "http://httpd.apache.org/docs/2.0/mod/core.html#MaxKeepAliveRequests", 448 | "is_container": "False" 449 | } 450 | }, 451 | 452 | 453 | { 454 | "model": "httpconfig.configdirective", 455 | "pk": 42, 456 | "fields": { 457 | "name": "NameVirtualHost", 458 | "documentation": "http://httpd.apache.org/docs/2.0/mod/core.html#NameVirtualHost", 459 | "is_container": "False" 460 | } 461 | }, 462 | 463 | 464 | { 465 | "model": "httpconfig.configdirective", 466 | "pk": 43, 467 | "fields": { 468 | "name": "Options", 469 | "documentation": "http://httpd.apache.org/docs/2.0/mod/core.html#Options", 470 | "is_container": "False" 471 | } 472 | }, 473 | 474 | 475 | { 476 | "model": "httpconfig.configdirective", 477 | "pk": 44, 478 | "fields": { 479 | "name": "Require", 480 | "documentation": "http://httpd.apache.org/docs/2.0/mod/core.html#Require", 481 | "is_container": "False" 482 | } 483 | }, 484 | 485 | 486 | { 487 | "model": "httpconfig.configdirective", 488 | "pk": 45, 489 | "fields": { 490 | "name": "RLimitCPU", 491 | "documentation": "http://httpd.apache.org/docs/2.0/mod/core.html#RLimitCPU", 492 | "is_container": "False" 493 | } 494 | }, 495 | 496 | 497 | { 498 | "model": "httpconfig.configdirective", 499 | "pk": 46, 500 | "fields": { 501 | "name": "RLimitMEM", 502 | "documentation": "http://httpd.apache.org/docs/2.0/mod/core.html#RLimitMEM", 503 | "is_container": "False" 504 | } 505 | }, 506 | 507 | 508 | { 509 | "model": "httpconfig.configdirective", 510 | "pk": 47, 511 | "fields": { 512 | "name": "RLimitNPROC", 513 | "documentation": "http://httpd.apache.org/docs/2.0/mod/core.html#RLimitNPROC", 514 | "is_container": "False" 515 | } 516 | }, 517 | 518 | 519 | { 520 | "model": "httpconfig.configdirective", 521 | "pk": 48, 522 | "fields": { 523 | "name": "Satisfy", 524 | "documentation": "http://httpd.apache.org/docs/2.0/mod/core.html#Satisfy", 525 | "is_container": "False" 526 | } 527 | }, 528 | 529 | 530 | { 531 | "model": "httpconfig.configdirective", 532 | "pk": 49, 533 | "fields": { 534 | "name": "ScriptInterpreterSource", 535 | "documentation": "http://httpd.apache.org/docs/2.0/mod/core.html#ScriptInterpreterSource", 536 | "is_container": "False" 537 | } 538 | }, 539 | 540 | 541 | { 542 | "model": "httpconfig.configdirective", 543 | "pk": 50, 544 | "fields": { 545 | "name": "ServerAdmin", 546 | "documentation": "http://httpd.apache.org/docs/2.0/mod/core.html#ServerAdmin", 547 | "is_container": "False" 548 | } 549 | }, 550 | 551 | 552 | { 553 | "model": "httpconfig.configdirective", 554 | "pk": 51, 555 | "fields": { 556 | "name": "ServerAlias", 557 | "documentation": "http://httpd.apache.org/docs/2.0/mod/core.html#ServerAlias", 558 | "is_container": "False" 559 | } 560 | }, 561 | 562 | 563 | { 564 | "model": "httpconfig.configdirective", 565 | "pk": 52, 566 | "fields": { 567 | "name": "ServerName", 568 | "documentation": "http://httpd.apache.org/docs/2.0/mod/core.html#ServerName", 569 | "is_container": "False" 570 | } 571 | }, 572 | 573 | 574 | { 575 | "model": "httpconfig.configdirective", 576 | "pk": 53, 577 | "fields": { 578 | "name": "ServerPath", 579 | "documentation": "http://httpd.apache.org/docs/2.0/mod/core.html#ServerPath", 580 | "is_container": "False" 581 | } 582 | }, 583 | 584 | 585 | { 586 | "model": "httpconfig.configdirective", 587 | "pk": 54, 588 | "fields": { 589 | "name": "ServerRoot", 590 | "documentation": "http://httpd.apache.org/docs/2.0/mod/core.html#ServerRoot", 591 | "is_container": "False" 592 | } 593 | }, 594 | 595 | 596 | { 597 | "model": "httpconfig.configdirective", 598 | "pk": 55, 599 | "fields": { 600 | "name": "ServerSignature", 601 | "documentation": "http://httpd.apache.org/docs/2.0/mod/core.html#ServerSignature", 602 | "is_container": "False" 603 | } 604 | }, 605 | 606 | 607 | { 608 | "model": "httpconfig.configdirective", 609 | "pk": 56, 610 | "fields": { 611 | "name": "ServerTokens", 612 | "documentation": "http://httpd.apache.org/docs/2.0/mod/core.html#ServerTokens", 613 | "is_container": "False" 614 | } 615 | }, 616 | 617 | 618 | { 619 | "model": "httpconfig.configdirective", 620 | "pk": 57, 621 | "fields": { 622 | "name": "SetHandler", 623 | "documentation": "http://httpd.apache.org/docs/2.0/mod/core.html#SetHandler", 624 | "is_container": "False" 625 | } 626 | }, 627 | 628 | 629 | { 630 | "model": "httpconfig.configdirective", 631 | "pk": 58, 632 | "fields": { 633 | "name": "SetInputFilter", 634 | "documentation": "http://httpd.apache.org/docs/2.0/mod/core.html#SetInputFilter", 635 | "is_container": "False" 636 | } 637 | }, 638 | 639 | 640 | { 641 | "model": "httpconfig.configdirective", 642 | "pk": 59, 643 | "fields": { 644 | "name": "SetOutputFilter", 645 | "documentation": "http://httpd.apache.org/docs/2.0/mod/core.html#SetOutputFilter", 646 | "is_container": "False" 647 | } 648 | }, 649 | 650 | 651 | { 652 | "model": "httpconfig.configdirective", 653 | "pk": 60, 654 | "fields": { 655 | "name": "TimeOut", 656 | "documentation": "http://httpd.apache.org/docs/2.0/mod/core.html#TimeOut", 657 | "is_container": "False" 658 | } 659 | }, 660 | 661 | 662 | { 663 | "model": "httpconfig.configdirective", 664 | "pk": 61, 665 | "fields": { 666 | "name": "TraceEnable", 667 | "documentation": "http://httpd.apache.org/docs/2.0/mod/core.html#TraceEnable", 668 | "is_container": "False" 669 | } 670 | }, 671 | 672 | 673 | { 674 | "model": "httpconfig.configdirective", 675 | "pk": 62, 676 | "fields": { 677 | "name": "UseCanonicalName", 678 | "documentation": "http://httpd.apache.org/docs/2.0/mod/core.html#UseCanonicalName", 679 | "is_container": "False" 680 | } 681 | } 682 | 683 | ] 684 | -------------------------------------------------------------------------------- /978-1-4842-0218-0_Source_Code_Ch05/www_example_com/httpconfig/models.py: -------------------------------------------------------------------------------- 1 | from django.db import models 2 | 3 | 4 | class ConfigDirective(models.Model): 5 | class Meta: 6 | verbose_name = 'Configuration Directive' 7 | verbose_name_plural = 'Configuration Directives' 8 | name = models.CharField(max_length=200) 9 | is_container = models.BooleanField(default=False) 10 | documentation = models.URLField( 11 | default='http://httpd.apache.org/docs/2.0/mod/core.html') 12 | 13 | def __unicode__(self): 14 | return self.name 15 | 16 | class VirtualHost(models.Model): 17 | class Meta: 18 | verbose_name = 'Virtual Host' 19 | verbose_name_plural = 'Virtual Hosts' 20 | is_default = models.BooleanField(default=False) 21 | is_template = models.BooleanField(default=False, 22 | help_text="""Template virtual hosts are 23 | commented out in the configuration 24 | and can be reused as templates""") 25 | description = models.CharField(max_length=200) 26 | bind_address = models.CharField(max_length=200) 27 | directives = models.ManyToManyField(ConfigDirective, through='VHostDirective') 28 | 29 | def __unicode__(self): 30 | default_mark = ' (*)' if self.is_default else '' 31 | return self.description + default_mark 32 | 33 | def domain_names(self): 34 | result = '' 35 | primary_domains = self.vhostdirective_set.filter(directive__name='ServerName') 36 | if primary_domains: 37 | result = "%(d)s" % {'d': primary_domains[0].value} 38 | else: 39 | result = 'No primary domain defined!' 40 | secondary_domains = self.vhostdirective_set.filter(directive__name='ServerAlias') 41 | if secondary_domains: 42 | result += ' (' 43 | for domain in secondary_domains: 44 | result += "%(d)s, " % {'d': domain.value} 45 | result = result[:-2] + ')' 46 | return result 47 | domain_names.allow_tags = True 48 | 49 | def code_snippet(self): 50 | return "View code snippet" % self.id 51 | code_snippet.allow_tags = True 52 | 53 | 54 | 55 | class VHostDirective(models.Model): 56 | class Meta: 57 | verbose_name = 'Virtual Host Directive' 58 | verbose_name_plural = 'Virtual Host Directives' 59 | directive = models.ForeignKey(ConfigDirective) 60 | vhost = models.ForeignKey(VirtualHost) 61 | parent = models.ForeignKey('self', blank=True, null=True, 62 | limit_choices_to={'directive__is_container': True}) 63 | value = models.CharField(max_length=200) 64 | 65 | def __unicode__(self): 66 | fmt_str = "<%s %s>" if self.directive.is_container else "%s %s" 67 | directive_name = self.directive.name.strip('<>') 68 | return fmt_str % (directive_name, self.value) 69 | 70 | def close_tag(self): 71 | return "" % self.directive.name.strip('<>') if self.directive.is_container else "" 72 | 73 | -------------------------------------------------------------------------------- /978-1-4842-0218-0_Source_Code_Ch05/www_example_com/httpconfig/templates/full_config.txt: -------------------------------------------------------------------------------- 1 | # Virtual host configuration section 2 | # automatically generated - do not edit 3 | 4 | {% for vhost in vhosts %} 5 | 6 | ## 7 | ## {{ vhost.vhost_data.description }} 8 | ## 9 | {% if vhost.vhost_data.is_template %}#{% endif %} 10 | {% if vhost.vhost_data.is_template %}#{% endif %} {% for orphan_directive in vhost.orphan_directives %} 11 | {% if vhost.vhost_data.is_template %}#{% endif %} {{ orphan_directive }} 12 | {% if vhost.vhost_data.is_template %}#{% endif %} {% endfor %} 13 | {% if vhost.vhost_data.is_template %}#{% endif %} {% for container in vhost.containers %} 14 | {% if vhost.vhost_data.is_template %}#{% endif %} {{ container.parent|safe }} 15 | {% if vhost.vhost_data.is_template %}#{% endif %} {% for child_dir in container.children %} 16 | {% if vhost.vhost_data.is_template %}#{% endif %} {{ child_dir }} 17 | {% if vhost.vhost_data.is_template %}#{% endif %} {% endfor %} 18 | {% if vhost.vhost_data.is_template %}#{% endif %} {{ container.parent.close_tag|safe }} 19 | {% if vhost.vhost_data.is_template %}#{% endif %} {% endfor %} 20 | {% if vhost.vhost_data.is_template %}#{% endif %} 21 | 22 | {% endfor %} 23 | -------------------------------------------------------------------------------- /978-1-4842-0218-0_Source_Code_Ch05/www_example_com/httpconfig/tests.py: -------------------------------------------------------------------------------- 1 | from django.test import TestCase 2 | 3 | # Create your tests here. 4 | -------------------------------------------------------------------------------- /978-1-4842-0218-0_Source_Code_Ch05/www_example_com/httpconfig/urls.py: -------------------------------------------------------------------------------- 1 | from django.conf.urls import patterns, include, url 2 | 3 | urlpatterns = patterns('httpconfig.views', 4 | url(r'^$', 'full_config'), 5 | url(r'^(?P\d+)/$', 'full_config'), 6 | ) 7 | 8 | -------------------------------------------------------------------------------- /978-1-4842-0218-0_Source_Code_Ch05/www_example_com/httpconfig/views.py: -------------------------------------------------------------------------------- 1 | from httpconfig.models import * 2 | from django.http import HttpResponse, HttpResponseRedirect 3 | from django.shortcuts import render_to_response, get_object_or_404 4 | 5 | # Create your views here. 6 | 7 | def full_config(request, object_id=None): 8 | if not object_id: 9 | vhosts = VirtualHost.objects.all() 10 | else: 11 | vhosts = VirtualHost.objects.filter(id=object_id) 12 | vhosts_list = [] 13 | for vhost in vhosts: 14 | vhost_struct = {} 15 | vhost_struct['vhost_data'] = vhost 16 | vhost_struct['orphan_directives'] = \ 17 | vhost.vhostdirective_set.filter(directive__is_container=False).filter(parent=None) 18 | vhost_struct['containers'] = [] 19 | for container_directive in \ 20 | vhost.vhostdirective_set.filter(directive__is_container=True): 21 | vhost_struct['containers'].append({'parent': container_directive, 22 | 'children': \ 23 | vhost.vhostdirective_set.filter(parent=container_directive), 24 | }) 25 | vhosts_list.append(vhost_struct) 26 | return render_to_response('full_config.txt', 27 | {'vhosts': vhosts_list}, 28 | mimetype='text/plain') 29 | 30 | -------------------------------------------------------------------------------- /978-1-4842-0218-0_Source_Code_Ch05/www_example_com/manage.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import os 3 | import sys 4 | 5 | if __name__ == "__main__": 6 | os.environ.setdefault("DJANGO_SETTINGS_MODULE", "www_example_com.settings") 7 | 8 | from django.core.management import execute_from_command_line 9 | 10 | execute_from_command_line(sys.argv) 11 | -------------------------------------------------------------------------------- /978-1-4842-0218-0_Source_Code_Ch05/www_example_com/www_example_com/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Apress/pro-python-system-admin-14/2b70cbe957969ce2f72e59858ffc4c0bbfe77a05/978-1-4842-0218-0_Source_Code_Ch05/www_example_com/www_example_com/__init__.py -------------------------------------------------------------------------------- /978-1-4842-0218-0_Source_Code_Ch05/www_example_com/www_example_com/settings.py: -------------------------------------------------------------------------------- 1 | """ 2 | Django settings for www_example_com project. 3 | 4 | For more information on this file, see 5 | https://docs.djangoproject.com/en/1.6/topics/settings/ 6 | 7 | For the full list of settings and their values, see 8 | https://docs.djangoproject.com/en/1.6/ref/settings/ 9 | """ 10 | 11 | # Build paths inside the project like this: os.path.join(BASE_DIR, ...) 12 | import os 13 | BASE_DIR = os.path.dirname(os.path.dirname(__file__)) 14 | 15 | 16 | # Quick-start development settings - unsuitable for production 17 | # See https://docs.djangoproject.com/en/1.6/howto/deployment/checklist/ 18 | 19 | # SECURITY WARNING: keep the secret key used in production secret! 20 | SECRET_KEY = 'x2k=%ee2@n)f1n3eqknd=ji5nj=_qe7mysls)kwzl^mz3ypl5k' 21 | 22 | # SECURITY WARNING: don't run with debug turned on in production! 23 | DEBUG = True 24 | 25 | TEMPLATE_DEBUG = True 26 | 27 | ALLOWED_HOSTS = [] 28 | 29 | 30 | # Application definition 31 | 32 | INSTALLED_APPS = ( 33 | 'django.contrib.admin', 34 | 'django.contrib.auth', 35 | 'django.contrib.contenttypes', 36 | 'django.contrib.sessions', 37 | 'django.contrib.messages', 38 | 'django.contrib.staticfiles', 39 | 'httpconfig', 40 | ) 41 | 42 | MIDDLEWARE_CLASSES = ( 43 | 'django.contrib.sessions.middleware.SessionMiddleware', 44 | 'django.middleware.common.CommonMiddleware', 45 | 'django.middleware.csrf.CsrfViewMiddleware', 46 | 'django.contrib.auth.middleware.AuthenticationMiddleware', 47 | 'django.contrib.messages.middleware.MessageMiddleware', 48 | 'django.middleware.clickjacking.XFrameOptionsMiddleware', 49 | ) 50 | 51 | ROOT_URLCONF = 'www_example_com.urls' 52 | 53 | WSGI_APPLICATION = 'www_example_com.wsgi.application' 54 | 55 | 56 | # Database 57 | # https://docs.djangoproject.com/en/1.6/ref/settings/#databases 58 | 59 | DATABASES = { 60 | 'default': { 61 | 'ENGINE': 'django.db.backends.sqlite3', 62 | 'NAME': os.path.join(BASE_DIR, 'db.sqlite3'), 63 | } 64 | } 65 | 66 | # Internationalization 67 | # https://docs.djangoproject.com/en/1.6/topics/i18n/ 68 | 69 | LANGUAGE_CODE = 'en-us' 70 | 71 | TIME_ZONE = 'UTC' 72 | 73 | USE_I18N = True 74 | 75 | USE_L10N = True 76 | 77 | USE_TZ = True 78 | 79 | 80 | # Static files (CSS, JavaScript, Images) 81 | # https://docs.djangoproject.com/en/1.6/howto/static-files/ 82 | 83 | STATIC_URL = '/static/' 84 | -------------------------------------------------------------------------------- /978-1-4842-0218-0_Source_Code_Ch05/www_example_com/www_example_com/urls.py: -------------------------------------------------------------------------------- 1 | from django.conf.urls import patterns, include, url 2 | 3 | from django.contrib import admin 4 | admin.autodiscover() 5 | 6 | urlpatterns = patterns('', 7 | # Examples: 8 | # url(r'^$', 'www_example_com.views.home', name='home'), 9 | # url(r'^blog/', include('blog.urls')), 10 | 11 | url(r'^admin/', include(admin.site.urls)), 12 | url(r'', include('httpconfig.urls')), 13 | ) 14 | -------------------------------------------------------------------------------- /978-1-4842-0218-0_Source_Code_Ch05/www_example_com/www_example_com/wsgi.py: -------------------------------------------------------------------------------- 1 | """ 2 | WSGI config for www_example_com project. 3 | 4 | It exposes the WSGI callable as a module-level variable named ``application``. 5 | 6 | For more information on this file, see 7 | https://docs.djangoproject.com/en/1.6/howto/deployment/wsgi/ 8 | """ 9 | 10 | import os 11 | os.environ.setdefault("DJANGO_SETTINGS_MODULE", "www_example_com.settings") 12 | 13 | from django.core.wsgi import get_wsgi_application 14 | application = get_wsgi_application() 15 | -------------------------------------------------------------------------------- /978-1-4842-0218-0_Source_Code_Ch06/http_log_parser.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import csv 4 | import re 5 | import os 6 | from manager import PluginManager 7 | 8 | DIRECTIVE_MAP = { 9 | '%h': 'remote_host', 10 | '%l': 'remote_logname', 11 | '%u': 'remote_user', 12 | '%t': 'time_stamp', 13 | '%r': 'request_line', 14 | '%>s': 'status', 15 | '%b': 'response_size', 16 | '%{Referer}i': 'referer_url', 17 | '%{User-Agent}i': 'user_agent', 18 | } 19 | 20 | 21 | class LogLineGenerator: 22 | def __init__(self, log_format=None, log_dir='logs'): 23 | # LogFormat "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\"" combined 24 | if not log_format: 25 | self.format_string = '%h %l %u %t %r %>s %b %{Referer}i %{User-Agent}i' 26 | else: 27 | self.format_string = log_format 28 | self.log_dir = log_dir 29 | self.re_tsquote = re.compile(r'(\[|\])') 30 | self.field_list = [] 31 | for directive in self.format_string.split(' '): 32 | self.field_list.append(DIRECTIVE_MAP[directive]) 33 | 34 | def _quote_translator(self, file_name): 35 | for line in open(file_name): 36 | yield self.re_tsquote.sub('"', line) 37 | 38 | def _file_list(self): 39 | for file in os.listdir(self.log_dir): 40 | file_name = "%s/%s" % (self.log_dir, file) 41 | if os.path.isfile(file_name): 42 | yield file_name 43 | 44 | def get_loglines(self): 45 | for file in self._file_list(): 46 | reader = csv.DictReader(self._quote_translator(file), fieldnames=self.field_list, delimiter=' ', quotechar='"') 47 | for line in reader: 48 | yield line 49 | 50 | def main(): 51 | plugin_manager = PluginManager() 52 | log_generator = LogLineGenerator() 53 | for log_line in log_generator.get_loglines(): 54 | plugin_manager.call_method('process', args=log_line) 55 | plugin_manager.call_method('report') 56 | 57 | if __name__ == '__main__': 58 | main() 59 | -------------------------------------------------------------------------------- /978-1-4842-0218-0_Source_Code_Ch06/manager.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import sys 4 | import os 5 | 6 | 7 | class Plugin(object): 8 | pass 9 | 10 | 11 | class PluginManager(): 12 | def __init__(self, path=None, plugin_init_args={}): 13 | if path: 14 | self.plugin_dir = path 15 | else: 16 | self.plugin_dir = os.path.dirname(__file__) + '/plugins/' 17 | self.plugins = {} 18 | self._load_plugins() 19 | self._register_plugins(**plugin_init_args) 20 | 21 | def _load_plugins(self): 22 | sys.path.append(self.plugin_dir) 23 | plugin_files = [fn for fn in os.listdir(self.plugin_dir) if fn.startswith('plugin_') and fn.endswith('.py')] 24 | plugin_modules = [m.split('.')[0] for m in plugin_files] 25 | for module in plugin_modules: 26 | m = __import__(module) 27 | 28 | def _register_plugins(self, **kwargs): 29 | for plugin in Plugin.__subclasses__(): 30 | obj = plugin(**kwargs) 31 | self.plugins[obj] = obj.keywords if hasattr(obj, 'keywords') else [] 32 | 33 | def call_method(self, method, args={}, keywords=[]): 34 | for plugin in self.plugins: 35 | if not keywords or (set(keywords) & set(self.plugins[plugin])): 36 | try: 37 | getattr(plugin, method)(**args) 38 | except: 39 | pass 40 | 41 | -------------------------------------------------------------------------------- /978-1-4842-0218-0_Source_Code_Ch06/plugins/plugin_count_200.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | from manager import Plugin 4 | 5 | class CountHTTP200(Plugin): 6 | 7 | def __init__(self, **kwargs): 8 | self.keywords = ['counter'] 9 | self.counter_200 = 0 10 | self.counter_total = 0 11 | 12 | def process(self, **kwargs): 13 | if 'status' in kwargs: 14 | self.counter_total += 1 15 | if kwargs['status'] == '200': 16 | self.counter_200 += 1 17 | 18 | def report(self, **kwargs): 19 | print '== HTTP code 200 counter ==' 20 | print "HTTP 200 responses: %d" % self.counter_200 21 | print "All responses: %d" % self.counter_total 22 | -------------------------------------------------------------------------------- /978-1-4842-0218-0_Source_Code_Ch06/plugins/plugin_geoip_stats.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | from manager import Plugin 4 | from operator import itemgetter 5 | import GeoIP 6 | import shapefile 7 | import numpy as np 8 | import matplotlib.pyplot as plt 9 | import matplotlib as mpl 10 | from mpl_toolkits.basemap import Basemap 11 | from matplotlib.collections import LineCollection 12 | from matplotlib import cm 13 | 14 | 15 | class GeoIPStats(Plugin): 16 | 17 | def __init__(self, **kwargs): 18 | self.gi = GeoIP.new(GeoIP.GEOIP_MEMORY_CACHE) 19 | self.countries = {} 20 | 21 | def process(self, **kwargs): 22 | if 'remote_host' in kwargs: 23 | country = self.gi.country_name_by_addr(kwargs['remote_host']) 24 | if country in self.countries: 25 | self.countries[country] += 1 26 | else: 27 | self.countries[country] = 1 28 | 29 | def report(self, **kwargs): 30 | print "== Requests by country ==" 31 | for (country, count) in sorted(self.countries.iteritems(), key=itemgetter(1), reverse=True): 32 | print " %10d: %s" % (count, country) 33 | generate_map(self.countries) 34 | 35 | 36 | def generate_map(countries): 37 | 38 | # Initialize plotting area, set the boundaries and add a sub-plot on which 39 | # we are going to plot the map 40 | fig = plt.figure(figsize=(11.7, 8.3)) 41 | plt.subplots_adjust(left=0.05,right=0.95,top=0.90,bottom=0.05,wspace=0.15,hspace=0.05) 42 | ax = plt.subplot(111) 43 | 44 | # Initialize the basemap, set the resolution, projection type and the viewport 45 | bm = Basemap(resolution='i', projection='robin', lon_0=0) 46 | 47 | # Tell basemap how to draw the countries (built-in shapes), draw parallels and meridians 48 | # and color in the water 49 | bm.drawcountries(linewidth=0.5) 50 | bm.drawparallels(np.arange(-90., 120., 30.)) 51 | bm.drawmeridians(np.arange(0., 360., 60.)) 52 | bm.drawmapboundary(fill_color='aqua') 53 | 54 | # Open the countries shapefile and read the shape and attribute information 55 | r = shapefile.Reader('world_borders/TM_WORLD_BORDERS-0.3.shp') 56 | shapes = r.shapes() 57 | records = r.records() 58 | 59 | # Iterate through all records (attributes) and shapes (countries) 60 | for record, shape in zip(records, shapes): 61 | 62 | # Extract longitude and latitude values into two separate arrays then 63 | # project the coordinates onto the map projection and transpose the array, so that 64 | # the data variable contains (lon, lat) pairs in the list. 65 | # Basically, the following two lines convert the initial data 66 | # [ [lon_original_1, lat_original_1], [lon_original_2, lat_original_2], ... ] 67 | # into projected data 68 | # [ [lon_projected_1, lat_projected_1, [lon_projected_2, lat_projected_2], ... ] 69 | # 70 | # Note: Calling baseshape object with the coordinates as an argument returns the 71 | # projection of those coordinates 72 | lon_array, lat_array = zip(*shape.points) 73 | data = np.array(bm(lon_array, lat_array)).T 74 | 75 | # Next we will create groups of points by splitting the shape.points according to 76 | # the indices provided in shape.parts 77 | 78 | if len(shape.parts) == 1: 79 | # If the shape has only one part, then we have only one group. Easy. 80 | groups = [data,] 81 | else: 82 | # If we have more than one part ... 83 | groups = [] 84 | for i in range(1, len(shape.parts)): 85 | # We iterate through all parts, and find their start and end positions 86 | index_start = shape.parts[i-1] 87 | index_end = shape.parts[i] 88 | # Then we copy all point between two indices into their own group and append 89 | # that group to the list 90 | groups.append(data[index_start:index_end]) 91 | # Last group starts at the last index and finishes at the end of the points list 92 | groups.append(data[index_end:]) 93 | 94 | # Create a collection of lines provided the group of points. Each group represents a line. 95 | lines = LineCollection(groups, antialiaseds=(1,)) 96 | # We then select a color from a color map (in this instance all Reds) 97 | # The intensity of the selected color is proportional to the number of requests. 98 | # Color map accepts values from 0 to 1, therefore we need to normalize our request count 99 | # figures, so that the max number of requests is 1, and the rest is proportionally spread 100 | # in the range from 0 to 1. 101 | max_value = float(max(countries.values())) 102 | country_name = record[4] 103 | 104 | requests = countries.get(country_name, 0) 105 | requests_norm = requests / max_value 106 | 107 | lines.set_facecolors(cm.Reds(requests_norm)) 108 | 109 | # Finally we set the border color to be black and add the shape to the sub-plot 110 | lines.set_edgecolors('k') 111 | lines.set_linewidth(0.1) 112 | ax.add_collection(lines) 113 | 114 | # Once we are ready, we save the resulting picture 115 | plt.savefig('requests_per_country.png', dpi=300) 116 | 117 | -------------------------------------------------------------------------------- /978-1-4842-0218-0_Source_Code_Ch06/plugins/world_borders/Readme.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Apress/pro-python-system-admin-14/2b70cbe957969ce2f72e59858ffc4c0bbfe77a05/978-1-4842-0218-0_Source_Code_Ch06/plugins/world_borders/Readme.txt -------------------------------------------------------------------------------- /978-1-4842-0218-0_Source_Code_Ch06/plugins/world_borders/TM_WORLD_BORDERS-0.3.dbf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Apress/pro-python-system-admin-14/2b70cbe957969ce2f72e59858ffc4c0bbfe77a05/978-1-4842-0218-0_Source_Code_Ch06/plugins/world_borders/TM_WORLD_BORDERS-0.3.dbf -------------------------------------------------------------------------------- /978-1-4842-0218-0_Source_Code_Ch06/plugins/world_borders/TM_WORLD_BORDERS-0.3.prj: -------------------------------------------------------------------------------- 1 | GEOGCS["GCS_WGS_1984",DATUM["D_WGS_1984",SPHEROID["WGS_1984",6378137.0,298.257223563]],PRIMEM["Greenwich",0.0],UNIT["Degree",0.0174532925199433]] -------------------------------------------------------------------------------- /978-1-4842-0218-0_Source_Code_Ch06/plugins/world_borders/TM_WORLD_BORDERS-0.3.shp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Apress/pro-python-system-admin-14/2b70cbe957969ce2f72e59858ffc4c0bbfe77a05/978-1-4842-0218-0_Source_Code_Ch06/plugins/world_borders/TM_WORLD_BORDERS-0.3.shp -------------------------------------------------------------------------------- /978-1-4842-0218-0_Source_Code_Ch06/plugins/world_borders/TM_WORLD_BORDERS-0.3.shx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Apress/pro-python-system-admin-14/2b70cbe957969ce2f72e59858ffc4c0bbfe77a05/978-1-4842-0218-0_Source_Code_Ch06/plugins/world_borders/TM_WORLD_BORDERS-0.3.shx -------------------------------------------------------------------------------- /978-1-4842-0218-0_Source_Code_Ch07/FileServer.java: -------------------------------------------------------------------------------- 1 | import java.io.*; 2 | import java.util.*; 3 | import java.text.*; 4 | import javax.servlet.*; 5 | import javax.servlet.http.*; 6 | 7 | public class FileServer extends HttpServlet { 8 | 9 | public void doGet(HttpServletRequest request, HttpServletResponse response) 10 | throws IOException, ServletException 11 | { 12 | response.setContentType("text/html"); 13 | PrintWriter out = response.getWriter(); 14 | 15 | out.println(System.getProperty("user.dir") + "
"); 16 | 17 | String fileName = request.getParameter("fn"); 18 | if (fileName != null) { 19 | out.println("File name: " + fileName); 20 | out.println("
"); 21 | out.println(readFile(fileName)); 22 | } else { 23 | out.println("No file specified"); 24 | } 25 | 26 | } 27 | 28 | private String readFile(String file) throws IOException { 29 | StringBuilder stringBuilder = new StringBuilder(); 30 | Scanner scanner = new Scanner(new BufferedReader(new FileReader(file))); 31 | 32 | try { 33 | while(scanner.hasNextLine()) { 34 | stringBuilder.append(scanner.nextLine() + "\n"); 35 | } 36 | } finally { 37 | scanner.close(); 38 | } 39 | return stringBuilder.toString(); 40 | } 41 | } 42 | 43 | -------------------------------------------------------------------------------- /978-1-4842-0218-0_Source_Code_Ch07/exctractor.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | ## 3 | ## Exctractor - Java exception extractor v0.1 4 | ## 5 | ## Usage: exctractor.py [options] LOG_DIR1 [LOG_DIR2 [LOG_DIR3 [...]]] 6 | ## 7 | ## options: 8 | ## -h, --help show this help message and exit 9 | ## -p FILE_PATTERN, --pattern=FILE_PATTERN Pattern for log files 10 | ## -f FORMAT, --format=FORMAT Output format: CSV (default) or TEXT 11 | ## -g Include group statistics in the report (TEXT mode only; enables TEXT mode) 12 | ## -v Verbose output (TEXT mode only; enables TEXT mode) 13 | ## 14 | ## Description: 15 | ## - Parses all log files (tries to differentiate between plain text and bzip files) 16 | ## - Looks for text blocks that potentially can be exceptions: 17 | ## . block starts with the timestamp 18 | ## . is followed by 1 or more lines without a timestamp 19 | ## - Block is validated to be an exception: 20 | ## . shall contain words 'exception' and 'java' 21 | ## - Block is then dissected into: 22 | ## . log line (one with the timestamp) 23 | ## . first exception line 24 | ## . exception body 25 | ## - If the dissected exception matches one of the grouping rules (as defined in rules definition file) it is recorded 26 | ## - Otherwise it'll get identified by generating MD5 hash to it's body contents: 27 | ## . Proxy objects removed 28 | ## . '(...text...)' lines removed 29 | ## 30 | ## Format of the rules file: 31 | ## 32 | ## 33 | ## 34 | ## 35 | ## 41 | ## 42 | ## 43 | ## 44 | ## Multiple rules can be grouped by using the same group name 45 | ## 46 | ## --------------------------------------------------------------------------------------------------------------- 47 | ## Exctractor - Java exception extractor 48 | ## Copyright (C) 2007 Rytis Sileika 49 | ## 50 | ## This program is free software; you can redistribute it and/or 51 | ## modify it under the terms of the GNU General Public License 52 | ## as published by the Free Software Foundation; either version 2 53 | ## of the License, or (at your option) any later version. 54 | ## 55 | ## This program is distributed in the hope that it will be useful, 56 | ## but WITHOUT ANY WARRANTY; without even the implied warranty of 57 | ## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 58 | ## GNU General Public License for more details. 59 | ## 60 | ## You should have received a copy of the GNU General Public License 61 | ## along with this program; if not, write to the Free Software 62 | ## Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 63 | ## 64 | 65 | import bz2, sys, os, re, md5, random, operator 66 | from xml.dom import minidom 67 | from optparse import OptionParser 68 | 69 | LOG_PATTERN = ".log" 70 | BZLOG_PATTERN = ".log.bz2" 71 | CONFIG_FILE = 'exctractor.xml' 72 | 73 | TPL_SUMMARY = {} 74 | TPL_SUMMARY['csv'] = "%(total)s, %(groups)s" 75 | TPL_SUMMARY['text'] = "=" * 80 + "\nTotal exceptions: %(total)s\nDifferent groups: %(groups)s" 76 | 77 | TS_RE_1 = re.compile("^\d\d\d\d.\d\d.\d\d") 78 | #TS_RE_2 = re.compile("^\d\d.....\d\d\d\d") 79 | #TS_RE_2 = re.compile("^... \d{2}, \d{4} \d{1-2}:\d{2}:\d{2}") 80 | TS_RE_2 = re.compile("^... \d\d, \d\d\d\d") 81 | 82 | OPTIONS = {} 83 | DIRS = [] 84 | 85 | class ExceptionContainer: 86 | def __init__(self): 87 | self.exceptions = {} 88 | self.filters = [] 89 | self.count = 0 90 | 91 | config = minidom.parse(CONFIG_FILE) 92 | 93 | for et in config.getElementsByTagName('exception_types'): 94 | for e in et.getElementsByTagName('exception'): 95 | m = md5.new() 96 | m.update(e.attributes['logline'].value) 97 | m.update(e.attributes['headline'].value) 98 | m.update(e.attributes['body'].value) 99 | self.filters.append({ 'id' : m.hexdigest(), 100 | 'll_re': re.compile(e.attributes['logline'].value), 101 | 'hl_re': re.compile(e.attributes['headline'].value), 102 | 'bl_re': re.compile(e.attributes['body'].value), 103 | 'group': e.attributes['group'].value, 104 | 'desc' : e.attributes['desc'].value, }) 105 | 106 | 107 | def insert(self, suspect_body, f_name=""): 108 | lines = suspect_body.strip().split("\n", 1) 109 | log_l = lines[0] 110 | if self.is_exception(lines[1]): 111 | self.count += 1 112 | try: 113 | hd_l, bd_l = lines[1].strip().split("\n", 1) 114 | except: 115 | hd_l, bd_l = (lines[1].strip(), "") 116 | 117 | logged = False 118 | 119 | for f in self.filters: 120 | if f['ll_re'].search(log_l) and f['hl_re'].search(hd_l) and f['bl_re'].search(bd_l): 121 | logged = True 122 | if f['id'] in self.exceptions: 123 | self.exceptions[f['id']]['count'] += 1 124 | else: 125 | self.exceptions[f['id']] = { 'count' : 1, 126 | 'log_line' : log_l, 127 | 'header' : hd_l, 128 | 'body' : bd_l, 129 | 'f_name' : f_name, 130 | 'desc' : f['desc'], 131 | 'group' : f['group'], } 132 | break 133 | 134 | if not logged: 135 | m = md5.new() 136 | try: 137 | m.update(log_l.split(" ", 3)[2]) 138 | except: 139 | pass 140 | m.update(hd_l) 141 | for ml in bd_l.strip().split("\n"): 142 | if ml: 143 | ml = re.sub("\(.*\)", "", ml) 144 | ml = re.sub("\$Proxy", "", ml) 145 | m.update(ml) 146 | if m.hexdigest() in self.exceptions: 147 | self.exceptions[m.hexdigest()]['count'] += 1 148 | else: 149 | self.exceptions[m.hexdigest()] = { 'count' : 1, 150 | 'log_line' : log_l, 151 | 'header' : hd_l, 152 | 'body' : bd_l, 153 | 'f_name' : f_name, 154 | 'desc' : 'NOT IDENTIFIED', 155 | 'group' : 'unrecognised_'+m.hexdigest(), } 156 | 157 | def is_exception(self, strace): 158 | if strace.lower().find('exception') != -1 and \ 159 | strace.lower().find('java') != -1: 160 | return True 161 | else: 162 | return False 163 | 164 | def print_status(self): 165 | categories = {} 166 | for e in self.exceptions: 167 | if self.exceptions[e]['group'] in categories: 168 | categories[self.exceptions[e]['group']] += self.exceptions[e]['count'] 169 | else: 170 | categories[self.exceptions[e]['group']] = self.exceptions[e]['count'] 171 | if OPTIONS.verbose: 172 | print '-' * 80 173 | print "Filter ID :", e 174 | print "Exception description :", self.exceptions[e]['desc'] 175 | print "Exception group :", self.exceptions[e]['group'] 176 | print "Exception count :", self.exceptions[e]['count'] 177 | print "First file :", self.exceptions[e]['f_name'] 178 | print "First occurrence log line :", self.exceptions[e]['log_line'] 179 | print "Stack trace headline :", self.exceptions[e]['header'] 180 | print "Stack trace :" 181 | print self.exceptions[e]['body'] 182 | 183 | if OPTIONS.verbose: 184 | print '=' * 80 185 | 186 | print TPL_SUMMARY[OPTIONS.format.lower()] % {'total': self.count, 'groups': len(categories)} 187 | 188 | if OPTIONS.groups: 189 | print '=' * 80 190 | for i in sorted(categories.iteritems(), key=operator.itemgetter(1), reverse=True): 191 | print "%8s (%6.2f%%) : %s" % (i[1], 100 * float(i[1]) / float(self.count), i[0] ) 192 | print '* Match first column (group name) with "Exception group" field in detailed list to get the exception details' 193 | 194 | 195 | def get_suspect(g): 196 | line = g.next() 197 | next_line = g.next() 198 | while 1: 199 | if not (TS_RE_1.search(next_line) or TS_RE_2.search(next_line)): 200 | suspect_body = line 201 | while not (TS_RE_1.search(next_line) or TS_RE_2.search(next_line)): 202 | suspect_body += next_line 203 | next_line = g.next() 204 | yield suspect_body 205 | else: 206 | try: 207 | line, next_line = next_line, g.next() 208 | except: 209 | raise StopIteration 210 | 211 | def parse_options(): 212 | global OPTIONS, DIRS 213 | cli_parser = OptionParser("Usage: %prog [options] LOG_DIR1 [LOG_DIR2 [LOG_DIR3 [...]]]") 214 | cli_parser.add_option("-p", "--pattern", dest="file_pattern", help="Pattern for log files", default="") 215 | cli_parser.add_option("-f", "--format", dest="format", help="Output format: CSV (default) or TEXT", default="csv") 216 | cli_parser.add_option("-g", action="store_true", dest="groups", default=False, help="Include group statistics in the report (TEXT mode only; enables TEXT mode)") 217 | cli_parser.add_option("-v", action="store_true", dest="verbose", default=False, help="Verbose output (TEXT mode only; enables TEXT mode)") 218 | 219 | (OPTIONS, ARGS) = cli_parser.parse_args() 220 | if OPTIONS.groups or OPTIONS.verbose: OPTIONS.format = 'text' 221 | if OPTIONS.verbose: OPTIONS.groups = True 222 | 223 | if not ARGS: 224 | cli_parser.print_help() 225 | sys.exit(1) 226 | else: 227 | for dir in ARGS: 228 | for root, dirs, files in os.walk(dir): 229 | DIRS.append(root) 230 | 231 | def main(): 232 | ec = ExceptionContainer() 233 | parse_options() 234 | 235 | for DIR in DIRS: 236 | for file in (DIR + "/" + f for f in os.listdir(DIR) if f.find(LOG_PATTERN) != -1 and f.find(OPTIONS.file_pattern) != -1 ): 237 | try: 238 | if file.find(BZLOG_PATTERN) != -1: 239 | fd = bz2.BZ2File(file, 'r') 240 | else: 241 | fd = open(file, 'r') 242 | g = (line for line in fd) 243 | suspects = get_suspect(g) 244 | for suspect_body in suspects: 245 | ec.insert(suspect_body, f_name=file) 246 | except: 247 | pass 248 | 249 | ec.print_status() 250 | 251 | if __name__ == '__main__': 252 | main() 253 | -------------------------------------------------------------------------------- /978-1-4842-0218-0_Source_Code_Ch07/exctractor.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 10 | 16 | 17 | 18 | -------------------------------------------------------------------------------- /978-1-4842-0218-0_Source_Code_Ch08/check_website_login.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import sys 4 | import urllib2, urllib 5 | import time 6 | from BeautifulSoup import BeautifulSoup 7 | from optparse import OptionParser 8 | 9 | NAGIOS_OK = 0 10 | NAGIOS_WARNING = 1 11 | NAGIOS_CRITICAL = 2 12 | WEBSITE_LOGON = 'https://auth.telegraph.co.uk/sam-ui/login.htm' 13 | WEBSITE_LOGOFF = 'https://auth.telegraph.co.uk/sam-ui/logoff.htm' 14 | WEBSITE_USER = 'user@example.com' 15 | WEBSITE_PASS = 'password' 16 | 17 | def test_logon_logoff(): 18 | opener = urllib2.build_opener(urllib2.HTTPCookieProcessor()) 19 | urllib2.install_opener(opener) 20 | data = urllib.urlencode({'email': WEBSITE_USER, 'password': WEBSITE_PASS}) 21 | status = [] 22 | try: 23 | # test logon 24 | result = opener.open(WEBSITE_LOGON, data) 25 | html_logon = result.read() 26 | soup_logon = BeautifulSoup(html_logon) 27 | logon_ok = validate_logon(soup_logon.head.title.text, result.geturl()) 28 | result.close() 29 | # test logoff 30 | result = opener.open(WEBSITE_LOGOFF) 31 | html_logoff = result.read() 32 | soup_logoff = BeautifulSoup(html_logoff) 33 | logoff_ok = validate_logoff(soup_logoff.head.title.text, result.geturl()) 34 | result.close() 35 | 36 | if logon_ok and logoff_ok: 37 | status = [NAGIOS_OK, 'Logon/logoff operation'] 38 | else: 39 | status = [NAGIOS_CRITICAL, 'ERROR: Failed to logon and then logoff to the web site'] 40 | except: 41 | status = [NAGIOS_CRITICAL, 'ERROR: Failure in the logon/logoff test'] 42 | return status 43 | 44 | def validate_logon(title, redirect_url): 45 | result = True 46 | if title.find('My Account') == -1: 47 | result = False 48 | if redirect_url != 'https://auth.telegraph.co.uk/customer-portal/myaccount/index.html': 49 | result = False 50 | return result 51 | 52 | def validate_logoff(title, redirect_url): 53 | result = True 54 | if title.find('My Account') != -1: 55 | result = False 56 | if redirect_url != 'http://www.telegraph.co.uk/': 57 | result = False 58 | return result 59 | 60 | def main(): 61 | parser = OptionParser() 62 | parser.add_option('-w', dest='time_warn', default=3.8, help="Warning threshold in seconds, defaul: %default") 63 | parser.add_option('-c', dest='time_crit', default=5.8, help="Critical threshold in seconds, default: %default") 64 | (options, args) = parser.parse_args() 65 | if float(options.time_crit) < float(options.time_warn): 66 | options.time_warn = options.time_crit 67 | start = time.time() 68 | code, message = test_logon_logoff() 69 | elapsed = time.time() - start 70 | if code != 0: 71 | print message 72 | sys.exit(code) 73 | else: 74 | if elapsed < float(options.time_warn): 75 | print "OK: Performed %s sucessfully in %f seconds" % (message, elapsed) 76 | sys.exit(NAGIOS_OK) 77 | elif elapsed < float(options.time_crit): 78 | print "WARNING: Performed %s sucessfully in %f seconds" % (message, elapsed) 79 | sys.exit(NAGIOS_WARNING) 80 | else: 81 | print "CRITICAL: Performed %s sucessfully in %f seconds" % (message, elapsed) 82 | sys.exit(NAGIOS_CRITICAL) 83 | 84 | 85 | if __name__ == '__main__': 86 | main() 87 | -------------------------------------------------------------------------------- /978-1-4842-0218-0_Source_Code_Ch08/check_website_login_requests.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import sys 4 | import urllib2, urllib 5 | import time 6 | import requests 7 | from BeautifulSoup import BeautifulSoup 8 | from optparse import OptionParser 9 | 10 | NAGIOS_OK = 0 11 | NAGIOS_WARNING = 1 12 | NAGIOS_CRITICAL = 2 13 | WEBSITE_LOGON = 'https://auth.telegraph.co.uk/sam-ui/login.htm' 14 | WEBSITE_LOGOFF = 'https://auth.telegraph.co.uk/sam-ui/logoff.htm' 15 | WEBSITE_USER = 'user@example.com' 16 | WEBSITE_PASS = 'password' 17 | 18 | def test_logon_logoff(): 19 | session = requests.Session() 20 | form_data = {'email': WEBSITE_USER, 'password': WEBSITE_PASS} 21 | status = [] 22 | try: 23 | # test logon 24 | result = session.post(WEBSITE_LOGON, data=form_data) 25 | html_logon = result.text 26 | soup_logon = BeautifulSoup(html_logon) 27 | logon_ok = validate_logon(soup_logon.head.title.text, result.url) 28 | # test logoff 29 | result = session.get(WEBSITE_LOGOFF) 30 | html_logoff = result.text 31 | soup_logoff = BeautifulSoup(html_logoff) 32 | logoff_ok = validate_logoff(soup_logoff.head.title.text, result.url) 33 | 34 | if logon_ok and logoff_ok: 35 | status = [NAGIOS_OK, 'Logon/logoff operation'] 36 | else: 37 | status = [NAGIOS_CRITICAL, 'ERROR: Failed to logon and then logoff to the web site'] 38 | except: 39 | status = [NAGIOS_CRITICAL, 'ERROR: Failure in the logon/logoff test'] 40 | import traceback 41 | traceback.print_exc() 42 | return status 43 | 44 | def validate_logon(title, redirect_url): 45 | result = True 46 | if title.find('My Account') == -1: 47 | result = False 48 | if redirect_url != 'https://auth.telegraph.co.uk/customer-portal/myaccount/index.html': 49 | result = False 50 | return result 51 | 52 | def validate_logoff(title, redirect_url): 53 | result = True 54 | if title.find('My Account') != -1: 55 | result = False 56 | if redirect_url != 'http://www.telegraph.co.uk': 57 | result = False 58 | return result 59 | 60 | def main(): 61 | parser = OptionParser() 62 | parser.add_option('-w', dest='time_warn', default=3.8, help="Warning threshold in seconds, defaul: %default") 63 | parser.add_option('-c', dest='time_crit', default=5.8, help="Critical threshold in seconds, default: %default") 64 | (options, args) = parser.parse_args() 65 | if float(options.time_crit) < float(options.time_warn): 66 | options.time_warn = options.time_crit 67 | start = time.time() 68 | code, message = test_logon_logoff() 69 | elapsed = time.time() - start 70 | if code != 0: 71 | print message 72 | sys.exit(code) 73 | else: 74 | if elapsed < float(options.time_warn): 75 | print "OK: Performed %s sucessfully in %f seconds" % (message, elapsed) 76 | sys.exit(NAGIOS_OK) 77 | elif elapsed < float(options.time_crit): 78 | print "WARNING: Performed %s sucessfully in %f seconds" % (message, elapsed) 79 | sys.exit(NAGIOS_WARNING) 80 | else: 81 | print "CRITICAL: Performed %s sucessfully in %f seconds" % (message, elapsed) 82 | sys.exit(NAGIOS_CRITICAL) 83 | 84 | 85 | if __name__ == '__main__': 86 | main() 87 | -------------------------------------------------------------------------------- /978-1-4842-0218-0_Source_Code_Ch08/check_website_navigation.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import re 4 | import sys 5 | import urllib2 6 | import urlparse 7 | import time 8 | from BeautifulSoup import BeautifulSoup 9 | from optparse import OptionParser 10 | import traceback 11 | 12 | NAGIOS_OK = 0 13 | NAGIOS_WARNING = 1 14 | NAGIOS_CRITICAL = 2 15 | WEBSITE_ROOT = 'http://www.bbc.co.uk/news/' 16 | 17 | def fetch_top_story(): 18 | status = [] 19 | try: 20 | result = urllib2.urlopen(WEBSITE_ROOT) 21 | html = result.read() 22 | soup = BeautifulSoup(html) 23 | # apparently there are two types of 'top stories' at BBC 24 | # one that spans across two columns with a large picture. this one is 'tshsplash' 25 | # another with the picture sharing the same space. this is called 'tsh' 26 | # either case the tag of class that starts with "tsh" is unique 27 | # i suspect the 'tsh' means TopStoryHeading or something similar. 28 | # the book text assumes 'tshsplash' scenario only! 29 | # this code however handles both cases correctly 30 | top_story_div = soup.find('div', {'id': 'top-story'}) 31 | h2_tag = top_story_div.find('h2', {'class': 'top-story-header '}) 32 | a_tag = h2_tag.find('a', {'class': 'story'}) 33 | story_heading = a_tag.text 34 | topstory_url = '' 35 | if a_tag.has_key('href'): 36 | topstory_url = urlparse.urljoin(WEBSITE_ROOT, a_tag['href']) 37 | else: 38 | status = [NAGIOS_CRITICAL, 'ERROR: Top story anchor tag has no link'] 39 | result = urllib2.urlopen(topstory_url) 40 | html = result.read() 41 | status = [NAGIOS_OK, story_heading] 42 | except: 43 | traceback.print_exc() 44 | status = [NAGIOS_CRITICAL, 'ERROR: Failed to retrieve the top story'] 45 | return status 46 | 47 | 48 | def main(): 49 | parser = OptionParser() 50 | parser.add_option('-w', dest='time_warn', default=1.8, help="Warning threshold in seconds, defaul: %default") 51 | parser.add_option('-c', dest='time_crit', default=3.8, help="Critical threshold in seconds, default: %default") 52 | (options, args) = parser.parse_args() 53 | if float(options.time_crit) < float(options.time_warn): 54 | options.time_warn = options.time_crit 55 | start = time.time() 56 | code, message = fetch_top_story() 57 | elapsed = time.time() - start 58 | if code != 0: 59 | print message 60 | sys.exit(code) 61 | else: 62 | if elapsed < float(options.time_warn): 63 | print "OK: Top story '%s' retrieved in %f seconds" % (message, elapsed) 64 | sys.exit(NAGIOS_OK) 65 | elif elapsed < float(options.time_crit): 66 | print "WARNING: Top story '%s' retrieved in %f seconds" % (message, elapsed) 67 | sys.exit(NAGIOS_WARNING) 68 | else: 69 | print "CRITICAL: Top story '%s' retrieved in %f seconds" % (message, elapsed) 70 | sys.exit(NAGIOS_CRITICAL) 71 | 72 | 73 | if __name__ == '__main__': 74 | main() 75 | -------------------------------------------------------------------------------- /978-1-4842-0218-0_Source_Code_Ch09/client-server/client/client.cfg: -------------------------------------------------------------------------------- 1 | [sensor] 2 | executable = check 3 | help = options 4 | path = sensors/ 5 | backup = sensors_backup/ 6 | 7 | [monitor] 8 | url = http://192.168.1.65:8081/xmlrpc/ 9 | 10 | -------------------------------------------------------------------------------- /978-1-4842-0218-0_Source_Code_Ch09/client-server/client/client_daemon.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import cherrypy 4 | import subprocess 5 | import os, sys 6 | import shutil 7 | import xmlrpclib 8 | import socket 9 | import tempfile 10 | import tarfile 11 | from cherrypy import _cptools 12 | from datetime import datetime 13 | from ConfigParser import SafeConfigParser 14 | 15 | 16 | class Root(_cptools.XMLRPCController): 17 | def __init__(self, conf_manager): 18 | self.cm = conf_manager 19 | 20 | @cherrypy.expose 21 | def cmd_submit_reading(self, ticket, sensor_name, arguments=None): 22 | cmd = ['%s/%s/%s' % (cm.sensor.path, sensor_name, cm.sensor.executable)] 23 | if arguments: 24 | cmd.extend([str(a) for a in arguments]) 25 | p = subprocess.Popen(cmd, stdout=subprocess.PIPE) 26 | resp_msg = p.communicate()[0] 27 | ret_code = p.returncode 28 | self.submit_reading(ticket, ret_code, resp_msg) 29 | return (ret_code, resp_msg) 30 | 31 | @cherrypy.expose 32 | def cmd_list_sensors(self): 33 | sensors = [d for d in os.listdir(self.cm.sensor.path) if 34 | os.path.isdir('%s/%s' % (self.cm.sensor.path, d)) and 35 | os.path.exists('%s/%s/%s' % (self.cm.sensor.path, d, self.cm.sensor.executable))] 36 | result = {} 37 | for s in sensors: 38 | cmd = '%(dir)s/%(sensor)s/%(exec)s %(opt)s' % { 'dir': self.cm.sensor.path, 39 | 'sensor': s, 40 | 'exec': self.cm.sensor.executable, 41 | 'opt': self.cm.sensor.help } 42 | p = subprocess.Popen(cmd.split(), stdout=subprocess.PIPE) 43 | result[s] = p.communicate()[0] 44 | return result 45 | 46 | @cherrypy.expose 47 | def cmd_register_new_server(self): 48 | hostname = socket.gethostname() 49 | proxy = xmlrpclib.ServerProxy(self.cm.monitor.url) 50 | url = proxy.cmd_get_new_monitor_url(hostname) 51 | try: 52 | proxy = xmlrpclib.ServerProxy(url) 53 | res = proxy.healthcheck() 54 | if res == 'OK': 55 | self.cm.monitor.url = url 56 | self.cm.save_config() 57 | except: 58 | pass 59 | return 'OK' 60 | 61 | @cherrypy.expose 62 | def cmd_update_sensor_code(self, sensor): 63 | # get the new file 64 | proxy = xmlrpclib.ServerProxy(self.cm.monitor.url) 65 | tmp_dir = tempfile.mkdtemp(dir='.') 66 | dst_file = "%s/%s.tar.bz2" % (tmp_dir, sensor) 67 | with open(dst_file, 'wb') as f: 68 | f.write(proxy.cmd_get_sensor_code(sensor).data) 69 | f.close() 70 | # unpack it 71 | arch = tarfile.open(dst_file) 72 | arch.extractall(path=tmp_dir) 73 | arch.close() 74 | # check it 75 | cmd = ["%s/%s/%s" % (tmp_dir, sensor, self.cm.sensor.executable), "options"] 76 | p = subprocess.Popen(cmd, stdout=subprocess.PIPE) 77 | p.communicate() 78 | if p.returncode != 0: 79 | # remove if fails 80 | shutil.rmtree(tmp_dir) 81 | else: 82 | # backup the existing package 83 | sens_dir = "%s/%s" % (self.cm.sensor.path, sensor) 84 | bck_dir = "%s/%s_%s" % (self.cm.sensor.backup, sensor, datetime.strftime(datetime.now(), '%Y-%m-%dT%H:%M:%S')) 85 | try: 86 | shutil.move(sens_dir, bck_dir) 87 | except: 88 | pass 89 | os.remove(dst_file) 90 | # replace with new 91 | shutil.move("%s/%s" % (tmp_dir, sensor), sens_dir) 92 | os.rmdir(tmp_dir) 93 | return 'OK' 94 | 95 | def submit_reading(self, ticket, ret_code, resp_msg): 96 | proxy = xmlrpclib.ServerProxy(self.cm.monitor.url) 97 | tstamp = datetime.now() 98 | res = proxy.cmd_store_probe_data(ticket, [ret_code, resp_msg], tstamp) 99 | return 100 | 101 | 102 | class ConfigManager(object): 103 | 104 | class Section: 105 | def __init__(self, name, parser): 106 | self.__dict__['name'] = name 107 | self.__dict__['parser'] = parser 108 | #reason for using __dict__ is that __setattr__ is called to initiate all local class attrs and so 109 | #declaring the below would cause __setattr__ to be called and obviously neither name nor parser exist yet 110 | #self.name = name 111 | #self.parser = parser 112 | 113 | def __setattr__(self, option, value): 114 | self.__dict__[option] = str(value) 115 | self.parser.set(self.name, option, str(value)) 116 | 117 | def __init__(self, file_name): 118 | self.parser = SafeConfigParser() 119 | self.parser.read(file_name) 120 | self.file_name = file_name 121 | for section in self.parser.sections(): 122 | setattr(self, section, self.Section(section, self.parser)) 123 | for option in self.parser.options(section): 124 | setattr(getattr(self, section), option, self.parser.get(section, option)) 125 | 126 | def __getattr__(self, section): 127 | self.parser.add_section(section) 128 | setattr(self, section, Section(section, self.parser)) 129 | return getattr(self, section) 130 | 131 | def save_config(self): 132 | f = open(self.file_name, 'w') 133 | self.parser.write(f) 134 | f.close() 135 | 136 | 137 | 138 | if __name__ == '__main__': 139 | cm = ConfigManager('client.cfg') 140 | cherrypy.quickstart(Root(cm), '/xmlrpc') 141 | -------------------------------------------------------------------------------- /978-1-4842-0218-0_Source_Code_Ch09/client-server/client/sensors/cpu_load/check: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | function list_options { 4 | echo 'idle - Idle CPU %' 5 | echo 'used - Total CPU used % ' 6 | echo 'user - CPU performing user tasks %' 7 | echo 'system - CPU performing system tasks %' 8 | echo 'iowait - CPU in IO wait state %' 9 | } 10 | 11 | #08:22:33 PM CPU %usr %nice %sys %iowait %irq %soft %steal %guest %idle 12 | #08:22:33 PM all 2.02 0.04 1.20 1.19 0.01 0.01 0.00 0.00 95.54 13 | 14 | if [ $# -eq 0 ] 15 | then 16 | RESULT=$(mpstat|awk '/all/ {print 100-$12}') 17 | else 18 | case $1 in 19 | 'options') 20 | list_options 21 | exit 0 22 | ;; 23 | 'used') 24 | RESULT=$(mpstat|awk '/all/ {print 100-$12}') 25 | ;; 26 | 'idle') 27 | RESULT=$(mpstat|awk '/all/ {print $12}') 28 | ;; 29 | 'user') 30 | RESULT=$(mpstat|awk '/all/ {print $4}') 31 | ;; 32 | 'system') 33 | RESULT=$(mpstat|awk '/all/ {print $6}') 34 | ;; 35 | 'iowait') 36 | RESULT=$(mpstat|awk '/all/ {print $7}') 37 | ;; 38 | *) 39 | RESULT=$(mpstat|awk '/all/ {print 100-$12}') 40 | ;; 41 | esac 42 | 43 | fi 44 | 45 | echo ${RESULT} 46 | 47 | -------------------------------------------------------------------------------- /978-1-4842-0218-0_Source_Code_Ch09/client-server/client/sensors/disk/check: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | function list_options { 4 | echo 'percent - free space %' 5 | echo 'used - used in KB' 6 | echo 'free - free in KB' 7 | } 8 | 9 | # Filesystem 1024-blocks Used Available Capacity Mounted on 10 | # /dev/mapper/vg_fedolin-lv_root 36220536 32341148 3511840 91% / 11 | # /dev/sda1 198337 64021 124076 35% /boot 12 | 13 | if [ $# -ne 2 ] 14 | then 15 | list_options 16 | else 17 | case $1 in 18 | 'options') 19 | list_options 20 | exit 0 21 | ;; 22 | 'percent') 23 | RESULT=$(df -kP $2|awk '/%/ {sub(/%/, "", $5); print $5}') 24 | ;; 25 | 'used') 26 | RESULT=$(df -kP $2|awk '/%/ {print $3}') 27 | ;; 28 | 'free') 29 | RESULT=$(df -kP $2|awk '/%/ {print $4}') 30 | ;; 31 | *) 32 | RESULT=$(df -kP --total|awk '/total/ {sub(/%/, "", $5); print $5}') 33 | ;; 34 | esac 35 | 36 | fi 37 | 38 | echo ${RESULT} 39 | 40 | -------------------------------------------------------------------------------- /978-1-4842-0218-0_Source_Code_Ch09/client-server/client/sensors/memory/check: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | function list_options { 4 | echo 'free_pct - Free memory, %' 5 | echo 'free - Free memory, in bytes' 6 | echo 'used_pct - Used memory, %' 7 | echo 'used - Used memory, in bytes' 8 | echo 'swap_used_pct - Used swap, %' 9 | } 10 | 11 | if [ $# -eq 0 ] 12 | then 13 | RESULT=$(mpstat|awk '/all/ {print 100-$11}') 14 | else 15 | case $1 in 16 | 'options') 17 | list_options 18 | exit 0 19 | ;; 20 | 'free_pct') 21 | RESULT=$(free|awk '/Mem/ {print $4/$2*100}') 22 | ;; 23 | 'free') 24 | RESULT=$(free|awk '/Mem/ {print $4}') 25 | ;; 26 | 'used_pct') 27 | RESULT=$(free|awk '/Mem/ {print $3/$2*100}') 28 | ;; 29 | 'used') 30 | RESULT=$(free|awk '/Mem/ {print $3}') 31 | ;; 32 | 'swap_used_pct') 33 | RESULT=$(free|awk '/Swap/ {print $3/$2*100}') 34 | ;; 35 | *) 36 | RESULT=$(mpstat|awk '/all/ {print 100-$11}') 37 | ;; 38 | esac 39 | 40 | fi 41 | 42 | echo ${RESULT} 43 | 44 | -------------------------------------------------------------------------------- /978-1-4842-0218-0_Source_Code_Ch09/client-server/client/sensors/processes/check: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | function list_options { 4 | echo 'load1 - Load average, 1 min' 5 | echo 'load5 - Load average, 5 min' 6 | echo 'load15 - Load average, 15 min' 7 | echo 'running - Number of running processes' 8 | echo 'total - Total number of processes' 9 | } 10 | 11 | if [ $# -eq 0 ] 12 | then 13 | RESULT=$(awk '{print $1}' /proc/loadavg) 14 | else 15 | case $1 in 16 | 'options') 17 | list_options 18 | exit 0 19 | ;; 20 | 'load1') 21 | RESULT=$(awk '{print $1}' /proc/loadavg) 22 | ;; 23 | 'load5') 24 | RESULT=$(awk '{print $2}' /proc/loadavg) 25 | ;; 26 | 'load15') 27 | RESULT=$(awk '{print $3}' /proc/loadavg) 28 | ;; 29 | 'running') 30 | RESULT=$(awk '{print $4}' /proc/loadavg|awk -F / '{print $1}') 31 | ;; 32 | 'total') 33 | RESULT=$(awk '{print $4}' /proc/loadavg|awk -F / '{print $2}') 34 | ;; 35 | *) 36 | RESULT=$(awk '{print $1}' /proc/loadavg) 37 | ;; 38 | esac 39 | 40 | fi 41 | 42 | echo ${RESULT} 43 | 44 | -------------------------------------------------------------------------------- /978-1-4842-0218-0_Source_Code_Ch09/client-server/client/sensors_backup/disk_2010-03-07T21_24_33/check: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | function list_options { 4 | echo 'percent - free space %' 5 | echo 'used - used in KB' 6 | echo 'free - free in KB' 7 | } 8 | 9 | # Filesystem 1024-blocks Used Available Capacity Mounted on 10 | # /dev/mapper/vg_fedolin-lv_root 36220536 32341148 3511840 91% / 11 | # /dev/sda1 198337 64021 124076 35% /boot 12 | 13 | if [ $# -ne 2 ] 14 | then 15 | list_options 16 | else 17 | case $1 in 18 | 'options') 19 | list_options 20 | exit 0 21 | ;; 22 | 'percent') 23 | RESULT=$(df -kP $2|awk '/%/ {sub(/%/, "", $5); print $5}') 24 | ;; 25 | 'used') 26 | RESULT=$(df -kP $2|awk '/%/ {print $3}') 27 | ;; 28 | 'free') 29 | RESULT=$(df -kP $2|awk '/%/ {print $4}') 30 | ;; 31 | *) 32 | RESULT=$(df -kP --total|awk '/total/ {sub(/%/, "", $5); print $5}') 33 | ;; 34 | esac 35 | 36 | fi 37 | 38 | echo ${RESULT} 39 | 40 | -------------------------------------------------------------------------------- /978-1-4842-0218-0_Source_Code_Ch09/client-server/client/sensors_backup/disk_2010-03-07T21_25_16/check: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | function list_options { 4 | echo 'percent - free space %' 5 | echo 'used - used in KB' 6 | echo 'free - free in KB' 7 | } 8 | 9 | # Filesystem 1024-blocks Used Available Capacity Mounted on 10 | # /dev/mapper/vg_fedolin-lv_root 36220536 32341148 3511840 91% / 11 | # /dev/sda1 198337 64021 124076 35% /boot 12 | 13 | if [ $# -ne 2 ] 14 | then 15 | list_options 16 | else 17 | case $1 in 18 | 'options') 19 | list_options 20 | exit 0 21 | ;; 22 | 'percent') 23 | RESULT=$(df -kP $2|awk '/%/ {sub(/%/, "", $5); print $5}') 24 | ;; 25 | 'used') 26 | RESULT=$(df -kP $2|awk '/%/ {print $3}') 27 | ;; 28 | 'free') 29 | RESULT=$(df -kP $2|awk '/%/ {print $4}') 30 | ;; 31 | *) 32 | RESULT=$(df -kP --total|awk '/total/ {sub(/%/, "", $5); print $5}') 33 | ;; 34 | esac 35 | 36 | fi 37 | 38 | echo ${RESULT} 39 | 40 | -------------------------------------------------------------------------------- /978-1-4842-0218-0_Source_Code_Ch09/client-server/client/sensors_backup/disk_2010-03-07T21_27_07/check: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | function list_options { 4 | echo 'percent - free space %' 5 | echo 'used - used in KB' 6 | echo 'free - free in KB' 7 | } 8 | 9 | # Filesystem 1024-blocks Used Available Capacity Mounted on 10 | # /dev/mapper/vg_fedolin-lv_root 36220536 32341148 3511840 91% / 11 | # /dev/sda1 198337 64021 124076 35% /boot 12 | 13 | if [ $# -ne 2 ] 14 | then 15 | list_options 16 | else 17 | case $1 in 18 | 'options') 19 | list_options 20 | exit 0 21 | ;; 22 | 'percent') 23 | RESULT=$(df -kP $2|awk '/%/ {sub(/%/, "", $5); print $5}') 24 | ;; 25 | 'used') 26 | RESULT=$(df -kP $2|awk '/%/ {print $3}') 27 | ;; 28 | 'free') 29 | RESULT=$(df -kP $2|awk '/%/ {print $4}') 30 | ;; 31 | *) 32 | RESULT=$(df -kP --total|awk '/total/ {sub(/%/, "", $5); print $5}') 33 | ;; 34 | esac 35 | 36 | fi 37 | 38 | echo ${RESULT} 39 | 40 | -------------------------------------------------------------------------------- /978-1-4842-0218-0_Source_Code_Ch09/client-server/client/sensors_backup/disk_2010-03-07T21_32_32/check: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | function list_options { 4 | echo 'percent - free space %' 5 | echo 'used - used in KB' 6 | echo 'free - free in KB' 7 | } 8 | 9 | # Filesystem 1024-blocks Used Available Capacity Mounted on 10 | # /dev/mapper/vg_fedolin-lv_root 36220536 32341148 3511840 91% / 11 | # /dev/sda1 198337 64021 124076 35% /boot 12 | 13 | if [ $# -ne 2 ] 14 | then 15 | list_options 16 | else 17 | case $1 in 18 | 'options') 19 | list_options 20 | exit 0 21 | ;; 22 | 'percent') 23 | RESULT=$(df -kP $2|awk '/%/ {sub(/%/, "", $5); print $5}') 24 | ;; 25 | 'used') 26 | RESULT=$(df -kP $2|awk '/%/ {print $3}') 27 | ;; 28 | 'free') 29 | RESULT=$(df -kP $2|awk '/%/ {print $4}') 30 | ;; 31 | *) 32 | RESULT=$(df -kP --total|awk '/total/ {sub(/%/, "", $5); print $5}') 33 | ;; 34 | esac 35 | 36 | fi 37 | 38 | echo ${RESULT} 39 | 40 | -------------------------------------------------------------------------------- /978-1-4842-0218-0_Source_Code_Ch09/client-server/server/MonitorLib.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import xmlrpclib 4 | import sqlite3 5 | 6 | class MonitorConfig(): 7 | def __init__(self): 8 | self.COMMANDS = { 'read': ['address', 'sensor'], 9 | 'set_server': ['address'], 10 | 'update_sensor': ['address', 'sensor'], 11 | 'list_sensors': ['address'], 12 | } 13 | 14 | class MonitorClient(): 15 | 16 | def __init__(self, address, port='8080'): 17 | self.url = "http://%(address)s:%(port)s/xmlrpc/" % {'address': address, 'port': port} 18 | self.sensor = None 19 | self.sensor_opts = None 20 | self.proxy = self.get_xmlrpc_proxy() 21 | 22 | def set_sensor(self, name, options=None): 23 | self.sensor = name 24 | self.sensor_opts = options 25 | 26 | def get_xmlrpc_proxy(self): 27 | proxy = xmlrpclib.ServerProxy(self.url, allow_none=True) 28 | return proxy 29 | 30 | # All 'command' functions must have the following naming convention: 'execute_' 31 | def execute_read(self, hostprobe=None): 32 | # the code below tries to find hostprobe id by matching supplied 33 | # hostname, probe and options strings against the configuration DB 34 | # if found, it will then raise a request ticket and submit it to the client 35 | 36 | #if not hostprobe: 37 | # con = sqlite3.connect('monitor.db') 38 | # res = con.execute('SELECT * from 39 | 40 | # the following is for unconditional execution (used for testing only) 41 | return self.proxy.cmd_submit_reading(self.sensor, self.sensor_opts) 42 | 43 | def execute_list_sensors(self): 44 | return self.proxy.cmd_list_sensors() 45 | 46 | 47 | 48 | 49 | 50 | if __name__ == '__main__': 51 | pass 52 | -------------------------------------------------------------------------------- /978-1-4842-0218-0_Source_Code_Ch09/client-server/server/monitor.db: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Apress/pro-python-system-admin-14/2b70cbe957969ce2f72e59858ffc4c0bbfe77a05/978-1-4842-0218-0_Source_Code_Ch09/client-server/server/monitor.db -------------------------------------------------------------------------------- /978-1-4842-0218-0_Source_Code_Ch09/client-server/server/monitor_cli.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | from MonitorLib import * 4 | from optparse import OptionParser 5 | import sys 6 | 7 | OPTIONS = {} 8 | ARGS = [] 9 | 10 | 11 | def parse_options(cfg): 12 | global OPTIONS, ARGS 13 | 14 | usage = """Usage: %prog command [options]""" 15 | p = OptionParser(usage) 16 | p.add_option('-a', '--address', dest='address', default=None, metavar='ADDRESS', 17 | help='Connect to the client at ADDRESS') 18 | p.add_option('-p', '--port', dest='port', default='8080', metavar='PORT', 19 | help='Specify port number [default: %default]') 20 | p.add_option('-s', '--sensor', dest='sensor', default=None, metavar='SENSOR', 21 | help='Request readings from sensor named SENSOR') 22 | p.add_option('-o', '--options', dest='options', default=None, metavar='STRING', 23 | help='Pass STRING as a comma separated list to the sensor') 24 | p.add_option('-h', '--hostprobe', dest='hostprobe', default=None, metavar='HOSTPROBEID', 25 | help='Run specific hostprobe check, all options ignored and data from DB is used') 26 | 27 | (OPTIONS, ARGS) = p.parse_args() 28 | 29 | # do we have incorrect number of arguments? 30 | if len(ARGS) != 1: 31 | p.print_help() 32 | sys.exit(-1) 33 | # is this a valid command? 34 | if ARGS[0] not in cfg.COMMANDS: 35 | print "ERROR: Unknown command '%s'" % ARGS[0] 36 | print 'Valid commands are:' 37 | for c in cfg.COMMANDS: 38 | print c 39 | p.print_help() 40 | sys.exit(-1) 41 | # do we have all required vars for the command? 42 | for var in cfg.COMMANDS[ARGS[0]]: 43 | if not getattr(OPTIONS, var): 44 | print "ERROR: Missing required option: '%s'" % var 45 | p.print_help() 46 | sys.exit(-1) 47 | 48 | class CommandLineReport(): 49 | def __init__(self): 50 | pass 51 | 52 | def read(self, result): 53 | print result 54 | 55 | def list_sensors(self, sensor_list): 56 | print 'The following sensors are available:' 57 | for sensor_name, options in sensor_list.iteritems(): 58 | print '- %s' % sensor_name 59 | for opt in options.strip().split("\n"): 60 | print ' %s' % opt 61 | 62 | 63 | if __name__ == '__main__': 64 | cfg = MonitorConfig() 65 | parse_options(cfg) 66 | mc = MonitorClient(OPTIONS.address, OPTIONS.port) 67 | if OPTIONS.sensor: 68 | opt_list = [o for o in OPTIONS.options.split(',')] if OPTIONS.options else None 69 | mc.set_sensor(OPTIONS.sensor, opt_list) 70 | res = getattr(mc, "execute_%s" % ARGS[0])() 71 | clr = CommandLineReport() 72 | getattr(clr, ARGS[0])(res) 73 | -------------------------------------------------------------------------------- /978-1-4842-0218-0_Source_Code_Ch09/client-server/server/monitor_daemon.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import cherrypy 4 | import sqlite3 5 | import socket 6 | from cherrypy import _cptools 7 | from MonitorLib import * 8 | from ConfigParser import SafeConfigParser 9 | 10 | class Root(_cptools.XMLRPCController): 11 | 12 | def __init__(self, cm): 13 | self.cm = cm 14 | 15 | @cherrypy.expose 16 | def cmd_store_probe_data(self, ticket, probe, tstamp): 17 | # probe - [ret_code, data_string] 18 | self.store_reading(ticket, probe, tstamp) 19 | return 'OK' 20 | 21 | @cherrypy.expose 22 | def cmd_get_new_monitor_url(self, host): 23 | port = cherrypy.config.get('server.socket_port') if cherrypy.config.get('server.socket_port') else 8080 24 | host = cherrypy.config.get('server.socket_host') if cherrypy.config.get('server.socket_host') else '127.0.0.1' 25 | server_url = "http://%s:%s/xmlrpc/" % (host, str(port)) 26 | con = sqlite3.connect('monitor.db') 27 | res = con.execute("""SELECT hostparams.value 28 | FROM hostparams, host, systemparams 29 | WHERE host.id = hostparams.host_id 30 | AND systemparams.name = 'monitor_url' 31 | AND hostparams.param_id = systemparams.id 32 | AND host.address = ?""", (host,) ).fetchone() 33 | if not res: 34 | res = con.execute("SELECT value FROM systemparams WHERE name = 'monitor_url'").fetchone() 35 | if res: 36 | server_url = res[0] 37 | return server_url 38 | 39 | @cherrypy.expose 40 | def cmd_get_sensor_code(self, sensor): 41 | with open("%s/%s.tar.bz2" % (self.cm.sensor.source_dir, sensor), 'rb') as f: 42 | return xmlrpclib.Binary(f.read()) 43 | 44 | 45 | @cherrypy.expose 46 | def healthcheck(self): 47 | return 'OK' 48 | 49 | 50 | def store_reading(ticket, probe, tstamp): 51 | con = sqlite3.connect('monitor.db') 52 | res = [r[0] for r in con.execute('SELECT hostprobe_id FROM ticketqueue WHERE id=?', (ticket,) )][0] 53 | if res: 54 | con.execute('DELETE FROM ticketqueue WHERE id=?', (ticket,) ) 55 | con.execute('INSERT INTO probereading VALUES (NULL, ?, ?, ?, ?)', (res, str(tstamp), float(probe[1].strip()), int(probe[0]))) 56 | con.commit() 57 | else: 58 | print 'Ticket does not exist: %s' % str(ticket) 59 | 60 | class ConfigManager(object): 61 | 62 | class Section: 63 | def __init__(self, name, parser): 64 | self.__dict__['name'] = name 65 | self.__dict__['parser'] = parser 66 | #reason for using __dict__ is that __setattr__ is called to initiate all local class attrs and so 67 | #declaring the below would cause __setattr__ to be called and obviously neither name nor parser exist yet 68 | #self.name = name 69 | #self.parser = parser 70 | 71 | def __setattr__(self, option, value): 72 | self.__dict__[option] = str(value) 73 | self.parser.set(self.name, option, str(value)) 74 | 75 | def __init__(self, file_name): 76 | self.parser = SafeConfigParser() 77 | self.parser.read(file_name) 78 | self.file_name = file_name 79 | for section in self.parser.sections(): 80 | setattr(self, section, self.Section(section, self.parser)) 81 | for option in self.parser.options(section): 82 | setattr(getattr(self, section), option, self.parser.get(section, option)) 83 | 84 | def __getattr__(self, option): 85 | self.parser.add_section(option) 86 | setattr(self, option, Section(option, self.parser)) 87 | return getattr(self, option) 88 | 89 | def save_config(self): 90 | f = open(self.file_name, 'w') 91 | self.parser.write(f) 92 | f.close() 93 | 94 | 95 | 96 | if __name__ == '__main__': 97 | cm = ConfigManager('server.cfg') 98 | cherrypy.config.update({'server.socket_port': 8081}) 99 | cherrypy.config.update({'server.socket_host': socket.gethostname()}) 100 | cherrypy.quickstart(Root(cm), '/xmlrpc') 101 | 102 | -------------------------------------------------------------------------------- /978-1-4842-0218-0_Source_Code_Ch09/client-server/server/monitor_db_init.sql: -------------------------------------------------------------------------------- 1 | 2 | -- ****************************************** 3 | -- Table: SENSOR 4 | -- Description: List of all available sensors 5 | 6 | DROP TABLE IF EXISTS sensor; 7 | 8 | CREATE TABLE sensor ( 9 | id INTEGER PRIMARY KEY, 10 | name TEXT 11 | ); 12 | 13 | INSERT INTO sensor VALUES (1, 'cpu_load'); 14 | INSERT INTO sensor VALUES (2, 'memory'); 15 | INSERT INTO sensor VALUES (3, 'processes'); 16 | 17 | -- ****************************************** 18 | -- Table: PROBE 19 | -- Description: Adds parameter list to sensor command 20 | -- and defines default thresholds 21 | 22 | DROP TABLE IF EXISTS probe; 23 | 24 | CREATE TABLE probe ( 25 | id INTEGER PRIMARY KEY, 26 | sensor_id INTEGER, 27 | name TEXT, 28 | parameter TEXT, 29 | warning FLOAT, 30 | error FLOAT, 31 | FOREIGN KEY (sensor_id) REFERENCES sensor(id) 32 | ); 33 | 34 | INSERT INTO probe VALUES ( 1, 1, 'Idle CPU %', 'idle', NULL, NULL); 35 | INSERT INTO probe VALUES ( 2, 1, 'Used CPU %', 'used', NULL, NULL); 36 | INSERT INTO probe VALUES ( 3, 1, 'User CPU %', 'user', NULL, NULL); 37 | INSERT INTO probe VALUES ( 4, 1, 'System CPU %', 'system', NULL, NULL); 38 | INSERT INTO probe VALUES ( 5, 1, 'IO Wait CPU %', 'iowait', NULL, NULL); 39 | INSERT INTO probe VALUES ( 6, 2, 'Free memory, %', 'free_pct', NULL, NULL); 40 | INSERT INTO probe VALUES ( 7, 2, 'Free memory, in bytes', 'free', NULL, NULL); 41 | INSERT INTO probe VALUES ( 8, 2, 'Used memory, %', 'used_pct', NULL, NULL); 42 | INSERT INTO probe VALUES ( 9, 2, 'Used memory, in bytes', 'used', NULL, NULL); 43 | INSERT INTO probe VALUES (10, 2, 'Used swap, %', 'swap_used_pct', NULL, NULL); 44 | INSERT INTO probe VALUES (11, 3, '1 min load average', 'load1', NULL, NULL); 45 | INSERT INTO probe VALUES (12, 3, '5 min load average', 'load5', NULL, NULL); 46 | INSERT INTO probe VALUES (13, 3, '15 min load average', 'load15', NULL, NULL); 47 | INSERT INTO probe VALUES (14, 3, 'Running processes', 'running', NULL, NULL); 48 | INSERT INTO probe VALUES (15, 3, 'Total processes', 'total', NULL, NULL); 49 | 50 | -- ****************************************** 51 | -- Table: HOST 52 | -- Description: List of all monitoring agents 53 | 54 | DROP TABLE IF EXISTS host; 55 | 56 | CREATE TABLE host ( 57 | id INTEGER PRIMARY KEY, 58 | name TEXT, 59 | address TEXT, 60 | port TEXT 61 | ); 62 | 63 | INSERT INTO host VALUES (1, 'My laptop', 'localhost', '8080'); 64 | 65 | -- ****************************************** 66 | -- Table: HOSTPROBE 67 | -- Description: Maps available probes to the hosts 68 | -- overrides thresholds if required 69 | 70 | DROP TABLE IF EXISTS hostprobe; 71 | 72 | CREATE TABLE hostprobe ( 73 | id INTEGER PRIMARY KEY, 74 | probe_id INTEGER, 75 | host_id INTEGER, 76 | warning FLOAT, 77 | error FLOAT, 78 | FOREIGN KEY (probe_id) REFERENCES probe(id), 79 | FOREIGN KEY (host_id) REFERENCES host(id) 80 | ); 81 | 82 | 83 | INSERT INTO hostprobe VALUES ( 1, 1, 1, NULL, NULL); 84 | INSERT INTO hostprobe VALUES ( 2, 2, 1, NULL, NULL); 85 | INSERT INTO hostprobe VALUES ( 3, 3, 1, NULL, NULL); 86 | INSERT INTO hostprobe VALUES ( 4, 4, 1, NULL, NULL); 87 | INSERT INTO hostprobe VALUES ( 5, 5, 1, NULL, NULL); 88 | INSERT INTO hostprobe VALUES ( 6, 6, 1, NULL, NULL); 89 | INSERT INTO hostprobe VALUES ( 7, 7, 1, NULL, NULL); 90 | INSERT INTO hostprobe VALUES ( 8, 8, 1, NULL, NULL); 91 | INSERT INTO hostprobe VALUES ( 9, 9, 1, NULL, NULL); 92 | INSERT INTO hostprobe VALUES (10, 10, 1, NULL, NULL); 93 | INSERT INTO hostprobe VALUES (11, 11, 1, NULL, NULL); 94 | INSERT INTO hostprobe VALUES (12, 12, 1, NULL, NULL); 95 | INSERT INTO hostprobe VALUES (13, 13, 1, NULL, NULL); 96 | INSERT INTO hostprobe VALUES (14, 14, 1, NULL, NULL); 97 | INSERT INTO hostprobe VALUES (15, 15, 1, NULL, NULL); 98 | 99 | -- ****************************************** 100 | -- Table: TICKETQUEUE 101 | -- Description: Holds all pendiing and sent tickets 102 | -- tickets are removed when the sensor reading arrive 103 | 104 | DROP TABLE IF EXISTS ticketqueue; 105 | 106 | CREATE TABLE ticketqueue ( 107 | id INTEGER PRIMARY KEY, 108 | hostprobe_id INTEGER, 109 | timestamp TEXT, 110 | dispatched INTEGER, 111 | FOREIGN KEY (hostprobe_id) REFERENCES hostprobe(id) 112 | ); 113 | 114 | -- ****************************************** 115 | -- Table: PROBEREADING 116 | -- Description: Stores all readings obtained from the monitoring agents 117 | 118 | DROP TABLE IF EXISTS probereading; 119 | 120 | CREATE TABLE probereading ( 121 | id INTEGER PRIMARY KEY, 122 | hostprobe_id INTEGER, 123 | timestamp TEXT, 124 | probe_value FLOAT, 125 | ret_code INTEGER, 126 | FOREIGN KEY (hostprobe_id) REFERENCES hostprobe(id) 127 | ); 128 | 129 | -- ****************************************** 130 | -- Table: PROBINGSCHEDULE 131 | -- Description: Defines execution intervals for the probes 132 | 133 | DROP TABLE IF EXISTS probingschedule; 134 | 135 | CREATE TABLE probingschedule ( 136 | id INTEGER PRIMARY KEY, 137 | hostprobe_id INTEGER, 138 | probeinterval INTEGER, 139 | FOREIGN KEY (hostprobe_id) REFERENCES hostprobe(id) 140 | ); 141 | 142 | INSERT INTO probingschedule VALUES (1, 11, 1); 143 | INSERT INTO probingschedule VALUES (2, 15, 1); 144 | INSERT INTO probingschedule VALUES (3, 8, 5); 145 | INSERT INTO probingschedule VALUES (4, 10, 5); 146 | 147 | -- ****************************************** 148 | -- Table: SYSTEMPARAMS 149 | -- Description: Defines system configuration parameters 150 | 151 | DROP TABLE IF EXISTS systemparams; 152 | 153 | CREATE TABLE systemparams ( 154 | id INTEGER PRIMARY KEY, 155 | name TEXT, 156 | value TEXT 157 | ); 158 | 159 | INSERT INTO systemparams VALUES (1, 'monitor_url', 'http://localhost:8081/xmlrpc/'); 160 | 161 | -- ****************************************** 162 | -- Table: HOSTPARAMS 163 | -- Description: Assigns system parameters to the hosts 164 | -- allows to override the default values 165 | 166 | DROP TABLE IF EXISTS hostparams; 167 | 168 | CREATE TABLE hostparams ( 169 | id INTEGER PRIMARY KEY, 170 | host_id INTEGER, 171 | param_id INTEGER, 172 | value TEXT, 173 | FOREIGN KEY (host_id) REFERENCES host(id), 174 | FOREIGN KEY (param_id) REFERENCES systemparams(id) 175 | ); 176 | 177 | INSERT INTO hostparams VALUES (1, 1, 1, 'http://localhost:8081/xmlrpc/'); 178 | 179 | -------------------------------------------------------------------------------- /978-1-4842-0218-0_Source_Code_Ch09/client-server/server/monitor_scheduler.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import sqlite3 4 | import time 5 | import sys 6 | import multiprocessing 7 | import xmlrpclib 8 | from datetime import datetime 9 | 10 | 11 | class Oscillator(multiprocessing.Process): 12 | 13 | def __init__(self, event, period): 14 | self.period = period 15 | self.event = event 16 | super(Oscillator, self).__init__() 17 | 18 | def run(self): 19 | try: 20 | while True: 21 | self.event.clear() 22 | time.sleep(self.period) 23 | self.event.set() 24 | except KeyboardInterrupt: 25 | pass 26 | 27 | 28 | class TicketScheduler(multiprocessing.Process): 29 | 30 | def __init__(self, event): 31 | self.event = event 32 | self.con = sqlite3.connect('monitor.db') 33 | super(TicketScheduler, self).__init__() 34 | 35 | def run(self): 36 | try: 37 | from datetime import datetime 38 | while True: 39 | self.event.wait() 40 | res = [r[0] for r in self.con.execute("""SELECT hostprobe_id 41 | FROM probingschedule 42 | WHERE (strftime('%s', 'now')/60) % probingschedule.probeinterval = 0;""")] 43 | for probe_id in res: 44 | self.con.execute("INSERT INTO ticketqueue VALUES (NULL, ?, datetime('now'), 0)", (probe_id,)) 45 | self.con.commit() 46 | except KeyboardInterrupt: 47 | pass 48 | 49 | 50 | class TicketDispatcher(multiprocessing.Process): 51 | 52 | def __init__(self, max_delay=10): 53 | self.delay = 1 54 | self.max_delay = max_delay 55 | self.con = sqlite3.connect('monitor.db') 56 | super(TicketDispatcher, self).__init__() 57 | 58 | def run(self): 59 | try: 60 | while True: 61 | time.sleep(self.delay) 62 | pending_tickets = [r for r in self.con.execute("SELECT id, hostprobe_id FROM ticketqueue WHERE dispatched = 0")] 63 | if not pending_tickets and self.delay < self.max_delay: 64 | self.delay += 1 65 | elif self.delay > 1: 66 | self.delay -= 1 67 | for (ticket_id, hostprobe_id) in pending_tickets: 68 | res = [r for r in self.con.execute("""SELECT host.address, host.port, sensor.name, probe.parameter 69 | FROM hostprobe, host, probe, sensor 70 | WHERE hostprobe.id=? 71 | AND hostprobe.host_id = host.id 72 | AND hostprobe.probe_id = probe.id 73 | AND probe.sensor_id = sensor.id""", (hostprobe_id,) )][0] 74 | self._send_request(ticket_id, res[0], res[1], res[2], res[3]) 75 | self.con.execute("UPDATE ticketqueue SET dispatched=1 WHERE id=?", (ticket_id,)) 76 | self.con.commit() 77 | except KeyboardInterrupt: 78 | pass 79 | 80 | def _send_request(self, ticket, address, port, sensor, parameter_string=None): 81 | url = "http://%s:%s/xmlrpc/" % (address, port) 82 | proxy = xmlrpclib.ServerProxy(url, allow_none=True) 83 | if parameter_string: 84 | parameter = parameter_string.split(',') 85 | else: 86 | parameter = None 87 | print ticket 88 | print sensor 89 | print parameter 90 | res = proxy.cmd_submit_reading(ticket, sensor, parameter) 91 | return 92 | 93 | 94 | def start_processes(): 95 | clock = multiprocessing.Event() 96 | o = Oscillator(clock, 60) 97 | s = TicketScheduler(clock) 98 | d = TicketDispatcher() 99 | o.start() 100 | s.start() 101 | d.start() 102 | try: 103 | while True: 104 | time.sleep(1) 105 | if len(multiprocessing.active_children()) == 0: 106 | break 107 | except KeyboardInterrupt: 108 | pass 109 | 110 | 111 | def register_new_server(host, port): 112 | proxy = xmlrpclib.ServerProxy('http://%s:%s/xmlrpc/' % (host, str(port))) 113 | res = proxy.cmd_register_new_server() 114 | 115 | def update_sensor_code(host, port, sensor): 116 | proxy = xmlrpclib.ServerProxy('http://%s:%s/xmlrpc/' % (host, str(port))) 117 | res = proxy.cmd_update_sensor_code(sensor) 118 | 119 | if __name__ == '__main__': 120 | update_sensor_code('localhost', 8080, 'disk') 121 | sys.exit() 122 | start_processes() 123 | -------------------------------------------------------------------------------- /978-1-4842-0218-0_Source_Code_Ch09/client-server/server/sensors/cpu_load.tar.bz2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Apress/pro-python-system-admin-14/2b70cbe957969ce2f72e59858ffc4c0bbfe77a05/978-1-4842-0218-0_Source_Code_Ch09/client-server/server/sensors/cpu_load.tar.bz2 -------------------------------------------------------------------------------- /978-1-4842-0218-0_Source_Code_Ch09/client-server/server/sensors/disk.tar.bz2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Apress/pro-python-system-admin-14/2b70cbe957969ce2f72e59858ffc4c0bbfe77a05/978-1-4842-0218-0_Source_Code_Ch09/client-server/server/sensors/disk.tar.bz2 -------------------------------------------------------------------------------- /978-1-4842-0218-0_Source_Code_Ch09/client-server/server/sensors/memory.tar.bz2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Apress/pro-python-system-admin-14/2b70cbe957969ce2f72e59858ffc4c0bbfe77a05/978-1-4842-0218-0_Source_Code_Ch09/client-server/server/sensors/memory.tar.bz2 -------------------------------------------------------------------------------- /978-1-4842-0218-0_Source_Code_Ch09/client-server/server/sensors/processes.tar.bz2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Apress/pro-python-system-admin-14/2b70cbe957969ce2f72e59858ffc4c0bbfe77a05/978-1-4842-0218-0_Source_Code_Ch09/client-server/server/sensors/processes.tar.bz2 -------------------------------------------------------------------------------- /978-1-4842-0218-0_Source_Code_Ch09/client-server/server/server.cfg: -------------------------------------------------------------------------------- 1 | [sensor] 2 | source_dir = sensors/ 3 | -------------------------------------------------------------------------------- /978-1-4842-0218-0_Source_Code_Ch09/example_cherrypy.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import cherrypy 4 | from cherrypy import _cptools 5 | 6 | class Root(_cptools.XMLRPCController): 7 | @cherrypy.expose 8 | def hello(selfi, name): 9 | return "Hello, %s" % name 10 | 11 | cherrypy.quickstart(Root(), '/') 12 | -------------------------------------------------------------------------------- /978-1-4842-0218-0_Source_Code_Ch09/example_oscillator.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import multiprocessing 4 | import time 5 | from datetime import datetime 6 | 7 | class Oscillator(multiprocessing.Process): 8 | 9 | def __init__(self, event, period): 10 | self.period = period 11 | self.event = event 12 | super(Oscillator, self).__init__() 13 | 14 | def run(self): 15 | try: 16 | while True: 17 | self.event.clear() 18 | time.sleep(self.period) 19 | self.event.set() 20 | except KeyboardInterrupt: 21 | pass 22 | 23 | class Scheduler(multiprocessing.Process): 24 | 25 | def __init__(self, event): 26 | self.event = event 27 | super(Scheduler, self).__init__() 28 | 29 | def run(self): 30 | try: 31 | while True: 32 | self.event.wait() 33 | print datetime.now() 34 | except KeyboardInterrupt: 35 | pass 36 | 37 | mgr = multiprocessing.Manager() 38 | e = mgr.Event() 39 | o = Oscillator(e, 60) 40 | s = Scheduler(e) 41 | o.start() 42 | s.start() 43 | try: 44 | while len(multiprocessing.active_children()) != 0: 45 | time.sleep(1) 46 | except KeyboardInterrupt: 47 | o.terminate() 48 | s.terminate() 49 | o.join() 50 | s.join() 51 | -------------------------------------------------------------------------------- /978-1-4842-0218-0_Source_Code_Ch09/example_processes.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import multiprocessing 3 | import time 4 | 5 | def sleeper(timeout): 6 | try: 7 | print "function: I am a sleeper function and going to sleep for %s seconds" % timeout 8 | time.sleep(timeout) 9 | print "function: I'm done!" 10 | except KeyboardInterrupt: 11 | print "function: I have received a signal to stop, exiting..." 12 | 13 | class SleeperClass(multiprocessing.Process): 14 | def __init__(self, timeout): 15 | self.timeout = timeout 16 | print "Class: I am a class and can do initialisation tasks before starting" 17 | super(SleeperClass, self).__init__() 18 | 19 | def run(self): 20 | try: 21 | print "Class: I have been told to run now" 22 | print "Class: So I'm going to sleep for %s seconds" % self.timeout 23 | time.sleep(self.timeout) 24 | print "Class: I'm done." 25 | except KeyboardInterrupt: 26 | print "Class: I must stop now, exiting..." 27 | 28 | p1 = multiprocessing.Process(target=sleeper, args=(50,)) 29 | p2 = SleeperClass(100) 30 | p1.start() 31 | p2.start() 32 | try: 33 | while len(multiprocessing.active_children()) != 0: 34 | time.sleep(1) 35 | except KeyboardInterrupt: 36 | p1.terminate() 37 | p2.terminate() 38 | p1.join() 39 | p2.join() 40 | 41 | -------------------------------------------------------------------------------- /978-1-4842-0218-0_Source_Code_Ch10/ctrl_example.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import subprocess 4 | import time 5 | from datetime import datetime 6 | 7 | p = subprocess.Popen('sleep 60', shell=True) 8 | 9 | while True: 10 | rc = p.poll() 11 | if rc is None: 12 | print "[%s] Process with PID: %d is still running..." % (datetime.now(), p.pid) 13 | time.sleep(10) 14 | p.kill() 15 | else: 16 | print "[%s] Process with PID: %d has terminated. Exit code: %d" % (datetime.now(), p.pid, rc) 17 | break 18 | -------------------------------------------------------------------------------- /978-1-4842-0218-0_Source_Code_Ch10/example.cfg: -------------------------------------------------------------------------------- 1 | [database] 2 | ip=192.168.1.1 3 | name=my_database 4 | user=myuser 5 | password=mypassword 6 | 7 | [tables] 8 | use_prefix=no 9 | prefix=mytables 10 | user_table=%(prefix)s_users 11 | mailbox_table=%(prefix)s_mailboxes 12 | -------------------------------------------------------------------------------- /978-1-4842-0218-0_Source_Code_Ch10/example2.cfg: -------------------------------------------------------------------------------- 1 | [tasks] 2 | step_1="+10" 3 | step_2="*5" 4 | step_3="-12" 5 | step_4="/3" 6 | step_5="+45" 7 | -------------------------------------------------------------------------------- /978-1-4842-0218-0_Source_Code_Ch10/example3.cfg: -------------------------------------------------------------------------------- 1 | [section] 2 | key2 = hello 3 | key1 = 1 4 | 5 | -------------------------------------------------------------------------------- /978-1-4842-0218-0_Source_Code_Ch10/fd_example.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import subprocess 4 | import os 5 | 6 | #f = os.open('out.txt', os.O_CREAT|os.O_WRONLY) 7 | #f = open('out.txt', 'w') 8 | subprocess.Popen('date', stdout=f) 9 | -------------------------------------------------------------------------------- /978-1-4842-0218-0_Source_Code_Ch10/setsid_example.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import subprocess 4 | import os 5 | 6 | print "I am running with the following user id: ", os.getuid() 7 | subprocess.Popen(('/bin/sh', '-c', 'echo "I am an external shell process with effective user id:"; id'), 8 | preexec_fn=os.setuid(501)) 9 | -------------------------------------------------------------------------------- /978-1-4842-0218-0_Source_Code_Ch11/sensor-db/generate-data.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | from datetime import datetime 4 | import sqlite3 5 | import numpy as np 6 | #import matplotlib.pyplot as plt 7 | 8 | ################### 9 | ### Laptop CPU data 10 | 11 | # periodic amplification (laptop) 12 | # 0 -> 1 13 | pl = np.sin(2*np.pi*np.arange(100800)/1440) 14 | pl[pl<0] = np.random.rand(1440/2)*0.1 15 | # data (laptop) 16 | # 0 -> ~3 17 | cl = np.random.pareto(10, size=100800) * 2 18 | # result 19 | cpu_l = cl * pl 20 | 21 | ################### 22 | ### Server CPU data 23 | 24 | # periodic amplification (server) 25 | # 0 -> 1 26 | ps = np.sin(2*np.pi*np.arange(100800)/1440) 27 | ps[ps<0.5] = np.sin(2*np.pi*np.arange(50, 1440/2 - 50)/(1440))*0.3 + 0.3 28 | # data (server) 29 | # ~1 -> ~7 30 | cs = np.random.normal(4, 0.9, 100800) 31 | cs[cs<0] = 0 32 | # trend function 33 | tp = np.sin(2*np.pi*np.arange(100800)/(1440*7))*0.1+0.1 34 | tl = np.arange(100800.)/(100800./1.) 35 | tf = tp + tl + 1 36 | # result 37 | cpu_s = cs * ps * tf 38 | 39 | ######################## 40 | ### Server HTTP req data 41 | 42 | # periodic amplification (server) 43 | # 0 -> 1 44 | psh = np.sin(2*np.pi*np.arange(100800)/1440) 45 | psh[psh>0.7] = 0.9/np.linspace(1., 2.2, 100800) 46 | psh[psh<0.5] = np.sin(2*np.pi*np.arange(50, 1440/2 - 50)/(1440))*0.3 + 0.3 47 | # data (server) 48 | # ~1 -> ~7 49 | csh = np.random.normal(400, 100, 100800) 50 | csh[csh<0] = 0 51 | # trend function 52 | tph = np.sin(2*np.pi*np.arange(100800)/(1440*7))*0.1+0.1 53 | tlh = np.arange(100800.)/(100800./1.) 54 | tfh = tph + tlh + 1 55 | # result 56 | http_s = csh * psh * tfh * 0.7 + 200 57 | 58 | con = sqlite3.connect('monitor.db') 59 | 60 | # datetime.fromtimestamp(1260999020).isoformat() 61 | 62 | # cpu_l 63 | # hostprobe_id = 1 64 | 65 | for i in range(len(cpu_l)): 66 | con.execute("INSERT INTO probereading (hostprobe_id, timestamp, probe_value, ret_code) VALUES (?, ?, ?, ?);", 67 | (1, datetime.fromtimestamp(1260999020+i*60).isoformat(), cpu_l[i], 0) 68 | ) 69 | con.commit() 70 | 71 | # cpu_s 72 | # hostprobe_id = 2 73 | 74 | for i in range(len(cpu_s)): 75 | con.execute("INSERT INTO probereading (hostprobe_id, timestamp, probe_value, ret_code) VALUES (?, ?, ?, ?);", 76 | (2, datetime.fromtimestamp(1260999020+i*60).isoformat(), cpu_s[i], 0) 77 | ) 78 | con.commit() 79 | # http_s 80 | # hostprobe_id = 3 81 | 82 | for i in range(0, len(http_s), 5): 83 | con.execute("INSERT INTO probereading (hostprobe_id, timestamp, probe_value, ret_code) VALUES (?, ?, ?, ?);", 84 | (3, datetime.fromtimestamp(1260999020+i*60).isoformat(), http_s[i], 0) 85 | ) 86 | con.commit() 87 | 88 | #plt.plot(http_s) 89 | #plt.show() 90 | -------------------------------------------------------------------------------- /978-1-4842-0218-0_Source_Code_Ch11/sensor-db/monitor.db: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Apress/pro-python-system-admin-14/2b70cbe957969ce2f72e59858ffc4c0bbfe77a05/978-1-4842-0218-0_Source_Code_Ch11/sensor-db/monitor.db -------------------------------------------------------------------------------- /978-1-4842-0218-0_Source_Code_Ch11/sensor-db/regen_data.sh: -------------------------------------------------------------------------------- 1 | rm monitor.db 2 | sqlite3 -init sample_data.sql monitor.db 3 | ./generate-data.py 4 | -------------------------------------------------------------------------------- /978-1-4842-0218-0_Source_Code_Ch11/sensor-db/sample_data.sql: -------------------------------------------------------------------------------- 1 | 2 | -- ****************************************** 3 | -- Table: SENSOR 4 | -- Description: List of all available sensors 5 | 6 | DROP TABLE IF EXISTS sensor; 7 | 8 | CREATE TABLE sensor ( 9 | id INTEGER PRIMARY KEY, 10 | name TEXT 11 | ); 12 | 13 | INSERT INTO sensor VALUES (1, 'cpu_load'); 14 | INSERT INTO sensor VALUES (2, 'web_server'); 15 | 16 | -- ****************************************** 17 | -- Table: PROBE 18 | -- Description: Adds parameter list to sensor command 19 | -- and defines default thresholds 20 | 21 | DROP TABLE IF EXISTS probe; 22 | 23 | CREATE TABLE probe ( 24 | id INTEGER PRIMARY KEY, 25 | sensor_id INTEGER, 26 | name TEXT, 27 | parameter TEXT, 28 | warning FLOAT, 29 | error FLOAT, 30 | FOREIGN KEY (sensor_id) REFERENCES sensor(id) 31 | ); 32 | 33 | INSERT INTO probe VALUES ( 1, 1, 'Used CPU %', 'used', 1.1, 1.3); 34 | INSERT INTO probe VALUES ( 2, 2, 'Received HTTP requests', 'receiver_req', 400, 500); 35 | 36 | -- ****************************************** 37 | -- Table: HOST 38 | -- Description: List of all monitoring agents 39 | 40 | DROP TABLE IF EXISTS host; 41 | 42 | CREATE TABLE host ( 43 | id INTEGER PRIMARY KEY, 44 | name TEXT, 45 | address TEXT, 46 | port TEXT 47 | ); 48 | 49 | INSERT INTO host VALUES (1, 'My laptop', 'localhost', '8080'); 50 | INSERT INTO host VALUES (2, 'My server', 'localhost', '8080'); 51 | 52 | -- ****************************************** 53 | -- Table: HOSTPROBE 54 | -- Description: Maps available probes to the hosts 55 | -- overrides thresholds if required 56 | 57 | DROP TABLE IF EXISTS hostprobe; 58 | 59 | CREATE TABLE hostprobe ( 60 | id INTEGER PRIMARY KEY, 61 | probe_id INTEGER, 62 | host_id INTEGER, 63 | warning FLOAT, 64 | error FLOAT, 65 | FOREIGN KEY (probe_id) REFERENCES probe(id), 66 | FOREIGN KEY (host_id) REFERENCES host(id) 67 | ); 68 | 69 | 70 | INSERT INTO hostprobe VALUES ( 1, 1, 1, NULL, NULL); 71 | INSERT INTO hostprobe VALUES ( 2, 1, 2, 6.5, 8.5); 72 | INSERT INTO hostprobe VALUES ( 3, 2, 2, 680, 780); 73 | 74 | -- ****************************************** 75 | -- Table: TICKETQUEUE 76 | -- Description: Holds all pendiing and sent tickets 77 | -- tickets are removed when the sensor reading arrive 78 | 79 | DROP TABLE IF EXISTS ticketqueue; 80 | 81 | CREATE TABLE ticketqueue ( 82 | id INTEGER PRIMARY KEY, 83 | hostprobe_id INTEGER, 84 | timestamp TEXT, 85 | dispatched INTEGER, 86 | FOREIGN KEY (hostprobe_id) REFERENCES hostprobe(id) 87 | ); 88 | 89 | -- ****************************************** 90 | -- Table: PROBEREADING 91 | -- Description: Stores all readings obtained from the monitoring agents 92 | 93 | DROP TABLE IF EXISTS probereading; 94 | 95 | CREATE TABLE probereading ( 96 | id INTEGER PRIMARY KEY, 97 | hostprobe_id INTEGER, 98 | timestamp TEXT, 99 | probe_value FLOAT, 100 | ret_code INTEGER, 101 | FOREIGN KEY (hostprobe_id) REFERENCES hostprobe(id) 102 | ); 103 | 104 | -- ****************************************** 105 | -- Table: PROBINGSCHEDULE 106 | -- Description: Defines execution intervals for the probes 107 | 108 | DROP TABLE IF EXISTS probingschedule; 109 | 110 | CREATE TABLE probingschedule ( 111 | id INTEGER PRIMARY KEY, 112 | hostprobe_id INTEGER, 113 | probeinterval INTEGER, 114 | FOREIGN KEY (hostprobe_id) REFERENCES hostprobe(id) 115 | ); 116 | 117 | INSERT INTO probingschedule VALUES (1, 1, 1); 118 | INSERT INTO probingschedule VALUES (2, 2, 1); 119 | INSERT INTO probingschedule VALUES (3, 3, 5); 120 | 121 | -- ****************************************** 122 | -- Table: SYSTEMPARAMS 123 | -- Description: Defines system configuration parameters 124 | 125 | DROP TABLE IF EXISTS systemparams; 126 | 127 | CREATE TABLE systemparams ( 128 | id INTEGER PRIMARY KEY, 129 | name TEXT, 130 | value TEXT 131 | ); 132 | 133 | INSERT INTO systemparams VALUES (1, 'monitor_url', 'http://localhost:8081/xmlrpc/'); 134 | 135 | -- ****************************************** 136 | -- Table: HOSTPARAMS 137 | -- Description: Assigns system parameters to the hosts 138 | -- allows to override the default values 139 | 140 | DROP TABLE IF EXISTS hostparams; 141 | 142 | CREATE TABLE hostparams ( 143 | id INTEGER PRIMARY KEY, 144 | host_id INTEGER, 145 | param_id INTEGER, 146 | value TEXT, 147 | FOREIGN KEY (host_id) REFERENCES host(id), 148 | FOREIGN KEY (param_id) REFERENCES systemparams(id) 149 | ); 150 | 151 | INSERT INTO hostparams VALUES (1, 1, 1, 'http://localhost:8081/xmlrpc/'); 152 | INSERT INTO hostparams VALUES (2, 2, 1, 'http://localhost:8081/xmlrpc/'); 153 | 154 | -------------------------------------------------------------------------------- /978-1-4842-0218-0_Source_Code_Ch11/sensor-db/stats_generator.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import sqlite3 4 | import dateutil 5 | 6 | import numpy as np 7 | import matplotlib 8 | matplotlib.use('Agg') 9 | import matplotlib.pyplot as plt 10 | 11 | from jinja2 import Environment, FileSystemLoader 12 | 13 | 14 | DATABASE='monitor.db' 15 | #LOCATION='/home/rytis/public_html/' 16 | LOCATION='/Users/rytis/Sites/' 17 | TIMESCALES=(1, 7, 30) 18 | 19 | class SiteGenerator: 20 | def __init__(self, db_name, location, tpl_env): 21 | self.location = location 22 | self.tpl_env = tpl_env 23 | self.db_name = db_name 24 | self.conn = sqlite3.connect(self.db_name) 25 | self.hosts = [] 26 | self._get_all_hosts() 27 | 28 | def generate_site_pages(self): 29 | self._generate_hosts_view() 30 | self._generate_host_details_contents() 31 | 32 | def _get_all_hosts(self): 33 | for h in self.conn.execute("SELECT * FROM host"): 34 | host_entry = list(h) 35 | query_str = """ SELECT hostprobe.id, 36 | probe.name, 37 | COALESCE(hostprobe.warning, probe.warning), 38 | COALESCE(hostprobe.error, probe.error) 39 | FROM probe, 40 | hostprobe 41 | WHERE probe.id = hostprobe.probe_id AND 42 | hostprobe.host_id = ? 43 | """ 44 | probes = self.conn.execute(query_str, (h[0],)).fetchall() 45 | host_entry.append(probes) 46 | self.hosts.append(host_entry) 47 | 48 | def _generate_hosts_view(self): 49 | t = self.tpl_env.get_template('index.template') 50 | f = open("%s/index.html" % self.location, 'w') 51 | f.write(t.render({'hosts': self.hosts})) 52 | f.close() 53 | 54 | def _generate_host_details_contents(self): 55 | for host in self.hosts: 56 | # host[0] - id 57 | # host[1] - name 58 | # host[2] - address 59 | # host[3] - port 60 | # host[4] - probelist 61 | for probe in host[4]: 62 | # probe[0] - (hostprobe_)id 63 | # probe[1] - probe name 64 | # probe[2] - warning threshold 65 | # probe[3] - error threshold 66 | sampling_rate = self.conn.execute("SELECT probeinterval FROM probingschedule WHERE hostprobe_id=?", 67 | (probe[0],)).fetchone()[0] 68 | for scale in TIMESCALES: 69 | self._plot_time_graph(probe[0], 70 | 24 * 60 * scale, 71 | sampling_rate, 72 | "%s - %s (scale: %s day(s))" % (host[1], probe[1], scale), 73 | "plot_%s_%s.png" % (probe[0], scale), 74 | warn = probe[2], 75 | err = probe[3] 76 | ) 77 | self._generate_host_probe_details(host, probe) 78 | for scale in TIMESCALES: 79 | self._generate_host_scale_details(host, scale) 80 | self._generate_host_toc(host) 81 | 82 | 83 | def _generate_host_toc(self, host): 84 | probe_sa = {} 85 | for probe in host[4]: 86 | probe_sa[probe[1]] = {} 87 | for scale in TIMESCALES: 88 | probe_sa[probe[1]][scale] = self._calculate_service_availability(probe, scale) 89 | t = self.tpl_env.get_template('host.template') 90 | f = open("%s/host_%s_details.html" % (self.location, host[0]), 'w') 91 | f.write(t.render({ 'host': host, 92 | 'timescales': TIMESCALES, 93 | 'probe_sa': probe_sa, 94 | })) 95 | f.close() 96 | 97 | 98 | def _calculate_service_availability(self, probe, scale): 99 | sa_warn = None 100 | sa_err = None 101 | sampling_rate = self.conn.execute("SELECT probeinterval FROM probingschedule WHERE hostprobe_id=?", 102 | (probe[0],)).fetchone()[0] 103 | records_to_read = int(24 * 60 * scale / sampling_rate) 104 | query_str = """SELECT count(*) 105 | FROM (SELECT probe_value 106 | FROM probereading 107 | WHERE hostprobe_id=? 108 | LIMIT ?) 109 | WHERE probe_value > ?""" 110 | if probe[2]: # calculate only if the warning threshold is set 111 | warning_hits = self.conn.execute(query_str, (probe[0], records_to_read, probe[2],)).fetchone()[0] 112 | sa_warn = float(warning_hits) / records_to_read 113 | if probe[3]: # calculate only if the error threshold is set 114 | error_hits = self.conn.execute(query_str, (probe[0], records_to_read, probe[3],)).fetchone()[0] 115 | sa_err = float(error_hits) / records_to_read 116 | return (sa_warn, sa_err) 117 | 118 | 119 | def _generate_host_probe_details(self, host_struct, probe_struct): 120 | # 1) host -> probe -> all scales (host_probe_details) 121 | t = self.tpl_env.get_template('host_probe_details.template') 122 | f = open("%s/hpd_%s.html" % (self.location, probe_struct[0]), 'w') 123 | images = [] 124 | for scale in TIMESCALES: 125 | images.append([ scale, 126 | "plot_%s_%s.png" % (probe_struct[0], scale), 127 | ]) 128 | f.write(t.render({'host': host_struct, 129 | 'probe': probe_struct, 130 | 'images': images, 131 | })) 132 | f.close() 133 | 134 | def _generate_host_scale_details(self, host_struct, scale): 135 | # 2) host -> scale -> all probes (host_scale_details) 136 | t = self.tpl_env.get_template('host_scale_details.template') 137 | f = open("%s/hsd_%s_%s.html" % (self.location, host_struct[0], scale), 'w') 138 | images = [] 139 | for probe in host_struct[4]: 140 | images.append([ probe[1], 141 | "plot_%s_%s.png" % (probe[0], scale), 142 | ]) 143 | f.write(t.render({'host': host_struct, 144 | 'scale': scale, 145 | 'images': images, 146 | })) 147 | f.close() 148 | 149 | def _plot_time_graph(self, hostprobe_id, time_window, sampling_rate, plot_title, plot_file_name, warn=None, err=None): 150 | records_to_read = int(time_window / sampling_rate) 151 | records = self.conn.execute("SELECT timestamp, probe_value FROM probereading WHERE hostprobe_id=? LIMIT ?", 152 | (hostprobe_id, records_to_read)).fetchall() 153 | time_array, val_array = zip(*records) 154 | 155 | mean = np.mean(val_array) 156 | std = np.std(val_array) 157 | warning_val = mean + 3*std 158 | error_val = mean + 4*std 159 | 160 | data_y = np.array(val_array) 161 | data_x = np.arange(len(data_y)) 162 | print dateutil.parser 163 | data_time = [dateutil.parser.parse(s) for s in time_array] 164 | data_xtime = matplotlib.dates.date2num(data_time) 165 | a, b = np.polyfit(data_x, data_y, 1) 166 | matplotlib.rcParams['font.size'] = 10 167 | fig = plt.figure(figsize=(8,4)) 168 | ax = fig.add_subplot(1, 1, 1) 169 | ax.set_title(plot_title + "\nMean: %.2f, Std Dev: %.2f, Warn Lvl: %.2f, Err Lvl: %.2f" % (mean, std, warning_val, error_val)) 170 | ax.plot_date(data_xtime, data_y, 'b') 171 | ax.plot_date(data_xtime, data_x * a + b, color='black', linewidth=3, marker='None', linestyle='-', alpha=0.5) 172 | fig.autofmt_xdate() 173 | if warn: 174 | ax.axhline(warn, color='orange', linestyle='--', linewidth=2, alpha=0.7) 175 | if err: 176 | ax.axhline(err, color='red', linestyle='--', linewidth=2, alpha=0.7) 177 | ax.grid(True) 178 | plt.savefig("%s/%s" % (self.location, plot_file_name)) 179 | 180 | 181 | def main(): 182 | fs_loader = FileSystemLoader('templates/') 183 | tpl_env = Environment(loader=fs_loader) 184 | sg = SiteGenerator(DATABASE, LOCATION, tpl_env) 185 | sg.generate_site_pages() 186 | 187 | 188 | if __name__ == '__main__': 189 | main() 190 | -------------------------------------------------------------------------------- /978-1-4842-0218-0_Source_Code_Ch11/sensor-db/templates/host.template: -------------------------------------------------------------------------------- 1 |

Host details: {{ host[1] }}

2 |

Views grouped by the timescales

3 |

Here you'll find all available probes for this host on the same timescale.

4 | 9 |

Views grouped by the probes

10 |

Here you'll find all available time scale views of the same probe

11 |
    12 | {% for probe in host[4] %} 13 |
  • {{ probe[1] }}
    • 14 | {% endfor %} 15 |
16 |

Host statistics

17 |

Service availability details

18 | {% for probe in probe_sa %} 19 |

Availability of the "{{ probe }}" check

20 |
    21 | {% for scale in probe_sa[probe] %} 22 |
  • On a {{ scale }} day(s) scale: 23 |
      24 |
    • Warning: {{ probe_sa[probe][scale][0]|round(3) }}%
      • 25 |
    • Error: {{ probe_sa[probe][scale][1]|round(3) }}%
      • 26 |
    27 |
    • 28 | {% endfor %} 29 |
30 | {% endfor %} 31 | -------------------------------------------------------------------------------- /978-1-4842-0218-0_Source_Code_Ch11/sensor-db/templates/host_probe_details.template: -------------------------------------------------------------------------------- 1 |

Host: {{ host[1] }}

2 |

Probe: {{ probe[1] }}

3 | 4 | {% for image in images %} 5 |

Time scale: {{ image[0] }} day(s)

6 | 7 | 8 | 9 | {% endfor %} 10 | -------------------------------------------------------------------------------- /978-1-4842-0218-0_Source_Code_Ch11/sensor-db/templates/host_scale_details.template: -------------------------------------------------------------------------------- 1 |

Host: {{ host[1] }}

2 |

Scale: {{ scale }} day(s)

3 | 4 | {% for image in images %} 5 |

{{ image[0] }}

6 | 7 | {% endfor %} 8 | -------------------------------------------------------------------------------- /978-1-4842-0218-0_Source_Code_Ch11/sensor-db/templates/index.template: -------------------------------------------------------------------------------- 1 |

Hosts

2 |
    3 | {% for host in hosts %} 4 |
  • {{ host[1] }} ({{ host[2] }}:{{ host[3] }})
    • 5 | {% endfor %} 6 |
7 | -------------------------------------------------------------------------------- /978-1-4842-0218-0_Source_Code_Ch13/mysql_db.cfg: -------------------------------------------------------------------------------- 1 | [main] 2 | user=root 3 | passwd=password 4 | host=localhost 5 | -------------------------------------------------------------------------------- /978-1-4842-0218-0_Source_Code_Ch13/mysql_inspector.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import re 4 | import os, sys 5 | from ConfigParser import SafeConfigParser 6 | import MySQLdb 7 | from plugin_manager import PluginManager 8 | 9 | 10 | def main(): 11 | cfg = SafeConfigParser() 12 | cfg.read('mysql_db.cfg') 13 | plugin_manager = PluginManager() 14 | connection = MySQLdb.connect(user=cfg.get('main', 'user'), 15 | passwd=cfg.get('main', 'passwd'), 16 | host=cfg.get('main', 'host')) 17 | env_vars = plugin_manager.call_method('generate', keywords=['provider'], args={'connection': connection}) 18 | plugin_manager.call_method('process', keywords=['consumer'], args={'connection': connection, 'env_vars': env_vars}) 19 | plugin_manager.call_method('report') 20 | 21 | if __name__ == '__main__': 22 | main() 23 | -------------------------------------------------------------------------------- /978-1-4842-0218-0_Source_Code_Ch13/plugin_manager.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import sys 4 | import os 5 | import traceback 6 | 7 | 8 | class Plugin(object): 9 | pass 10 | 11 | 12 | class PluginManager(): 13 | def __init__(self, path=None, plugin_init_args={}): 14 | if path: 15 | self.plugin_dir = path 16 | else: 17 | self.plugin_dir = os.path.dirname(__file__) + '/plugins/' 18 | self.plugins = {} 19 | self._load_plugins() 20 | self._register_plugins(**plugin_init_args) 21 | 22 | def _load_plugins(self): 23 | sys.path.append(self.plugin_dir) 24 | plugin_files = [fn for fn in os.listdir(self.plugin_dir) if fn.startswith('plugin_') and fn.endswith('.py')] 25 | plugin_modules = [m.split('.')[0] for m in plugin_files] 26 | for module in plugin_modules: 27 | m = __import__(module) 28 | 29 | def _register_plugins(self, **kwargs): 30 | for plugin in Plugin.__subclasses__(): 31 | obj = plugin(**kwargs) 32 | self.plugins[obj] = obj.keywords if hasattr(obj, 'keywords') else [] 33 | 34 | def call_method(self, method, args={}, keywords=[]): 35 | result = {} 36 | for plugin in self.plugins: 37 | if not keywords or (set(keywords) & set(self.plugins[plugin])): 38 | try: 39 | name_space = plugin.__class__.__name__ 40 | result[name_space] = getattr(plugin, method)(**args) 41 | except AttributeError: 42 | pass 43 | return result 44 | 45 | -------------------------------------------------------------------------------- /978-1-4842-0218-0_Source_Code_Ch13/plugins/plugin_advisor.py: -------------------------------------------------------------------------------- 1 | 2 | from plugin_manager import Plugin 3 | import urllib2 4 | from BeautifulSoup import BeautifulSoup 5 | 6 | class PerformanceAdvisor(Plugin): 7 | 8 | def __init__(self, **kwargs): 9 | self.keywords = ['consumer'] 10 | print self.__class__.__name__, 'initialising...' 11 | 12 | def process(self, **kwargs): 13 | print self.__class__.__name__, 'processing data...' 14 | 15 | def report(self, **kwargs): 16 | print self.__class__.__name__, 'reporting...' 17 | 18 | 19 | class KeyBufferSizeAdvisor(Plugin): 20 | 21 | def __init__(self, **kwargs): 22 | self.keywords = ['consumer'] 23 | self.physical_mem = 0 24 | self.key_buffer = 0 25 | self.ratio = 0.0 26 | self.recommended_buffer = 0 27 | self.recommended_ratio = 0.4 28 | 29 | def process(self, **kwargs): 30 | self.key_buffer = int(kwargs['env_vars']['ServerSystemVariables']['key_buffer_size']) 31 | self.physical_mem = int(kwargs['env_vars']['HostProperties']['mem_phys_total']) 32 | self.ratio = float(self.key_buffer) / self.physical_mem 33 | self.recommended_buffer = int(self.physical_mem * self.recommended_ratio) 34 | 35 | def report(self, **kwargs): 36 | print self.__class__.__name__, 'reporting...' 37 | print "The key buffer size currently is %d" % self.key_buffer 38 | if self.ratio < self.recommended_ratio: 39 | print "This setting seems to be too small for the amount of memory installed: %d" % self.physical_mem 40 | else: 41 | print "You may have allocated too much memory for the key buffer" 42 | print "You currently have %d, you must free up some memory" 43 | print "Consider setting key_buffer_size to %d, if the difference is too high" % self.recommended_buffer 44 | 45 | 46 | class SlowQueriesAdvisor(Plugin): 47 | 48 | def __init__(self, **kwargs): 49 | self.keywords = ['consumer'] 50 | self.log_slow = False 51 | self.long_query_time = 0 52 | self.total_slow_queries = 0 53 | self.total_requests = 0 54 | self.long_qry_ratio = 0.0 # in % 55 | self.threshold = 0.0001 # in % 56 | self.advise = '' 57 | 58 | def process(self, **kwargs): 59 | if kwargs['env_vars']['ServerSystemVariables']['log_slow_queries'] == 'ON': 60 | self.log_slow = True 61 | self.long_query_time = float(kwargs['env_vars']['ServerSystemVariables']['long_query_time']) 62 | self.total_slow_queries = int(kwargs['env_vars']['ServerStatusVariables']['Slow_queries']) 63 | self.total_requests = int(kwargs['env_vars']['ServerStatusVariables']['Questions']) 64 | self.long_qry_ratio = (100. * self.total_slow_queries) / self.total_requests 65 | 66 | def report(self, **kwargs): 67 | print self.__class__.__name__, 'reporting...' 68 | if self.log_slow: 69 | print "There are %d slow requests out of total %d, which is %f%%" % (self.total_slow_queries, 70 | self.total_requests, 71 | self.long_qry_ratio) 72 | print "Currently all queries taking longer than %f are considered slow" % self.long_query_time 73 | if self.long_qry_ratio < self.threshold: 74 | print 'The current slow queries ratio seems to be reasonable' 75 | else: 76 | print 'You seem to have lots of slow queries, investigate them and possibly increase long_query_time' 77 | else: 78 | print 'The slow queries are not logged, set log_slow_queries to ON for tracking' 79 | 80 | 81 | class MySQLVersionAdvisor(Plugin): 82 | 83 | def __init__(self, **kwargs): 84 | self.keywords = ['consumer'] 85 | self.advices = [] 86 | self.installed_release = None 87 | self.latest_release = None 88 | 89 | def _check_latest_ga_release(self): 90 | html = urllib2.urlopen('http://www.mysql.com/downloads/mysql/') 91 | soup = BeautifulSoup(html) 92 | tags = soup.findAll('h1') 93 | version_str = tags[1].string.split()[-1] 94 | (major, minor, release) = [int(i) for i in version_str.split('.')] 95 | return (major, minor, release) 96 | 97 | 98 | def process(self, **kwargs): 99 | version = kwargs['env_vars']['ServerSystemVariables']['version'].split('-')[0] 100 | (major, minor, release) = [int(i) for i in version.split('.')] 101 | latest_major, latest_minor, latest_rel = self._check_latest_ga_release() 102 | self.installed_release = (major, minor, release) 103 | self.latest_release = (latest_major, latest_minor, latest_rel) 104 | if major < latest_major: 105 | self.advices.append(('CRITICAL', 'There is a newer major release available, you should upgrade')) 106 | elif major == latest_major and minor < latest_minor: 107 | self.advices.append(('WARNING', 'There is a newer minor release available, consider an upgrade')) 108 | elif major == latest_major and minor == latest_minor and release < latest_rel: 109 | self.advices.append(('NOTE', 'There is a newer update release available, consider a patch')) 110 | else: 111 | self.advices.append(('OK', 'Your installation is up to date')) 112 | 113 | def report(self, **kwargs): 114 | print self.__class__.__name__, 'reporting...' 115 | print "The running server version is: %d.%d.%d" % self.installed_release 116 | print "The latest available GA release is: %d.%d.%d" % self.latest_release 117 | for rec in self.advices: 118 | print "%10s: %s" % (rec[0], rec[1]) 119 | 120 | -------------------------------------------------------------------------------- /978-1-4842-0218-0_Source_Code_Ch13/plugins/plugin_system_query.py: -------------------------------------------------------------------------------- 1 | 2 | from plugin_manager import Plugin 3 | import psutil 4 | 5 | 6 | class ServerStatusVariables(Plugin): 7 | 8 | def __init__(self, **kwargs): 9 | self.keywords = ['provider'] 10 | print self.__class__.__name__, 'initialising...' 11 | 12 | def generate(self, **kwargs): 13 | cursor = kwargs['connection'].cursor() 14 | cursor.execute('SHOW /*!50002 GLOBAL */ STATUS') 15 | result = {} 16 | for k, v in cursor.fetchall(): 17 | result[k] = v 18 | cursor.close() 19 | return result 20 | 21 | 22 | class ServerSystemVariables(Plugin): 23 | 24 | def __init__(self, **kwargs): 25 | self.keywords = ['provider'] 26 | print self.__class__.__name__, 'initialising...' 27 | 28 | def generate(self, **kwargs): 29 | cursor = kwargs['connection'].cursor() 30 | cursor.execute('SHOW GLOBAL VARIABLES') 31 | result = {} 32 | for k, v in cursor.fetchall(): 33 | result[k] = v 34 | cursor.close() 35 | return result 36 | 37 | 38 | class HostProperties(Plugin): 39 | 40 | def __init__(self, **kwargs): 41 | self.keywords = ['provider'] 42 | print self.__class__.__name__, 'initialising...' 43 | 44 | def _get_total_cores(self): 45 | f = open('/proc/cpuinfo', 'r') 46 | c_cpus = 0 47 | for line in f.readlines(): 48 | if line.startswith('processor'): 49 | c_cpus += 1 50 | f.close() 51 | return c_cpus 52 | 53 | def generate(self, **kwargs): 54 | result = { 'mem_phys_total': psutil.TOTAL_PHYMEM, 55 | 'mem_phys_avail': psutil.avail_phymem(), 56 | 'mem_phys_used' : psutil.used_phymem(), 57 | 'mem_virt_total': psutil.total_virtmem(), 58 | 'mem_virt_avail': psutil.avail_virtmem(), 59 | 'mem_virt_used' : psutil.used_virtmem(), 60 | 'cpu_cores' : self._get_total_cores(), 61 | } 62 | return result 63 | -------------------------------------------------------------------------------- /978-1-4842-0218-0_Source_Code_Ch14/backup.cfg: -------------------------------------------------------------------------------- 1 | [main] 2 | volume_id=vol-******** 3 | vol_device=/dev/sdf 4 | mount_dir=/mysql-db 5 | image_id=ami-******** 6 | key_name=******** 7 | key_location=/home/rytis/EC2/ 8 | security_grp=database 9 | 10 | -------------------------------------------------------------------------------- /978-1-4842-0218-0_Source_Code_Ch14/db_backup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import sys 4 | import logging 5 | import time 6 | import subprocess 7 | import boto 8 | import boto.ec2 9 | from ConfigParser import SafeConfigParser 10 | import MySQLdb 11 | from datetime import datetime 12 | 13 | CFG_FILE = 'backup.cfg' 14 | 15 | class BackupManager: 16 | 17 | def __init__(self, cfg_file=CFG_FILE, logger=None): 18 | self.logger = logger 19 | self.config = SafeConfigParser() 20 | self.config.read(cfg_file) 21 | self.aws_access_key = boto.config.get('Credentials', 'aws_access_key_id') 22 | self.aws_secret_key = boto.config.get('Credentials', 'aws_secret_access_key') 23 | self.ec2conn = boto.ec2.connection.EC2Connection(self.aws_access_key, self.aws_secret_key) 24 | self.image = self.ec2conn.get_image(self.config.get('main', 'image_id')) 25 | self.volume = self.ec2conn.get_all_volumes([self.config.get('main', 'volume_id')])[0] 26 | self.reservation = None 27 | self.ssh_cmd = [] 28 | 29 | 30 | def _init_remote_cmd_args(self): 31 | key_file = "%s/%s.pem" % (self.config.get('main', 'key_location'), self.config.get('main', 'key_name')) 32 | remote_user = 'root' 33 | remote_host = self.reservation.instances[0].public_dns_name 34 | #remote_host = 'ec2-174-129-144-95.compute-1.amazonaws.com' 35 | remote_resource = "%s@%s" % (remote_user, remote_host) 36 | self.ssh_cmd = ['ssh', 37 | '-o', 'StrictHostKeyChecking=no', 38 | '-i', key_file, 39 | remote_resource] 40 | 41 | def _start_instance(self): 42 | self.logger.debug('Starting new instance...') 43 | self.reservation = self.image.run(key_name=self.config.get('main', 'key_name'), 44 | security_groups=[self.config.get('main', 'security_grp')], 45 | placement=self.volume.zone) 46 | instance = self.reservation.instances[0] 47 | while instance.state != u'running': 48 | time.sleep(60) 49 | instance.update() 50 | self.logger.debug("instance state: %s" % instance.state) 51 | self.logger.debug("Instance %s is running and available at %s" % (instance.id, instance.public_dns_name)) 52 | 53 | def _attach_volume(self, volume=None): 54 | if not volume: 55 | volume_to_attach = self.volume 56 | else: 57 | volume_to_attach = volume 58 | instance_id = self.reservation.instances[0].id 59 | self.logger.debug("Attaching volume %s to instance %s as %s" % (volume_to_attach.id, 60 | instance_id, 61 | self.config.get('main', 'vol_device'))) 62 | volume_to_attach.attach(instance_id, self.config.get('main', 'vol_device')) 63 | while volume_to_attach.attachment_state() != u'attached': 64 | time.sleep(20) 65 | volume_to_attach.update() 66 | self.logger.debug("volume status: %s", volume_to_attach.attachment_state()) 67 | time.sleep(10) # give it some extra time, aws sometimes is mis-reporting the volume state 68 | self.logger.debug("Finished attaching volume") 69 | 70 | 71 | def _detach_volume(self, volume=None): 72 | if not volume: 73 | volume_to_detach = self.volume 74 | else: 75 | volume_to_detach = volume 76 | self.logger.debug("Detaching volume %s" % volume_to_detach.id) 77 | volume_to_detach.detach() 78 | while volume_to_detach.attachment_state() == u'attached': 79 | time.sleep(20) 80 | volume_to_detach.update() 81 | self.logger.debug("volume status: %s", volume_to_detach.attachment_state()) 82 | self.logger.debug('done') 83 | 84 | 85 | def _mount_volume(self): 86 | self.logger.debug("Mounting %s on %s" % (self.config.get('main', 'vol_device'), 87 | self.config.get('main', 'mount_dir'))) 88 | remote_command = "mount %(dev)s %(mp)s && df -h %(mp)s" % {'dev': self.config.get('main', 'vol_device'), 89 | 'mp': self.config.get('main', 'mount_dir')} 90 | rc = subprocess.call(self.ssh_cmd + [remote_command]) 91 | self.logger.debug('done') 92 | 93 | def _copy_db(self): 94 | self.logger.debug('Backing up the DB...') 95 | time.sleep(60) 96 | #for i in range(5): 97 | # self.logger.debug("%d minute(s) left" % (5-i)) 98 | # time.sleep(60) 99 | 100 | def _control_mysql(self, command): 101 | self.logger.debug("Sending MySQL DB daemon command to: %s" % command) 102 | remote_command = "/sbin/service mysqld %s; pgrep mysqld" % command 103 | rc = subprocess.call(self.ssh_cmd + [remote_command]) 104 | self.logger.debug('done') 105 | 106 | 107 | def _unmount_volume(self): 108 | self.logger.debug("Unmounting %s" % self.config.get('main', 'mount_dir')) 109 | remote_command = "sync; sync; umount %(mp)s; df -h %(mp)s" % {'mp':self.config.get('main', 'mount_dir')} 110 | rc = subprocess.call(self.ssh_cmd + [remote_command]) 111 | self.logger.debug('done') 112 | 113 | 114 | def _create_snapshot(self, volume=None): 115 | if not volume: 116 | volume_to_snapshot = self.volume 117 | else: 118 | volume_to_snapshot = volume 119 | self.logger.debug("Taking a snapshot of %s" % volume_to_snapshot.id) 120 | volume_to_snapshot.create_snapshot(description="Snapshot created on %s" % datetime.isoformat(datetime.now())) 121 | self.logger.debug('done') 122 | 123 | def _terminate_instance(self): 124 | instance = self.reservation.instances[0] 125 | self.logger.debug("Terminating instance %s" % instance.id) 126 | instance.stop() 127 | while instance.state != u'terminated': 128 | time.sleep(60) 129 | instance.update() 130 | self.logger.debug("instance state: %s" % instance.state) 131 | self.logger.debug('done') 132 | 133 | 134 | def main(): 135 | console = logging.StreamHandler() 136 | logger = logging.getLogger('DB_Backup') 137 | logger.addHandler(console) 138 | logger.setLevel(logging.DEBUG) 139 | bck = BackupManager(logger=logger) 140 | bck._start_instance() 141 | bck._init_remote_cmd_args() 142 | bck._attach_volume() 143 | bck._mount_volume() 144 | bck._control_mysql('start') 145 | bck._copy_db() 146 | bck._control_mysql('stop') 147 | bck._unmount_volume() 148 | bck._detach_volume() 149 | bck._create_snapshot() 150 | bck._terminate_instance() 151 | 152 | 153 | 154 | if __name__ == '__main__': 155 | main() 156 | -------------------------------------------------------------------------------- /9781484202180.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Apress/pro-python-system-admin-14/2b70cbe957969ce2f72e59858ffc4c0bbfe77a05/9781484202180.jpg -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Apress/pro-python-system-admin-14/2b70cbe957969ce2f72e59858ffc4c0bbfe77a05/LICENSE.txt -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Apress Source Code 2 | 3 | This repository accompanies [*Pro Python System Administration*](http://www.apress.com/9781484202180) by Rytis Sileika (Apress, 2014). 4 | 5 | ![Cover image](9781484202180.jpg) 6 | 7 | Download the files as a zip using the green button, or clone the repository to your machine using Git. 8 | 9 | ## Releases 10 | 11 | Release v1.0 corresponds to the code in the published book, without corrections or updates. 12 | 13 | ## Contributions 14 | 15 | See the file Contributing.md for more information on how you can contribute to this repository. 16 | -------------------------------------------------------------------------------- /contributing.md: -------------------------------------------------------------------------------- 1 | # Contributing to Apress Source Code 2 | 3 | Copyright for Apress source code belongs to the author(s). However, under fair use you are encouraged to fork and contribute minor corrections and updates for the benefit of the author(s) and other readers. 4 | 5 | ## How to Contribute 6 | 7 | 1. Make sure you have a GitHub account. 8 | 2. Fork the repository for the relevant book. 9 | 3. Create a new branch on which to make your change, e.g. 10 | `git checkout -b my_code_contribution` 11 | 4. Commit your change. Include a commit message describing the correction. Please note that if your commit message is not clear, the correction will not be accepted. 12 | 5. Submit a pull request. 13 | 14 | Thank you for your contribution! --------------------------------------------------------------------------------