├── ElasticsearchPlugin.py
├── README.md
├── libnmap
    ├── __init__.py
    ├── diff.py
    ├── objects
    │   ├── __init__.py
    │   ├── cpe.py
    │   ├── host.py
    │   ├── os.py
    │   ├── report.py
    │   └── service.py
    ├── parser.py
    ├── plugins
    │   ├── __init__.py
    │   ├── backend_host.py
    │   ├── backend_service.py
    │   ├── backendplugin.py
    │   ├── backendpluginFactory.py
    │   ├── es.py
    │   ├── mongodb.py
    │   ├── s3.py
    │   └── sql.py
    ├── process.py
    └── reportjson.py
├── run.py
├── src
    ├── flower.tar.gz
    ├── supervisord_client.conf
    └── supervisord_server.conf
├── tasks.py
├── wyfunc.py
└── wyportmap.py


/ElasticsearchPlugin.py:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/env python
 2 | 
 3 | from libnmap.parser import NmapParser
 4 | from libnmap.reportjson import ReportDecoder
 5 | from libnmap.plugins.es import NmapElasticsearchPlugin
 6 | from datetime import datetime
 7 | import json
 8 | 
 9 | nmap_report = NmapParser.parse_fromfile('libnmap/test/files/1_hosts.xml')
10 | mindex = datetime.fromtimestamp(nmap_report.started).strftime('%Y-%m-%d')
11 | db = NmapElasticsearchPlugin(index=mindex)
12 | dbid = db.insert(nmap_report)
13 | nmap_json = db.get(dbid)
14 | 
15 | nmap_obj = json.loads(json.dumps(nmap_json), cls=ReportDecoder)
16 | print(nmap_obj)
17 | #print(db.getall())
18 | 
19 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
  1 | # thorns
  2 | thorns_project 分布式异步队列系统
  3 | 
  4 | 运行流程
  5 | -----------------------------------
  6 | * 启动redis内存服务器，作为队列存储数据库使用
  7 | * 配置芹菜(celery)运行环境，并连接redis队列内存，读取执行任务，并返回结果存储到后端MySQL数据库
  8 | * 配置任务控制台花花(flower)，并连接redis队列内存，管理所有worker客户端与执行的任务队列
  9 | * 通过run.py脚本调用celery向队列压入任务
 10 | * 通过flower的http api脚本调用api向队列压入任务
 11 | * 任务执行的结果自动存入后端数据库
 12 | 
 13 | 反馈
 14 | -----------------------------------
 15 | > 微博：http://weibo.com/ringzero<br />
 16 | > 邮箱：ringzero@0x557.org<br />
 17 | 
 18 | 运行环境
 19 | -----------------------------------
 20 | * CentOS、Kali Linux、Ubuntu、Debian
 21 | * Python 2.7.x
 22 | * Redis
 23 | * MysQL
 24 | * Celery
 25 | * Tornado
 26 | * Supervisord
 27 | 
 28 | 安装配置说明
 29 | -----------------------------------
 30 | ## CentOS 服务端
 31 | 
 32 | #### 安装 Redis-Server
 33 | 	$ wget http://download.redis.io/releases/redis-2.8.19.tar.gz
 34 | 	$ tar xzf redis-2.8.19.tar.gz
 35 | 	$ cd redis-2.8.19
 36 | 	$ make
 37 | 	$ sudo cp src/redis-server /usr/bin/
 38 | 	$ sudo cp redis.conf /etc/redis.conf
 39 | 	/* 修改 /etc/redis.conf 37行，将daemonize no改为daemonize yes，让redis后台运行 */
 40 | 	$ sudo vim /etc/redis.conf
 41 | 	daemonize yes
 42 | 	# 启动Redis-Server
 43 | 	$ sudo redis-server /etc/redis.conf
 44 | 
 45 | #### 安装 pip
 46 | 	$ wget https://pypi.python.org/packages/source/p/pip/pip-6.0.8.tar.gz
 47 | 	$ tar zvxf pip-6.0.8.tar.gz
 48 | 	$ cd pip-6.0.8
 49 | 	$ sudo python setup.py install
 50 | 
 51 | #### 安装 MySQL-python
 52 | 	$ sudo yum -y install python-devel mysql-devel subversion-devel
 53 | 	$ sudo pip install MySQL-python SQLAlchemy
 54 | 
 55 | #### 安装 Celery
 56 | 	$ sudo pip install -U celery[redis]
 57 | 
 58 | #### 安装 Flower & 下载thorns运行环境代码
 59 | 	$ cd /home/
 60 | 	$ sudo yum -y install git
 61 | 	$ git clone https://github.com/ring04h/thorns.git
 62 | 	$ cd /home/thorns/src
 63 | 	$ tar zvxf flower.tar.gz
 64 | 	$ cd flower-0.7.3
 65 | 	$ python setup.py install
 66 | 	/* 启动Flower 这里的redis ip可以配置为你的外网的IP */
 67 | 	$ celery flower --port=8080 --broker=redis://127.0.0.1:6379/0 &
 68 | 	建议使用Supervisord的守护进程来启动Flower，确保系统7*24小时的稳定性
 69 | 
 70 | #### 安装 Supervisord
 71 | 	$ sudo pip install supervisor
 72 | 	$ sudo cp /home/thorns/src/supervisord_server.conf /etc/supervisord.conf
 73 | 	/* 修改 /etc/supervisord.conf 141行 修改redis ip为你自己的ip --broker=redis://127.0.0.1:6379/0 */
 74 | 	/* 修改 /etc/supervisord.conf 153行 修改programe为你想定义的worker名称 [program:worker-ringzero] */
 75 | 	$ sudo vim /etc/supervisord.conf
 76 | 	/* 启动 supervisord */
 77 | 	$ supervisord -c /etc/supervisord.conf
 78 | 	http://127.0.0.1:9001/ 可以在线守护管理thorns的进程，实现远程重启
 79 | 
 80 | #### 检查各服务是否正常启动后，开始配置客户端任务脚本
 81 | 	1、http://youip:8080/  thorns 控制台
 82 | 	2、http://youip:9001/  supervisord 控制台
 83 | 	3、修改tasks.py内的芹菜配置
 84 | 	对应你自己的redis-server服务器IP
 85 | 	BROKER_URL = 'redis://120.132.54.90:6379/0',
 86 | 	对应你自己的MySQL-server服务器IP
 87 | 	CELERY_RESULT_BACKEND = 'db+mysql://celery:celery1@3Wscan@42.62.52.62:443/wscan',
 88 | 
 89 | 	配置完毕后，就可以部署多台客户端进行分布式任务执行了
 90 | 
 91 | ## CentOS 客户端（建议大规模部署）
 92 | 	# 安装 git & 下载 thorns_project
 93 | 	$ sudo yum -y install git
 94 | 	$ cd /home/
 95 | 	$ git clone https://github.com/ring04h/thorns.git
 96 | 
 97 | 	# 安装 pip
 98 | 	$ wget https://pypi.python.org/packages/source/p/pip/pip-6.0.8.tar.gz
 99 | 	$ tar zvxf pip-6.0.8.tar.gz
100 | 	$ cd pip-6.0.8
101 | 	$ sudo python setup.py install
102 | 
103 | 	# 安装 MySQL-python
104 | 	$ sudo yum -y install python-devel mysql-devel subversion-devel
105 | 	$ sudo pip install MySQL-python SQLAlchemy
106 | 
107 | 	# 安装 nmap
108 | 	# 32位系统
109 | 	$ sudo rpm -vhU https://nmap.org/dist/nmap-6.47-1.i386.rpm
110 | 	# 64位系统
111 | 	$ sudo rpm -vhU https://nmap.org/dist/nmap-6.47-1.x86_64.rpm
112 | 
113 | 	# 安装 Celery
114 | 	$ sudo pip install -U celery[redis]
115 | 
116 | 	# 安装 Supervisord
117 | 	$ sudo pip install supervisor
118 | 	$ sudo cp /home/thorns/src/supervisord_client.conf /etc/supervisord.conf
119 | 	/* 修改 /etc/supervisord.conf 140行 修改programe为你想定义的worker名称 [program:worker-ringzero] */
120 | 	$ sudo vim /etc/supervisord.conf
121 | 	/* 启动 supervisord */
122 | 	$ supervisord -c /etc/supervisord.conf
123 | 	http://127.0.0.1:9001/ 可以在线守护管理thorns的进程，实现远程重启
124 | 
125 | 	# 修改tasks.py内的芹菜配置（分布式任务关键配置项）
126 | 	对应你自己的redis-server服务器IP
127 | 	BROKER_URL = 'redis://120.132.54.90:6379/0',
128 | 	对应你自己的MySQL-server服务器IP
129 | 	CELERY_RESULT_BACKEND = 'db+mysql://celery:celery1@3Wscan@42.62.52.62:443/wscan',
130 | 
131 | 	# 环境搭建完毕，这时候访问thorns project的控制台，就会发现worker客户端已经出现在那里
132 | 	演示地址：http://thorns.wuyun.org:8080/
133 | 	你的请访问：http://youip:8080/
134 | 
135 | 
136 | 使用说明(可客户端发起任务也可http api发起任务)
137 | -----------------------------------
138 | #### 命令行调用
139 | 	在你的任意一台worker客户端，或者thorns服务端
140 | 	$ cd /home/thorns/
141 | 	$ python run.py 42.62.52.1-42.62.62.254 188
142 | 	$ python run.py 42.62.52.1-254 189
143 | 	均可以向redis压入nmap扫描任务，worker客户端的分布式集群会自动分发任务执行，并存储到后台数据库
144 | 	记得修改wyportmap.py里面的扫描结果，存到你自己的数据库
145 | 	
146 | 	reinhard-mbp:thorns reinhard$ python run.py 42.62.52.1-254 189
147 | 	--------------------------------------------------
148 | 	* push 42.62.52.1 to Redis
149 | 	* AsyncResult:23147d02-294d-41e5-84e5-5e1b15e72fc4
150 | 	--------------------------------------------------
151 | 	* push 42.62.52.2 to Redis
152 | 	* AsyncResult:542984a4-4434-475f-9a62-bfc81206ea57
153 | 	--------------------------------------------------
154 | 	* push 42.62.52.3 to Redis
155 | 	* AsyncResult:7d005661-d719-41ef-babc-4c853b2c49cc
156 | 	--------------------------------------------------
157 | 	* push 42.62.52.4 to Redis
158 | 	* AsyncResult:ddcf9486-09d9-4dd2-9bb4-2618e6a161b8
159 | 	--------------------------------------------------
160 | 
161 | 	wyportmap相关帮助: https://github.com/ring04h/wyportmap
162 | 
163 | #### HTTP API 远程调用
164 |     重启 worker 线程池:
165 |     $ curl -X POST http://thorns.wuyun.org:8080/api/worker/pool/restart/myworker
166 |     
167 |     远程调用HTTP API启动一个nmap扫描任务：
168 |     $ curl -X POST -d '{"args":["42.62.52.62",2222]}' http://thorns.wuyun.org:8080/api/task/send-task/tasks.nmap_dispath
169 | 
170 |     强制结束一个正在执行的任务：
171 |     $ curl -X POST -d 'terminate=True' http://thorns.wuyun.org:8088/api/task/revoke/a9361c1b-fd1d-4f48-9be2-8656a57e906b
172 | 
173 | 


--------------------------------------------------------------------------------
/libnmap/__init__.py:
--------------------------------------------------------------------------------
1 | # -*- coding: utf-8 -*-
2 | 
3 | __author__ = 'Ronald Bister, Mike Boutillier'
4 | __credits__ = ['Ronald Bister', 'Mike Boutillier']
5 | __maintainer__ = 'Ronald Bister'
6 | __email__ = 'mini.pelle@gmail.com'
7 | __license__ = 'CC-BY'
8 | __version__ = '0.6.1'
9 | 


--------------------------------------------------------------------------------
/libnmap/diff.py:
--------------------------------------------------------------------------------
 1 | # -*- coding: utf-8 -*-
 2 | 
 3 | 
 4 | class DictDiffer(object):
 5 |     """
 6 |         Calculate the difference between two dictionaries as:
 7 |         (1) items added
 8 |         (2) items removed
 9 |         (3) keys same in both but changed values
10 |         (4) keys same in both and unchanged values
11 |     """
12 |     def __init__(self, current_dict, past_dict):
13 |         self.current_dict = current_dict
14 |         self.past_dict = past_dict
15 |         self.set_current = set(current_dict.keys())
16 |         self.set_past = set(past_dict.keys())
17 |         self.intersect = self.set_current.intersection(self.set_past)
18 | 
19 |     def added(self):
20 |         return self.set_current - self.intersect
21 | 
22 |     def removed(self):
23 |         return self.set_past - self.intersect
24 | 
25 |     def changed(self):
26 |         return (set(o for o in self.intersect
27 |                 if self.past_dict[o] != self.current_dict[o]))
28 | 
29 |     def unchanged(self):
30 |         return (set(o for o in self.intersect
31 |                 if self.past_dict[o] == self.current_dict[o]))
32 | 
33 | 
34 | class NmapDiff(DictDiffer):
35 |     """
36 |         NmapDiff compares two objects of same type to enable the user to check:
37 | 
38 |         - what has changed
39 |         - what has been added
40 |         - what has been removed
41 |         - what was kept unchanged
42 | 
43 |         NmapDiff inherit from DictDiffer which makes the actual comparaison.
44 |         The different methods from DictDiffer used by NmapDiff are the
45 |         following:
46 | 
47 |         - NmapDiff.changed()
48 |         - NmapDiff.added()
49 |         - NmapDiff.removed()
50 |         - NmapDiff.unchanged()
51 | 
52 |         Each of the returns a python set() of key which have changed in the
53 |         compared objects. To check the different keys that could be returned,
54 |         refer to the get_dict() method of the objects you which to
55 |         compare (i.e: libnmap.objects.NmapHost, NmapService,...).
56 |     """
57 |     def __init__(self, nmap_obj1, nmap_obj2):
58 |         """
59 |             Constructor of NmapDiff:
60 | 
61 |             - Checks if the two objects are of the same class
62 |             - Checks if the objects are "comparable" via a call to id() (dirty)
63 |             - Inherits from DictDiffer and
64 |         """
65 |         if(nmap_obj1.__class__ != nmap_obj2.__class__ or
66 |            nmap_obj1.id != nmap_obj2.id):
67 |             raise NmapDiffException("Comparing objects with non-matching id")
68 | 
69 |         self.object1 = nmap_obj1.get_dict()
70 |         self.object2 = nmap_obj2.get_dict()
71 | 
72 |         DictDiffer.__init__(self, self.object1, self.object2)
73 | 
74 |     def __repr__(self):
75 |         return ("added: [{0}] -- changed: [{1}] -- "
76 |                 "unchanged: [{2}] -- removed [{3}]".format(self.added(),
77 |                                                            self.changed(),
78 |                                                            self.unchanged(),
79 |                                                            self.removed()))
80 | 
81 | 
82 | class NmapDiffException(Exception):
83 |     def __init__(self, msg):
84 |         self.msg = msg
85 | 


--------------------------------------------------------------------------------
/libnmap/objects/__init__.py:
--------------------------------------------------------------------------------
1 | # -*- coding: utf-8 -*-
2 | 
3 | from libnmap.objects.report import NmapReport
4 | from libnmap.objects.host import NmapHost
5 | from libnmap.objects.service import NmapService
6 | 
7 | __all__ = ['NmapReport', 'NmapHost', 'NmapService']
8 | 


--------------------------------------------------------------------------------
/libnmap/objects/cpe.py:
--------------------------------------------------------------------------------
 1 | # -*- coding: utf-8 -*-
 2 | 
 3 | 
 4 | class CPE(object):
 5 |     """
 6 |         CPE class offers an API for basic CPE objects.
 7 |         These objects could be found in NmapService or in <os> tag
 8 |         within NmapHost.
 9 | 
10 |         :todo: interpret CPE string and provide appropriate API
11 |     """
12 |     def __init__(self, cpestring):
13 |         self._cpestring = cpestring
14 |         self.cpedict = {}
15 | 
16 |         zk = ['cpe', 'part', 'vendor', 'product', 'version',
17 |               'update', 'edition', 'language']
18 |         self._cpedict = dict((k, '') for k in zk)
19 |         splitup = cpestring.split(':')
20 |         self._cpedict.update(dict(zip(zk, splitup)))
21 | 
22 |     @property
23 |     def cpestring(self):
24 |         """
25 |             Accessor for the full CPE string.
26 |         """
27 |         return self._cpestring
28 | 
29 |     def __repr__(self):
30 |         return self._cpestring
31 | 
32 |     def get_part(self):
33 |         """
34 |             Returns the cpe part (/o, /h, /a)
35 |         """
36 |         return self._cpedict['part']
37 | 
38 |     def get_vendor(self):
39 |         """
40 |             Returns the vendor name
41 |         """
42 |         return self._cpedict['vendor']
43 | 
44 |     def get_product(self):
45 |         """
46 |             Returns the product name
47 |         """
48 |         return self._cpedict['product']
49 | 
50 |     def get_version(self):
51 |         """
52 |             Returns the version of the cpe
53 |         """
54 |         return self._cpedict['version']
55 | 
56 |     def get_update(self):
57 |         """
58 |             Returns the update version
59 |         """
60 |         return self._cpedict['update']
61 | 
62 |     def get_edition(self):
63 |         """
64 |             Returns the cpe edition
65 |         """
66 |         return self._cpedict['edition']
67 | 
68 |     def get_language(self):
69 |         """
70 |             Returns the cpe language
71 |         """
72 |         return self._cpedict['language']
73 | 
74 |     def is_application(self):
75 |         """
76 |             Returns True if cpe describes an application
77 |         """
78 |         return (self.get_part() == '/a')
79 | 
80 |     def is_hardware(self):
81 |         """
82 |             Returns True if cpe describes a hardware
83 |         """
84 |         return (self.get_part() == '/h')
85 | 
86 |     def is_operating_system(self):
87 |         """
88 |             Returns True if cpe describes an operating system
89 |         """
90 |         return (self.get_part() == '/o')
91 | 


--------------------------------------------------------------------------------
/libnmap/objects/host.py:
--------------------------------------------------------------------------------
  1 | # -*- coding: utf-8 -*-
  2 | 
  3 | from libnmap.diff import NmapDiff
  4 | from libnmap.objects.os import NmapOSFingerprint
  5 | 
  6 | 
  7 | class NmapHost(object):
  8 |     """
  9 |         NmapHost is a class representing a host object of NmapReport
 10 |     """
 11 |     def __init__(self, starttime='', endtime='', address=None, status=None,
 12 |                  hostnames=None, services=None, extras=None):
 13 |         """
 14 |             NmapHost constructor
 15 |             :param starttime: unix timestamp of when the scan against
 16 |             that host started
 17 |             :type starttime: string
 18 |             :param endtime: unix timestamp of when the scan against
 19 |             that host ended
 20 |             :type endtime: string
 21 |             :param address: dict ie :{'addr': '127.0.0.1', 'addrtype': 'ipv4'}
 22 |             :param status: dict ie:{'reason': 'localhost-response',
 23 |                                     'state': 'up'}
 24 |             :return: NmapHost:
 25 |         """
 26 |         self._starttime = starttime
 27 |         self._endtime = endtime
 28 |         self._hostnames = hostnames if hostnames is not None else []
 29 |         self._status = status if status is not None else {}
 30 |         self._services = services if services is not None else []
 31 |         self._extras = extras if extras is not None else {}
 32 |         self._osfingerprinted = False
 33 |         self.os = None
 34 |         if 'os' in self._extras:
 35 |             self.os = NmapOSFingerprint(self._extras['os'])
 36 |             self._osfingerprinted = True
 37 |         else:
 38 |             self.os = NmapOSFingerprint({})
 39 | 
 40 |         self._ipv4_addr = None
 41 |         self._ipv6_addr = None
 42 |         self._mac_addr = None
 43 |         self._vendor = None
 44 |         for addr in address:
 45 |             if addr['addrtype'] == "ipv4":
 46 |                 self._ipv4_addr = addr['addr']
 47 |             elif addr['addrtype'] == 'ipv6':
 48 |                 self._ipv6_addr = addr['addr']
 49 |             elif addr['addrtype'] == 'mac':
 50 |                 self._mac_addr = addr['addr']
 51 |             if 'vendor' in addr:
 52 |                 self._vendor = addr['vendor']
 53 | 
 54 |         self._main_address = self._ipv4_addr or self._ipv6_addr or ''
 55 |         self._address = address
 56 | 
 57 |     def __eq__(self, other):
 58 |         """
 59 |             Compare eq NmapHost based on :
 60 | 
 61 |                 - hostnames
 62 |                 - address
 63 |                 - if an associated services has changed
 64 | 
 65 |             :return: boolean
 66 |         """
 67 |         rval = False
 68 |         if(self.__class__ == other.__class__ and self.id == other.id):
 69 |             rval = (self.changed(other) == 0)
 70 |         return rval
 71 | 
 72 |     def __ne__(self, other):
 73 |         """
 74 |             Compare ne NmapHost based on:
 75 | 
 76 |                 - hostnames
 77 |                 - address
 78 |                 - if an associated services has changed
 79 | 
 80 |             :return: boolean
 81 |         """
 82 |         rval = True
 83 |         if(self.__class__ == other.__class__ and self.id == other.id):
 84 |             rval = (self.changed(other) > 0)
 85 |         return rval
 86 | 
 87 |     def __repr__(self):
 88 |         """
 89 |             String representing the object
 90 |             :return: string
 91 |         """
 92 |         return "{0}: [{1} ({2}) - {3}]".format(self.__class__.__name__,
 93 |                                                self.address,
 94 |                                                " ".join(self._hostnames),
 95 |                                                self.status)
 96 | 
 97 |     def __hash__(self):
 98 |         """
 99 |             Hash is needed to be able to use our object in sets
100 |             :return: hash
101 |         """
102 |         return (hash(self.status) ^ hash(self.address) ^
103 |                 hash(frozenset(self._services)) ^
104 |                 hash(frozenset(" ".join(self._hostnames))))
105 | 
106 |     def changed(self, other):
107 |         """
108 |             return the number of attribute who have changed
109 |             :param other: NmapHost object to compare
110 |             :return int
111 |         """
112 |         return len(self.diff(other).changed())
113 | 
114 |     def save(self, backend):
115 |         
116 |         if backend is not None:
117 |             _id = backend.insert(self)
118 |         else:
119 |             raise RuntimeError
120 |         return _id
121 | 
122 |     @property
123 |     def starttime(self):
124 |         """
125 |             Accessor for the unix timestamp of when the scan was started
126 | 
127 |             :return: string
128 |         """
129 |         return self._starttime
130 | 
131 |     @property
132 |     def endtime(self):
133 |         """
134 |             Accessor for the unix timestamp of when the scan ended
135 | 
136 |             :return: string
137 |         """
138 |         return self._endtime
139 | 
140 |     @property
141 |     def address(self):
142 |         """
143 |             Accessor for the IP address of the scanned host
144 | 
145 |             :return: IP address as a string
146 |         """
147 |         return self._main_address
148 | 
149 |     @address.setter
150 |     def address(self, addrdict):
151 |         """
152 |             Setter for the address dictionnary.
153 | 
154 |             :param addrdict: valid dict is {'addr': '1.1.1.1',
155 |                                             'addrtype': 'ipv4'}
156 |         """
157 |         if addrdict['addrtype'] == 'ipv4':
158 |             self._ipv4_addr = addrdict['addr']
159 |         elif addrdict['addrtype'] == 'ipv6':
160 |             self._ipv6_addr = addrdict['addr']
161 |         elif addrdict['addrtype'] == 'mac':
162 |             self._mac_addr = addrdict['addr']
163 |         if 'vendor' in addrdict:
164 |             self._vendor = addrdict['vendor']
165 | 
166 |         self._main_address = self._ipv4_addr or self._ipv6_addr or ''
167 |         self._address = addrdict
168 | 
169 |     @property
170 |     def ipv4(self):
171 |         """
172 |             Accessor for the IPv4 address of the scanned host
173 | 
174 |             :return: IPv4 address as a string
175 |         """
176 |         return self._ipv4_addr or ''
177 | 
178 |     @property
179 |     def mac(self):
180 |         """
181 |             Accessor for the MAC address of the scanned host
182 | 
183 |             :return: MAC address as a string
184 |         """
185 |         return self._mac_addr or ''
186 | 
187 |     @property
188 |     def vendor(self):
189 |         """
190 |             Accessor for the vendor attribute of the scanned host
191 | 
192 |             :return: string (vendor) of empty string if no vendor defined
193 |         """
194 |         return self._vendor or ''
195 | 
196 |     @property
197 |     def ipv6(self):
198 |         """
199 |             Accessor for the IPv6 address of the scanned host
200 | 
201 |             :return: IPv6 address as a string
202 |         """
203 |         return self._ipv6_addr or ''
204 | 
205 |     @property
206 |     def status(self):
207 |         """
208 |             Accessor for the host's status (up, down, unknown...)
209 | 
210 |             :return: string
211 |         """
212 |         return self._status['state']
213 | 
214 |     @status.setter
215 |     def status(self, statusdict):
216 |         """
217 |             Setter for the status dictionnary.
218 | 
219 |             :param statusdict: valid dict is {"state": "open",
220 |                                               "reason": "syn-ack",
221 |                                               "reason_ttl": "0"}
222 |                                 'state' is the only mandatory key.
223 |         """
224 |         self._status = statusdict
225 | 
226 |     def is_up(self):
227 |         """
228 |             method to determine if host is up or not
229 | 
230 |             :return: bool
231 |         """
232 |         rval = False
233 |         if self.status == 'up':
234 |             rval = True
235 |         return rval
236 | 
237 |     @property
238 |     def hostnames(self):
239 |         """
240 |             Accessor returning the list of hostnames (array of strings).
241 | 
242 |             :return: array of string
243 |         """
244 |         return self._hostnames
245 | 
246 |     @property
247 |     def services(self):
248 |         """
249 |             Accessor for the array of scanned services for that host.
250 | 
251 |             An array of NmapService objects is returned.
252 | 
253 |             :return: array of NmapService
254 |         """
255 |         return self._services
256 | 
257 |     def get_ports(self):
258 |         """
259 |             Retrieve a list of the port used by each service of the NmapHost
260 | 
261 |             :return: list: of tuples (port,'proto') ie:[(22,'tcp'),(25, 'tcp')]
262 |         """
263 |         return [(p.port, p.protocol) for p in self._services]
264 | 
265 |     def get_open_ports(self):
266 |         """
267 |             Same as get_ports() but only for open ports
268 | 
269 |             :return: list: of tuples (port,'proto') ie:[(22,'tcp'),(25, 'tcp')]
270 |         """
271 |         return ([(p.port, p.protocol)
272 |                 for p in self._services if p.state == 'open'])
273 | 
274 |     def get_service(self, portno, protocol='tcp'):
275 |         """
276 |             :param portno: int the portnumber
277 |             :param protocol='tcp': string ('tcp','udp')
278 | 
279 |             :return: NmapService or None
280 |         """
281 |         plist = [p for p in self._services if
282 |                  p.port == portno and p.protocol == protocol]
283 |         if len(plist) > 1:
284 |             raise Exception("Duplicate services found in NmapHost object")
285 |         return plist.pop() if len(plist) else None
286 | 
287 |     def get_service_byid(self, service_id):
288 |         """
289 |             Returns a NmapService by providing its id.
290 | 
291 |             The id of a nmap service is a python tupl made of (protocol, port)
292 |         """
293 |         rval = None
294 |         for _tmpservice in self._services:
295 |             if _tmpservice.id == service_id:
296 |                 rval = _tmpservice
297 |         return rval
298 | 
299 |     def os_class_probabilities(self):
300 |         """
301 |             Returns an array of possible OS class detected during
302 |             the OS fingerprinting.
303 | 
304 |             :return: Array of NmapOSClass objects
305 |         """
306 |         rval = []
307 |         if self.os is not None:
308 |             rval = self.os.osclasses
309 |         return rval
310 | 
311 |     def os_match_probabilities(self):
312 |         """
313 |             Returns an array of possible OS match detected during
314 |             the OS fingerprinting
315 | 
316 |             :return: array of NmapOSMatches objects
317 |         """
318 |         rval = []
319 |         if self.os is not None:
320 |             rval = self.os.osmatches
321 |         return rval
322 | 
323 |     @property
324 |     def os_fingerprinted(self):
325 |         """
326 |             Specify if the host has OS fingerprint data available
327 | 
328 |             :return: Boolean
329 |         """
330 |         return self._osfingerprinted
331 | 
332 |     @property
333 |     def os_fingerprint(self):
334 |         """
335 |             Returns the fingerprint of the scanned system.
336 | 
337 |             :return: string
338 |         """
339 |         rval = ''
340 |         if self.os is not None:
341 |             rval = "\n".join(self.os.fingerprints)
342 |         return rval
343 | 
344 |     def os_ports_used(self):
345 |         """
346 |             Returns an array of the ports used for OS fingerprinting
347 | 
348 |             :return: array of ports used: [{'portid': '22',
349 |                                             'proto': 'tcp',
350 |                                             'state': 'open'},]
351 |         """
352 |         rval = []
353 |         try:
354 |             rval = self._extras['os']['ports_used']
355 |         except (KeyError, TypeError):
356 |             pass
357 |         return rval
358 | 
359 |     @property
360 |     def tcpsequence(self):
361 |         """
362 |             Returns the difficulty to determine remotely predict
363 |             the tcp sequencing.
364 | 
365 |             return: string
366 |         """
367 |         rval = ''
368 |         try:
369 |             rval = self._extras['tcpsequence']['difficulty']
370 |         except (KeyError, TypeError):
371 |             pass
372 |         return rval
373 | 
374 |     @property
375 |     def ipsequence(self):
376 |         """
377 |             Return the class of ip sequence of the remote hosts.
378 | 
379 |             :return: string
380 |         """
381 |         rval = ''
382 |         try:
383 |             rval = self._extras['ipidsequence']['class']
384 |         except (KeyError, TypeError):
385 |             pass
386 |         return rval
387 | 
388 |     @property
389 |     def uptime(self):
390 |         """
391 |             uptime of the remote host (if nmap was able to determine it)
392 | 
393 |             :return: string (in seconds)
394 |         """
395 |         rval = 0
396 |         try:
397 |             rval = int(self._extras['uptime']['seconds'])
398 |         except (KeyError, TypeError):
399 |             pass
400 |         return rval
401 | 
402 |     @property
403 |     def lastboot(self):
404 |         """
405 |             Since when the host was booted.
406 | 
407 |             :return: string
408 |         """
409 |         rval = ''
410 |         try:
411 |             rval = self._extras['uptime']['lastboot']
412 |         except (KeyError, TypeError):
413 |             pass
414 |         return rval
415 | 
416 |     @property
417 |     def distance(self):
418 |         """
419 |             Number of hops to host
420 | 
421 |             :return: int
422 |         """
423 |         rval = 0
424 |         try:
425 |             rval = int(self._extras['distance']['value'])
426 |         except (KeyError, TypeError):
427 |             pass
428 |         return rval
429 | 
430 |     @property
431 |     def scripts_results(self):
432 |         """
433 |             Scripts results specific to the scanned host
434 | 
435 |             :return: array of <script> dictionary
436 |         """
437 |         rval = {}
438 |         try:
439 |             rval = self._extras['hostscript']
440 |         except (KeyError, TypeError):
441 |             pass
442 |         return rval
443 | 
444 |     @property
445 |     def id(self):
446 |         """
447 |             id of the host. Used for diff()ing NmapObjects
448 | 
449 |             :return: string
450 |         """
451 |         return self.address
452 | 
453 |     @property
454 |     def extraports_state(self):
455 |         """
456 |             dictionnary containing state and amount of extra ports scanned
457 |             for which a common state, usually, closed was discovered.
458 | 
459 |             :return: dict with keys 'state' and 'count' or None
460 |         """
461 |         _xtrports = self._extras.get('extraports', None)
462 | 
463 |         if _xtrports is None:
464 |             return None
465 | 
466 |         return {'state': _xtrports['state'], 'count': _xtrports['count']}
467 | 
468 |     @property
469 |     def extraports_reasons(self):
470 |         """
471 |             dictionnary containing reasons why extra ports scanned
472 |             for which a common state, usually, closed was discovered.
473 | 
474 |             :return: array of dict containing keys 'state' and 'count' or None
475 |         """
476 |         r = self._extras.get('extraports', {})
477 | 
478 |         return r.get('reasons', None)
479 | 
480 |     def get_dict(self):
481 |         """
482 |             Return a dict representation of the object.
483 | 
484 |             This is needed by NmapDiff to allow comparaison
485 | 
486 |             :return dict
487 |         """
488 |         d = dict([("{0}::{1}".format(s.__class__.__name__, str(s.id)),
489 |                    hash(s))
490 |                  for s in self.services])
491 | 
492 |         d.update({'address': self.address, 'status': self.status,
493 |                   'hostnames': " ".join(self._hostnames)})
494 |         return d
495 | 
496 |     def diff(self, other):
497 |         """
498 |             Calls NmapDiff to check the difference between self and
499 |             another NmapHost object.
500 | 
501 |             Will return a NmapDiff object.
502 | 
503 |             This objects return python set() of keys describing the elements
504 |             which have changed, were added, removed or kept unchanged.
505 | 
506 |             :param other: NmapHost to diff with
507 | 
508 |             :return: NmapDiff object
509 |         """
510 |         return NmapDiff(self, other)
511 | 


--------------------------------------------------------------------------------
/libnmap/objects/os.py:
--------------------------------------------------------------------------------
  1 | # -*- coding: utf-8 -*-
  2 | 
  3 | import warnings
  4 | from libnmap.objects.cpe import CPE
  5 | 
  6 | 
  7 | class OSFPPortUsed(object):
  8 |     """
  9 |         Port used class: this enables the user of NmapOSFingerprint class
 10 |         to have a common and clear interface to access portused data which
 11 |         were collected and used during os fingerprint scan
 12 |     """
 13 |     def __init__(self, port_used_dict):
 14 |         try:
 15 |             self._state = port_used_dict['state']
 16 |             self._proto = port_used_dict['proto']
 17 |             self._portid = port_used_dict['portid']
 18 |         except KeyError:
 19 |             raise Exception("Cannot create OSFPPortUsed: missing required key")
 20 | 
 21 |     @property
 22 |     def state(self):
 23 |         """
 24 |             Accessor for the portused state (closed, open,...)
 25 |         """
 26 |         return self._state
 27 | 
 28 |     @property
 29 |     def proto(self):
 30 |         """
 31 |             Accessor for the portused protocol (tcp, udp,...)
 32 |         """
 33 |         return self._proto
 34 | 
 35 |     @property
 36 |     def portid(self):
 37 |         """
 38 |             Accessor for the referenced port number used
 39 |         """
 40 |         return self._portid
 41 | 
 42 | 
 43 | class NmapOSMatch(object):
 44 |     """
 45 |         NmapOSMatch is an internal class used for offering results
 46 |         from an nmap os fingerprint. This common interfaces makes
 47 |         a compatibility between old nmap xml (<1.04) and new nmap
 48 |         xml versions (used in nmapv6 for instance).
 49 | 
 50 |         In previous xml version, osclass tags from nmap fingerprints
 51 |         were not directly mapped to a osmatch. In new xml version,
 52 |         osclass could be embedded in osmatch tag.
 53 | 
 54 |         The approach to solve this is to create a common class
 55 |         which will, for older xml version, match based on the accuracy
 56 |         osclass to an osmatch. If no match, an osmatch will be made up
 57 |         from a concat of os class attributes: vendor and osfamily.
 58 |         Unmatched osclass will have a line attribute of -1.
 59 | 
 60 |         More info, see issue #26 or http://seclists.org/nmap-dev/2012/q2/252
 61 |     """
 62 |     def __init__(self, osmatch_dict):
 63 |         _osmatch_dict = osmatch_dict['osmatch']
 64 |         if('name' not in _osmatch_dict or
 65 |            'line' not in _osmatch_dict or
 66 |            'accuracy' not in _osmatch_dict):
 67 |             raise Exception("Cannot create NmapOSClass: missing required key")
 68 | 
 69 |         self._name = _osmatch_dict['name']
 70 |         self._line = _osmatch_dict['line']
 71 |         self._accuracy = _osmatch_dict['accuracy']
 72 | 
 73 |         # create osclass list
 74 |         self._osclasses = []
 75 |         try:
 76 |             for _osclass in osmatch_dict['osclasses']:
 77 |                 try:
 78 |                     _osclassobj = NmapOSClass(_osclass)
 79 |                 except:
 80 |                     raise Exception("Could not create NmapOSClass object")
 81 |                 self._osclasses.append(_osclassobj)
 82 |         except KeyError:
 83 |             pass
 84 | 
 85 |     def add_osclass(self, osclass_obj):
 86 | 
 87 |         """
 88 |             Add a NmapOSClass object to the OSMatch object. This method is
 89 |             useful to implement compatibility with older versions of NMAP
 90 |             by providing a common interface to access os fingerprint data.
 91 |         """
 92 |         self._osclasses.append(osclass_obj)
 93 | 
 94 |     @property
 95 |     def osclasses(self):
 96 |         """
 97 |             Accessor for all NmapOSClass objects matching with this OS Match
 98 |         """
 99 |         return self._osclasses
100 | 
101 |     @property
102 |     def name(self):
103 |         """
104 |             Accessor for name attribute (e.g.: Linux 2.4.26 (Slackware 10.0.0))
105 |         """
106 |         return self._name
107 | 
108 |     @property
109 |     def line(self):
110 |         """
111 |             Accessor for line attribute as integer. value equals -1 if this
112 |             osmatch holds orphans NmapOSClass objects. This could happen with
113 |             older version of nmap xml engine (<1.04 (e.g: nmapv6)).
114 | 
115 |             :return: int
116 |         """
117 |         return int(self._line)
118 | 
119 |     @property
120 |     def accuracy(self):
121 |         """
122 |             Accessor for accuracy
123 | 
124 |             :return: int
125 |         """
126 |         return int(self._accuracy)
127 | 
128 |     def get_cpe(self):
129 |         """
130 |             This method return a list of cpe stings and not CPE objects as
131 |             the NmapOSClass.cpelist property. This method is a helper to
132 |             simplify data management.
133 | 
134 |             For more advanced handling of CPE data, use NmapOSClass.cpelist
135 |             and use the methods from CPE class
136 |         """
137 |         _cpelist = []
138 |         for osc in self.osclasses:
139 |             for cpe in osc.cpelist:
140 |                 _cpelist.append(cpe.cpestring)
141 |         return _cpelist
142 | 
143 |     def __repr__(self):
144 |         rval = "{0}: {1}".format(self.name, self.accuracy)
145 |         for _osclass in self._osclasses:
146 |             rval += "\r\n  |__ os class: {0}".format(str(_osclass))
147 |         return rval
148 | 
149 | 
150 | class NmapOSClass(object):
151 |     """
152 |         NmapOSClass offers an unified API to access data from analysed
153 |         osclass tag. As implemented in libnmap and newer version of nmap,
154 |         osclass objects will always be embedded in a NmapOSMatch.
155 |         Unmatched NmapOSClass will be stored in "dummy" NmapOSMatch objects
156 |         which will have the particularity of have a line attribute of -1.
157 |         On top of this, NmapOSClass will have optional CPE objects
158 |         embedded.
159 |     """
160 |     def __init__(self, osclass_dict):
161 |         _osclass = osclass_dict['osclass']
162 |         if('vendor' not in _osclass or
163 |            'osfamily' not in _osclass or
164 |            'accuracy' not in _osclass):
165 |             raise Exception("Wrong osclass structure: missing required key")
166 | 
167 |         self._vendor = _osclass['vendor']
168 |         self._osfamily = _osclass['osfamily']
169 |         self._accuracy = _osclass['accuracy']
170 | 
171 |         self._osgen = ''
172 |         self._type = ''
173 |         # optional data
174 |         if 'osgen' in _osclass:
175 |             self._osgen = _osclass['osgen']
176 |         if 'type' in _osclass:
177 |             self._type = _osclass['type']
178 | 
179 |         self._cpelist = []
180 |         for _cpe in osclass_dict['cpe']:
181 |             self._cpelist.append(CPE(_cpe))
182 | 
183 |     @property
184 |     def cpelist(self):
185 |         """
186 |             Returns a list of CPE Objects matching with this os class
187 | 
188 |             :return: list of CPE objects
189 |             :rtype: Array
190 |         """
191 |         return self._cpelist
192 | 
193 |     @property
194 |     def vendor(self):
195 |         """
196 |             Accessor for vendor information (Microsoft, Linux,...)
197 | 
198 |             :return: string
199 |         """
200 |         return self._vendor
201 | 
202 |     @property
203 |     def osfamily(self):
204 |         """
205 |             Accessor for OS family information (Windows, Linux,...)
206 | 
207 |             :return: string
208 |         """
209 |         return self._osfamily
210 | 
211 |     @property
212 |     def accuracy(self):
213 |         """
214 |             Accessor for OS class detection accuracy (int)
215 | 
216 |             :return: int
217 |         """
218 |         return int(self._accuracy)
219 | 
220 |     @property
221 |     def osgen(self):
222 |         """
223 |             Accessor for OS class generation (7, 8, 2.4.X,...).
224 | 
225 |             :return: string
226 |         """
227 |         return self._osgen
228 | 
229 |     @property
230 |     def type(self):
231 |         """
232 |             Accessor for OS class type (general purpose,...)
233 | 
234 |             :return: string
235 |         """
236 |         return self._type
237 | 
238 |     @property
239 |     def description(self):
240 |         """
241 |             Accessor helper which returns a concataned string of
242 |             the valuable attributes from NmapOSClass object
243 | 
244 |             :return: string
245 |         """
246 |         rval = "{0}: {1}, {2}".format(self.type, self.vendor, self.osfamily)
247 |         if len(self.osgen):
248 |             rval += "({0})".format(self.osgen)
249 |         return rval
250 | 
251 |     def __repr__(self):
252 |         rval = "{0}: {1}, {2}".format(self.type, self.vendor, self.osfamily)
253 |         if len(self.osgen):
254 |             rval += "({0})".format(self.osgen)
255 |         for _cpe in self._cpelist:
256 |             rval += "\r\n    |__ {0}".format(str(_cpe))
257 |         return rval
258 | 
259 | 
260 | class NmapOSFingerprint(object):
261 |     """
262 |         NmapOSFingerprint is a easier API for using os fingerprinting.
263 |         Data for OS fingerprint (<os> tag) is instanciated from
264 |         a NmapOSFingerprint which is accessible in NmapHost via NmapHost.os
265 |     """
266 |     def __init__(self, osfp_data):
267 |         self.__osmatches = []
268 |         self.__ports_used = []
269 |         self.__fingerprints = []
270 | 
271 |         if 'osmatches' in osfp_data:
272 |             for _osmatch in osfp_data['osmatches']:
273 |                 _osmatch_obj = NmapOSMatch(_osmatch)
274 |                 self.__osmatches.append(_osmatch_obj)
275 |         if 'osclasses' in osfp_data:
276 |             for _osclass in osfp_data['osclasses']:
277 |                 _osclass_obj = NmapOSClass(_osclass)
278 |                 _osmatched = self.get_osmatch(_osclass_obj)
279 |                 if _osmatched is not None:
280 |                     _osmatched.add_osclass(_osclass_obj)
281 |                 else:
282 |                     self._add_dummy_osmatch(_osclass_obj)
283 |         if 'osfingerprints' in osfp_data:
284 |             for _osfp in osfp_data['osfingerprints']:
285 |                 if 'fingerprint' in _osfp:
286 |                     self.__fingerprints.append(_osfp['fingerprint'])
287 |         if 'ports_used' in osfp_data:
288 |             for _pused_dict in osfp_data['ports_used']:
289 |                 _pused = OSFPPortUsed(_pused_dict)
290 |                 self.__ports_used.append(_pused)
291 | 
292 |     def get_osmatch(self, osclass_obj):
293 |         """
294 |             This function enables NmapOSFingerprint to determine if an
295 |             NmapOSClass object could be attached to an existing NmapOSMatch
296 |             object in order to respect the common interface for
297 |             the nmap xml version < 1.04 and >= 1.04
298 | 
299 |             This method will return an NmapOSMatch object matching with
300 |             the NmapOSClass provided in parameter
301 |             (match is performed based on accuracy)
302 | 
303 |             :return: NmapOSMatch object
304 |         """
305 |         rval = None
306 |         for _osmatch in self.__osmatches:
307 |             if _osmatch.accuracy == osclass_obj.accuracy:
308 |                 rval = _osmatch
309 |                 break  # sorry
310 |         return rval
311 | 
312 |     def _add_dummy_osmatch(self, osclass_obj):
313 |         """
314 |             This functions creates a dummy NmapOSMatch object in order to
315 |             encapsulate an NmapOSClass object which was not matched with an
316 |             existing NmapOSMatch object
317 |         """
318 |         _dname = "{0}:{1}:{2}".format(osclass_obj.type,
319 |                                       osclass_obj.vendor,
320 |                                       osclass_obj.osfamily)
321 |         _dummy_dict = {'osmatch': {'name': _dname,
322 |                                    'accuracy': osclass_obj.accuracy,
323 |                                    'line': -1},
324 |                        'osclasses': []}
325 |         _dummy_osmatch = NmapOSMatch(_dummy_dict)
326 |         self.__osmatches.append(_dummy_osmatch)
327 | 
328 |     @property
329 |     def osmatches(self, min_accuracy=0):
330 |         _osmatches = []
331 | 
332 |         for _osmatch in self.__osmatches:
333 |             if _osmatch.accuracy >= min_accuracy:
334 |                 _osmatches.append(_osmatch)
335 | 
336 |         return _osmatches
337 | 
338 |     @property
339 |     def fingerprint(self):
340 |         return "\r\n".join(self.__fingerprints)
341 | 
342 |     @property
343 |     def fingerprints(self):
344 |         return self.__fingerprints
345 | 
346 |     @property
347 |     def ports_used(self):
348 |         """
349 |             Return an array of OSFPPortUsed object with the ports used to
350 |             perform the os fingerprint. This dict might contain another dict
351 |             embedded containing the ports_reason values.
352 |         """
353 |         return self.__ports_used
354 | 
355 |     def osmatch(self, min_accuracy=90):
356 |         warnings.warn("NmapOSFingerprint.osmatch is deprecated: "
357 |                       "use NmapOSFingerprint.osmatches", DeprecationWarning)
358 |         os_array = []
359 |         for _osmatch in self.__osmatches:
360 |             if _osmatch.accuracy >= min_accuracy:
361 |                 os_array.append(_osmatch.name)
362 |         return os_array
363 | 
364 |     def osclass(self, min_accuracy=90):
365 |         warnings.warn("NmapOSFingerprint.osclass() is deprecated: "
366 |                       "use NmapOSFingerprint.osclasses() if applicable",
367 |                       DeprecationWarning)
368 |         os_array = []
369 |         for osmatch_entry in self.osmatches():
370 |             if osmatch_entry.accuracy >= min_accuracy:
371 |                 for oclass in osmatch_entry.osclasses:
372 |                     _ftstr = "type:{0}|vendor:{1}|osfamily{2}".format(
373 |                              oclass.type,
374 |                              oclass.vendor,
375 |                              oclass.osfamily)
376 |                     os_array.append(_ftstr)
377 |         return os_array
378 | 
379 |     def os_cpelist(self):
380 |         cpelist = []
381 |         for _osmatch in self.osmatches:
382 |             for oclass in _osmatch.osclasses:
383 |                 cpelist.extend(oclass.cpelist)
384 |         return cpelist
385 | 
386 |     def __repr__(self):
387 |         rval = ""
388 |         for _osmatch in self.osmatches:
389 |             rval += "\r\n{0}".format(_osmatch)
390 |         rval += "Fingerprints: ".format(self.fingerprint)
391 |         return rval
392 | 


--------------------------------------------------------------------------------
/libnmap/objects/report.py:
--------------------------------------------------------------------------------
  1 | # -*- coding: utf-8 -*-
  2 | from libnmap.diff import NmapDiff
  3 | 
  4 | 
  5 | class NmapReport(object):
  6 |     """
  7 |         NmapReport is the usual interface for the end user to
  8 |         read scans output.
  9 | 
 10 |         A NmapReport as the following structure:
 11 | 
 12 |         - Scan headers data
 13 |         - A list of scanned hosts (NmapReport.hosts)
 14 |         - Scan footer data
 15 | 
 16 |         It is to note that each NmapHost comprised in NmapReport.hosts array
 17 |         contains also a list of scanned services (NmapService object).
 18 | 
 19 |         This means that if NmapParser.parse*() is the input interface for the
 20 |         end user of the lib. NmapReport is certainly the output interface for
 21 |         the end user of the lib.
 22 |     """
 23 |     def __init__(self, raw_data=None):
 24 |         """
 25 |             Constructor for NmapReport object.
 26 | 
 27 |             This is usually called by the NmapParser module.
 28 |         """
 29 |         self._nmaprun = {}
 30 |         self._scaninfo = {}
 31 |         self._hosts = []
 32 |         self._runstats = {}
 33 |         if raw_data is not None:
 34 |             self.__set_raw_data(raw_data)
 35 | 
 36 |     def save(self, backend):
 37 |         """
 38 |             This method gets a NmapBackendPlugin representing the backend.
 39 | 
 40 |             :param backend: libnmap.plugins.PluginBackend object.
 41 | 
 42 |             Object created by BackendPluginFactory and enabling nmap reports
 43 |             to be saved/stored in any type of backend implemented in plugins.
 44 | 
 45 |             The primary key of the stored object is returned.
 46 | 
 47 |             :return: str
 48 |         """
 49 |         if backend is not None:
 50 |             _id = backend.insert(self)
 51 |         else:
 52 |             raise RuntimeError
 53 |         return _id
 54 | 
 55 |     def diff(self, other):
 56 |         """
 57 |             Calls NmapDiff to check the difference between self and
 58 |             another NmapReport object.
 59 | 
 60 |             Will return a NmapDiff object.
 61 | 
 62 |             :return: NmapDiff object
 63 |             :todo: remove is_consistent approach, diff() should be generic.
 64 |         """
 65 |         if self.is_consistent() and other.is_consistent():
 66 |             _rdiff = NmapDiff(self, other)
 67 |         else:
 68 |             _rdiff = set()
 69 |         return _rdiff
 70 | 
 71 |     @property
 72 |     def started(self):
 73 |         """
 74 |             Accessor returning a unix timestamp of when the scan was started.
 75 | 
 76 |             :return: integer
 77 |         """
 78 |         rval = -1
 79 |         try:
 80 |             s_start = self._nmaprun['start']
 81 |             rval = int(s_start)
 82 |         except(KeyError, TypeError, ValueError):
 83 |             pass
 84 |         return rval
 85 | 
 86 |     @property
 87 |     def commandline(self):
 88 |         """
 89 |             Accessor returning the full nmap command line fired.
 90 | 
 91 |             :return: string
 92 |         """
 93 |         return self._nmaprun['args']
 94 | 
 95 |     @property
 96 |     def version(self):
 97 |         """
 98 |             Accessor returning the version of the
 99 |             nmap binary used to perform the scan.
100 | 
101 |             :return: string
102 |         """
103 |         return self._nmaprun['version']
104 | 
105 |     @property
106 |     def scan_type(self):
107 |         """
108 |             Accessor returning a string which identifies what type of scan
109 |             was launched (syn, ack, tcp,...).
110 | 
111 |             :return: string
112 |         """
113 |         return self._scaninfo['type']
114 | 
115 |     @property
116 |     def hosts(self):
117 |         """
118 |             Accessor returning an array of scanned hosts.
119 | 
120 |             Scanned hosts are NmapHost objects.
121 | 
122 |             :return: array of NmapHost
123 |         """
124 |         return self._hosts
125 | 
126 |     def get_host_byid(self, host_id):
127 |         """
128 |            Gets a NmapHost object directly from the host array
129 |            by looking it up by id.
130 | 
131 |            :param ip_addr: ip address of the host to lookup
132 |            :type ip_addr: string
133 | 
134 |            :return: NmapHost
135 |         """
136 |         rval = None
137 |         for _rhost in self._hosts:
138 |             if _rhost.address == host_id:
139 |                 rval = _rhost
140 |         return rval
141 | 
142 |     @property
143 |     def endtime(self):
144 |         """
145 |             Accessor returning a unix timestamp of when the scan ended.
146 | 
147 |             :return: integer
148 |         """
149 |         rval = -1
150 |         try:
151 |             rval = int(self._runstats['finished']['time'])
152 |         except(KeyError, TypeError, ValueError):
153 |             pass
154 |         return rval
155 | 
156 |     @property
157 |     def endtimestr(self):
158 |         """
159 |             Accessor returning a human readable time string
160 |             of when the scan ended.
161 | 
162 |             :return: string
163 |         """
164 |         rval = ''
165 |         try:
166 |             rval = self._runstats['finished']['timestr']
167 |         except(KeyError, TypeError, ValueError):
168 |             pass
169 |         return rval
170 | 
171 |     @property
172 |     def summary(self):
173 |         """
174 |             Accessor returning a string describing and
175 |             summarizing the scan.
176 | 
177 |             :return: string
178 |         """
179 |         rval = ''
180 |         try:
181 |             rval = self._runstats['finished']['summary']
182 |         except(KeyError, TypeError):
183 |             pass
184 | 
185 |         if len(rval) == 0:
186 |             rval = ("Nmap ended at {0} ; {1} IP addresses ({2} hosts up)"
187 |                     " scanned in {3} seconds".format(self.endtimestr,
188 |                                                      self.hosts_total,
189 |                                                      self.hosts_up,
190 |                                                      self.elapsed))
191 |         return rval
192 | 
193 |     @property
194 |     def elapsed(self):
195 |         """
196 |             Accessor returning the number of seconds the scan took
197 | 
198 |             :return: float (0 >= or -1)
199 |         """
200 |         rval = -1
201 |         try:
202 |             s_elapsed = self._runstats['finished']['elapsed']
203 |             rval = float(s_elapsed)
204 |         except (KeyError, TypeError, ValueError):
205 |             rval = -1
206 |         return rval
207 | 
208 |     @property
209 |     def hosts_up(self):
210 |         """
211 |             Accessor returning the numer of host detected
212 |             as 'up' during the scan.
213 | 
214 |             :return: integer (0 >= or -1)
215 |         """
216 |         rval = -1
217 |         try:
218 |             s_up = self._runstats['hosts']['up']
219 |             rval = int(s_up)
220 |         except (KeyError, TypeError, ValueError):
221 |             rval = -1
222 |         return rval
223 | 
224 |     @property
225 |     def hosts_down(self):
226 |         """
227 |             Accessor returning the numer of host detected
228 |             as 'down' during the scan.
229 | 
230 |             :return: integer (0 >= or -1)
231 |         """
232 |         rval = -1
233 |         try:
234 |             s_down = self._runstats['hosts']['down']
235 |             rval = int(s_down)
236 |         except (KeyError, TypeError, ValueError):
237 |             rval = -1
238 |         return rval
239 | 
240 |     @property
241 |     def hosts_total(self):
242 |         """
243 |             Accessor returning the number of hosts scanned in total.
244 | 
245 |             :return: integer (0 >= or -1)
246 |         """
247 |         rval = -1
248 |         try:
249 |             s_total = self._runstats['hosts']['total']
250 |             rval = int(s_total)
251 |         except (KeyError, TypeError, ValueError):
252 |             rval = -1
253 |         return rval
254 | 
255 |     def get_raw_data(self):
256 |         """
257 |             Returns a dict representing the NmapReport object.
258 | 
259 |             :return: dict
260 |             :todo: deprecate. get rid of this uglyness.
261 |         """
262 |         raw_data = {'_nmaprun': self._nmaprun,
263 |                     '_scaninfo': self._scaninfo,
264 |                     '_hosts': self._hosts,
265 |                     '_runstats': self._runstats}
266 |         return raw_data
267 | 
268 |     def __set_raw_data(self, raw_data):
269 |         self._nmaprun = raw_data['_nmaprun']
270 |         self._scaninfo = raw_data['_scaninfo']
271 |         self._hosts = raw_data['_hosts']
272 |         self._runstats = raw_data['_runstats']
273 | 
274 |     def is_consistent(self):
275 |         """
276 |             Checks if the report is consistent and can be diffed().
277 | 
278 |             This needs to be rewritten and removed: diff() should be generic.
279 | 
280 |             :return: boolean
281 |         """
282 |         rval = False
283 |         rdata = self.get_raw_data()
284 |         _consistent_keys = ['_nmaprun', '_scaninfo', '_hosts', '_runstats']
285 |         if(set(_consistent_keys) == set(rdata.keys()) and
286 |            len([dky for dky in rdata.keys() if rdata[dky] is not None]) == 4):
287 |             rval = True
288 |         return rval
289 | 
290 |     def get_dict(self):
291 |         """
292 |             Return a python dict representation of the NmapReport object.
293 |             This is used to diff() NmapReport objects via NmapDiff.
294 | 
295 |             :return: dict
296 |         """
297 |         rdict = dict([("{0}::{1}".format(_host.__class__.__name__,
298 |                                          str(_host.id)),
299 |                      hash(_host)) for _host in self.hosts])
300 |         rdict.update({'commandline': self.commandline,
301 |                       'version': self.version,
302 |                       'scan_type': self.scan_type,
303 |                       'elapsed': self.elapsed,
304 |                       'hosts_up': self.hosts_up,
305 |                       'hosts_down': self.hosts_down,
306 |                       'hosts_total': self.hosts_total})
307 |         return rdict
308 | 
309 |     @property
310 |     def id(self):
311 |         """
312 |             Dummy id() defined for reports.
313 |         """
314 |         return hash(1)
315 | 
316 |     def __eq__(self, other):
317 |         """
318 |             Compare eq NmapReport based on :
319 | 
320 |                 - create a diff obj and check the result
321 |                 report are equal if added&changed&removed are empty
322 | 
323 |             :return: boolean
324 |         """
325 |         rval = False
326 |         if(self.__class__ == other.__class__ and self.id == other.id):
327 |             diffobj = self.diff(other)
328 |             rval = (len(diffobj.changed()) == 0 and
329 |                     len(diffobj.added()) == 0 and
330 |                     len(diffobj.removed()) == 0
331 |                     )
332 |         return rval
333 | 
334 |     def __ne__(self, other):
335 |         """
336 |             Compare ne NmapReport based on:
337 | 
338 |                 - create a diff obj and check the result
339 |                 report are ne if added|changed|removed are not empty
340 | 
341 |             :return: boolean
342 |         """
343 |         rval = True
344 |         if(self.__class__ == other.__class__ and self.id == other.id):
345 |             diffobj = self.diff(other)
346 |             rval = (len(diffobj.changed()) != 0 or
347 |                     len(diffobj.added()) != 0 or
348 |                     len(diffobj.removed()) != 0
349 |                     )
350 |         return rval
351 | 
352 |     def __repr__(self):
353 |         return "{0}: started at {1} hosts up {2}/{3}".format(
354 |                self.__class__.__name__,
355 |                self.started,
356 |                self.hosts_up,
357 |                self.hosts_total)
358 | 


--------------------------------------------------------------------------------
/libnmap/objects/service.py:
--------------------------------------------------------------------------------
  1 | # -*- coding: utf-8 -*-
  2 | from libnmap.diff import NmapDiff
  3 | from libnmap.objects.os import CPE
  4 | 
  5 | 
  6 | class NmapService(object):
  7 |     """
  8 |         NmapService represents a nmap scanned service. Its id() is comprised
  9 |         of the protocol and the port.
 10 | 
 11 |         Depending on the scanning options, some additional details might be
 12 |         available or not. Like banner or extra datas from NSE (nmap scripts).
 13 |     """
 14 |     def __init__(self, portid, protocol='tcp', state=None,
 15 |                  service=None, owner=None, service_extras=None):
 16 |         """
 17 |             Constructor
 18 | 
 19 |             :param portid: port number
 20 |             :type portid: string
 21 |             :param protocol: protocol of port scanned (tcp, udp)
 22 |             :type protocol: string
 23 |             :param state: python dict describing the service status
 24 |             :type state: python dict
 25 |             :param service: python dict describing the service name and banner
 26 |             :type service: python dict
 27 |             :param service_extras: additional info about the tested service
 28 |             like scripts' data
 29 |         """
 30 |         try:
 31 |             self._portid = int(portid or -1)
 32 |         except (ValueError, TypeError):
 33 |             raise
 34 |         if self._portid < 0 or self._portid > 65535:
 35 |             raise ValueError
 36 | 
 37 |         self._protocol = protocol
 38 |         # self._taskid = taskid
 39 |         # self._address = address
 40 |         self._state = state if state is not None else {}
 41 |         self._service = service if service is not None else {}
 42 | 
 43 |         self._cpelist = []
 44 |         if 'cpelist' in self._service:
 45 |             for _cpe in self._service['cpelist']:
 46 |                 _cpeobj = CPE(_cpe)
 47 |                 self._cpelist.append(_cpeobj)
 48 | 
 49 |         self._owner = ''
 50 |         if owner is not None and 'name' in owner:
 51 |             self._owner = owner['name']
 52 | 
 53 |         self._reason = ''
 54 |         self._reason_ip = ''
 55 |         self._reason_ttl = ''
 56 |         self._servicefp = ''
 57 |         self._tunnel = ''
 58 | 
 59 |         if 'reason' in self._state:
 60 |             self._reason = self._state['reason']
 61 |         if 'reason_ttl' in self._state:
 62 |             self._reason_ttl = self._state['reason_ttl']
 63 |         if 'reason_ip' in self._state:
 64 |             self._reason_ip = self._state['reason_ip']
 65 | 
 66 |         if 'servicefp' in self._service:
 67 |             self._servicefp = self._service['servicefp']
 68 |         if 'tunnel' in self._service:
 69 |             self._tunnel = self._service['tunnel']
 70 | 
 71 |         self._service_extras = []
 72 |         if service_extras is not None:
 73 |             self._service_extras = service_extras
 74 | 
 75 |     def __eq__(self, other):
 76 |         """
 77 |             Compares two NmapService objects to see if they are the same or
 78 |             if one of them changed.
 79 | 
 80 |             :param other: NmapService
 81 | 
 82 |             :return: boolean
 83 |         """
 84 |         rval = False
 85 |         if(self.__class__ == other.__class__ and self.id == other.id):
 86 |             rval = (self.changed(other) == 0)
 87 |         return rval
 88 | 
 89 |     def __ne__(self, other):
 90 |         """
 91 |             Compares two NmapService objects to see if they are different
 92 |             if one of them changed.
 93 | 
 94 |             :param other: NmapService
 95 | 
 96 |             :return: boolean
 97 |         """
 98 |         rval = True
 99 |         if(self.__class__ == other.__class__ and self.id == other.id):
100 |             rval = (self.changed(other) > 0)
101 |         return rval
102 | 
103 |     def __repr__(self):
104 |         return "{0}: [{1} {2}/{3} {4} ({5})]".format(self.__class__.__name__,
105 |                                                      self.state,
106 |                                                      str(self.port),
107 |                                                      self.protocol,
108 |                                                      self.service,
109 |                                                      self.banner)
110 | 
111 |     def __hash__(self):
112 |         return (hash(self.port) ^ hash(self.protocol) ^ hash(self.state) ^
113 |                 hash(self.reason) ^ hash(self.service) ^ hash(self.banner))
114 | 
115 |     def changed(self, other):
116 |         """
117 |             Checks if a NmapService is different from another.
118 | 
119 |             :param other: NmapService
120 | 
121 |             :return: boolean
122 |         """
123 |         return len(self.diff(other).changed())
124 | 
125 |     def save(self, backend):
126 |         if backend is not None:
127 |             _id = backend.insert(self)
128 |         else:
129 |             raise RuntimeError
130 |         return _id
131 | 
132 |     @property
133 |     def port(self):
134 |         """
135 |             Accessor for port.
136 | 
137 |             :return: integer or -1
138 |         """
139 |         return self._portid
140 | 
141 |     @property
142 |     def protocol(self):
143 |         """
144 |             Accessor for protocol
145 | 
146 |             :return: string
147 |         """
148 |         return self._protocol
149 | 
150 |     @property
151 |     def state(self):
152 |         """
153 |             Accessor for service's state (open, filtered, closed,...)
154 | 
155 |             :return: string
156 |         """
157 |         return self._state['state'] if 'state' in self._state else None
158 | 
159 |     @property
160 |     def reason(self):
161 |         """
162 |             Accessor for service's state reason (syn-ack, filtered,...)
163 | 
164 |             :return: string or empty if not applicable
165 |         """
166 |         return self._reason
167 | 
168 |     @property
169 |     def reason_ip(self):
170 |         """
171 |             Accessor for service's state reason ip
172 | 
173 |             :return: string or empty if not applicable
174 |         """
175 |         return self._reason_ip
176 | 
177 |     @property
178 |     def reason_ttl(self):
179 |         """
180 |             Accessor for service's state reason ttl
181 | 
182 |             :return: string or empty if not applicable
183 |         """
184 |         return self._reason_ttl
185 | 
186 |     @property
187 |     def service(self):
188 |         """
189 |             Accessor for service name.
190 | 
191 |             :return: string or empty
192 |         """
193 |         return self._service['name'] if 'name' in self._service else ''
194 | 
195 |     @property
196 |     def service_dict(self):
197 |         """
198 |             Accessor for service dictionary.
199 | 
200 |             :return: dict or None
201 |         """
202 |         return self._service
203 | 
204 |     def open(self):
205 |         """
206 |             Tells if the port was open or not
207 | 
208 |             :return: boolean
209 |         """
210 |         return 'state' in self._state and self._state['state'] == 'open'
211 | 
212 |     @property
213 |     def product(self):
214 |         if 'product' in self._service.keys():
215 |             return self._service['product'].rstrip()
216 |         else:
217 |             return None
218 | 
219 |     @property
220 |     def product_version(self):
221 |         if 'version' in self._service.keys():
222 |             return self._service['version'].rstrip()
223 |         else:
224 |             return None
225 | 
226 |     @property
227 |     def product_extrainfo(self):
228 |         if 'extrainfo' in self._service.keys():
229 |             return self._service['extrainfo'].rstrip()
230 |         else:
231 |             return None
232 | 
233 |     @property
234 |     def owner(self):
235 |         """
236 |             Accessor for service owner if available
237 |         """
238 |         return self._owner
239 | 
240 |     @property
241 |     def banner(self):
242 |         """
243 |             Accessor for the service's banner. Only available
244 |             if the nmap option -sV or similar was used.
245 | 
246 |             :return: string
247 |         """
248 |         notrelevant = ['name', 'method', 'conf', 'cpelist',
249 |                        'servicefp', 'tunnel']
250 |         relevant = ['product', 'version', 'extrainfo']
251 |         b = ''
252 |         skeys = self._service.keys()
253 |         if 'method' in self._service and self._service['method'] == "probed":
254 |             for relk in relevant:
255 |                 if relk in skeys:
256 |                     b += '{0}: {1} '.format(relk, self._service[relk])
257 |             for mkey in skeys:
258 |                 if mkey not in notrelevant and mkey not in relevant:
259 |                     b += '{0}: {1} '.format(mkey, self._service[mkey])
260 |         return b.rstrip()
261 | 
262 |     @property
263 |     def cpelist(self):
264 |         """
265 |             Accessor for list of CPE for this particular service
266 |         """
267 |         return self._cpelist
268 | 
269 |     @property
270 |     def scripts_results(self):
271 |         """
272 |             Gives a python list of the nse scripts results.
273 | 
274 |             The dict key is the name (id) of the nse script and
275 |             the value is the output of the script.
276 | 
277 |             :return: dict
278 |         """
279 |         scripts_dict = None
280 |         try:
281 |             scripts_dict = self._service_extras['scripts']
282 |         except (KeyError, TypeError):
283 |             pass
284 |         return scripts_dict
285 | 
286 |     @property
287 |     def servicefp(self):
288 |         """
289 |             Accessor for the service's fingerprint
290 |             if the nmap option -sV or -A is used
291 | 
292 |             :return: string if available
293 |         """
294 |         return self._servicefp
295 | 
296 |     @property
297 |     def tunnel(self):
298 |         """
299 |             Accessor for the service's tunnel type
300 |             if applicable and available from scan
301 |             results
302 | 
303 |             :return: string if available
304 |         """
305 |         return self._tunnel
306 | 
307 |     @property
308 |     def id(self):
309 |         """
310 |             Accessor for the id() of the NmapService.
311 | 
312 |             This is used for diff()ing NmapService object via NmapDiff.
313 | 
314 |             :return: tuple
315 |         """
316 |         return "{0}.{1}".format(self.protocol, self.port)
317 | 
318 |     def get_dict(self):
319 |         """
320 |             Return a python dict representation of the NmapService object.
321 | 
322 |             This is used to diff() NmapService objects via NmapDiff.
323 | 
324 |             :return: dict
325 |         """
326 |         return ({'id': str(self.id), 'port': str(self.port),
327 |                  'protocol': self.protocol, 'banner': self.banner,
328 |                  'service': self.service, 'state': self.state,
329 |                  'reason': self.reason})
330 | 
331 |     def diff(self, other):
332 |         """
333 |             Calls NmapDiff to check the difference between self and
334 |             another NmapService object.
335 | 
336 |             Will return a NmapDiff object.
337 | 
338 |             This objects return python set() of keys describing the elements
339 |             which have changed, were added, removed or kept unchanged.
340 | 
341 |             :return: NmapDiff object
342 |         """
343 |         return NmapDiff(self, other)
344 | 


--------------------------------------------------------------------------------
/libnmap/parser.py:
--------------------------------------------------------------------------------
  1 | # -*- coding: utf-8 -*-
  2 | 
  3 | 
  4 | try:
  5 |     import xml.etree.cElementTree as ET
  6 | except ImportError:
  7 |     import xml.etree.ElementTree as ET
  8 | from libnmap.objects import NmapHost, NmapService, NmapReport
  9 | 
 10 | 
 11 | class NmapParser(object):
 12 |     @classmethod
 13 |     def parse(cls, nmap_data=None, data_type='XML', incomplete=False):
 14 |         """
 15 |             Generic class method of NmapParser class.
 16 | 
 17 |             The data to be parsed does not need to be a complete nmap
 18 |             scan report. You can possibly give <hosts>...</hosts>
 19 |             or <port> XML tags.
 20 | 
 21 |             :param nmap_data: any portion of nmap scan result. \
 22 |             nmap_data should always be a string representing a part \
 23 |             or a complete nmap scan report.
 24 |             :type nmap_data: string
 25 | 
 26 |             :param data_type: specifies the type of data to be parsed.
 27 |             :type data_type: string ("XML"|"JSON"|"YAML").
 28 | 
 29 |             :param incomplete: enable you to parse interrupted nmap scans \
 30 |             and/or incomplete nmap xml blocks by adding a </nmaprun> at \
 31 |             the end of the scan.
 32 |             :type incomplete: boolean
 33 | 
 34 |             As of today, only XML parsing is supported.
 35 | 
 36 |             :return: NmapObject (NmapHost, NmapService or NmapReport)
 37 |         """
 38 | 
 39 |         nmapobj = None
 40 |         if data_type == "XML":
 41 |             nmapobj = cls._parse_xml(nmap_data, incomplete)
 42 |         else:
 43 |             raise NmapParserException("Unknown data type provided. "
 44 |                                       "Please check documentation for "
 45 |                                       "supported data types.")
 46 |         return nmapobj
 47 | 
 48 |     @classmethod
 49 |     def _parse_xml(cls, nmap_data=None, incomplete=False):
 50 |         """
 51 |             Protected class method used to process a specific data type.
 52 |             In this case: XML. This method is called by cls.parse class
 53 |             method and receives nmap scan results data (in XML).
 54 | 
 55 |             :param nmap_data: any portion of nmap scan result can be given \
 56 |             as argument. nmap_data should always be a string representing \
 57 |             a part or a complete nmap scan report.
 58 |             :type nmap_data: string
 59 | 
 60 |             This method checks which portion of a nmap scan is given \
 61 |             as argument.
 62 |             It could be:
 63 | 
 64 |                 1. a full nmap scan report;
 65 |                 2. a scanned host: <host> tag in a nmap scan report
 66 |                 3. a scanned service: <port> tag
 67 |                 4. a list of hosts: <hosts/> tag (TODO)
 68 |                 5. a list of ports: <ports/> tag
 69 | 
 70 |             :param incomplete: enable you to parse interrupted nmap scans \
 71 |             and/or incomplete nmap xml blocks by adding a </nmaprun> at \
 72 |             the end of the scan.
 73 |             :type incomplete: boolean
 74 | 
 75 |             :return: NmapObject (NmapHost, NmapService or NmapReport) \
 76 |                     or a list of NmapObject
 77 |         """
 78 | 
 79 |         if not nmap_data:
 80 |             raise NmapParserException("No report data to parse: please "
 81 |                                       "provide a valid XML nmap report")
 82 |         elif not isinstance(nmap_data, str):
 83 |             raise NmapParserException("wrong nmap_data type given as "
 84 |                                       "argument: cannot parse data")
 85 | 
 86 |         if incomplete is True:
 87 |             nmap_data += "</nmaprun>"
 88 | 
 89 |         try:
 90 |             root = ET.fromstring(nmap_data)
 91 |         except:
 92 |             raise NmapParserException("Wrong XML structure: cannot parse data")
 93 | 
 94 |         nmapobj = None
 95 |         if root.tag == 'nmaprun':
 96 |             nmapobj = cls._parse_xml_report(root)
 97 |         elif root.tag == 'host':
 98 |             nmapobj = cls._parse_xml_host(root)
 99 |         elif root.tag == 'ports':
100 |             nmapobj = cls._parse_xml_ports(root)
101 |         elif root.tag == 'port':
102 |             nmapobj = cls._parse_xml_port(root)
103 |         else:
104 |             raise NmapParserException("Unpexpected data structure for XML "
105 |                                       "root node")
106 |         return nmapobj
107 | 
108 |     @classmethod
109 |     def _parse_xml_report(cls, root=None):
110 |         """
111 |             This method parses out a full nmap scan report from its XML root
112 |             node: <nmaprun>.
113 | 
114 |             :param root: Element from xml.ElementTree (top of XML the document)
115 |             :type root: Element
116 | 
117 |             :return: NmapReport object
118 |         """
119 | 
120 |         nmap_scan = {'_nmaprun': {}, '_scaninfo': {},
121 |                      '_hosts': [], '_runstats': {}}
122 | 
123 |         if root is None:
124 |             raise NmapParserException("No root node provided to parse XML "
125 |                                       "report")
126 | 
127 |         nmap_scan['_nmaprun'] = cls.__format_attributes(root)
128 |         for el in root:
129 |             if el.tag == 'scaninfo':
130 |                 nmap_scan['_scaninfo'] = cls.__parse_scaninfo(el)
131 |             elif el.tag == 'host':
132 |                 nmap_scan['_hosts'].append(cls._parse_xml_host(el))
133 |             elif el.tag == 'runstats':
134 |                 nmap_scan['_runstats'] = cls.__parse_runstats(el)
135 |             # else:
136 |             #    print "struct pparse unknown attr: {0} value: {1}".format(
137 |             #        el.tag,
138 |             #        el.get(el.tag))
139 |         return NmapReport(nmap_scan)
140 | 
141 |     @classmethod
142 |     def parse_fromstring(cls, nmap_data, data_type="XML", incomplete=False):
143 |         """
144 |             Call generic cls.parse() method and ensure that a string is \
145 |             passed on as argument. If not, an exception is raised.
146 | 
147 |             :param nmap_data: Same as for parse(), any portion of nmap scan. \
148 |             Reports could be passed as argument. Data type _must_ be a string.
149 | 
150 |             :type nmap_data: string
151 | 
152 |             :param data_type: Specifies the type of data passed on as argument.
153 | 
154 |             :param incomplete: enable you to parse interrupted nmap scans \
155 |             and/or incomplete nmap xml blocks by adding a </nmaprun> at \
156 |             the end of the scan.
157 |             :type incomplete: boolean
158 | 
159 |             :return: NmapObject
160 |         """
161 | 
162 |         if not isinstance(nmap_data, str):
163 |             raise NmapParserException("bad argument type for "
164 |                                       "xarse_fromstring(): should be a string")
165 |         return cls.parse(nmap_data, data_type, incomplete)
166 | 
167 |     @classmethod
168 |     def parse_fromfile(cls, nmap_report_path,
169 |                        data_type="XML",
170 |                        incomplete=False):
171 |         """
172 |             Call generic cls.parse() method and ensure that a correct file \
173 |             path is given as argument. If not, an exception is raised.
174 | 
175 |             :param nmap_data: Same as for parse(). \
176 |             Any portion of nmap scan reports could be passed as argument. \
177 |             Data type _must be a valid path to a file containing \
178 |             nmap scan results.
179 | 
180 |             :param data_type: Specifies the type of serialization in the file.
181 | 
182 |             :param incomplete: enable you to parse interrupted nmap scans \
183 |             and/or incomplete nmap xml blocks by adding a </nmaprun> at \
184 |             the end of the scan.
185 |             :type incomplete: boolean
186 | 
187 |             :return: NmapObject
188 |         """
189 | 
190 |         try:
191 |             with open(nmap_report_path, 'r') as fileobj:
192 |                 fdata = fileobj.read()
193 |                 rval = cls.parse(fdata, data_type, incomplete)
194 |         except IOError:
195 |             raise
196 |         return rval
197 | 
198 |     @classmethod
199 |     def parse_fromdict(cls, rdict):
200 |         """
201 |             Strange method which transforms a python dict \
202 |             representation of a NmapReport and turns it into an \
203 |             NmapReport object. \
204 |             Needs to be reviewed and possibly removed.
205 | 
206 |             :param rdict: python dict representation of an NmapReport
207 |             :type rdict: dict
208 | 
209 |             :return: NmapReport
210 |         """
211 | 
212 |         nreport = {}
213 |         if list(rdict.keys())[0] == '__NmapReport__':
214 |             r = rdict['__NmapReport__']
215 |             nreport['_runstats'] = r['_runstats']
216 |             nreport['_scaninfo'] = r['_scaninfo']
217 |             nreport['_nmaprun'] = r['_nmaprun']
218 |             hlist = []
219 |             for h in r['_hosts']:
220 |                 slist = []
221 |                 for s in h['__NmapHost__']['_services']:
222 |                     cname = '__NmapService__'
223 |                     slist.append(NmapService(portid=s[cname]['_portid'],
224 |                                              protocol=s[cname]['_protocol'],
225 |                                              state=s[cname]['_state'],
226 |                                              owner=s[cname]['_owner'],
227 |                                              service=s[cname]['_service']))
228 | 
229 |                 nh = NmapHost(starttime=h['__NmapHost__']['_starttime'],
230 |                               endtime=h['__NmapHost__']['_endtime'],
231 |                               address=h['__NmapHost__']['_address'],
232 |                               status=h['__NmapHost__']['_status'],
233 |                               hostnames=h['__NmapHost__']['_hostnames'],
234 |                               extras=h['__NmapHost__']['_extras'],
235 |                               services=slist)
236 |                 hlist.append(nh)
237 |             nreport['_hosts'] = hlist
238 |             nmapobj = NmapReport(nreport)
239 |         return nmapobj
240 | 
241 |     @classmethod
242 |     def __parse_scaninfo(cls, scaninfo_data):
243 |         """
244 |             Private method parsing a portion of a nmap scan result.
245 |             Receives a <scaninfo> XML tag.
246 | 
247 |             :param scaninfo_data: <scaninfo> XML tag from a nmap scan
248 |             :type scaninfo_data: xml.ElementTree.Element or a string
249 | 
250 |             :return: python dict representing the XML scaninfo tag
251 |         """
252 | 
253 |         xelement = cls.__format_element(scaninfo_data)
254 |         return cls.__format_attributes(xelement)
255 | 
256 |     @classmethod
257 |     def _parse_xml_host(cls, scanhost_data):
258 |         """
259 |             Protected method parsing a portion of a nmap scan result.
260 |             Receives a <host> XML tag representing a scanned host with
261 |             its services.
262 | 
263 |             :param scaninfo_data: <host> XML tag from a nmap scan
264 |             :type scaninfo_data: xml.ElementTree.Element or a string
265 | 
266 |             :return: NmapHost object
267 |         """
268 | 
269 |         xelement = cls.__format_element(scanhost_data)
270 |         _host_header = cls.__format_attributes(xelement)
271 |         _hostnames = []
272 |         _services = []
273 |         _status = {}
274 |         _addresses = []
275 |         _host_extras = {}
276 |         extra_tags = ['uptime', 'distance', 'tcpsequence',
277 |                       'ipidsequence', 'tcptssequence', 'times']
278 |         for xh in xelement:
279 |             if xh.tag == 'hostnames':
280 |                 for hostname in cls.__parse_hostnames(xh):
281 |                     _hostnames.append(hostname)
282 |             elif xh.tag == 'ports':
283 |                 ports_dict = cls._parse_xml_ports(xh)
284 |                 for port in ports_dict['ports']:
285 |                     _services.append(port)
286 |                 _host_extras['extraports'] = ports_dict['extraports']
287 |             elif xh.tag == 'status':
288 |                 _status = cls.__format_attributes(xh)
289 |             elif xh.tag == 'address':
290 |                 _addresses.append(cls.__format_attributes(xh))
291 |             elif xh.tag == 'os':
292 |                 _os_extra = cls.__parse_os_fingerprint(xh)
293 |                 _host_extras.update({'os': _os_extra})
294 |             elif xh.tag == 'hostscript':
295 |                 _host_scripts = cls.__parse_host_scripts(xh)
296 |                 _host_extras.update({'hostscript': _host_scripts})
297 |             elif xh.tag in extra_tags:
298 |                 _host_extras[xh.tag] = cls.__format_attributes(xh)
299 |             # else:
300 |             #    print "struct host unknown attr: %s value: %s" %
301 |             #           (h.tag, h.get(h.tag))
302 |         _stime = ''
303 |         _etime = ''
304 |         if 'starttime' in _host_header:
305 |             _stime = _host_header['starttime']
306 |         if 'endtime' in _host_header:
307 |             _etime = _host_header['endtime']
308 |         nhost = NmapHost(_stime,
309 |                          _etime,
310 |                          _addresses,
311 |                          _status,
312 |                          _hostnames,
313 |                          _services,
314 |                          _host_extras)
315 |         return nhost
316 | 
317 |     @classmethod
318 |     def __parse_hostnames(cls, scanhostnames_data):
319 |         """
320 |             Private method parsing the hostnames list within a <host> XML tag.
321 | 
322 |             :param scanhostnames_data: <hostnames> XML tag from a nmap scan
323 |             :type scanhostnames_data: xml.ElementTree.Element or a string
324 | 
325 |             :return: list of hostnames
326 |         """
327 | 
328 |         xelement = cls.__format_element(scanhostnames_data)
329 |         hostnames = []
330 |         for hname in xelement:
331 |             if hname.tag == 'hostname':
332 |                 hostnames.append(hname.get('name'))
333 |         return hostnames
334 | 
335 |     @classmethod
336 |     def _parse_xml_ports(cls, scanports_data):
337 |         """
338 |             Protected method parsing the list of scanned services from
339 |             a targeted host. This protected method cannot be called directly
340 |             with a string. A <ports/> tag can be directly passed to parse()
341 |             and the below method will be called and return a list of nmap
342 |             scanned services.
343 | 
344 |             :param scanports_data: <ports> XML tag from a nmap scan
345 |             :type scanports_data: xml.ElementTree.Element or a string
346 | 
347 |             :return: list of NmapService
348 |         """
349 | 
350 |         xelement = cls.__format_element(scanports_data)
351 | 
352 |         rdict = {'ports': [], 'extraports': None}
353 |         for xservice in xelement:
354 |             if xservice.tag == 'port':
355 |                 nport = cls._parse_xml_port(xservice)
356 |                 rdict['ports'].append(nport)
357 |             elif xservice.tag == 'extraports':
358 |                 extraports = cls.__parse_extraports(xservice)
359 |                 rdict['extraports'] = extraports
360 |             # else:
361 |             #    print "struct port unknown attr: %s value: %s" %
362 |             #           (h.tag, h.get(h.tag))
363 |         return rdict
364 | 
365 |     @classmethod
366 |     def _parse_xml_port(cls, scanport_data):
367 |         """
368 |             Protected method parsing a scanned service from a targeted host.
369 |             This protected method cannot be called directly.
370 |             A <port/> tag can be directly passed to parse() and the below
371 |             method will be called and return a NmapService object
372 |             representing the state of the service.
373 | 
374 |             :param scanport_data: <port> XML tag from a nmap scan
375 |             :type scanport_data: xml.ElementTree.Element or a string
376 | 
377 |             :return: NmapService
378 |         """
379 | 
380 |         xelement = cls.__format_element(scanport_data)
381 | 
382 |         _port = cls.__format_attributes(xelement)
383 |         _portid = _port['portid'] if 'portid' in _port else None
384 |         _protocol = _port['protocol'] if 'protocol' in _port else None
385 | 
386 |         _state = None
387 |         _service = None
388 |         _owner = None
389 |         _service_scripts = []
390 |         _service_extras = {}
391 |         for xport in xelement:
392 |             if xport.tag == 'state':
393 |                 _state = cls.__format_attributes(xport)
394 |             elif xport.tag == 'service':
395 |                 _service = cls.__parse_service(xport)
396 |             elif xport.tag == 'owner':
397 |                 _owner = cls.__format_attributes(xport)
398 |             elif xport.tag == 'script':
399 |                 _script_dict = cls.__parse_script(xport)
400 |                 _service_scripts.append(_script_dict)
401 |         _service_extras['scripts'] = _service_scripts
402 | 
403 |         if(_portid is None or _protocol is None or _state is None):
404 |             raise NmapParserException("XML <port> tag is incomplete. One "
405 |                                       "of the following tags is missing: "
406 |                                       "portid, protocol or state or tag.")
407 | 
408 |         nport = NmapService(_portid,
409 |                             _protocol,
410 |                             _state,
411 |                             _service,
412 |                             _owner,
413 |                             _service_extras)
414 |         return nport
415 | 
416 |     @classmethod
417 |     def __parse_service(cls, xserv):
418 |         """
419 |             Parse <service> tag to manage CPE object
420 |         """
421 |         _service = cls.__format_attributes(xserv)
422 |         _cpelist = []
423 |         for _servnode in xserv:
424 |             if _servnode.tag == 'cpe':
425 |                 _cpe_string = _servnode.text
426 |                 _cpelist.append(_cpe_string)
427 |         _service['cpelist'] = _cpelist
428 |         return _service
429 | 
430 |     @classmethod
431 |     def __parse_extraports(cls, extraports_data):
432 |         """
433 |             Private method parsing the data from extra scanned ports.
434 |             X extraports were in state "closed" server returned "conn-refused"
435 |             tag: <extraports>
436 | 
437 |             :param extraports_data: XML data for extraports
438 |             :type extraports_data: xml.ElementTree.Element or a string
439 | 
440 |             :return: python dict with following keys: state, count, reason
441 |         """
442 |         rdict = {'state': '', 'count': '', 'reasons': []}
443 |         xelement = cls.__format_element(extraports_data)
444 |         extraports_dict = cls.__format_attributes(xelement)
445 | 
446 |         if 'state' in extraports_dict:
447 |             rdict['state'] = extraports_dict
448 |         if 'count' in extraports_dict:
449 |             rdict['count'] = extraports_dict
450 |         for xelt in xelement:
451 |             if xelt.tag == 'extrareasons':
452 |                 extrareasons_dict = cls.__format_attributes(xelt)
453 |                 rdict['reasons'].append(extrareasons_dict)
454 |         return rdict
455 | 
456 |     @classmethod
457 |     def __parse_script(cls, script_data):
458 |         """
459 |             Private method parsing the data from NSE scripts output
460 | 
461 |             :param script_data: portion of XML describing the results of the
462 |             script data
463 |             :type script_data: xml.ElementTree.Element or a string
464 | 
465 |             :return: python dict holding scripts output
466 |         """
467 |         _script_dict = cls.__format_attributes(script_data)
468 | 
469 |         _elt_dict = {}
470 |         for script_elem in script_data:
471 |             if script_elem.tag == 'elem':
472 |                 _elt_dict.update({script_elem.get('key'): script_elem.text})
473 |             elif script_elem.tag == 'table':
474 |                 tdict = {}
475 |                 for telem in script_elem:
476 |                     tdict[telem.get('key')] = telem.text
477 |                 _elt_dict[script_elem.get('key')] = tdict
478 |         _script_dict['elements'] = _elt_dict
479 |         return _script_dict
480 | 
481 |     @classmethod
482 |     def __parse_host_scripts(cls, scripts_data):
483 |         """
484 |             Private method parsing the data from scripts affecting
485 |             the target host.
486 |             Contents of <hostscript> is returned as a list of dict.
487 | 
488 |             :param scripts_data: portion of XML describing the results of the
489 |             scripts data
490 |             :type scripts_data: xml.ElementTree.Element or a string
491 | 
492 |             :return: python list holding scripts output in a dict
493 |         """
494 |         _host_scripts = []
495 |         for xscript in scripts_data:
496 |             if xscript.tag == 'script':
497 |                 _script_dict = cls.__parse_script(xscript)
498 |             _host_scripts.append(_script_dict)
499 |         return _host_scripts
500 | 
501 |     @classmethod
502 |     def __parse_os_fingerprint(cls, os_data):
503 |         """
504 |             Private method parsing the data from an OS fingerprint (-O).
505 |             Contents of <os> is returned as a dict.
506 | 
507 |             :param os_data: portion of XML describing the results of the
508 |             os fingerprinting attempt
509 |             :type os_data: xml.ElementTree.Element or a string
510 | 
511 |             :return: python dict representing the XML os tag
512 |         """
513 |         rdict = {}
514 |         xelement = cls.__format_element(os_data)
515 | 
516 |         os_class_probability = []
517 |         os_match_probability = []
518 |         os_ports_used = []
519 |         os_fingerprints = []
520 |         for xos in xelement:
521 |             # for nmap xml version < 1.04, osclass is not
522 |             # embedded in osmatch
523 |             if xos.tag == 'osclass':
524 |                 os_class_proba = cls.__parse_osclass(xos)
525 |                 os_class_probability.append(os_class_proba)
526 |             elif xos.tag == 'osmatch':
527 |                 os_match_proba = cls.__parse_osmatch(xos)
528 |                 os_match_probability.append(os_match_proba)
529 |             elif xos.tag == 'portused':
530 |                 os_portused = cls.__format_attributes(xos)
531 |                 os_ports_used.append(os_portused)
532 |             elif xos.tag == 'osfingerprint':
533 |                 os_fp_dict = cls.__format_attributes(xos)
534 |                 os_fingerprints.append(os_fp_dict)
535 | 
536 |         rdict['osmatches'] = os_match_probability
537 |         rdict['osclasses'] = os_class_probability
538 |         rdict['ports_used'] = os_ports_used
539 |         rdict['osfingerprints'] = os_fingerprints
540 | 
541 |         return rdict
542 | 
543 |     @classmethod
544 |     def __parse_osmatch(cls, osmatch_data):
545 |         """
546 |             This methods parses osmatch data and returns a dict. Depending
547 |             on the nmap xml version, osmatch could contain an osclass
548 |             dict.
549 | 
550 |             :param osmatch_data: <osmatch> XML tag from a nmap scan
551 |             :type osmatch_data: xml.ElementTree.Element or a string
552 | 
553 |             :return: python dict representing the XML osmatch tag
554 |         """
555 |         rdict = {}
556 |         xelement = cls.__format_element(osmatch_data)
557 |         rdict['osmatch'] = cls.__format_attributes(xelement)
558 |         rdict['osclasses'] = []
559 |         for xmltag in xelement:
560 |             if xmltag.tag == 'osclass':
561 |                 _osclass_dict = cls.__parse_osclass(xmltag)
562 |                 rdict['osclasses'].append(_osclass_dict)
563 |             else:
564 |                 exmsg = "Unexcepted node in <osmatch>: {0}".format(xmltag.tag)
565 |                 raise NmapParserException(exmsg)
566 |         return rdict
567 | 
568 |     @classmethod
569 |     def __parse_osclass(cls, osclass_data):
570 |         """
571 |             This methods parses osclass data and returns a dict. Depending
572 |             on the nmap xml version, osclass could contain a cpe
573 |             dict.
574 | 
575 |             :param osclass_data: <osclass> XML tag from a nmap scan
576 |             :type osclass_data: xml.ElementTree.Element or a string
577 | 
578 |             :return: python dict representing the XML osclass tag
579 |         """
580 |         rdict = {}
581 |         xelement = cls.__format_element(osclass_data)
582 |         rdict['osclass'] = cls.__format_attributes(xelement)
583 |         rdict['cpe'] = []
584 |         for xmltag in xelement:
585 |             if xmltag.tag == 'cpe':
586 |                 _cpe_string = xmltag.text
587 |                 rdict['cpe'].append(_cpe_string)
588 |             else:
589 |                 exmsg = "Unexcepted node in <osclass>: {0}".format(xmltag.tag)
590 |                 raise NmapParserException(exmsg)
591 |         return rdict
592 | 
593 |     @classmethod
594 |     def __parse_runstats(cls, scanrunstats_data):
595 |         """
596 |             Private method parsing a portion of a nmap scan result.
597 |             Receives a <runstats> XML tag.
598 | 
599 |             :param scanrunstats_data: <runstats> XML tag from a nmap scan
600 |             :type scanrunstats_data: xml.ElementTree.Element or a string
601 | 
602 |             :return: python dict representing the XML runstats tag
603 |         """
604 | 
605 |         xelement = cls.__format_element(scanrunstats_data)
606 | 
607 |         rdict = {}
608 |         for xmltag in xelement:
609 |             if xmltag.tag in ['finished', 'hosts']:
610 |                 rdict[xmltag.tag] = cls.__format_attributes(xmltag)
611 |             else:
612 |                 exmsg = "Unexcepted node in <runstats>: {0}".format(xmltag.tag)
613 |                 raise NmapParserException(exmsg)
614 | 
615 |         return rdict
616 | 
617 |     @staticmethod
618 |     def __format_element(elt_data):
619 |         """
620 |             Private method which ensures that a XML portion to be parsed is
621 |             of type xml.etree.ElementTree.Element.
622 |             If elt_data is a string, then it is converted to an
623 |             XML Element type.
624 | 
625 |             :param elt_data: XML Element to be parsed or string
626 |             to be converted to a XML Element
627 | 
628 |             :return: Element
629 |         """
630 |         if isinstance(elt_data, str):
631 |             try:
632 |                 xelement = ET.fromstring(elt_data)
633 |             except:
634 |                 raise NmapParserException("Error while trying "
635 |                                           "to instanciate XML Element from "
636 |                                           "string {0}".format(elt_data))
637 |         elif ET.iselement(elt_data):
638 |             xelement = elt_data
639 |         else:
640 |             raise NmapParserException("Error while trying to parse supplied "
641 |                                       "data: unsupported format")
642 |         return xelement
643 | 
644 |     @staticmethod
645 |     def __format_attributes(elt_data):
646 |         """
647 |             Private method which converts a single XML tag to a python dict.
648 |             It also checks that the elt_data given as argument is of type
649 |             xml.etree.ElementTree.Element
650 | 
651 |             :param elt_data: XML Element to be parsed or string
652 |             to be converted to a XML Element
653 | 
654 |             :return: Element
655 |         """
656 | 
657 |         rval = {}
658 |         if not ET.iselement(elt_data):
659 |             raise NmapParserException("Error while trying to parse supplied "
660 |                                       "data attributes: format is not XML or "
661 |                                       "XML tag is empty")
662 |         try:
663 |             for dkey in elt_data.keys():
664 |                 rval[dkey] = elt_data.get(dkey)
665 |                 if rval[dkey] is None:
666 |                     raise NmapParserException("Error while trying to build-up "
667 |                                               "element attributes: empty "
668 |                                               "attribute {0}".format(dkey))
669 |         except:
670 |             raise
671 |         return rval
672 | 
673 | 
674 | class NmapParserException(Exception):
675 |     def __init__(self, msg):
676 |         self.msg = msg
677 | 
678 |     def __str__(self):
679 |         return self.msg
680 | 


--------------------------------------------------------------------------------
/libnmap/plugins/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/ring04h/thorns/65dda993878e3d96fa412a6576fe44dd3d61328a/libnmap/plugins/__init__.py


--------------------------------------------------------------------------------
/libnmap/plugins/backend_host.py:
--------------------------------------------------------------------------------
  1 | #!/usr/bin/env python
  2 | from sqlalchemy import create_engine
  3 | from sqlalchemy.schema import Column
  4 | from sqlalchemy.types import Integer, DateTime, LargeBinary, Text, String, Boolean
  5 | from sqlalchemy.ext.declarative import declarative_base
  6 | from sqlalchemy.orm import sessionmaker
  7 | 
  8 | from libnmap.plugins.backendplugin import NmapBackendPlugin
  9 | from libnmap.reportjson import ReportEncoder, ReportDecoder
 10 | 
 11 | import json
 12 | from datetime import datetime
 13 | 
 14 | Base = declarative_base()
 15 | 
 16 | class NmapSqlPlugin(NmapBackendPlugin):
 17 |     """
 18 |         This class handle the persistence of NmapRepport object in SQL backend
 19 |         Implementation is made using sqlalchemy(0.8.1)
 20 |         usage :
 21 | 
 22 |         #get a nmapReport object
 23 |         from libnmap.parser import NmapParser
 24 |         from libnmap.reportjson import ReportDecoder, ReportEncoder
 25 |         import json
 26 |         nmap_report_obj = NmapParser.parse_fromfile(
 27 |                '/home/vagrant/python-nmap-lib/libnmap/test/files/1_hosts.xml')
 28 | 
 29 |          #get a backend with in memory sqlite
 30 |          from libnmap.plugins.backendpluginFactory import BackendPluginFactory
 31 |          mybackend_mem = BackendPluginFactory.create(plugin_name='sql',
 32 |                                                      url='sqlite://',
 33 |                                                      echo=True)
 34 | 
 35 |          mybackend_mysql = BackendPluginFactory.create(plugin_name='sql',
 36 |                             url='mysql+mysqldb://scott:tiger@localhost/foo',
 37 |                             echo=True)
 38 |          mybackend = BackendPluginFactory.create(plugin_name='sql',
 39 |                                         url='sqlite:////tmp/reportdb.sql',
 40 |                                         echo=True)
 41 |          #lets save!!
 42 |          nmap_report_obj.save(mybackend)
 43 |          mybackend.getall()
 44 |          mybackend.get(1)
 45 |     """
 46 |     class Reports(Base):
 47 |         """
 48 |             Embeded class for ORM map NmapReport to a
 49 |             simple three column table
 50 |         """
 51 |         __tablename__ = 'result_ip'
 52 | 
 53 |         id = Column('id', Integer, primary_key=True)
 54 |         taskid = Column('taskid', Integer)
 55 |         inserted = Column('inserted', DateTime(), default='now')
 56 |         domain = Column('domain', String(256))
 57 |         address = Column('address', String(32))
 58 |         is_up = Column('is_up', Boolean)
 59 |         os = Column('os', String(256))
 60 |     
 61 |         def __init__(self, obj_NmapReport):
 62 |             self.inserted = datetime.fromtimestamp(int(obj_NmapReport.endtime))
 63 |             self.taskid = obj_NmapReport.taskid
 64 |             self.is_up = obj_NmapReport.is_up()
 65 | 
 66 |             if len(obj_NmapReport.hostnames) > 0:
 67 |                 self.domain = obj_NmapReport.hostnames[0]
 68 |             else:
 69 |                 self.domain = None
 70 | 
 71 |             self.address = obj_NmapReport.address
 72 |             
 73 |             if len(obj_NmapReport.os.osmatch()) > 0:
 74 |                 self.os = obj_NmapReport.os.osmatch()[0]
 75 |             else:
 76 |                 self.os = None
 77 | 
 78 |         def decode(self):
 79 |             json_decoded = self.report_json.decode('utf-8')
 80 |             nmap_report_obj = json.loads(json_decoded,
 81 |                                          cls=ReportDecoder)
 82 |             return nmap_report_obj
 83 | 
 84 |     def __init__(self, **kwargs):
 85 |         """
 86 |             constructor receive a **kwargs as the **kwargs in the sqlalchemy
 87 |             create_engine() method (see sqlalchemy docs)
 88 |             You must add to this **kwargs an 'url' key with the url to your
 89 |             database
 90 |             This constructor will :
 91 |             - create all the necessary obj to discuss with the DB
 92 |             - create all the mapping(ORM)
 93 | 
 94 |             todo : suport the : sqlalchemy.engine_from_config
 95 | 
 96 |             :param **kwargs:
 97 |             :raises: ValueError if no url is given,
 98 |                     all exception sqlalchemy can throw
 99 |             ie sqlite in memory url='sqlite://' echo=True
100 |             ie sqlite file on hd url='sqlite:////tmp/reportdb.sql' echo=True
101 |             ie mysql url='mysql+mysqldb://scott:tiger@localhost/foo'
102 |         """
103 |         NmapBackendPlugin.__init__(self)
104 |         self.engine = None
105 |         self.url = None
106 |         self.Session = sessionmaker()
107 | 
108 |         if 'url' not in kwargs:
109 |             raise ValueError
110 |         self.url = kwargs['url']
111 |         del kwargs['url']
112 |         try:
113 |             self.engine = create_engine(self.url, **kwargs)
114 |             Base.metadata.create_all(bind=self.engine, checkfirst=True)
115 |             self.Session.configure(bind=self.engine)
116 |         except:
117 |             raise
118 | 
119 |     def insert(self, nmap_report):
120 |         """
121 |            insert NmapReport in the backend
122 | 
123 |            :param NmapReport:
124 | 
125 |            :returns: the ident of the object in the backend for future usage \
126 |            or None
127 |         """
128 |         sess = self.Session()
129 |         report = NmapSqlPlugin.Reports(nmap_report)
130 |         sess.add(report)
131 |         sess.commit()
132 |         reportid = report.id
133 |         sess.close()
134 |         return reportid if reportid else None
135 | 
136 |     def get(self, report_id=None):
137 |         """
138 |             retreive a NmapReport from the backend
139 | 
140 |             :param id: str
141 | 
142 |             :returns: NmapReport
143 |         """
144 |         if report_id is None:
145 |             raise ValueError
146 |         sess = self.Session()
147 |         our_report = (
148 |             sess.query(NmapSqlPlugin.Reports).filter_by(id=report_id).first())
149 |         sess.close()
150 |         return our_report.decode() if our_report else None
151 | 
152 |     def getall(self):
153 |         """
154 |             :param filter: Nice to have implement a filter capability
155 | 
156 |             :returns: collection of tuple (id,NmapReport)
157 |         """
158 |         sess = self.Session()
159 |         nmapreportList = []
160 |         for report in (
161 |                 sess.query(NmapSqlPlugin.Reports).
162 |                 order_by(NmapSqlPlugin.Reports.inserted)):
163 |             nmapreportList.append((report.id, report.decode()))
164 |         sess.close()
165 |         return nmapreportList
166 | 
167 |     def delete(self, report_id=None):
168 |         """
169 |             Remove a report from the backend
170 | 
171 |             :param id: str
172 | 
173 |             :returns: The number of rows deleted
174 |         """
175 |         if report_id is None:
176 |             raise ValueError
177 |         nb_line = 0
178 |         sess = self.Session()
179 |         nb_line = sess.query(NmapSqlPlugin.Reports).\
180 |             filter_by(id=report_id).delete()
181 |         sess.commit()
182 |         sess.close()
183 |         return nb_line
184 | 


--------------------------------------------------------------------------------
/libnmap/plugins/backend_service.py:
--------------------------------------------------------------------------------
  1 | #!/usr/bin/env python
  2 | from sqlalchemy import create_engine
  3 | from sqlalchemy.schema import Column
  4 | from sqlalchemy.types import Integer, DateTime, LargeBinary, Text, String
  5 | from sqlalchemy.ext.declarative import declarative_base
  6 | from sqlalchemy.orm import sessionmaker
  7 | 
  8 | from libnmap.plugins.backendplugin import NmapBackendPlugin
  9 | from libnmap.reportjson import ReportEncoder, ReportDecoder
 10 | 
 11 | import json
 12 | from datetime import datetime
 13 | import binascii
 14 | 
 15 | Base = declarative_base()
 16 | 
 17 | class NmapSqlPlugin(NmapBackendPlugin):
 18 |     """
 19 |         This class handle the persistence of NmapRepport object in SQL backend
 20 |         Implementation is made using sqlalchemy(0.8.1)
 21 |         usage :
 22 | 
 23 |         #get a nmapReport object
 24 |         from libnmap.parser import NmapParser
 25 |         from libnmap.reportjson import ReportDecoder, ReportEncoder
 26 |         import json
 27 |         nmap_report_obj = NmapParser.parse_fromfile(
 28 |                '/home/vagrant/python-nmap-lib/libnmap/test/files/1_hosts.xml')
 29 | 
 30 |          #get a backend with in memory sqlite
 31 |          from libnmap.plugins.backendpluginFactory import BackendPluginFactory
 32 |          mybackend_mem = BackendPluginFactory.create(plugin_name='sql',
 33 |                                                      url='sqlite://',
 34 |                                                      echo=True)
 35 | 
 36 |          mybackend_mysql = BackendPluginFactory.create(plugin_name='sql',
 37 |                             url='mysql+mysqldb://scott:tiger@localhost/foo',
 38 |                             echo=True)
 39 |          mybackend = BackendPluginFactory.create(plugin_name='sql',
 40 |                                         url='sqlite:////tmp/reportdb.sql',
 41 |                                         echo=True)
 42 |          #lets save!!
 43 |          nmap_report_obj.save(mybackend)
 44 |          mybackend.getall()
 45 |          mybackend.get(1)
 46 |     """
 47 |     class Reports(Base):
 48 |         """
 49 |             Embeded class for ORM map NmapReport to a
 50 |             simple three column table
 51 |         """
 52 |         __tablename__ = 'result_ports'
 53 | 
 54 |         id = Column('id', Integer, primary_key=True)
 55 |         taskid = Column('taskid', Integer)
 56 |         inserted = Column('inserted', DateTime(), default='now')
 57 |         address = Column('address', String(256))
 58 |         port = Column('port', Integer)
 59 |         service = Column('service', String(256))
 60 |         state = Column('state', String(12))
 61 |         protocol = Column('protocol', String(12))
 62 |         product = Column('product', String(64))
 63 |         product_version = Column('product_version', String(64))
 64 |         product_extrainfo = Column('product_extrainfo', String(128))
 65 |         # banner = Column('banner', String(256))
 66 |         scripts_results = Column('scripts_results', Text)
 67 | 
 68 |         def __init__(self, obj_NmapReport):
 69 |             self.inserted = datetime.fromtimestamp(int(obj_NmapReport.endtime))
 70 |             self.taskid = obj_NmapReport.taskid
 71 |             self.address = obj_NmapReport.address
 72 |             self.port = obj_NmapReport.port
 73 |             self.service = obj_NmapReport.service
 74 |             self.state = obj_NmapReport.state
 75 |             self.protocol = str(obj_NmapReport.protocol)
 76 |             self.product = str(obj_NmapReport.product)
 77 |             self.product_version = str(obj_NmapReport.product_version)
 78 |             self.product_extrainfo = str(obj_NmapReport.product_extrainfo)
 79 |             # self.banner = str(obj_NmapReport.banner)
 80 |             # self.scripts_results = binascii.b2a_hex(str(obj_NmapReport.scripts_results))
 81 | 
 82 |             if len(obj_NmapReport.scripts_results) > 0:                
 83 |                 self.scripts_results = obj_NmapReport.scripts_results[0]['output']
 84 |             else:
 85 |                 self.scripts_results = None
 86 | 
 87 | 
 88 |         def decode(self):
 89 |             json_decoded = self.report_json.decode('utf-8')
 90 |             nmap_report_obj = json.loads(json_decoded,
 91 |                                          cls=ReportDecoder)
 92 |             return nmap_report_obj
 93 | 
 94 |     def __init__(self, **kwargs):
 95 |         """
 96 |             constructor receive a **kwargs as the **kwargs in the sqlalchemy
 97 |             create_engine() method (see sqlalchemy docs)
 98 |             You must add to this **kwargs an 'url' key with the url to your
 99 |             database
100 |             This constructor will :
101 |             - create all the necessary obj to discuss with the DB
102 |             - create all the mapping(ORM)
103 | 
104 |             todo : suport the : sqlalchemy.engine_from_config
105 | 
106 |             :param **kwargs:
107 |             :raises: ValueError if no url is given,
108 |                     all exception sqlalchemy can throw
109 |             ie sqlite in memory url='sqlite://' echo=True
110 |             ie sqlite file on hd url='sqlite:////tmp/reportdb.sql' echo=True
111 |             ie mysql url='mysql+mysqldb://scott:tiger@localhost/foo'
112 |         """
113 |         NmapBackendPlugin.__init__(self)
114 |         self.engine = None
115 |         self.url = None
116 |         self.Session = sessionmaker()
117 | 
118 |         if 'url' not in kwargs:
119 |             raise ValueError
120 |         self.url = kwargs['url']
121 |         del kwargs['url']
122 |         try:
123 |             self.engine = create_engine(self.url, **kwargs)
124 |             Base.metadata.create_all(bind=self.engine, checkfirst=True)
125 |             self.Session.configure(bind=self.engine)
126 |         except:
127 |             raise
128 | 
129 |     def insert(self, nmap_report):
130 |         """
131 |            insert NmapReport in the backend
132 | 
133 |            :param NmapReport:
134 | 
135 |            :returns: the ident of the object in the backend for future usage \
136 |            or None
137 |         """
138 |         sess = self.Session()
139 |         report = NmapSqlPlugin.Reports(nmap_report)
140 |         sess.add(report)
141 |         sess.commit()
142 |         reportid = report.id
143 |         sess.close()
144 |         return reportid if reportid else None
145 | 
146 |     def get(self, report_id=None):
147 |         """
148 |             retreive a NmapReport from the backend
149 | 
150 |             :param id: str
151 | 
152 |             :returns: NmapReport
153 |         """
154 |         if report_id is None:
155 |             raise ValueError
156 |         sess = self.Session()
157 |         our_report = (
158 |             sess.query(NmapSqlPlugin.Reports).filter_by(id=report_id).first())
159 |         sess.close()
160 |         return our_report.decode() if our_report else None
161 | 
162 |     def getall(self):
163 |         """
164 |             :param filter: Nice to have implement a filter capability
165 | 
166 |             :returns: collection of tuple (id,NmapReport)
167 |         """
168 |         sess = self.Session()
169 |         nmapreportList = []
170 |         for report in (
171 |                 sess.query(NmapSqlPlugin.Reports).
172 |                 order_by(NmapSqlPlugin.Reports.inserted)):
173 |             nmapreportList.append((report.id, report.decode()))
174 |         sess.close()
175 |         return nmapreportList
176 | 
177 |     def delete(self, report_id=None):
178 |         """
179 |             Remove a report from the backend
180 | 
181 |             :param id: str
182 | 
183 |             :returns: The number of rows deleted
184 |         """
185 |         if report_id is None:
186 |             raise ValueError
187 |         nb_line = 0
188 |         sess = self.Session()
189 |         nb_line = sess.query(NmapSqlPlugin.Reports).\
190 |             filter_by(id=report_id).delete()
191 |         sess.commit()
192 |         sess.close()
193 |         return nb_line
194 | 


--------------------------------------------------------------------------------
/libnmap/plugins/backendplugin.py:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/env python
 2 | 
 3 | 
 4 | class NmapBackendPlugin(object):
 5 |     """
 6 |         Abstract class showing to the minimal implementation for a plugin
 7 |         All subclass MUST at least implement the following methods
 8 |     """
 9 |     def __init__(self):
10 |         self.dbname = 'nmapdb'
11 |         self.store = 'reports'
12 | 
13 |     def insert(self, NmapReport):
14 |         """
15 |             insert NmapReport in the backend
16 |             :param NmapReport:
17 |             :return: str the ident of the object in the backend for
18 |             future usage
19 |             or None
20 |         """
21 |         raise NotImplementedError
22 | 
23 |     def delete(self, id):
24 |         """
25 |             delete NmapReport if the backend
26 |             :param id: str
27 |         """
28 |         raise NotImplementedError
29 | 
30 |     def get(self, id):
31 |         """
32 |             retreive a NmapReport from the backend
33 |             :param id: str
34 |             :return: NmapReport
35 |         """
36 |         raise NotImplementedError
37 | 
38 |     def getall(self, filter):
39 |         """
40 |             :return: collection of tuple (id,NmapReport)
41 |             :param filter: Nice to have implement a filter capability
42 |         """
43 |         raise NotImplementedError
44 | 


--------------------------------------------------------------------------------
/libnmap/plugins/backendpluginFactory.py:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/env python
 2 | import sys
 3 | import inspect
 4 | 
 5 | 
 6 | class BackendPluginFactory(object):
 7 |     """
 8 |         This is a backend plugin factory a backend instance MUST be
 9 |         created via the static method create()
10 |         ie : mybackend = BackendPluginFactory.create()
11 |     """
12 |     @classmethod
13 |     def create(cls, plugin_name="mongodb", **kwargs):
14 |         """Import the needed lib and return an object NmapBackendPlugin
15 |            representing the backend of your desire.
16 |            NmapBackendPlugin is an abstract class, to know what argument
17 |            need to be given, review the code of the subclass you need
18 |            :param plugin_name: str : name of the py file without .py
19 |            :return: NmapBackend (abstract class on top of all plugin)
20 |         """
21 |         backendplugin = None
22 |         plugin_path = "libnmap.plugins.{0}".format(plugin_name)
23 |         __import__(plugin_path)
24 |         pluginobj = sys.modules[plugin_path]
25 |         pluginclasses = inspect.getmembers(pluginobj, inspect.isclass)
26 |         for classname, classobj in pluginclasses:
27 |             if inspect.getmodule(classobj).__name__.find(plugin_path) == 0:
28 |                 try:
29 |                     backendplugin = classobj(**kwargs)
30 |                 except Exception as error:
31 |                     raise Exception("Cannot create Backend: {0}".format(error))
32 |         return backendplugin
33 | 


--------------------------------------------------------------------------------
/libnmap/plugins/es.py:
--------------------------------------------------------------------------------
 1 | # -*- coding: utf-8 -*-
 2 | 
 3 | import json
 4 | from libnmap.reportjson import ReportEncoder
 5 | from libnmap.plugins.backendplugin import NmapBackendPlugin
 6 | from elasticsearch import Elasticsearch
 7 | from datetime import datetime
 8 | 
 9 | 
10 | class NmapElasticsearchPlugin(NmapBackendPlugin):
11 |     """
12 |         This class enables the user to store and manipulate nmap reports \
13 |         in a elastic search db.
14 |     """
15 |     def __init__(self, index=None):
16 |         if index is None:
17 |             self.index = "nmap.{0}".format(datetime.now().strftime('%Y-%m-%d'))
18 |         else:
19 |             self.index = index
20 |         self._esapi = Elasticsearch()
21 | 
22 |     def insert(self, report, doc_type=None):
23 |         """
24 |             insert NmapReport in the backend
25 |             :param NmapReport:
26 |             :return: str the ident of the object in the backend for
27 |             future usage
28 |             or None
29 |         """
30 |         if doc_type is None:
31 |             doc_type = 'NmapReport'
32 |         j = json.dumps(report, cls=ReportEncoder)
33 |         res = self._esapi.index(
34 |             index=self.index,
35 |             doc_type=doc_type,
36 |             body=json.loads(j))
37 |         rc = res['_id']
38 |         return rc
39 | 
40 |     def delete(self, id):
41 |         """
42 |             delete NmapReport if the backend
43 |             :param id: str
44 |         """
45 |         raise NotImplementedError
46 | 
47 |     def get(self, id):
48 |         """
49 |             retreive a NmapReport from the backend
50 |             :param id: str
51 |             :return: NmapReport
52 |         """
53 |         res = self._esapi.get(index=self.index,
54 |                               doc_type="NmapReport",
55 |                               id=id)['_source']
56 |         return res
57 | 
58 |     def getall(self, filter=None):
59 |         """
60 |             :return: collection of tuple (id,NmapReport)
61 |             :param filter: Nice to have implement a filter capability
62 |         """
63 |         rsearch = self._esapi.search(index=self.index,
64 |                                      body={"query": {"match_all": {}}})
65 |         print("--------------------")
66 |         print(type(rsearch))
67 |         print(rsearch)
68 |         print("------------")
69 | 


--------------------------------------------------------------------------------
/libnmap/plugins/mongodb.py:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/env python
 2 | import json
 3 | from pymongo import MongoClient
 4 | from bson.objectid import ObjectId
 5 | 
 6 | from libnmap.reportjson import ReportEncoder
 7 | from libnmap.parser import NmapParser
 8 | from libnmap.plugins.backendplugin import NmapBackendPlugin
 9 | 
10 | 
11 | class NmapMongodbPlugin(NmapBackendPlugin):
12 |     """
13 |         This class handle the persistence of NmapRepport object in mongodb
14 |         Implementation is made using pymongo
15 |         Object of this class must be create via the
16 |         BackendPluginFactory.create(**url) where url is a named dict like
17 |         {'plugin_name': "mongodb"} this dict may reeive all the param
18 |         MongoClient() support
19 |     """
20 |     def __init__(self, dbname=None, store=None, **kwargs):
21 |         NmapBackendPlugin.__init__(self)
22 |         if dbname is not None:
23 |             self.dbname = dbname
24 |         if store is not None:
25 |             self.store = store
26 |         self.dbclient = MongoClient(**kwargs)
27 |         self.collection = self.dbclient[self.dbname][self.store]
28 | 
29 |     def insert(self, report):
30 |         """
31 |             create a json object from an NmapReport instance
32 |             :param NmapReport: obj to insert
33 |             :return: str id
34 |         """
35 |         j = json.dumps(report, cls=ReportEncoder)
36 |         try:
37 |             oid = self.collection.insert(json.loads(j))
38 |         except:
39 |             raise Exception("Failed to insert nmap object in MongoDB")
40 |         return str(oid)
41 | 
42 |     def get(self, str_report_id=None):
43 |         """ select a NmapReport by Id
44 |             :param str: id
45 |             :return: NmapReport object
46 |         """
47 |         rid = str_report_id
48 |         nmapreport = None
49 |         if str_report_id is not None and isinstance(str_report_id, str):
50 |             rid = ObjectId(str_report_id)
51 | 
52 |         if isinstance(rid, ObjectId):
53 |             # get a specific report by mongo's id
54 |             resultset = self.collection.find({'_id': rid})
55 |             if resultset.count() == 1:
56 |                 # search by id means only one in the iterator
57 |                 record = resultset[0]
58 |                 # remove mongo's id to recreate the NmapReport Obj
59 |                 del record['_id']
60 |                 nmapreport = NmapParser.parse_fromdict(record)
61 |         return nmapreport
62 | 
63 |     def getall(self, dict_filter=None):
64 |         """return a list of tuple (id,NmapReport) saved in the backend
65 |            TODO : add a filter capability
66 |         """
67 |         nmapreportlist = []
68 |         resultset = self.collection.find()
69 |         for report in resultset:
70 |             oid = report['_id']
71 |             del report['_id']
72 |             nmapreport = NmapParser.parse_fromdict(report)
73 |             nmapreportlist.append((oid, nmapreport))
74 |         return nmapreportlist
75 | 
76 |     def delete(self, report_id=None):
77 |         """
78 |             delete an obj from the backend
79 |             :param str: id
80 |             :return: dict document with result or None
81 |         """
82 |         if report_id is not None and isinstance(report_id, str):
83 |             return self.collection.remove({'_id': ObjectId(report_id)})
84 |         else:
85 |             return self.collection.remove({'_id': report_id})
86 | 


--------------------------------------------------------------------------------
/libnmap/plugins/s3.py:
--------------------------------------------------------------------------------
  1 | #!/usr/bin/env python
  2 | """
  3 | :mod:`libnmap.plugin.s3` -- S3 Backend Plugin
  4 | =============================================
  5 | 
  6 | .. module:: libnmap.plugin.s3
  7 | 
  8 | :platform: Linux
  9 | :synopsis: a plugin is representation of a S3 backend using boto
 10 | 
 11 | .. moduleauthor:: Ronald Bister
 12 | .. moduleauthor:: Mike Boutillier
 13 | """
 14 | import json
 15 | from bson.objectid import ObjectId
 16 | from boto.s3.connection import S3Connection, OrdinaryCallingFormat
 17 | from boto.s3.key import Key
 18 | from boto.s3.bucketlistresultset import bucket_lister
 19 | from boto.exception import S3ResponseError
 20 | from libnmap.reportjson import ReportEncoder
 21 | from libnmap.parser import NmapParser
 22 | from libnmap.plugins.backendplugin import NmapBackendPlugin
 23 | 
 24 | 
 25 | class NmapS3Plugin(NmapBackendPlugin):
 26 |     """
 27 |         This plugin save the reports on S3 and compatible.
 28 |     """
 29 |     def __init__(self, **kwargs):
 30 |         """
 31 |             - create the conn object
 32 |             - create the bucket (if it doesn't exist)
 33 |                 - if not given, awsaccessKey_nmapreport
 34 |             - may raise exception (ie in case of conflict bucket name)
 35 |             - sample :
 36 |             To connect to walrus:
 37 |             from libnmap.plugins.backendpluginFactory import
 38 |                             BackendPluginFactory
 39 |             walrusBackend =
 40 |               BackendPluginFactory.create(
 41 |                     plugin_name='s3',
 42 |                     host="walrus.ecc.eucalyptus.com",
 43 |                     path="/services/Walrus",port=8773,
 44 |                     is_secure=False,
 45 |                     aws_access_key_id='UU72FLVJCAYRATLXI70YH',
 46 |                     aws_secret_access_key=
 47 |                                'wFg7gP5YFHjVlxakw1g1uCC8UR2xVW5ax9ErZCut')
 48 |            To connect to S3:
 49 |            mybackend_S3 =
 50 |              BackendPluginFactory.create(
 51 |                 plugin_name='s3',
 52 |                 is_secure=True,
 53 |                 aws_access_key_id='MYACCESSKEY',
 54 |                 aws_secret_access_key='MYSECRET')
 55 |         """
 56 |         NmapBackendPlugin.__init__(self)
 57 |         try:
 58 |             calling_format = OrdinaryCallingFormat()
 59 |             if 'bucket' not in kwargs:
 60 |                 self.bucket_name = ''.join(
 61 |                     [kwargs['aws_access_key_id'].lower(),
 62 |                      "_nmapreport"])
 63 |             else:
 64 |                 self.bucket_name = kwargs['bucket']
 65 |                 del kwargs['bucket']
 66 |             kwargs['calling_format'] = calling_format
 67 |             self.conn = S3Connection(**kwargs)
 68 |             self.bucket = self.conn.lookup(self.bucket_name)
 69 |             if self.bucket is None:
 70 |                 self.bucket = self.conn.create_bucket(self.bucket_name)
 71 |         except:
 72 |             raise
 73 | 
 74 |     def insert(self, report):
 75 |         """
 76 |             create a json string from an NmapReport instance
 77 |             and push it to S3 bucket.
 78 | 
 79 |             :param NmapReport: obj to insert
 80 |             :rtype: string
 81 |             :return: str id
 82 |             :todo: Add tagging option
 83 |         """
 84 |         try:
 85 |             oid = ObjectId()
 86 |             mykey = Key(self.bucket)
 87 |             mykey.key = str(oid)
 88 |             strjsonnmapreport = json.dumps(report, cls=ReportEncoder)
 89 |             mykey.set_contents_from_string(strjsonnmapreport)
 90 |         except:
 91 |             raise Exception("Failed to add nmap object in s3 bucket")
 92 |         return str(oid)
 93 | 
 94 |     def get(self, str_report_id=None):
 95 |         """
 96 |             select a NmapReport by Id.
 97 | 
 98 |             :param str: id
 99 |             :rtype: NmapReport
100 |             :return: NmapReport object
101 |         """
102 |         nmapreport = None
103 |         if str_report_id is not None and isinstance(str_report_id, str):
104 |             try:
105 |                 mykey = Key(self.bucket)
106 |                 mykey.key = str_report_id
107 |                 nmapreportjson = json.loads(mykey.get_contents_as_string())
108 |                 nmapreport = NmapParser.parse_fromdict(nmapreportjson)
109 |             except S3ResponseError:
110 |                 pass
111 |         return nmapreport
112 | 
113 |     def getall(self, dict_filter=None):
114 |         """
115 |             :rtype: List of tuple
116 |             :return: list of key/report
117 |             :todo: add a filter capability
118 |         """
119 |         nmapreportlist = []
120 |         for key in bucket_lister(self.bucket):
121 |             if isinstance(key, Key):
122 |                 nmapreportjson = json.loads(key.get_contents_as_string())
123 |                 nmapreport = NmapParser.parse_fromdict(nmapreportjson)
124 |                 nmapreportlist.append((key.key, nmapreport))
125 |         return nmapreportlist
126 | 
127 |     def delete(self, report_id=None):
128 |         """
129 |             delete an obj from the backend
130 | 
131 |             :param str: id
132 |             :return: dict document with result or None
133 |         """
134 |         rcode = None
135 |         if report_id is not None and isinstance(report_id, str):
136 |             rcode = self.bucket.delete_key(report_id)
137 |         return rcode
138 | 


--------------------------------------------------------------------------------
/libnmap/plugins/sql.py:
--------------------------------------------------------------------------------
  1 | #!/usr/bin/env python
  2 | from sqlalchemy import create_engine
  3 | from sqlalchemy.schema import Column
  4 | from sqlalchemy.types import Integer, DateTime, LargeBinary, Text, String
  5 | from sqlalchemy.ext.declarative import declarative_base
  6 | from sqlalchemy.orm import sessionmaker
  7 | 
  8 | from libnmap.plugins.backendplugin import NmapBackendPlugin
  9 | from libnmap.reportjson import ReportEncoder, ReportDecoder
 10 | 
 11 | import json
 12 | from datetime import datetime
 13 | 
 14 | Base = declarative_base()
 15 | 
 16 | class NmapSqlPlugin(NmapBackendPlugin):
 17 |     """
 18 |         This class handle the persistence of NmapRepport object in SQL backend
 19 |         Implementation is made using sqlalchemy(0.8.1)
 20 |         usage :
 21 | 
 22 |         #get a nmapReport object
 23 |         from libnmap.parser import NmapParser
 24 |         from libnmap.reportjson import ReportDecoder, ReportEncoder
 25 |         import json
 26 |         nmap_report_obj = NmapParser.parse_fromfile(
 27 |                '/home/vagrant/python-nmap-lib/libnmap/test/files/1_hosts.xml')
 28 | 
 29 |          #get a backend with in memory sqlite
 30 |          from libnmap.plugins.backendpluginFactory import BackendPluginFactory
 31 |          mybackend_mem = BackendPluginFactory.create(plugin_name='sql',
 32 |                                                      url='sqlite://',
 33 |                                                      echo=True)
 34 | 
 35 |          mybackend_mysql = BackendPluginFactory.create(plugin_name='sql',
 36 |                             url='mysql+mysqldb://scott:tiger@localhost/foo',
 37 |                             echo=True)
 38 |          mybackend = BackendPluginFactory.create(plugin_name='sql',
 39 |                                         url='sqlite:////tmp/reportdb.sql',
 40 |                                         echo=True)
 41 |          #lets save!!
 42 |          nmap_report_obj.save(mybackend)
 43 |          mybackend.getall()
 44 |          mybackend.get(1)
 45 |     """
 46 |     class Reports(Base):
 47 |         """
 48 |             Embeded class for ORM map NmapReport to a
 49 |             simple three column table
 50 |         """
 51 |         __tablename__ = 'reports'
 52 | 
 53 |         id = Column('report_id', Integer, primary_key=True)
 54 |         inserted = Column('inserted', DateTime(), default='now')
 55 |         host = Column('host', String(256))
 56 |         command_line = Column('command_line', String(256))
 57 |         report_json = Column('report_json', LargeBinary())
 58 | 
 59 |         def __init__(self, obj_NmapReport):
 60 |             # self.inserted = datetime.fromtimestamp(obj_NmapReport.endtime)
 61 |             self.command_line = str(obj_NmapReport.status)
 62 | 
 63 |             self.host = obj_NmapReport.address
 64 |             self.command_line = 
 65 | 
 66 |             dumped_json = json.dumps(obj_NmapReport,
 67 |                                      cls=ReportEncoder)
 68 |             self.report_json = bytes(dumped_json.encode('UTF-8'))
 69 | 
 70 |         def decode(self):
 71 |             json_decoded = self.report_json.decode('utf-8')
 72 |             nmap_report_obj = json.loads(json_decoded,
 73 |                                          cls=ReportDecoder)
 74 |             return nmap_report_obj
 75 | 
 76 |     def __init__(self, **kwargs):
 77 |         """
 78 |             constructor receive a **kwargs as the **kwargs in the sqlalchemy
 79 |             create_engine() method (see sqlalchemy docs)
 80 |             You must add to this **kwargs an 'url' key with the url to your
 81 |             database
 82 |             This constructor will :
 83 |             - create all the necessary obj to discuss with the DB
 84 |             - create all the mapping(ORM)
 85 | 
 86 |             todo : suport the : sqlalchemy.engine_from_config
 87 | 
 88 |             :param **kwargs:
 89 |             :raises: ValueError if no url is given,
 90 |                     all exception sqlalchemy can throw
 91 |             ie sqlite in memory url='sqlite://' echo=True
 92 |             ie sqlite file on hd url='sqlite:////tmp/reportdb.sql' echo=True
 93 |             ie mysql url='mysql+mysqldb://scott:tiger@localhost/foo'
 94 |         """
 95 |         NmapBackendPlugin.__init__(self)
 96 |         self.engine = None
 97 |         self.url = None
 98 |         self.Session = sessionmaker()
 99 | 
100 |         if 'url' not in kwargs:
101 |             raise ValueError
102 |         self.url = kwargs['url']
103 |         del kwargs['url']
104 |         try:
105 |             self.engine = create_engine(self.url, **kwargs)
106 |             Base.metadata.create_all(bind=self.engine, checkfirst=True)
107 |             self.Session.configure(bind=self.engine)
108 |         except:
109 |             raise
110 | 
111 |     def insert(self, nmap_report):
112 |         """
113 |            insert NmapReport in the backend
114 | 
115 |            :param NmapReport:
116 | 
117 |            :returns: the ident of the object in the backend for future usage \
118 |            or None
119 |         """
120 |         sess = self.Session()
121 |         report = NmapSqlPlugin.Reports(nmap_report)
122 |         sess.add(report)
123 |         sess.commit()
124 |         reportid = report.id
125 |         sess.close()
126 |         return reportid if reportid else None
127 | 
128 |     def get(self, report_id=None):
129 |         """
130 |             retreive a NmapReport from the backend
131 | 
132 |             :param id: str
133 | 
134 |             :returns: NmapReport
135 |         """
136 |         if report_id is None:
137 |             raise ValueError
138 |         sess = self.Session()
139 |         our_report = (
140 |             sess.query(NmapSqlPlugin.Reports).filter_by(id=report_id).first())
141 |         sess.close()
142 |         return our_report.decode() if our_report else None
143 | 
144 |     def getall(self):
145 |         """
146 |             :param filter: Nice to have implement a filter capability
147 | 
148 |             :returns: collection of tuple (id,NmapReport)
149 |         """
150 |         sess = self.Session()
151 |         nmapreportList = []
152 |         for report in (
153 |                 sess.query(NmapSqlPlugin.Reports).
154 |                 order_by(NmapSqlPlugin.Reports.inserted)):
155 |             nmapreportList.append((report.id, report.decode()))
156 |         sess.close()
157 |         return nmapreportList
158 | 
159 |     def delete(self, report_id=None):
160 |         """
161 |             Remove a report from the backend
162 | 
163 |             :param id: str
164 | 
165 |             :returns: The number of rows deleted
166 |         """
167 |         if report_id is None:
168 |             raise ValueError
169 |         nb_line = 0
170 |         sess = self.Session()
171 |         nb_line = sess.query(NmapSqlPlugin.Reports).\
172 |             filter_by(id=report_id).delete()
173 |         sess.commit()
174 |         sess.close()
175 |         return nb_line
176 | 


--------------------------------------------------------------------------------
/libnmap/process.py:
--------------------------------------------------------------------------------
  1 | #!/usr/bin/env python
  2 | # -*- coding: utf-8 -*-
  3 | 
  4 | import os
  5 | import pwd
  6 | import shlex
  7 | import subprocess
  8 | import multiprocessing
  9 | from threading import Thread
 10 | from xml.dom import pulldom
 11 | import warnings
 12 | 
 13 | try:
 14 |     from Queue import Empty, Full
 15 | except ImportError:
 16 |     from queue import Empty, Full
 17 | 
 18 | __all__ = [
 19 |     'NmapProcess'
 20 | ]
 21 | 
 22 | 
 23 | class NmapTask(object):
 24 |     """
 25 |     NmapTask is a internal class used by process. Each time nmap
 26 |     starts a new task during the scan, a new class will be instanciated.
 27 |     Classes examples are: "Ping Scan", "NSE script", "DNS Resolve",..
 28 |     To each class an estimated time to complete is assigned and updated
 29 |     at least every second within the NmapProcess.
 30 |     A property NmapProcess.current_task points to the running task at
 31 |     time T and a dictionnary NmapProcess.tasks with "task name" as key
 32 |     is built during scan execution
 33 |     """
 34 |     def __init__(self, name, starttime=0, extrainfo=''):
 35 |         self.name = name
 36 |         self.etc = 0
 37 |         self.progress = 0
 38 |         self.percent = 0
 39 |         self.remaining = 0
 40 |         self.status = 'started'
 41 |         self.starttime = starttime
 42 |         self.endtime = 0
 43 |         self.extrainfo = extrainfo
 44 |         self.updated = 0
 45 | 
 46 | 
 47 | class NmapProcess(Thread):
 48 |     """
 49 |     NmapProcess is a class which wraps around the nmap executable.
 50 | 
 51 |     Consequently, in order to run an NmapProcess, nmap should be installed
 52 |     on the host running the script. By default NmapProcess will produce
 53 |     the output of the nmap scan in the nmap XML format. This could be then
 54 |     parsed out via the NmapParser class from libnmap.parser module.
 55 |     """
 56 |     def __init__(self, targets="127.0.0.1",
 57 |                  options="-sT", event_callback=None, safe_mode=True, fqp=None):
 58 |         """
 59 |         Constructor of NmapProcess class.
 60 | 
 61 |         :param targets: hosts to be scanned. Could be a string of hosts \
 62 |         separated with a coma or a python list of hosts/ip.
 63 |         :type targets: string or list
 64 | 
 65 |         :param options: list of nmap options to be applied to scan. \
 66 |         These options are all documented in nmap's man pages.
 67 | 
 68 |         :param event_callback: callable function which will be ran \
 69 |         each time nmap process outputs data. This function will receive \
 70 |         two parameters:
 71 | 
 72 |             1. the nmap process object
 73 |             2. the data produced by nmap process. See readme for examples.
 74 | 
 75 |         :param safe_mode: parameter to protect unsafe options like -oN, -oG, \
 76 |         -iL, -oA,...
 77 | 
 78 |         :param fqp: full qualified path, if None, nmap will be searched \
 79 |         in the PATH
 80 | 
 81 |         :return: NmapProcess object
 82 | 
 83 |         """
 84 |         Thread.__init__(self)
 85 |         unsafe_opts = set(['-oG', '-oN', '-iL', '-oA', '-oS', '-oX',
 86 |                            '--iflist', '--resume', '--stylesheet',
 87 |                            '--datadir'])
 88 |         if fqp:
 89 |             if os.path.isfile(fqp) and os.access(fqp, os.X_OK):
 90 |                 self.__nmap_binary = fqp
 91 |             else:
 92 |                 raise EnvironmentError(1, "wrong path or not executable", fqp)
 93 |         else:
 94 |             nmap_binary_name = "nmap"
 95 |             self.__nmap_binary = self._whereis(nmap_binary_name)
 96 |         # self.__nmap_fixed_options = "-oX - -vvv --stats-every 1s"
 97 |         self.__nmap_fixed_options = "-oX -"
 98 | 
 99 |         if self.__nmap_binary is None:
100 |             raise EnvironmentError(1, "nmap is not installed or could "
101 |                                       "not be found in system path")
102 | 
103 |         if isinstance(targets, str):
104 |             self.__nmap_targets = targets.replace(" ", "").split(',')
105 |         elif isinstance(targets, list):
106 |             self.__nmap_targets = targets
107 |         else:
108 |             raise Exception("Supplied target list should be either a "
109 |                             "string or a list")
110 | 
111 |         self._nmap_options = set(options.split())
112 |         if safe_mode and not self._nmap_options.isdisjoint(unsafe_opts):
113 |             raise Exception("unsafe options activated while safe_mode "
114 |                             "is set True")
115 |         self.__nmap_dynamic_options = options
116 |         self.__sudo_run = ''
117 |         self.__nmap_command_line = self.get_command_line()
118 | 
119 |         if event_callback and callable(event_callback):
120 |             self.__nmap_event_callback = event_callback
121 |         else:
122 |             self.__nmap_event_callback = None
123 |         (self.DONE, self.READY, self.RUNNING,
124 |          self.CANCELLED, self.FAILED) = range(5)
125 |         self._run_init()
126 | 
127 |     def _run_init(self):
128 |         self.__process_killed = multiprocessing.Event()
129 |         self.__nmap_command_line = self.get_command_line()
130 |         # API usable in callback function
131 |         self.__nmap_proc = None
132 |         self.__qout = None
133 |         self.__nmap_rc = 0
134 |         self.__state = self.RUNNING
135 |         self.__starttime = 0
136 |         self.__endtime = 0
137 |         self.__version = ''
138 |         self.__elapsed = ''
139 |         self.__summary = ''
140 |         self.__stdout = ''
141 |         self.__stderr = ''
142 |         self.__current_task = ''
143 |         self.__nmap_tasks = {}
144 | 
145 |     def _whereis(self, program):
146 |         """
147 |         Protected method enabling the object to find the full path of a binary
148 |         from its PATH environment variable.
149 | 
150 |         :param program: name of a binary for which the full path needs to
151 |         be discovered.
152 | 
153 |         :return: the full path to the binary.
154 | 
155 |         :todo: add a default path list in case PATH is empty.
156 |         """
157 |         for path in os.environ.get('PATH', '').split(':'):
158 |             if (os.path.exists(os.path.join(path, program)) and not
159 |                os.path.isdir(os.path.join(path, program))):
160 |                 return os.path.join(path, program)
161 |         return None
162 | 
163 |     def get_command_line(self):
164 |         """
165 |         Public method returning the reconstructed command line ran via the lib
166 | 
167 |         :return: the full nmap command line to run
168 |         :rtype: string
169 |         """
170 |         return ("{0} {1} {2} {3} {4}".format(self.__sudo_run,
171 |                                              self.__nmap_binary,
172 |                                              self.__nmap_fixed_options,
173 |                                              self.__nmap_dynamic_options,
174 |                                              " ".join(self.__nmap_targets)))
175 | 
176 |     def sudo_run(self, run_as='root'):
177 |         """
178 |         Public method enabling the library's user to run the scan with
179 |         priviledges via sudo. The sudo configuration should be set manually
180 |         on the local system otherwise sudo will prompt for a password.
181 |         This method alters the command line by prefixing the sudo command to
182 |         nmap and will then call self.run()
183 | 
184 |         :param run_as: user name to which the lib needs to sudo to run the scan
185 | 
186 |         :return: return code from nmap execution
187 |         """
188 |         sudo_user = run_as.split().pop()
189 |         try:
190 |             pwd.getpwnam(sudo_user).pw_uid
191 |         except KeyError:
192 |             _exmsg = ("Username {0} does not exists. Please supply"
193 |                       " a valid username".format(run_as))
194 |             raise EnvironmentError(_exmsg)
195 | 
196 |         sudo_path = self._whereis("sudo")
197 |         if sudo_path is None:
198 |             raise EnvironmentError(2, "sudo is not installed or "
199 |                                       "could not be found in system path: "
200 |                                       "cannot run nmap with sudo")
201 | 
202 |         self.__sudo_run = "{0} -u {1}".format(sudo_path, sudo_user)
203 |         rc = self.run()
204 |         self.__sudo_run = ""
205 | 
206 |         return rc
207 | 
208 |     def sudo_run_background(self, run_as='root'):
209 |         """
210 |         Public method enabling the library's user to run in background a
211 |         nmap scan with priviledges via sudo.
212 |         The sudo configuration should be set manually on the local system
213 |         otherwise sudo will prompt for a password.
214 |         This method alters the command line by prefixing the sudo command to
215 |         nmap and will then call self.run()
216 | 
217 |         :param run_as: user name to which the lib needs to sudo to run the scan
218 | 
219 |         :return: return code from nmap execution
220 |         """
221 |         sudo_user = run_as.split().pop()
222 |         try:
223 |             pwd.getpwnam(sudo_user).pw_uid
224 |         except KeyError:
225 |             _exmsg = ("Username {0} does not exists. Please supply"
226 |                       " a valid username".format(run_as))
227 |             raise EnvironmentError(_exmsg)
228 | 
229 |         sudo_path = self._whereis("sudo")
230 |         if sudo_path is None:
231 |             raise EnvironmentError(2, "sudo is not installed or "
232 |                                       "could not be found in system path: "
233 |                                       "cannot run nmap with sudo")
234 | 
235 |         self.__sudo_run = "{0} -u {1}".format(sudo_path, sudo_user)
236 |         super(NmapProcess, self).start()
237 | 
238 |     def run(self):
239 |         """
240 |         Public method which is usually called right after the constructor
241 |         of NmapProcess. This method starts the nmap executable's subprocess.
242 |         It will also bind a Process that will read from subprocess' stdout
243 |         and stderr and push the lines read in a python queue for futher
244 |         processing. This processing is waken-up each time data is pushed
245 |         from the nmap binary into the stdout reading routine. Processing
246 |         could be performed by a user-provided callback. The whole
247 |         NmapProcess object could be accessible asynchroneously.
248 | 
249 |         return: return code from nmap execution
250 |         """
251 |         def ioreader_routine(proc_stdout, io_queue, data_pushed, producing):
252 |             """
253 |             local function that will read lines from a file descriptor
254 |             and put the data in a python queue for futher processing.
255 | 
256 |             :param proc_stdout: file descriptor to read lines from.
257 |             :param io_queue: queue in which read lines will be pushed.
258 |             :param data_pushed: queue used to push data read from the
259 |             nmap stdout back into the parent process
260 |             :param producing: shared variable to notify the parent process
261 |             that processing is either running, either over.
262 |             """
263 |             producing.value = 1
264 |             for streamline in iter(proc_stdout.readline, b''):
265 |                 if self.__process_killed.is_set():
266 |                     break
267 |                 if streamline is not None:
268 |                     try:
269 |                         io_queue.put(str(streamline.decode()))
270 |                     except Full:
271 |                         pass
272 |                     data_pushed.set()
273 |             producing.value = 0
274 |             data_pushed.set()
275 | 
276 |         self._run_init()
277 |         producing = multiprocessing.Value('i', 1)
278 |         data_pushed = multiprocessing.Event()
279 |         self.__qout = multiprocessing.Queue()
280 | 
281 |         _tmp_cmdline = shlex.split(self.__nmap_command_line)
282 |         try:
283 |             self.__nmap_proc = subprocess.Popen(args=_tmp_cmdline,
284 |                                                 stdout=subprocess.PIPE,
285 |                                                 stderr=subprocess.PIPE,
286 |                                                 bufsize=0)
287 |             ioreader = multiprocessing.Process(target=ioreader_routine,
288 |                                                args=(self.__nmap_proc.stdout,
289 |                                                      self.__qout,
290 |                                                      data_pushed,
291 |                                                      producing))
292 |             # ioreader.daemon=False
293 |             ioreader.start()
294 |             self.__state = self.RUNNING
295 |         except OSError:
296 |             self.__state = self.FAILED
297 |             raise EnvironmentError(1, "nmap is not installed or could "
298 |                                       "not be found in system path")
299 | 
300 |         thread_stream = ''
301 |         while(self.__nmap_proc.poll() is None or producing.value == 1):
302 |             if self.__process_killed.is_set():
303 |                 break
304 |             if producing.value == 1 and self.__qout.empty():
305 |                 try:
306 |                     data_pushed.wait()
307 |                 except KeyboardInterrupt:
308 |                     break
309 |             try:
310 |                 thread_stream = self.__qout.get_nowait()
311 |             except Empty:
312 |                 pass
313 |             except KeyboardInterrupt:
314 |                 break
315 |             else:
316 |                 self.__stdout += thread_stream
317 |                 evnt = self.__process_event(thread_stream)
318 |                 if self.__nmap_event_callback and evnt:
319 |                     self.__nmap_event_callback(self)
320 |             data_pushed.clear()
321 |         ioreader.join()
322 |         # queue clean-up
323 |         while not self.__qout.empty():
324 |             self.__stdout += self.__qout.get_nowait()
325 |         self.__stderr += str(self.__nmap_proc.stderr.read().decode())
326 | 
327 |         self.__nmap_rc = self.__nmap_proc.poll()
328 |         if self.rc is None:
329 |             self.__state = self.CANCELLED
330 |         elif self.rc == 0:
331 |             self.__state = self.DONE
332 |             if self.current_task:
333 |                 self.__nmap_tasks[self.current_task.name].progress = 100
334 |         else:
335 |             self.__state = self.FAILED
336 |         return self.rc
337 | 
338 |     def run_background(self):
339 |         """
340 |         run nmap scan in background as a thread.
341 |         For privileged scans, consider NmapProcess.sudo_run_background()
342 |         """
343 |         self.__state = self.RUNNING
344 |         super(NmapProcess, self).start()
345 | 
346 |     def is_running(self):
347 |         """
348 |         Checks if nmap is still running.
349 | 
350 |         :return: True if nmap is still running
351 |         """
352 |         return self.state == self.RUNNING
353 | 
354 |     def has_terminated(self):
355 |         """
356 |         Checks if nmap has terminated. Could have failed or succeeded
357 | 
358 |         :return: True if nmap process is not running anymore.
359 |         """
360 |         return (self.state == self.DONE or self.state == self.FAILED
361 |                 or self.state == self.CANCELLED)
362 | 
363 |     def has_failed(self):
364 |         """
365 |         Checks if nmap has failed.
366 | 
367 |         :return: True if nmap process errored.
368 |         """
369 |         return self.state == self.FAILED
370 | 
371 |     def is_successful(self):
372 |         """
373 |         Checks if nmap terminated successfully.
374 | 
375 |         :return: True if nmap terminated successfully.
376 |         """
377 |         return self.state == self.DONE
378 | 
379 |     def stop(self):
380 |         """
381 |         Send KILL -15 to the nmap subprocess and gently ask the threads to
382 |         stop.
383 |         """
384 |         self.__state = self.CANCELLED
385 |         if self.__nmap_proc.poll() is None:
386 |             self.__nmap_proc.kill()
387 |         self.__qout.cancel_join_thread()
388 |         self.__process_killed.set()
389 | 
390 |     def __process_event(self, eventdata):
391 |         """
392 |         Private method called while nmap process is running. It enables the
393 |         library to handle specific data/events produced by nmap process.
394 |         So far, the following events are supported:
395 | 
396 |         1. task progress: updates estimated time to completion and percentage
397 |            done while scan is running. Could be used in combination with a
398 |            callback function which could then handle this data while scan is
399 |            running.
400 |         2. nmap run: header of the scan. Usually displayed when nmap is started
401 |         3. finished: when nmap scan ends.
402 | 
403 |         :return: True is event is known.
404 | 
405 |         :todo: handle parsing directly via NmapParser.parse()
406 |         """
407 |         rval = False
408 |         try:
409 |             edomdoc = pulldom.parseString(eventdata)
410 |             for xlmnt, xmlnode in edomdoc:
411 |                 if xlmnt is not None and xlmnt == pulldom.START_ELEMENT:
412 |                     if (xmlnode.nodeName == 'taskbegin' and
413 |                             xmlnode.attributes.keys()):
414 |                         xt = xmlnode.attributes
415 |                         taskname = xt['task'].value
416 |                         starttime = xt['time'].value
417 |                         xinfo = ''
418 |                         if 'extrainfo' in xt.keys():
419 |                             xinfo = xt['extrainfo'].value
420 |                         newtask = NmapTask(taskname, starttime, xinfo)
421 |                         self.__nmap_tasks[newtask.name] = newtask
422 |                         self.__current_task = newtask.name
423 |                         rval = True
424 |                     elif (xmlnode.nodeName == 'taskend' and
425 |                             xmlnode.attributes.keys()):
426 |                         xt = xmlnode.attributes
427 |                         tname = xt['task'].value
428 |                         xinfo = ''
429 |                         self.__nmap_tasks[tname].endtime = xt['time'].value
430 |                         if 'extrainfo' in xt.keys():
431 |                             xinfo = xt['extrainfo'].value
432 |                         self.__nmap_tasks[tname].extrainfo = xinfo
433 |                         self.__nmap_tasks[tname].status = "ended"
434 |                         rval = True
435 |                     elif (xmlnode.nodeName == 'taskprogress' and
436 |                             xmlnode.attributes.keys()):
437 |                         xt = xmlnode.attributes
438 |                         tname = xt['task'].value
439 |                         percent = xt['percent'].value
440 |                         etc = xt['etc'].value
441 |                         remaining = xt['remaining'].value
442 |                         updated = xt['time'].value
443 |                         self.__nmap_tasks[tname].percent = percent
444 |                         self.__nmap_tasks[tname].progress = percent
445 |                         self.__nmap_tasks[tname].etc = etc
446 |                         self.__nmap_tasks[tname].remaining = remaining
447 |                         self.__nmap_tasks[tname].updated = updated
448 |                         rval = True
449 |                     elif (xmlnode.nodeName == 'nmaprun' and
450 |                             xmlnode.attributes.keys()):
451 |                         self.__starttime = xmlnode.attributes['start'].value
452 |                         self.__version = xmlnode.attributes['version'].value
453 |                         rval = True
454 |                     elif (xmlnode.nodeName == 'finished' and
455 |                             xmlnode.attributes.keys()):
456 |                         self.__endtime = xmlnode.attributes['time'].value
457 |                         self.__elapsed = xmlnode.attributes['elapsed'].value
458 |                         self.__summary = xmlnode.attributes['summary'].value
459 |                         rval = True
460 |         except:
461 |             pass
462 |         return rval
463 | 
464 |     @property
465 |     def command(self):
466 |         """
467 |         return the constructed nmap command or empty string if not
468 |         constructed yet.
469 | 
470 |         :return: string
471 |         """
472 |         return self.__nmap_command_line or ''
473 | 
474 |     @property
475 |     def targets(self):
476 |         """
477 |         Provides the list of targets to scan
478 | 
479 |         :return: list of string
480 |         """
481 |         return self.__nmap_targets
482 | 
483 |     @property
484 |     def options(self):
485 |         """
486 |         Provides the list of options for that scan
487 | 
488 |         :return: list of string (nmap options)
489 |         """
490 |         return self._nmap_options
491 | 
492 |     @property
493 |     def state(self):
494 |         """
495 |         Accessor for nmap execution state. Possible states are:
496 | 
497 |         - self.READY
498 |         - self.RUNNING
499 |         - self.FAILED
500 |         - self.CANCELLED
501 |         - self.DONE
502 | 
503 |         :return: integer (from above documented enum)
504 |         """
505 |         return self.__state
506 | 
507 |     @property
508 |     def starttime(self):
509 |         """
510 |         Accessor for time when scan started
511 | 
512 |         :return: string. Unix timestamp
513 |         """
514 |         return self.__starttime
515 | 
516 |     @property
517 |     def endtime(self):
518 |         """
519 |         Accessor for time when scan ended
520 | 
521 |         :return: string. Unix timestamp
522 |         """
523 |         warnings.warn("data collected from finished events are deprecated."
524 |                       "Use NmapParser.parse()", DeprecationWarning)
525 |         return self.__endtime
526 | 
527 |     @property
528 |     def elapsed(self):
529 |         """
530 |         Accessor returning for how long the scan ran (in seconds)
531 | 
532 |         :return: string
533 |         """
534 |         warnings.warn("data collected from finished events are deprecated."
535 |                       "Use NmapParser.parse()", DeprecationWarning)
536 |         return self.__elapsed
537 | 
538 |     @property
539 |     def summary(self):
540 |         """
541 |         Accessor returning a short summary of the scan's results
542 | 
543 |         :return: string
544 |         """
545 |         warnings.warn("data collected from finished events are deprecated."
546 |                       "Use NmapParser.parse()", DeprecationWarning)
547 |         return self.__summary
548 | 
549 |     @property
550 |     def tasks(self):
551 |         """
552 |         Accessor returning for the list of tasks ran during nmap scan
553 | 
554 |         :return: dict of NmapTask object
555 |         """
556 |         return self.__nmap_tasks
557 | 
558 |     @property
559 |     def version(self):
560 |         """
561 |         Accessor for nmap binary version number
562 | 
563 |         :return: version number of nmap binary
564 |         :rtype: string
565 |         """
566 |         return self.__version
567 | 
568 |     @property
569 |     def current_task(self):
570 |         """
571 |         Accessor for the current NmapTask beeing run
572 | 
573 |         :return: NmapTask or None if no task started yet
574 |         """
575 |         rval = None
576 |         if len(self.__current_task):
577 |             rval = self.tasks[self.__current_task]
578 |         return rval
579 | 
580 |     @property
581 |     def etc(self):
582 |         """
583 |         Accessor for estimated time to completion
584 | 
585 |         :return:  estimated time to completion
586 |         """
587 |         rval = 0
588 |         if self.current_task:
589 |             rval = self.current_task.etc
590 |         return rval
591 | 
592 |     @property
593 |     def progress(self):
594 |         """
595 |         Accessor for progress status in percentage
596 | 
597 |         :return: percentage of job processed.
598 |         """
599 |         rval = 0
600 |         if self.current_task:
601 |             rval = self.current_task.progress
602 |         return rval
603 | 
604 |     @property
605 |     def rc(self):
606 |         """
607 |         Accessor for nmap execution's return code
608 | 
609 |         :return: nmap execution's return code
610 |         """
611 |         return self.__nmap_rc
612 | 
613 |     @property
614 |     def stdout(self):
615 |         """
616 |         Accessor for nmap standart output
617 | 
618 |         :return: output from nmap scan in XML
619 |         :rtype: string
620 |         """
621 |         return self.__stdout
622 | 
623 |     @property
624 |     def stderr(self):
625 |         """
626 |         Accessor for nmap standart error
627 | 
628 |         :return: output from nmap when errors occured.
629 |         :rtype: string
630 |         """
631 |         return self.__stderr
632 | 
633 | 
634 | def main():
635 |     def mycallback(nmapscan=None):
636 |         if nmapscan.is_running() and nmapscan.current_task:
637 |             ntask = nmapscan.current_task
638 |             print("Task {0} ({1}): ETC: {2} DONE: {3}%".format(ntask.name,
639 |                                                                ntask.status,
640 |                                                                ntask.etc,
641 |                                                                ntask.progress))
642 |     nm = NmapProcess("scanme.nmap.org",
643 |                      options="-A",
644 |                      event_callback=mycallback)
645 |     rc = nm.run()
646 |     if rc == 0:
647 |         print("Scan started at {0} nmap version: {1}").format(nm.starttime,
648 |                                                               nm.version)
649 |         print("state: {0} (rc: {1})").format(nm.state, nm.rc)
650 |         print("results size: {0}").format(len(nm.stdout))
651 |         print("Scan ended {0}: {1}").format(nm.endtime, nm.summary)
652 |     else:
653 |         print("state: {0} (rc: {1})").format(nm.state, nm.rc)
654 |         print("Error: {stderr}").format(stderr=nm.stderr)
655 |         print("Result: {0}").format(nm.stdout)
656 | 
657 | if __name__ == '__main__':
658 |     main()
659 | 


--------------------------------------------------------------------------------
/libnmap/reportjson.py:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/env python
 2 | # -*- coding: utf-8 -*-
 3 | 
 4 | import json
 5 | from libnmap.objects import NmapHost, NmapService, NmapReport
 6 | from libnmap.objects.os import NmapOSFingerprint, NmapOSMatch, NmapOSClass
 7 | from libnmap.objects.os import CPE, OSFPPortUsed
 8 | from libnmap.parser import NmapParser
 9 | 
10 | 
11 | class ReportEncoder(json.JSONEncoder):
12 |     def default(self, obj):
13 |         otype = {'NmapHost': NmapHost,
14 |                  'NmapOSFingerprint': NmapOSFingerprint,
15 |                  'NmapOSMatch': NmapOSMatch,
16 |                  'NmapOSClass': NmapOSClass,
17 |                  'CPE': CPE,
18 |                  'OSFPPortUsed': OSFPPortUsed,
19 |                  'NmapService': NmapService,
20 |                  'NmapReport': NmapReport}
21 |         if isinstance(obj, tuple(otype.values())):
22 |             key = ('__{0}__').format(obj.__class__.__name__)
23 |             return {key: obj.__dict__}
24 |         return json.JSONEncoder.default(self, obj)
25 | 
26 | 
27 | class ReportDecoder(json.JSONDecoder):
28 |     def decode(self, json_str):
29 |         r = NmapParser.parse_fromdict(json.loads(json_str))
30 |         return r
31 | 


--------------------------------------------------------------------------------
/run.py:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/env python
 2 | # encoding: utf-8
 3 | # tasks.py
 4 | # email: ringzero@0x557.org
 5 | 
 6 | import sys
 7 | from tasks import *
 8 | from wyfunc import make_target_list
 9 | 
10 | def start_nmap_dispath(targets, taskid=None):
11 | 	print '-' * 50
12 | 	target_list = make_target_list(targets)
13 | 
14 | 	for target in target_list:
15 | 		print '-' * 50
16 | 		print '* push %s to Redis' % target
17 | 		print '* AsyncResult:%s' % nmap_dispath.delay(target,taskid=taskid)
18 | 
19 | 	print '-' * 50
20 | 	print '* Push nmapscan tasks complete.'
21 | 	print '-' * 50
22 | 
23 | if __name__ == "__main__":
24 | 	if len(sys.argv) == 2:
25 | 		start_nmap_dispath(sys.argv[1])
26 | 		sys.exit(0)
27 | 	elif len(sys.argv) == 3:
28 | 		start_nmap_dispath(sys.argv[1], sys.argv[2])
29 | 	else:
30 | 		print ("usage: %s targets taskid" % sys.argv[0])
31 | 		sys.exit(-1)


--------------------------------------------------------------------------------
/src/flower.tar.gz:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/ring04h/thorns/65dda993878e3d96fa412a6576fe44dd3d61328a/src/flower.tar.gz


--------------------------------------------------------------------------------
/src/supervisord_client.conf:
--------------------------------------------------------------------------------
  1 | ; Sample supervisor config file.
  2 | ;
  3 | ; For more information on the config file, please see:
  4 | ; http://supervisord.org/configuration.html
  5 | ;
  6 | ; Notes:
  7 | ;  - Shell expansion ("~" or "$HOME") is not supported.  Environment
  8 | ;    variables can be expanded using this syntax: "%(ENV_HOME)s".
  9 | ;  - Comments must have a leading space: "a=b ;comment" not "a=b;comment".
 10 | 
 11 | [unix_http_server]
 12 | file=/tmp/supervisor.sock   ; (the path to the socket file)
 13 | ;chmod=0700                 ; socket file mode (default 0700)
 14 | ;chown=nobody:nogroup       ; socket file uid:gid owner
 15 | ;username=user              ; (default is no username (open server))
 16 | ;password=123               ; (default is no password (open server))
 17 | 
 18 | [inet_http_server]         ; inet (TCP) server disabled by default
 19 | port=0.0.0.0:9001        ; (ip_address:port specifier, *:port for all iface)
 20 | ;username=user              ; (default is no username (open server))
 21 | ;password=123               ; (default is no password (open server))
 22 | 
 23 | [supervisord]
 24 | logfile=/tmp/supervisord.log ; (main log file;default $CWD/supervisord.log)
 25 | logfile_maxbytes=50MB        ; (max main logfile bytes b4 rotation;default 50MB)
 26 | logfile_backups=10           ; (num of main logfile rotation backups;default 10)
 27 | loglevel=info                ; (log level;default info; others: debug,warn,trace)
 28 | pidfile=/tmp/supervisord.pid ; (supervisord pidfile;default supervisord.pid)
 29 | nodaemon=false               ; (start in foreground if true;default false)
 30 | minfds=1024                  ; (min. avail startup file descriptors;default 1024)
 31 | minprocs=200                 ; (min. avail process descriptors;default 200)
 32 | ;umask=022                   ; (process file creation umask;default 022)
 33 | ;user=chrism                 ; (default is current user, required if root)
 34 | ;identifier=supervisor       ; (supervisord identifier, default is 'supervisor')
 35 | ;directory=/tmp              ; (default is not to cd during start)
 36 | ;nocleanup=true              ; (don't clean up tempfiles at start;default false)
 37 | ;childlogdir=/tmp            ; ('AUTO' child log dir, default $TEMP)
 38 | ;environment=KEY="value"     ; (key value pairs to add to environment)
 39 | ;strip_ansi=false            ; (strip ansi escape codes in logs; def. false)
 40 | 
 41 | ; the below section must remain in the config file for RPC
 42 | ; (supervisorctl/web interface) to work, additional interfaces may be
 43 | ; added by defining them in separate rpcinterface: sections
 44 | [rpcinterface:supervisor]
 45 | supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface
 46 | 
 47 | [supervisorctl]
 48 | serverurl=unix:///tmp/supervisor.sock ; use a unix:// URL  for a unix socket
 49 | ;serverurl=http://127.0.0.1:9001 ; use an http:// url to specify an inet socket
 50 | ;username=chris              ; should be same as http_username if set
 51 | ;password=123                ; should be same as http_password if set
 52 | ;prompt=mysupervisor         ; cmd line prompt (default "supervisor")
 53 | ;history_file=~/.sc_history  ; use readline history if available
 54 | 
 55 | ; The below sample program section shows all possible program subsection values,
 56 | ; create one or more 'real' program: sections to be able to control them under
 57 | ; supervisor.
 58 | 
 59 | ;[program:theprogramname]
 60 | ;command=/bin/cat              ; the program (relative uses PATH, can take args)
 61 | ;process_name=%(program_name)s ; process_name expr (default %(program_name)s)
 62 | ;numprocs=1                    ; number of processes copies to start (def 1)
 63 | ;directory=/tmp                ; directory to cwd to before exec (def no cwd)
 64 | ;umask=022                     ; umask for process (default None)
 65 | ;priority=999                  ; the relative start priority (default 999)
 66 | ;autostart=true                ; start at supervisord start (default: true)
 67 | ;autorestart=unexpected        ; whether/when to restart (default: unexpected)
 68 | ;startsecs=1                   ; number of secs prog must stay running (def. 1)
 69 | ;startretries=3                ; max # of serial start failures (default 3)
 70 | ;exitcodes=0,2                 ; 'expected' exit codes for process (default 0,2)
 71 | ;stopsignal=QUIT               ; signal used to kill process (default TERM)
 72 | ;stopwaitsecs=10               ; max num secs to wait b4 SIGKILL (default 10)
 73 | ;stopasgroup=false             ; send stop signal to the UNIX process group (default false)
 74 | ;killasgroup=false             ; SIGKILL the UNIX process group (def false)
 75 | ;user=chrism                   ; setuid to this UNIX account to run the program
 76 | ;redirect_stderr=true          ; redirect proc stderr to stdout (default false)
 77 | ;stdout_logfile=/a/path        ; stdout log path, NONE for none; default AUTO
 78 | ;stdout_logfile_maxbytes=1MB   ; max # logfile bytes b4 rotation (default 50MB)
 79 | ;stdout_logfile_backups=10     ; # of stdout logfile backups (default 10)
 80 | ;stdout_capture_maxbytes=1MB   ; number of bytes in 'capturemode' (default 0)
 81 | ;stdout_events_enabled=false   ; emit events on stdout writes (default false)
 82 | ;stderr_logfile=/a/path        ; stderr log path, NONE for none; default AUTO
 83 | ;stderr_logfile_maxbytes=1MB   ; max # logfile bytes b4 rotation (default 50MB)
 84 | ;stderr_logfile_backups=10     ; # of stderr logfile backups (default 10)
 85 | ;stderr_capture_maxbytes=1MB   ; number of bytes in 'capturemode' (default 0)
 86 | ;stderr_events_enabled=false   ; emit events on stderr writes (default false)
 87 | ;environment=A="1",B="2"       ; process environment additions (def no adds)
 88 | ;serverurl=AUTO                ; override serverurl computation (childutils)
 89 | 
 90 | ; The below sample eventlistener section shows all possible
 91 | ; eventlistener subsection values, create one or more 'real'
 92 | ; eventlistener: sections to be able to handle event notifications
 93 | ; sent by supervisor.
 94 | 
 95 | ;[eventlistener:theeventlistenername]
 96 | ;command=/bin/eventlistener    ; the program (relative uses PATH, can take args)
 97 | ;process_name=%(program_name)s ; process_name expr (default %(program_name)s)
 98 | ;numprocs=1                    ; number of processes copies to start (def 1)
 99 | ;events=EVENT                  ; event notif. types to subscribe to (req'd)
100 | ;buffer_size=10                ; event buffer queue size (default 10)
101 | ;directory=/tmp                ; directory to cwd to before exec (def no cwd)
102 | ;umask=022                     ; umask for process (default None)
103 | ;priority=-1                   ; the relative start priority (default -1)
104 | ;autostart=true                ; start at supervisord start (default: true)
105 | ;autorestart=unexpected        ; whether/when to restart (default: unexpected)
106 | ;startsecs=1                   ; number of secs prog must stay running (def. 1)
107 | ;startretries=3                ; max # of serial start failures (default 3)
108 | ;exitcodes=0,2                 ; 'expected' exit codes for process (default 0,2)
109 | ;stopsignal=QUIT               ; signal used to kill process (default TERM)
110 | ;stopwaitsecs=10               ; max num secs to wait b4 SIGKILL (default 10)
111 | ;stopasgroup=false             ; send stop signal to the UNIX process group (default false)
112 | ;killasgroup=false             ; SIGKILL the UNIX process group (def false)
113 | ;user=chrism                   ; setuid to this UNIX account to run the program
114 | ;redirect_stderr=true          ; redirect proc stderr to stdout (default false)
115 | ;stdout_logfile=/a/path        ; stdout log path, NONE for none; default AUTO
116 | ;stdout_logfile_maxbytes=1MB   ; max # logfile bytes b4 rotation (default 50MB)
117 | ;stdout_logfile_backups=10     ; # of stdout logfile backups (default 10)
118 | ;stdout_events_enabled=false   ; emit events on stdout writes (default false)
119 | ;stderr_logfile=/a/path        ; stderr log path, NONE for none; default AUTO
120 | ;stderr_logfile_maxbytes=1MB   ; max # logfile bytes b4 rotation (default 50MB)
121 | ;stderr_logfile_backups        ; # of stderr logfile backups (default 10)
122 | ;stderr_events_enabled=false   ; emit events on stderr writes (default false)
123 | ;environment=A="1",B="2"       ; process environment additions
124 | ;serverurl=AUTO                ; override serverurl computation (childutils)
125 | 
126 | ; The below sample group section shows all possible group values,
127 | ; create one or more 'real' group: sections to create "heterogeneous"
128 | ; process groups.
129 | 
130 | ;[group:thegroupname]
131 | ;programs=progname1,progname2  ; each refers to 'x' in [program:x] definitions
132 | ;priority=999                  ; the relative start priority (default 999)
133 | 
134 | ; The [include] section can just contain the "files" setting.  This
135 | ; setting can list multiple files (separated by whitespace or
136 | ; newlines).  It can also contain wildcards.  The filenames are
137 | ; interpreted as relative to this file.  Included files *cannot*
138 | ; include files themselves.
139 | 
140 | [program:worker-ringzero]
141 | command=celery -A tasks worker --loglevel=info --workdir=/home/thorns --concurrency=10 --hostname=%(program_name)s%(process_num)02d
142 | directory=/home/thorns
143 | user=root
144 | process_name=%(program_name)s_%(process_num)02d
145 | numprocs=1
146 | autostart=true
147 | autorestart=true
148 | startetries=10
149 | exitcodes=0
150 | stopsignal=KILL
151 | stopwaitsecs=10
152 | redirect_stderr=true
153 | 
154 | ;[include]
155 | ;files = relative/directory/*.ini
156 | 


--------------------------------------------------------------------------------
/src/supervisord_server.conf:
--------------------------------------------------------------------------------
  1 | ; Sample supervisor config file.
  2 | ;
  3 | ; For more information on the config file, please see:
  4 | ; http://supervisord.org/configuration.html
  5 | ;
  6 | ; Notes:
  7 | ;  - Shell expansion ("~" or "$HOME") is not supported.  Environment
  8 | ;    variables can be expanded using this syntax: "%(ENV_HOME)s".
  9 | ;  - Comments must have a leading space: "a=b ;comment" not "a=b;comment".
 10 | 
 11 | [unix_http_server]
 12 | file=/tmp/supervisor.sock   ; (the path to the socket file)
 13 | ;chmod=0700                 ; socket file mode (default 0700)
 14 | ;chown=nobody:nogroup       ; socket file uid:gid owner
 15 | ;username=user              ; (default is no username (open server))
 16 | ;password=123               ; (default is no password (open server))
 17 | 
 18 | [inet_http_server]         ; inet (TCP) server disabled by default
 19 | port=0.0.0.0:9001        ; (ip_address:port specifier, *:port for all iface)
 20 | ;username=user              ; (default is no username (open server))
 21 | ;password=123               ; (default is no password (open server))
 22 | 
 23 | [supervisord]
 24 | logfile=/tmp/supervisord.log ; (main log file;default $CWD/supervisord.log)
 25 | logfile_maxbytes=50MB        ; (max main logfile bytes b4 rotation;default 50MB)
 26 | logfile_backups=10           ; (num of main logfile rotation backups;default 10)
 27 | loglevel=info                ; (log level;default info; others: debug,warn,trace)
 28 | pidfile=/tmp/supervisord.pid ; (supervisord pidfile;default supervisord.pid)
 29 | nodaemon=false               ; (start in foreground if true;default false)
 30 | minfds=1024                  ; (min. avail startup file descriptors;default 1024)
 31 | minprocs=200                 ; (min. avail process descriptors;default 200)
 32 | ;umask=022                   ; (process file creation umask;default 022)
 33 | ;user=chrism                 ; (default is current user, required if root)
 34 | ;identifier=supervisor       ; (supervisord identifier, default is 'supervisor')
 35 | ;directory=/tmp              ; (default is not to cd during start)
 36 | ;nocleanup=true              ; (don't clean up tempfiles at start;default false)
 37 | ;childlogdir=/tmp            ; ('AUTO' child log dir, default $TEMP)
 38 | ;environment=KEY="value"     ; (key value pairs to add to environment)
 39 | ;strip_ansi=false            ; (strip ansi escape codes in logs; def. false)
 40 | 
 41 | ; the below section must remain in the config file for RPC
 42 | ; (supervisorctl/web interface) to work, additional interfaces may be
 43 | ; added by defining them in separate rpcinterface: sections
 44 | [rpcinterface:supervisor]
 45 | supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface
 46 | 
 47 | [supervisorctl]
 48 | serverurl=unix:///tmp/supervisor.sock ; use a unix:// URL  for a unix socket
 49 | ;serverurl=http://127.0.0.1:9001 ; use an http:// url to specify an inet socket
 50 | ;username=chris              ; should be same as http_username if set
 51 | ;password=123                ; should be same as http_password if set
 52 | ;prompt=mysupervisor         ; cmd line prompt (default "supervisor")
 53 | ;history_file=~/.sc_history  ; use readline history if available
 54 | 
 55 | ; The below sample program section shows all possible program subsection values,
 56 | ; create one or more 'real' program: sections to be able to control them under
 57 | ; supervisor.
 58 | 
 59 | ;[program:theprogramname]
 60 | ;command=/bin/cat              ; the program (relative uses PATH, can take args)
 61 | ;process_name=%(program_name)s ; process_name expr (default %(program_name)s)
 62 | ;numprocs=1                    ; number of processes copies to start (def 1)
 63 | ;directory=/tmp                ; directory to cwd to before exec (def no cwd)
 64 | ;umask=022                     ; umask for process (default None)
 65 | ;priority=999                  ; the relative start priority (default 999)
 66 | ;autostart=true                ; start at supervisord start (default: true)
 67 | ;autorestart=unexpected        ; whether/when to restart (default: unexpected)
 68 | ;startsecs=1                   ; number of secs prog must stay running (def. 1)
 69 | ;startretries=3                ; max # of serial start failures (default 3)
 70 | ;exitcodes=0,2                 ; 'expected' exit codes for process (default 0,2)
 71 | ;stopsignal=QUIT               ; signal used to kill process (default TERM)
 72 | ;stopwaitsecs=10               ; max num secs to wait b4 SIGKILL (default 10)
 73 | ;stopasgroup=false             ; send stop signal to the UNIX process group (default false)
 74 | ;killasgroup=false             ; SIGKILL the UNIX process group (def false)
 75 | ;user=chrism                   ; setuid to this UNIX account to run the program
 76 | ;redirect_stderr=true          ; redirect proc stderr to stdout (default false)
 77 | ;stdout_logfile=/a/path        ; stdout log path, NONE for none; default AUTO
 78 | ;stdout_logfile_maxbytes=1MB   ; max # logfile bytes b4 rotation (default 50MB)
 79 | ;stdout_logfile_backups=10     ; # of stdout logfile backups (default 10)
 80 | ;stdout_capture_maxbytes=1MB   ; number of bytes in 'capturemode' (default 0)
 81 | ;stdout_events_enabled=false   ; emit events on stdout writes (default false)
 82 | ;stderr_logfile=/a/path        ; stderr log path, NONE for none; default AUTO
 83 | ;stderr_logfile_maxbytes=1MB   ; max # logfile bytes b4 rotation (default 50MB)
 84 | ;stderr_logfile_backups=10     ; # of stderr logfile backups (default 10)
 85 | ;stderr_capture_maxbytes=1MB   ; number of bytes in 'capturemode' (default 0)
 86 | ;stderr_events_enabled=false   ; emit events on stderr writes (default false)
 87 | ;environment=A="1",B="2"       ; process environment additions (def no adds)
 88 | ;serverurl=AUTO                ; override serverurl computation (childutils)
 89 | 
 90 | ; The below sample eventlistener section shows all possible
 91 | ; eventlistener subsection values, create one or more 'real'
 92 | ; eventlistener: sections to be able to handle event notifications
 93 | ; sent by supervisor.
 94 | 
 95 | ;[eventlistener:theeventlistenername]
 96 | ;command=/bin/eventlistener    ; the program (relative uses PATH, can take args)
 97 | ;process_name=%(program_name)s ; process_name expr (default %(program_name)s)
 98 | ;numprocs=1                    ; number of processes copies to start (def 1)
 99 | ;events=EVENT                  ; event notif. types to subscribe to (req'd)
100 | ;buffer_size=10                ; event buffer queue size (default 10)
101 | ;directory=/tmp                ; directory to cwd to before exec (def no cwd)
102 | ;umask=022                     ; umask for process (default None)
103 | ;priority=-1                   ; the relative start priority (default -1)
104 | ;autostart=true                ; start at supervisord start (default: true)
105 | ;autorestart=unexpected        ; whether/when to restart (default: unexpected)
106 | ;startsecs=1                   ; number of secs prog must stay running (def. 1)
107 | ;startretries=3                ; max # of serial start failures (default 3)
108 | ;exitcodes=0,2                 ; 'expected' exit codes for process (default 0,2)
109 | ;stopsignal=QUIT               ; signal used to kill process (default TERM)
110 | ;stopwaitsecs=10               ; max num secs to wait b4 SIGKILL (default 10)
111 | ;stopasgroup=false             ; send stop signal to the UNIX process group (default false)
112 | ;killasgroup=false             ; SIGKILL the UNIX process group (def false)
113 | ;user=chrism                   ; setuid to this UNIX account to run the program
114 | ;redirect_stderr=true          ; redirect proc stderr to stdout (default false)
115 | ;stdout_logfile=/a/path        ; stdout log path, NONE for none; default AUTO
116 | ;stdout_logfile_maxbytes=1MB   ; max # logfile bytes b4 rotation (default 50MB)
117 | ;stdout_logfile_backups=10     ; # of stdout logfile backups (default 10)
118 | ;stdout_events_enabled=false   ; emit events on stdout writes (default false)
119 | ;stderr_logfile=/a/path        ; stderr log path, NONE for none; default AUTO
120 | ;stderr_logfile_maxbytes=1MB   ; max # logfile bytes b4 rotation (default 50MB)
121 | ;stderr_logfile_backups        ; # of stderr logfile backups (default 10)
122 | ;stderr_events_enabled=false   ; emit events on stderr writes (default false)
123 | ;environment=A="1",B="2"       ; process environment additions
124 | ;serverurl=AUTO                ; override serverurl computation (childutils)
125 | 
126 | ; The below sample group section shows all possible group values,
127 | ; create one or more 'real' group: sections to create "heterogeneous"
128 | ; process groups.
129 | 
130 | ;[group:thegroupname]
131 | ;programs=progname1,progname2  ; each refers to 'x' in [program:x] definitions
132 | ;priority=999                  ; the relative start priority (default 999)
133 | 
134 | ; The [include] section can just contain the "files" setting.  This
135 | ; setting can list multiple files (separated by whitespace or
136 | ; newlines).  It can also contain wildcards.  The filenames are
137 | ; interpreted as relative to this file.  Included files *cannot*
138 | ; include files themselves.
139 | 
140 | [program:flower]
141 | command=celery flower --port=8080 --broker=redis://127.0.0.1:6379/0
142 | directory=/home/thorns
143 | process_name=%(program_name)s_%(process_num)02d
144 | numprocs=1
145 | autostart=true
146 | autorestart=true
147 | startetries=10
148 | exitcodes=0
149 | stopsignal=KILL
150 | stopwaitsecs=10
151 | redirect_stderr=true
152 | 
153 | [program:worker-ringzero]
154 | command=celery -A tasks worker --loglevel=info --workdir=/home/thorns --concurrency=10 --hostname=%(program_name)s%(process_num)02d
155 | directory=/home/thorns
156 | user=root
157 | process_name=%(program_name)s_%(process_num)02d
158 | numprocs=1
159 | autostart=true
160 | autorestart=true
161 | startetries=10
162 | exitcodes=0
163 | stopsignal=KILL
164 | stopwaitsecs=10
165 | redirect_stderr=true
166 | 
167 | ;[include]
168 | ;files = relative/directory/*.ini
169 | 


--------------------------------------------------------------------------------
/tasks.py:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/env python
 2 | # encoding: utf-8
 3 | # tasks.py
 4 | # email: ringzero@0x557.org
 5 | 
 6 | '''
 7 | 	Thorns Project 分布式任务控制脚本
 8 | 	tasks
 9 | 		-- nmap_dispath			# nmap 扫描调度函数
10 | 		-- hydra_dispath 		# hydra 暴力破解调度函数
11 | 		-- medusa_dispath 		# medusa 暴力破解调度函数
12 | 
13 | 	worker run()
14 | 		--workdir=/home/thorns
15 | '''
16 | 
17 | import subprocess
18 | from celery import Celery, platforms 
19 | 
20 | # 初始化芹菜对象
21 | app = Celery()
22 | 
23 | # 允许celery以root权限启动
24 | platforms.C_FORCE_ROOT = True
25 | 
26 | # 修改celery的全局配置
27 | app.conf.update(
28 | 	CELERY_IMPORTS = ("tasks", ),
29 | 	BROKER_URL = 'redis://120.132.54.90:6379/0',
30 | 	CELERY_RESULT_BACKEND = 'db+mysql://celery:celery1@3Wscan@42.62.52.62:443/wscan',
31 | 	CELERY_TASK_SERIALIZER='json',
32 | 	CELERY_RESULT_SERIALIZER='json',
33 | 	CELERY_TIMEZONE='Asia/Shanghai',
34 | 	CELERY_ENABLE_UTC=True,
35 | 	CELERY_REDIS_MAX_CONNECTIONS=5000, # Redis 最大连接数
36 | 	BROKER_TRANSPORT_OPTIONS = {'visibility_timeout': 3600}, # 如果任务没有在 可见性超时 内确认接收，任务会被重新委派给另一个Worker并执行  默认1 hour.
37 | 	# BROKER_TRANSPORT_OPTIONS = {'fanout_prefix': True},		# 设置一个传输选项来给消息加上前缀
38 | )
39 | 
40 | # 失败任务重启休眠时间300秒，最大重试次数5次
41 | # @app.task(bind=True, default_retry_delay=300, max_retries=5)
42 | 
43 | @app.task
44 | def nmap_dispath(targets, taskid=None):
45 | 	# nmap环境参数配置
46 | 	run_script_path = '/home/thorns'
47 | 	if taskid == None:
48 | 		cmdline = 'python wyportmap.py %s' % targets
49 | 	else: 
50 | 		cmdline = 'python wyportmap.py %s %s' % (targets, taskid)
51 | 	nmap_proc = subprocess.Popen(cmdline,shell=True,stdout=subprocess.PIPE,stderr=subprocess.PIPE)
52 | 	process_output = nmap_proc.stdout.readlines()
53 | 	return process_output
54 | 
55 | @app.task
56 | def hydra_dispath(targets, protocol, userdic, passdic, taskid=None):
57 | 	# 命令执行环境参数配置
58 | 	run_script_path = '/home/thorns/script/hydra'
59 | 	run_env = '{"LD_LIBRARY_PATH": "/home/thorns/libs/"}'
60 | 
61 | 	if taskid == None:
62 | 		cmdline = 'python hydra.py %s %s %s %s' % (target, protocol, userdic, passdic)
63 | 	else:
64 | 		cmdline = 'python hydra.py %s %s %s %s %s' % (target, protocol, userdic, passdic, taskid)
65 | 
66 | 	nmap_proc = subprocess.Popen(cmdline,shell=True,stdout=subprocess.PIPE,stderr=subprocess.PIPE,cwd=run_script_path,env=run_env)
67 | 
68 | 	process_output = nmap_proc.stdout.readlines()
69 | 	return process_output
70 | 
71 | @app.task
72 | def medusa_dispath(targets, protocol, userdic, passdic, taskid=None):
73 | 	# 命令执行环境参数配置
74 | 	run_script_path = '/home/thorns/script/medusa'
75 | 	run_env = '{"LD_LIBRARY_PATH": "/home/thorns/libs/"}'
76 | 
77 | 	if taskid == None:
78 | 		cmdline = 'python medusa.py %s %s %s %s' % (target, protocol, userdic, passdic)
79 | 	else:
80 | 		cmdline = 'python medusa.py %s %s %s %s %s' % (target, protocol, userdic, passdic, taskid)
81 | 
82 | 	nmap_proc = subprocess.Popen(cmdline,shell=True,stdout=subprocess.PIPE,stderr=subprocess.PIPE,cwd=run_script_path,env=run_env)
83 | 
84 | 	process_output = nmap_proc.stdout.readlines()
85 | 	return process_output
86 | 
87 | 
88 | 
89 | 
90 | 
91 | 


--------------------------------------------------------------------------------
/wyfunc.py:
--------------------------------------------------------------------------------
  1 | #!/usr/bin/env python
  2 | # encoding: utf-8
  3 | # wyfunc.py
  4 | # email: ringzero@0x557.org
  5 | 
  6 | def ip2num(ip):
  7 | 	ip = [int(x) for x in ip.split('.')]
  8 | 	return ip[0] << 24 | ip[1] << 16 | ip[2] << 8 | ip[3]
  9 | 
 10 | def num2ip(num):
 11 | 	return '%s.%s.%s.%s' % (
 12 | 		(num & 0xff000000) >> 24,
 13 | 		(num & 0x00ff0000) >> 16,
 14 | 		(num & 0x0000ff00) >> 8,
 15 | 		num & 0x000000ff
 16 | 	)
 17 |  
 18 | def gen_ips(start, end):
 19 | 	"""生成IP地址"""
 20 | 	# if num & 0xff 过滤掉 最后一段为 0 的IP
 21 | 	return [num2ip(num) for num in range(start, end + 1) if num & 0xff]
 22 | 
 23 | def make_ips_c_block(ipaddr):
 24 | 	address = {}
 25 | 	ipaddr = ipaddr.split('.')
 26 | 	ipaddr[3] = '0'
 27 | 	ipaddr = '.'.join(ipaddr)
 28 | 
 29 | 	address[ipaddr] = gen_ips(ip2num(ipaddr),ip2num(ipaddr) + 254)
 30 | 	return address
 31 | 
 32 | def ip_check(ip):
 33 | 	q = ip.split('.')
 34 | 	return len(q) == 4 and len(filter(lambda x: x >= 0 and x <= 255, \
 35 | 		map(int, filter(lambda x: x.isdigit(), q)))) == 4
 36 | 
 37 | def make_target_list(targets):
 38 | 	target_list = []
 39 | 	process_nmap_count = 1
 40 | 	startip, endip = targets.split('-')
 41 | 	if not ip_check(startip):
 42 | 		print '* StartIP format error'
 43 | 	else:
 44 | 		endip_len = len(endip.split('.'))
 45 | 		startip_num = ip2num(startip)
 46 | 		if endip_len == 1: # 结束IP为数字的场景
 47 | 			startip_endnum = startip.split('.')[3]
 48 | 			target_count =  int(endip) - int(startip_endnum)
 49 | 			if target_count >= 0:
 50 | 				target_list.append(num2ip(startip_num))
 51 | 				# 每10个IP分配到一个wyportmap子进程上
 52 | 				# 更新：为每个独立的IP创建一个单独的任务，并删除掉开始IP，修改的时候注释掉上面那行
 53 | 				for i in xrange(0,(target_count+process_nmap_count-1)/process_nmap_count):
 54 | 					ip_count = (i * process_nmap_count) + process_nmap_count
 55 | 					remaining_count = target_count - ip_count
 56 | 					if remaining_count > 0:
 57 | 						scan_startip = num2ip(startip_num)
 58 | 						endip_num = startip_num + process_nmap_count
 59 | 						startip_num = endip_num
 60 | 						# target_option = '%s-%s' % (scan_startip, num2ip(endip_num))
 61 | 						target_option = num2ip(endip_num)
 62 | 						target_list.append(target_option)
 63 | 					else:
 64 | 						scan_startip = num2ip(startip_num)
 65 | 						endip_num = startip_num + process_nmap_count + remaining_count
 66 | 						startip_num = endip_num
 67 | 						# target_option = '%s-%s' % (scan_startip, num2ip(endip_num))
 68 | 						target_option = num2ip(endip_num)
 69 | 						target_list.append(target_option)
 70 | 			else:
 71 | 				print '* EndIP Less than StartIP'
 72 | 		elif endip_len == 4:
 73 | 			if ip_check(endip):
 74 | 				startip_num = ip2num(startip)
 75 | 				endip_num = ip2num(endip)
 76 | 				if startip_num <= endip_num:
 77 | 					target_count =  endip_num - startip_num
 78 | 					if target_count >= 0:
 79 | 						target_list.append(num2ip(startip_num))
 80 | 						# 每10个IP分配到一个wyportmap子进程上
 81 | 						# 更新：为每个独立的IP创建一个单独的任务，并删除掉开始IP，修改的时候注释掉上面那行
 82 | 						for i in xrange(0,(target_count+process_nmap_count-1)/process_nmap_count):
 83 | 							ip_count = (i * process_nmap_count) + process_nmap_count
 84 | 							remaining_count = target_count - ip_count
 85 | 							if remaining_count > 0:
 86 | 								scan_startip = num2ip(startip_num)
 87 | 								endip_num = startip_num + process_nmap_count
 88 | 								startip_num = endip_num
 89 | 								# target_option = '%s-%s' % (scan_startip, num2ip(endip_num))
 90 | 								target_option = num2ip(endip_num)
 91 | 								target_list.append(target_option)
 92 | 							else:
 93 | 								scan_startip = num2ip(startip_num)
 94 | 								endip_num = startip_num + process_nmap_count + remaining_count
 95 | 								startip_num = endip_num
 96 | 								# target_option = '%s-%s' % (scan_startip, num2ip(endip_num))
 97 | 								target_option = num2ip(endip_num)
 98 | 								target_list.append(target_option)
 99 | 				else:
100 | 					print '* EndIP Less than StartIP'
101 | 			else:
102 | 				print '* EndIP format error'	
103 | 		else:
104 | 			print '* EndIP format error'
105 | 
106 | 	return target_list
107 | 
108 | 
109 | 


--------------------------------------------------------------------------------
/wyportmap.py:
--------------------------------------------------------------------------------
  1 | #!/usr/bin/env python
  2 | # encoding: utf-8
  3 | # mail: ringzero@0x557.org
  4 | 
  5 | import json
  6 | import sys
  7 | from time import sleep
  8 | from libnmap.process import NmapProcess
  9 | from libnmap.reportjson import ReportDecoder, ReportEncoder
 10 | from libnmap.parser import NmapParser, NmapParserException
 11 | from libnmap.plugins.backendpluginFactory import BackendPluginFactory
 12 | 
 13 | # 重试次数 & 超时时间(s)
 14 | retrycnt = 3
 15 | timeout = 3600
 16 | 
 17 | # 数据库连接 & 全局扫描参数
 18 | global_dbcoon = 'mysql+mysqldb://celery:celery1@3Wscan@42.62.52.62:443/wscan'
 19 | # global_dbcoon = 'mysql+mysqldb://用户名:密码@数据库服务器IP:数据库端口/数据库名称'
 20 | global_options = '-sT -P0 -sV -O --script=banner -p T:21-25,80-89,110,143,443,513,873,1080,1433,1521,1158,3306-3308,3389,3690,5900,6379,7001,8000-8090,9000,9418,27017-27019,50060,111,11211,2049'
 21 | 
 22 | # 处理端口状态
 23 | global_log_states = ['open'] # open, filtered, closed, unfiltered
 24 | 
 25 | def do_nmap_scan(targets, options=global_options):
 26 | 	# 运行次数初始化
 27 | 	trycnt = 0
 28 | 
 29 | 	while True:
 30 | 		# 运行时间初始化
 31 | 		runtime = 0
 32 | 
 33 | 		if trycnt >= retrycnt:
 34 | 			# print '-' * 50
 35 | 			return 'retry overflow'
 36 | 
 37 | 		try:
 38 | 			nmap_proc = NmapProcess(targets=targets, options=options, safe_mode=False)
 39 | 			nmap_proc.run_background()
 40 | 
 41 | 			while nmap_proc.is_running():
 42 | 				if runtime >= timeout:	# 运行超时，结束掉任务，休息1分钟, 再重启这个nmap任务
 43 | 					# print '-' * 50
 44 | 					# print "* timeout. terminate it..."
 45 | 					nmap_proc.stop()
 46 | 					# 休眠时间
 47 | 					sleep(60)
 48 | 					trycnt += 1
 49 | 					break
 50 | 				else:
 51 | 					# print 'running[%ss]:%s' % (runtime, nmap_proc.command)
 52 | 					sleep(5)
 53 | 					runtime += 5
 54 | 			if nmap_proc.is_successful():
 55 | 				# print '-' * 50
 56 | 				print nmap_proc.summary
 57 | 				return nmap_proc.stdout
 58 | 
 59 | 		except Exception, e:
 60 | 			# raise e
 61 | 			print e
 62 | 			trycnt += 1
 63 | 			if trycnt >= retrycnt:
 64 | 				# print '-' * 50
 65 | 				# print '* retry overflow'
 66 | 				return e
 67 | 
 68 | def parse_nmap_report(nmap_stdout, taskid=None):
 69 | 	try:
 70 | 		# 处理结果并写入后台数据库
 71 | 		nmap_report = NmapParser.parse(nmap_stdout)
 72 | 
 73 | 		# 声明后台对应的ORM数据库处理模型
 74 | 		my_services_backend = BackendPluginFactory.create(plugin_name='backend_service', url=global_dbcoon, echo=False, encoding='utf-8', pool_timeout=3600)
 75 | 		my_hosts_backend = BackendPluginFactory.create(plugin_name='backend_host', url=global_dbcoon, echo=False, encoding='utf-8', pool_timeout=3600)
 76 | 
 77 | 		# 开始处理扫描结果
 78 | 		for host in nmap_report.hosts:
 79 | 
 80 | 				# print("Nmap scan : {0}".format(host.address))
 81 | 				host.taskid = taskid
 82 | 
 83 | 				# 处理主机开放的服务和端口
 84 | 				for serv in host.services:
 85 | 					serv.address = host.address
 86 | 					serv.taskid = taskid
 87 | 					serv.endtime = host.endtime
 88 | 
 89 | 					if serv.state in global_log_states:
 90 | 						serv.save(my_services_backend)
 91 | 
 92 | 				host.save(my_hosts_backend)
 93 | 
 94 | 		return 'Scan finished'
 95 | 
 96 | 	except Exception, e:
 97 | 		# 处理报表出错，返回错误结果
 98 | 		return e
 99 | 
100 | def run_wyportmap(targets, taskid=None):
101 | 	# print '-' * 50
102 | 	# print '* Starting id:(%s) [%s] portmap scan' % (taskid, targets)
103 | 	# print '-' * 50
104 | 	nmap_result = do_nmap_scan(targets)
105 | 	# print '-' * 50
106 | 	return parse_nmap_report(nmap_result,taskid)
107 | 
108 | if __name__ == "__main__":
109 | 	if len(sys.argv) == 2:
110 | 		print run_wyportmap(sys.argv[1])
111 | 		sys.exit(0)
112 | 	elif len(sys.argv) == 3:
113 | 		print run_wyportmap(sys.argv[1], sys.argv[2])
114 | 	else:
115 | 		print ("usage: %s targets taskid" % sys.argv[0])
116 | 		sys.exit(-1)
117 | 
118 | 
119 | 
120 | 
121 | 
122 | 


--------------------------------------------------------------------------------