├── .gitignore
├── AUTHORS
├── COPYING
├── Dockerfile
├── README.md
├── bind_nsec3_maxiterations.patch
├── doc
    ├── n3map-hashcatify.1
    ├── n3map-johnify.1
    ├── n3map-nsec3-lookup.1
    └── n3map.1
├── hashcatify.py
├── johnify.py
├── map.py
├── n3map.bash_completion
├── n3map
    ├── __init__.py
    ├── exception.py
    ├── hashcatify.py
    ├── johnify.py
    ├── log.py
    ├── map.py
    ├── name.py
    ├── nsec3chain.py
    ├── nsec3hash.c
    ├── nsec3lookup.py
    ├── nsec3walker.py
    ├── nsecwalker.py
    ├── predict.py
    ├── prehash.py
    ├── query.py
    ├── queryprovider.py
    ├── rrfile.py
    ├── rrtypes
    │   ├── __init__.py
    │   ├── nsec.py
    │   ├── nsec3.py
    │   └── rr.py
    ├── statusline.py
    ├── tree
    │   ├── __init__.py
    │   ├── bstree.py
    │   ├── nsec3tree.py
    │   └── rbtree.py
    ├── util.py
    ├── vis.py
    └── walker.py
├── nsec3-lookup.py
├── nsec3_gen_fmt_plug.c
├── pyproject.toml
├── screenshot.png
└── setup.py


/.gitignore:
--------------------------------------------------------------------------------
 1 | build/
 2 | dist/
 3 | 
 4 | *.swp
 5 | *~
 6 | 
 7 | *.so
 8 | *.o
 9 | *.pyc
10 | 
11 | tags
12 | TAGS
13 | *.egg-info
14 | 


--------------------------------------------------------------------------------
/AUTHORS:
--------------------------------------------------------------------------------
1 | Ralf Sager <nsec3map at 3fnc dot org>
2 | Giuliano Grassi
3 | 


--------------------------------------------------------------------------------
/Dockerfile:
--------------------------------------------------------------------------------
 1 | FROM debian:bookworm
 2 | 
 3 | WORKDIR /usr/src/nsec3map
 4 | COPY . /usr/src/nsec3map
 5 | 
 6 | RUN apt-get -y update
 7 | RUN apt-get install -y libssl3 libssl-dev \
 8 | 	python3 \
 9 | 	python3-dev \
10 | 	python3-pip \
11 | 	python3-dnspython \
12 | 	python3-numpy \
13 | 	python3-scipy
14 | 
15 | RUN pip install .[predict]
16 | 
17 | WORKDIR /host
18 | ENTRYPOINT ["n3map"]
19 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
  1 | nsec3map - DNSSEC Zone Enumerator
  2 | =================================
  3 | 
  4 | `n3map` is a tool that can enumerate DNS zone entries based on DNSSEC
  5 | [NSEC][NSEC] or [NSEC3][NSEC3] record chains.  It can be used to discover hosts
  6 | in a DNS zone quickly and with a minimum amount of queries if said zone is
  7 | DNSSEC-enabled.
  8 | 
  9 | `n3map` was written primarily to show that NSEC3 does not offer meaningful
 10 | protection against zone enumeration.
 11 | Although originally only intended as a PoC and written in Python, it is
 12 | actually quite fast and able to enumerate even large zones (with a million or
 13 | more entries) in a short time given adequate hardware.
 14 | 
 15 | It also includes a simple [John the Ripper][JtR] plugin that can be used to crack the
 16 | obtained NSEC3 hashes.
 17 | 
 18 | ![n3map screenshot](screenshot.png)
 19 | 
 20 | Usage Examples
 21 | --------------
 22 | 
 23 | Some typical usage examples are shown below. For a more detailed documentation,
 24 | refer to the man pages or the output of `n3map --help`.
 25 | 
 26 | ### NSEC Zone Walking
 27 | 
 28 | The most basic example is to enumerate a particular zone (e.g. example.com) and
 29 | store the retrieved NSEC/NSEC3 records in a file example.com.zone:
 30 | 
 31 | 	$ n3map -v -o example.com.zone example.com
 32 | 	n3map 0.4.0: starting mapping of example.com
 33 | 	looking up nameservers for zone example.com.
 34 | 	using nameserver: 199.43.133.53:53 (b.iana-servers.net.)
 35 | 	using nameserver: 199.43.132.53:53 (a.iana-servers.net.)
 36 | 	checking SOA...
 37 | 	detecting zone type...
 38 | 	zone uses NSEC records
 39 | 	starting enumeration in mixed query mode...
 40 | 	discovered owner: example.com.	A NS SOA TXT AAAA RRSIG NSEC DNSKEY
 41 | 	discovered owner: www.example.com.	A TXT AAAA RRSIG NSEC
 42 | 	;; walking example.com.: records =   2; queries =   4; ............. q/s = 11 ;;
 43 | 	finished mapping of example.com. in 0:00:00.196471
 44 | 
 45 | The `-v` switch is only used for more verbosity and not generally needed. With
 46 | no further arguments, `nsec3map` detects automatically whether the zone uses
 47 | NSEC or NSEC3 and uses the corresponding enumeration method. It also looks up
 48 | the zone's nameservers by itself.
 49 | 
 50 | Some nameservers do not accept NSEC queries. In such a case, `--query-mode A`
 51 | (short `-A`) can be used instead. For example, to enumerate the root zone, one
 52 | could run the command:
 53 | 
 54 | 	n3map -v -A --output root.zone  .
 55 | 
 56 | #### Avoiding Sub-Zones
 57 | 
 58 | Note that the above command will likely print a lot of warnings about sub-zones
 59 | (children of the zone that we want to enumerate). `n3map` tries its best to
 60 | avoid descending into sub-zones and instead tries to jump over them
 61 | automatically.
 62 | If you wish to avoid most of these warnings, you can tell `n3map` to never add
 63 | prefix labels to the queries it sends using the `--no-prefix-labels` option.
 64 | For example:
 65 | 
 66 |     n3map -vA --no-prefix-labels -o root.zone .
 67 | 
 68 | This option is particularly useful to enumerate top-level domain (TLD) zones.
 69 | Note however that using it can sometimes lead to a less complete enumeration
 70 | for zones with nested subdomains.
 71 | 
 72 | Alternatively, you can try to find nameservers that respond to
 73 | direct NSEC queries (find them e.g. by trying `--query-mode=NSEC`) and tell
 74 | `n3map` to only use those:
 75 | 
 76 |     n3map -vo example.com.zone goodns{1,2}.example.com example.com
 77 | 
 78 | ### NSEC3 Zone Enumeration
 79 | 
 80 | The following example shows the enumeration of a NSEC3 chain at example.com
 81 | using a nameserver at 192.168.1.37. It also shows the NSEC3 zone size
 82 | prediction and progress indicator (enabled using the `-p` switch).
 83 | 
 84 | 	$ n3map -3po example.com.zone 192.168.1.37 example.com
 85 | 	;; mapping example.com.: 79% [===========================================================================                   ] ;;
 86 | 	;; records = 797; queries = 802; hashes = 3840; predicted zone size = 1003; ............... q/s = 513; coverage =  95.677595% ;;
 87 | 	
 88 | 	received SIGINT, terminating
 89 | 
 90 | Note that the enumeration will proceed slower towards the end as it becomes
 91 | harder to find domain names that are not covered by any retrieved NSEC3
 92 | records. Therefore, finishing the enumeration of a large zone can take quite
 93 | some time and computing resources. It is advisable to manually cancel the
 94 | enumeration once the query rate drops under a certain limit.
 95 | 
 96 | You should also make use of the `--limit-rate` option to reduce stress on the
 97 | nameservers. If you think the enumeration is too slow because of a high
 98 | round-trip time to the nameservers, you can also use a more aggressive mode
 99 | which sends multiple queries simultaneously (`--aggressive` option). The
100 | following example shows how to use these options:
101 | 
102 | 	n3map -3pvo example.com.zone --aggressive 16 --limit-rate 100/s example.com
103 | 
104 | This will cause nsec3map to send a maximum of 16 queries in parallel while at
105 | the same time keeping the query rate at or below roughly 100 queries per
106 | second.
107 | 
108 | It is also possible to continue the enumeration from a partially obtained NSEC3
109 | (or NSEC) chain, as long as the zone's NSEC3 parameters (salt, iteration count)
110 | have not been changed:
111 | 
112 | 	n3map -3pv --input example.com.partial --output example.com.zone --ignore-overlapping example.com
113 | 
114 | This will first read the NSEC3 records from example.com.partial and then
115 | continue the enumeration, saving the NSEC3 chain to example.com.zone.
116 | The `--ignore-overlapping` option should be used for large zones, or if it is
117 | otherwise likely that changes are made to the zone during the enumeration.  If
118 | specified, nsec3map will not abort the enumeration when it receives an NSEC3
119 | record which overlaps with another record that was received earlier. Note
120 | however that you will not get a completely consistent view of the NSEC3 chain
121 | if you use this option.
122 | 
123 | ### Cracking NSEC3 Hashes
124 | 
125 | Once you obtained some NSEC3 records from a particular zone, you can (try to)
126 | crack them using John the Ripper and the supplied NSEC3 patch (see *John the
127 | Ripper Plugin* below on how to install it).
128 | 
129 | First, the NSEC3 records need to be converted to a different format used by the
130 | JtR patch:
131 | 
132 | 	n3map-johnify example.com.zone example.com.john
133 | 
134 | The records can then be cracked simply by running  `john` on the resulting file:
135 | 
136 | 	john example.com.john
137 | 
138 | Refer to the JtR documentation for more information on how to make use of
139 | john's different cracking modes, wordlist rules and so on. It is probably a
140 | good idea to adapt the wordlist and mangling rules to the kind of zone you are
141 | trying to map.
142 | 
143 | You can also try to crack NSEC3 records using [hashcat][hashcat],
144 | using hashes converted to a slightly different format:
145 | 
146 | 	n3map-hashcatify example.com.zone example.com.hashcat
147 | 
148 | The records can then be cracked simply by running `hashcat` on the resulting file:
149 | 
150 | 	hashcat -m 8300 example.com.hashcat
151 | 
152 | 
153 | 
154 | Installation
155 | ------------
156 | 
157 | ### From PyPI
158 | 
159 | The PyPI package still needs to compile the C extension module for faster hashing,
160 | which means you need a C compiler as well as the necessary header files for
161 | Python and libcrypto (OpenSSL) installed.
162 | 
163 | For Debian-based systems:
164 | 
165 |     sudo apt-get install python3 python3-pip python3-dev gcc libssl3 libssl-dev
166 | 
167 | To then install nsec3map from PyPI, simply run:
168 | 
169 |     python3 -m pip install n3map[predict]
170 | 
171 | If you do not care about NSEC3 zone size prediction and don't want
172 | numpy and scipy installed, you can use:
173 | 
174 |     python3 -m pip install n3map
175 | 
176 | #### Installing into a Virtual Environment
177 | 
178 | It may be advisable to install n3map into a Python venv, especially if you are
179 | faced with any dependency problems:
180 | 
181 |     mkdir venv
182 |     python3 -m venv venv
183 |     source venv/bin/activate
184 |     python3 -m pip install n3map[predict]
185 | 
186 | More conveniently, you can also use [pipx](https://github.com/pypa/pipx):
187 | 
188 |     pipx install n3map[predict]
189 | 
190 | Note that you still need libssl, libssl-dev, gcc and python3-dev.
191 | 
192 | ### From Git Repository
193 | 
194 | Dependencies:
195 | 
196 |   * Python >= 3.9
197 |   * dnspython >= 2.0
198 |   * OpenSSL (libcrypto) >= 3.0.0
199 |   * Optionally numpy and scipy for zone size prediction (recommended)
200 | 
201 | Additionally, pip, setuptools and GCC (for the extension module) are required
202 | during setup.
203 | 
204 | On a Debian system, just run
205 | 
206 | 	sudo apt-get install python3 python3-dev gcc python3-pip \
207 |          python3-setuptools python3-dnspython libssl3 libssl-dev \
208 |          python3-numpy python3-scipy
209 | 
210 | Installation:
211 | 
212 | After cloning the repositry / unpacking the tarball, cd into the project
213 | directory and run:
214 | 
215 | 	python3 -m pip install .[predict]
216 | 
217 | This will compile the extension modules(s) and install the scripts, python
218 | modules as well as the man pages.
219 | It will make a user install if you are not root.
220 | 
221 | If you do not care about NSEC3 zone size prediction and don't want
222 | numpy and scipy installed, you can use:
223 | 
224 | 	python3 -m pip install .
225 | 
226 | Alternatively, you can install it w/o pip:
227 | 
228 | 	sudo python3 setup.py install
229 | 
230 | #### Running directly from Source Directory
231 | 
232 | Alternatively, you can also run nsec3map directly from the source directory
233 | without installing it:
234 | 
235 |     ./map.py [options]
236 | 
237 | If you want to use OpenSSL accelerated
238 | hashing however, you still need to build the extension module:
239 | 
240 | 	python3 setup.py build_ext
241 | 
242 | This should compile a shared object nsec3hash.so in the build/ directory. You
243 | can then copy this file to the n3map/ directory.
244 | 
245 | ### John the Ripper Plugin
246 | 
247 | **Update**: The latest version of [John the Ripper jumbo][JtR] includes the NSEC3
248 | cracking patch from this project. There is no need to install it separately,
249 | just follow the build instructions for JtR-Jumbo. Using the latest source
250 | version is recommended.
251 | 
252 | Alternatively, you can also use [hashcat][hashcat].
253 | 
254 | Docker
255 | --------
256 | 
257 | Building the docker container.
258 | 
259 | 	docker build -t nsec3map .
260 | 
261 | Running n3map or e.g. n3map-hashcatify:
262 | 
263 | 	docker run -it --rm -v "${PWD}:/host" nsec3map -v -o example.com.zone example.com
264 | 	docker run -it --entrypoint n3map-hashcatify --rm -v "${PWD}:/host" nsec3map example.com.zone example.com.hashcat
265 | 
266 | 
267 | Limitations
268 | -----------
269 | 
270 | * Many DNS errors are not handled correctly
271 | * No automatic parallelization of NSEC walking (though it is possible to do this manually by partitioning the namespace)
272 | * High memory usage (mostly as a result of using CPython)
273 | * ...
274 | 
275 | (remember that nsec3map is still mostly a PoC tool...)
276 | 
277 | [NSEC]: https://www.ietf.org/rfc/rfc4034.txt "Resource Records for the DNS Security Extensions"
278 | [NSEC3]: https://www.ietf.org/rfc/rfc5155.txt "DNS Security (DNSSEC) Hashed Authenticated Denial of Existence"
279 | [JtR]: https://github.com/openwall/john "John the Ripper (Jumbo)"
280 | [hashcat]: https://hashcat.net/hashcat/ "hashcat"
281 | 


--------------------------------------------------------------------------------
/bind_nsec3_maxiterations.patch:
--------------------------------------------------------------------------------
 1 | --- a/lib/dns/nsec3.c
 2 | +++ b/lib/dns/nsec3.c
 3 | @@ -1784,7 +1784,7 @@ dns_nsec3_maxiterations(dns_db_t *db, dns_dbversion_t *version,
 4 |  	dst_key_t *key = NULL;
 5 |  	isc_buffer_t buffer;
 6 |  	isc_result_t result;
 7 | -	isc_uint16_t bits, minbits = 4096;
 8 | +	unsigned int bits, minbits = 4096;
 9 |  
10 |  	result = dns_db_getoriginnode(db, &node);
11 |  	if (result != ISC_R_SUCCESS)
12 | @@ -1811,7 +1811,7 @@ dns_nsec3_maxiterations(dns_db_t *db, dns_dbversion_t *version,
13 |  		isc_buffer_add(&buffer, rdata.length);
14 |  		CHECK(dst_key_fromdns(dns_db_origin(db), rdataset.rdclass,
15 |  				      &buffer, mctx, &key));
16 | -		bits = dst_key_getbits(key);
17 | +		bits = dst_key_size(key);
18 |  		dst_key_free(&key);
19 |  		if (minbits > bits)
20 |  			minbits = bits;
21 | 


--------------------------------------------------------------------------------
/doc/n3map-hashcatify.1:
--------------------------------------------------------------------------------
 1 | .TH N3MAP-HASHCATIFY 1 "2017-06-10" "n3map v.0.2.14"
 2 | .SH NAME
 3 | n3map-hashcatify \- convert NSEC3 records to a hashcat-friendly format.
 4 | .SH SYNOPSIS
 5 | .B n3map-hashcatify
 6 | file [outfile]
 7 | .SH DESCRIPTION
 8 | .B n3map-hashcatify 
 9 | reads NSEC3 records from a file, and writes them to standard output in a format
10 | readable by hashcat. If outfile is specified, the records are written to
11 | outfile instead of standard output.
12 | 
13 | .SH "SEE ALSO"
14 | \fBn3map\fR(1),
15 | \fBn3map-nsec3-lookup\fR(1)
16 | 


--------------------------------------------------------------------------------
/doc/n3map-johnify.1:
--------------------------------------------------------------------------------
 1 | .TH N3MAP-JOHNIFY 1 "2011-12-05" "n3map v.0.2.14"
 2 | .SH NAME
 3 | n3map-johnify \- convert NSEC3 records to a \fBjohn\fR(8)-friendly format.
 4 | .SH SYNOPSIS
 5 | .B n3map-johnify
 6 | file [outfile]
 7 | .SH DESCRIPTION
 8 | .B n3map-johnify 
 9 | reads NSEC3 records from a file, and writes them to standard output in a format
10 | readable by \fRjohn\fR(8). If outfile is specified, the records are written to
11 | outfile instead of standard output.
12 | 
13 | .SH NOTES
14 | \fRjohn\fR(8) needs a special patch that is included with n3map to be able to
15 | crack NSEC3 records.
16 | 
17 | .SH "SEE ALSO"
18 | \fBn3map\fR(1),
19 | \fBjohn\fR(8),
20 | \fBn3map-nsec3-lookup\fR(1)
21 | 


--------------------------------------------------------------------------------
/doc/n3map-nsec3-lookup.1:
--------------------------------------------------------------------------------
 1 | .TH N3MAP-NSEC3-LOOKUP 1 "2011-12-05" "n3map v.0.2.14"
 2 | .SH NAME
 3 | n3map-nsec3-lookup \- lookup NSEC3 records from a file
 4 | .SH SYNOPSIS
 5 | .B n3map-nsec3-lookup 
 6 | file [-o outfile] [-z zone] [-v]
 7 | .SH DESCRIPTION
 8 | .B n3map-nsec3-lookup
 9 | first loads an NSEC3 chain from the specified file.
10 | Then, every domain name read from standard input is hashed and an NSEC3 record
11 | whose hashed owner name matches the hash is searched. If a record is found, the
12 | cleartext domain name and the coresponding record is written to standard output.
13 | 
14 | .SS Options
15 | .TP 
16 | \fB\-o\fR \fIoutfile\fR
17 | Write the found records to \fIoutfile\fR instead of standard output.
18 | 
19 | .TP 
20 | \fB\-z\fR \fIzone\fR
21 | If this option is used, every domain name read from standard input is
22 | interpreted relative to \fIzone\fR. 
23 | 
24 | .TP 
25 | \fB\-v\fR
26 | Be more verbose.
27 | 
28 | .SH NOTES
29 | This tool could also be used to crack NSEC3 records by passing guesses for
30 | domain names to standard input. However, this is not recommended since it is in
31 | no way optimised for performance. 
32 | If you want to crack NSEC3 records, you should use a tool more suitable to do
33 | the job, e.g. \fBjohn\fR(8) or hashcat.
34 | 
35 | .SH EXAMPLES
36 | .PP
37 | Read NSEC3 records from records.nsec3 read domain names relative to example.com
38 | from standard input:
39 | .PP
40 | .RS
41 | $ n3map-nsec3-lookup records.nsec3 -z example.com
42 | .RE
43 | .PP
44 | Read cleartext fully-qualified domain names from names and write all found NSEC3
45 | records to out.
46 | .PP
47 | .RS
48 | $ n3map-nsec3-lookup records.nsec3 -o out < names
49 | .RE
50 | .PP
51 | .SH "SEE ALSO"
52 | \fBn3map\fR(1),
53 | \fBn3map-hashcatify\fR(1),
54 | \fBn3map-johnify\fR(1),
55 | \fBjohn\fR(8)
56 | 
57 | .SH BUGS
58 | .PP
59 | lot's of em. 
60 | .PP
61 | It may use a huge amount of memory depending on the number of NSEC3 records.
62 | .PP
63 | Reading large lists of records is slow.
64 | 
65 | 


--------------------------------------------------------------------------------
/doc/n3map.1:
--------------------------------------------------------------------------------
  1 | .TH N3MAP 1 "2011-12-05" "n3map v.0.3"
  2 | .SH NAME
  3 | n3map \- Enumerate DNSSEC-enabled DNS zones 
  4 | .SH SYNOPSIS
  5 | .B n3map 
  6 | [options...] [-o file] [nameserver[:port]]... zone
  7 | .SH DESCRIPTION
  8 | Enumerate DNSSEC-enabled DNS zones that use either NSEC or NSEC3 records.
  9 | .SS Enumeration Options
 10 | .TP 
 11 | \fB\-a\fR, \fB\-\-auto\fR
 12 | Automatically detect whether the zone uses NSEC or NSEC3 records (default).
 13 | 
 14 | .TP 
 15 | \fB\-3\fR, \fB\-\-nsec3\fR
 16 | Use NSEC3 Enumeration.
 17 | .TP 
 18 | \fB\-n\fR, \fB\-\-nsec\fR
 19 | Use NSEC Enumeration.
 20 | .TP 
 21 | \fB\-o\fR, \fB\-\-output\fR=\fIFILE\fR
 22 | Write the received records to FILE. If FILE is -, the records are written to
 23 | standard output.
 24 | .TP 
 25 | \fB\-i\fR, \fB\-\-input\fR=\fIFILE\fR
 26 | Read a list of records from FILE and continue the enumeration. Such a file was
 27 | typically generated by the \fB\-\-output\fR option. 
 28 | In NSEC3 mode, the label counter value is also restored from FILE if
 29 | present (see the \fI\-\-label-counter\fR option).
 30 | .TP 
 31 | \fB\-c\fR, \fB\-\-continue\fR=\fIFILE\fR
 32 | Same as --input FILE --output FILE, but will preserve FILE as a backup file
 33 | until the enumeration is finished. Will create FILE if it does not exist yet.
 34 | 
 35 | Note that the original input file is regenerated which means that any additional
 36 | comments you may have made in FILE will be lost.
 37 | 
 38 | .SS NSEC Options
 39 | .TP 
 40 | \fB\-m\fR, \fB\-\-query-mode\fR=\fIMODE\fR
 41 | Change the NSEC query mode/strategy. MODE may be one of the following:
 42 | .IP 
 43 | \fIA\fR:
 44 | In this mode, only A queries are used. This mode is quite reliable, however, it
 45 | may require a lot of additional queries in some situations.
 46 | 
 47 | Since the direct querying of NSEC records is not needed for normal operations,
 48 | some nameservers do not allow it all. In those cases, this mode can be used to
 49 | circumvent that restriction. For the same reason enumeration attempts using
 50 | this only A queries may also appear slightly less obvious (especially when used
 51 | together with --ldh).
 52 | .IP 
 53 | \fINSEC\fR:
 54 | This mode only uses NSEC queries. It uses only a minimum amount of queries. The
 55 | downside is that this mode cannot be used on zones with subzones wich are served
 56 | by the same nameserver since this would lead to an endless loop. If this
 57 | situation is detected the enumeration will be aborted.
 58 | .IP 
 59 | \fImixed\fR:
 60 | This mode is the same as \fINSEC\fR mode, however, it may use A queries to step
 61 | over subzones. It uses less queries than the A mode and is currently the most
 62 | reliable NSEC query mode. By default, this mode is used.
 63 | 
 64 | .TP 
 65 | \fB\-M\fR, \fB\-\-mixed\fR
 66 | Same as --query-mode=mixed.
 67 | .TP 
 68 | \fB\-A\fR
 69 | Same as --query-mode=A.
 70 | .TP 
 71 | \fB\-N\fR
 72 | Same as --query-mode=NSEC.
 73 | .TP 
 74 | \fB\-b\fR, \fB\-\-binary\fR
 75 | Use all possible binary values as domain names in queries (default). 
 76 | .TP
 77 | .B " "
 78 | The DNS protocol allows any byte value in query names. However,
 79 | queries containing non-printable characters are quite unusual and may easily be
 80 | detected as an anomaly. See also the \fI\-\-ldh\fR option.
 81 | .TP 
 82 | \fB\-l\fR, \fB\-\-ldh\fR
 83 | Use only lowercase ASCII characters, digits and the hyphen (-) in query names
 84 | (known as the LDH rule).
 85 | .TP
 86 | .B " "
 87 | Using this option makes enumeration look less suspicious. The drawback is that
 88 | it may step over (miss) domain names that use other characters.
 89 | This option has no effect if the query mode is \fINSEC\fR (see the
 90 | \fI\-\-query-mode\fR option for details).
 91 | .TP 
 92 | \fB\-s\fR, \fB\-\-start\fR=\fINAME\fR
 93 | Instead of walking through the zone from the beginning, start at the domain NAME.
 94 | NAME is interpreted relative to the zone name. 
 95 | .TP
 96 | .B " "
 97 | Note that in \fINSEC\fR and \fImixed\fR query modes, the NAME has to actually
 98 | exist in the zone. Otherwise it can be an arbitrary domain name. See also the
 99 | \fI\-\-query-mode\fR option.
100 | .TP 
101 | \fB\-e\fR, \fB\-\-end\fR=\fINAME\fR
102 | End as soon as enumeration reaches the domain NAME. NAME is interpreted relative
103 | to the zone name. The NAME may be an arbitrary domain name that does not need to
104 | actually exist in the zone.
105 | 
106 | .TP 
107 | \fB\-\-no\-prefix\-labels\fR
108 | do not add leading labels ("\\x00.") to increment query names. Applies to 'A'
109 | query mode.  This will result in incomplete enumeration for many zones, but is
110 | useful to avoid descending into subzones when querying non-authoritative
111 | namservers or to speed up the enumeration of TLDs.
112 | 
113 | .SS NSEC3 Options
114 | .TP
115 | \fB\-f\fR, \fB\-\-aggressive\fR=\fIN\fR
116 | send up to N queries in parallel. This may speed up the enumeration
117 | significantly if the DNS server's round-trip time is high. However, it will also
118 | cause n3map to make more queries than usual because it cannot completely avoid
119 | queries which resolve to the same NSEC3 records. Use with caution.
120 | .TP
121 | \fB\-\-ignore-overlapping\fR
122 | Do not abort enumeration if overlapping NSEC3 records are received. 
123 | .TP
124 | .B " "
125 | Normally, overlapping NSEC3 records are a sign that the zone has changed during
126 | the enumeration process. If you specify this option you may end up with a
127 | NSEC3 chain that is not entirely correct when the enumeration ends (i.e., some
128 | records may be missing and others may have been deleted from the zone since).
129 | .TP 
130 | .B " "
131 | Note however that this option is especially useful when enumerating very large zones
132 | which change frequently (during enumeration) and whose enumeration would thus
133 | likely fail.
134 | .TP 
135 | \fB\-p\fR, \fB\-\-predict\fR
136 | try to predict the size of the zone based on the records already received.
137 | Note that this option might slow down the enumeration process (experimental)
138 | .TP 
139 | \fB\-\-processes\fR=\fIN\fR
140 | Specifies the number of NSEC3 hash calculation processes to use.
141 | By default the number is the number of CPUs - 1 (minimum 1).
142 | .TP
143 | \fB\-\-hashlimit\fR=\fIN[K|M|G|T]\fR
144 | Stop the enumeration after checking N hashes, even if it is not finished.
145 | Use this option to prevent n3map from wasting cpu cycles in a (possibly futile)
146 | attempt to extract the last few records of a zone.
147 | Default = 0 (unlimited).
148 | 
149 | .SS Advanced NSEC3 Options
150 | These options are for advanced users and are rarely needed. They should be used
151 | with caution.
152 | .TP 
153 | \fB\-\-label-counter\fR=\fIN\fR
154 | Set the initial label counter to N. The label counter is used to generate the
155 | next possible query name in NSEC3 mode. When continuing an enumeration, it 
156 | is important to set the initial label counter to the last value tried
157 | before. Otherwise, it may take a long time to retry all the values which have
158 | already been tried in the previous run. However, you normally don't need to
159 | specify the label counter since it is automatically restored from the input file
160 | when present (see the \fI\-\-input\fR option).
161 | 
162 | .TP 
163 | \fB\-\-queue-element-size\fR=\fIN\fR
164 | Set the number of (domainname, hash) pairs in a single queue element. Setting this
165 | option on a low value increases synchronisation overhead. Setting it too high may
166 | result in a long delay before the enumeration is started, especially on older
167 | systems.
168 | .TP 
169 | \fB\-\-no-openssl\fR
170 | do not use OpenSSL for hashing. This is slower particularily for zones that use
171 | a high iteration count.
172 | 
173 | .SS General Options
174 | .TP
175 | \fB\-q\fR, \fB\-\-quiet\fR
176 | do not display progress information during enumeration
177 | .TP
178 | \fB\-\-limit-rate\fR=\fIrate{/s|/m|/h}\fR
179 | Limit the maximum query rate. The Rate may be any positive floating-point number
180 | followed by a mandatory `/s', `/m' or `/h' suffix.
181 | .TP
182 | \fB\-\-timeout\fR=\fITIME\fR
183 | Specifies how long to wait for a response from a DNS server, in milliseconds.
184 | .TP
185 | \fB\-\-max-retries\fR=\fIN\fR
186 | Specifies how many times to repeat a query if the first attempt has failed due
187 | to a response timeout.
188 | .TP
189 | \fB\-\-max-errors\fR=\fIN\fR
190 | Specifies how many consecutive errors/wrongful responses  a server may return
191 | before it is removed from the nameserver list.
192 | .TP
193 | \fB\-\-detection-attempts\fR=\fIN\fR
194 | Specifies how many times to try zone type (NSEC/NSEC3) detection. N=0 specifies
195 | an unlimited number of attempts.
196 | .TP
197 | .B \-\-detect-only
198 | detect zone type, write it to stdout and exit. Don't enumerate the zone.
199 | .TP
200 | .B \-\-omit-soa-check
201 | Do not check the SOA record of the zone before starting the enumeration. This
202 | option may be used if you wish to perform no unnecessary queries. However, it
203 | should be used with caution, especially when enumerating an NSEC zone.
204 | .TP 
205 | .B \-\-omit-dnskey-check
206 | Do not check the DNSKEY record of the zone before starting the enumeration. This
207 | option may be used if you wish to perform no unnecessary queries. However, it
208 | should be used with caution, as it may lead to problems when trying to
209 | enumerate a zone that is not actually DNSSEC-enabled.
210 | .TP 
211 | \fB\-h\fR, \fB\-\-help\fR
212 | Display a help message on standard output and exit successfully.
213 | .TP
214 | .B \-\-version
215 | Display version information on standard output and exit successfully.
216 | .TP 
217 | \fB\-v\fR, \fB\-\-verbose\fR
218 | increase verbosity level (use multiple times for greater effect) 
219 | .TP 
220 | \fB\-\-color\fR=\fIWHEN\fR
221 | colorize output; WHEN can be 'auto' (default), 'always' or 'never'.
222 | .TP 
223 | \fB\-4\fR
224 | Use IPv4 only.
225 | .TP 
226 | \fB\-6\fR
227 | Use IPv6 only.
228 | 
229 | .SH EXIT STATUS
230 | .TP
231 | 0
232 | if OK
233 | .TP
234 | 1
235 | if an error occurred
236 | .TP
237 | 2
238 | if a serious error occurred (e.g. error parsing the command line arguments)
239 | 
240 | .SH EXAMPLES
241 | .PP
242 | A simple example:
243 | .PP
244 | .RS
245 | $ n3map -p ns1.example.com. example.com.
246 | .RE
247 | .PP
248 | This will enumerate the zone example.com (if it is DNSSEC-enabled) using the
249 | nameserver ns1.example.com. It will automatically determine whether the zone
250 | uses NSEC or NSEC3 records. We also used the \fI-p\fR option so we can see the
251 | progress of the enumeration.
252 | If we want to see what happens in more detail, we can increase the verbosity
253 | using the \fI-v\fR option:
254 | .PP
255 | .RS
256 | $ n3map -pv ns1.example.com. example.com.
257 | .RE
258 | .PP
259 | Next, we want to save all received NSEC or NSEC3 records to a file (which is
260 | what you usually want). The file will be called `records':
261 | .PP
262 | .RS
263 | .nf
264 | $ n3map -pv ns1.example.com. example.com. -o records
265 | .fi
266 | .RE
267 | .PP
268 | You can always interrupt the enumeration by sending the SIGINT signal to the
269 | main process (usually this is achieved by pressing CTRl-C).
270 | If you have interrupted a session and want to continue where it stopped later,
271 | you may used the \fI-c\fR option:
272 | .PP
273 | .RS
274 | .nf
275 | $ n3map -pv ns1.example.com. example.com. -o records
276 | [interrupt by pressing CTRL-C]
277 | $ n3map -pv ns1.example.com. example.com. -c records
278 | .fi
279 | .RE
280 | .PP
281 | The next example is a bit more sophisticated:
282 | .PP
283 | .RS
284 | .nf
285 | $ n3map ns1.example.com. ns2.example.com:5353  example.com. -pv3o records.nsec3  --limit-rate 10/s
286 | .fi
287 | .RE
288 | .PP
289 | This command forces NSEC3 enumeration (the \fI-3\fR option) and limits the query
290 | rate to a maximum of 10 queries / second. Note that we also specified a second
291 | nameserver using a different port (5353).
292 | 
293 | .SH "SEE ALSO"
294 | \fBn3map-nsec3-lookup\fR(1),
295 | \fBn3map-hashcatify\fR(1),
296 | \fBn3map-johnify\fR(1),
297 | \fBdig(1)\fR
298 | 
299 | .SH BUGS
300 | .PP
301 | lot's of em. 
302 | .PP
303 | It may use a huge amount of memory when enumerating large zones. 
304 | .PP
305 | Reading large lists of records is slow.
306 | 
307 | 


--------------------------------------------------------------------------------
/hashcatify.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env python3
2 | 
3 | import n3map.hashcatify
4 | 
5 | if __name__ == '__main__':
6 |     n3map.hashcatify.main()
7 | 
8 | 


--------------------------------------------------------------------------------
/johnify.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env python3
2 | 
3 | import n3map.johnify
4 | 
5 | if __name__ == '__main__':
6 |     n3map.johnify.main()
7 | 
8 | 


--------------------------------------------------------------------------------
/map.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env python3
2 | 
3 | import n3map.map
4 | 
5 | if __name__ == '__main__':
6 |     n3map.map.main()
7 | 
8 | 


--------------------------------------------------------------------------------
/n3map.bash_completion:
--------------------------------------------------------------------------------
 1 | __n3map_complete()
 2 | {
 3 | 	local cur prev
 4 | 	cur="${COMP_WORDS[COMP_CWORD]}"
 5 | 	prev="${COMP_WORDS[COMP_CWORD-1]}"
 6 | 
 7 | 	case "$prev" in
 8 | 	--query-mode)
 9 | 		COMPREPLY=( $(compgen -W "A mixed NSEC" -- "$cur") )
10 | 		return 0
11 | 		;;
12 | 	-i|--input|-o|--output|-c|--continue)
13 | 		COMPREPLY=( $(compgen -f "$cur") )	
14 | 		return 0
15 | 		;;
16 | 	esac
17 | 	case "$cur" in
18 | 	-*)
19 | 		COMPREPLY=( $(compgen -W "--aggressive --auto --binary \
20 | 			--continue --end --help --ignore-overlapping --input \
21 | 			--label-counter --ldh --limit-rate --max-retries \
22 | 			--mixed --no-openssl --nsec --nsec3 --omit-soa-check \
23 | 			--output --predict --processes --query-mode \
24 | 			--queue-element-size --quiet --start --timeout \
25 | 			--verbose --version -3 -A -M -N -a -b -c -e -f -h -i \
26 | 			-l -m -n -o -p -q -s -v --" -- "$cur" ) )
27 | 		return 0
28 | 		;;
29 | 	esac
30 | }
31 | complete -o default -F __n3map_complete n3map
32 | 


--------------------------------------------------------------------------------
/n3map/__init__.py:
--------------------------------------------------------------------------------
1 | 
2 | __version__ = "0.8.2"
3 | 
4 | 


--------------------------------------------------------------------------------
/n3map/exception.py:
--------------------------------------------------------------------------------
 1 | 
 2 | class N3MapError(Exception):
 3 |     def __str__(self):
 4 |         return ''.join(map(str, self.args))
 5 | 
 6 | class NSECError(N3MapError):
 7 |     pass
 8 | 
 9 | class NSECWalkError(N3MapError):
10 |     pass
11 | 
12 | class NSEC3Error(N3MapError):
13 |     pass
14 | 
15 | class NSEC3WalkError(N3MapError):
16 |     pass
17 | 
18 | class HashLimitReached(N3MapError):
19 |     pass
20 | 
21 | class ZoneChangedError(N3MapError):
22 |     def __str__(self):
23 |         return ''.join(map(str, self.args)) + '\nzone may have been modified'
24 | 
25 | class InvalidPortError(N3MapError):
26 |     def __str__(self):
27 |         return "invalid port specified: " + str(self.args[0])
28 | 
29 | class InvalidAddressError(N3MapError):
30 |     def __str__(self):
31 |         return "invalid address specified: " + str(self.args[0])
32 | 
33 | class NameResolutionError(N3MapError):
34 |     pass
35 | 
36 | class TimeOutError(N3MapError):
37 |     def __str__(self):
38 |         return 'timeout: ' + ''.join(map(str, self.args))
39 | 
40 | class UnexpectedResponseStatus(N3MapError):
41 |     def __init__(self, status):
42 |         self.status = status
43 | 
44 |     def __str__(self):
45 |         return 'received unexpected response status ' + str(self.status)
46 | 
47 | 
48 | class MaxRetriesError(N3MapError):
49 |     def __str__(self):
50 |         return 'timeout: ' + ''.join(map(str, self.args))
51 | 
52 | class MaxNsErrors(N3MapError):
53 |     pass
54 | 
55 | class QueryError(N3MapError):
56 |     def __str__(self):
57 |         return 'received bad response'
58 |     pass
59 | 
60 | class InvalidDomainNameError(N3MapError):
61 |     def __str__(self):
62 |         return "invalid domain name: " + ''.join(map(str, self.args))
63 | 
64 | class MaxLabelLengthError(N3MapError):
65 |     def __str__(self):
66 |         return "maximum domain name label length exceeded"
67 | 
68 | class MaxLabelValueError(N3MapError):
69 |     def __str__(self):
70 |         return "maximum domain name label value exceeded"
71 | 
72 | class MaxDomainNameLengthError(N3MapError):
73 |     def __str__(self):
74 |         return "maximum domain name length exceeded"
75 | 
76 | class ParseError(N3MapError):
77 |     pass
78 | 
79 | class FileParseError(N3MapError):
80 |     def __init__(self, filename, line, msg):
81 |         super(FileParseError, self).__init__(filename, line, msg)
82 |         self.filename = filename
83 |         self.line = line
84 |         self.msg = msg
85 |     def __str__(self):
86 |         return self.filename + ':' + str(self.line) + ": " + str(self.msg)
87 | 
88 | 


--------------------------------------------------------------------------------
/n3map/hashcatify.py:
--------------------------------------------------------------------------------
 1 | import sys
 2 | import os
 3 | import re
 4 | 
 5 | from . import log
 6 | from . import rrfile
 7 | from . import util
 8 | from .exception import N3MapError
 9 | 
10 | def usage(argv):
11 |     sys.stderr.write("usage: " + os.path.basename(argv[0]) + " file [outfile]\n")
12 |     sys.exit(2)
13 | 
14 | def hashcatify_main(argv):
15 |     log.logger = log.Logger()
16 |     try:
17 |         if len(argv) < 2:
18 |             usage(argv)
19 |         if len(argv) == 3:
20 |             out = open(argv[2], "w")
21 |         else:
22 |             out = sys.stdout
23 | 
24 |         records_file = rrfile.open_input_rrfile(argv[1])
25 | 
26 |         for nsec3 in records_file.nsec3_reader():
27 |             nsec3_hash = util.base32_ext_hex_encode(nsec3.hashed_owner).lower()
28 |             nsec3_hash = nsec3_hash.decode()
29 |             zone = str(nsec3.zone)
30 |             zone = re.sub('\.$', '', zone)
31 |             iterations = "{0:d}".format(nsec3.iterations)
32 |             salt = nsec3.salt.hex()
33 |             out.write(":".join((nsec3_hash, "." + zone, salt, iterations))
34 |                     + "\n")
35 |     except (IOError, N3MapError) as e:
36 |         log.fatal(e)
37 | 
38 | 
39 | def main():
40 |     try:
41 |         sys.exit(hashcatify_main(sys.argv))
42 |     except KeyboardInterrupt:
43 |         sys.stderr.write("\nreceived SIGINT, terminating\n")
44 |         sys.exit(3)
45 | 
46 | 


--------------------------------------------------------------------------------
/n3map/johnify.py:
--------------------------------------------------------------------------------
 1 | import sys
 2 | import os
 3 | 
 4 | from . import log
 5 | from . import rrfile
 6 | from . import util
 7 | from .exception import N3MapError
 8 | 
 9 | def usage(argv):
10 |     sys.stderr.write("usage: " + os.path.basename(argv[0]) + " file [outfile]\n")
11 |     sys.exit(2)
12 | 
13 | def johnify_main(argv):
14 |     log.logger = log.Logger()
15 |     try:
16 |         if len(argv) < 2:
17 |             usage(argv)
18 |         if len(argv) == 3:
19 |             out = open(argv[2], "w")
20 |         else:
21 |             out = sys.stdout
22 | 
23 |         records_file = rrfile.open_input_rrfile(argv[1])
24 | 
25 |         for nsec3 in records_file.nsec3_reader():
26 |             nsec3_hash = nsec3.hashed_owner.hex()
27 |             zone = str(nsec3.zone)
28 |             iterations = "{0:d}".format(nsec3.iterations)
29 |             salt = nsec3.salt.hex()
30 |             out.write("$NSEC3$" + "$".join((iterations, salt,
31 |                 nsec3_hash, zone)) + "\n")
32 |     except (IOError, N3MapError) as e:
33 |         log.fatal(e)
34 | 
35 | 
36 | def main():
37 |     try:
38 |         sys.exit(johnify_main(sys.argv))
39 |     except KeyboardInterrupt:
40 |         sys.stderr.write("\nreceived SIGINT, terminating\n")
41 |         sys.exit(3)
42 | 
43 | 


--------------------------------------------------------------------------------
/n3map/log.py:
--------------------------------------------------------------------------------
  1 | import sys
  2 | import os
  3 | import fcntl
  4 | import array
  5 | import termios
  6 | 
  7 | import signal
  8 | import time
  9 | import collections
 10 | 
 11 | LOG_FATAL = -2
 12 | LOG_ERROR = -1
 13 | LOG_WARN = 0
 14 | LOG_INFO = 1
 15 | LOG_DEBUG1 = 2
 16 | LOG_DEBUG2 = 3
 17 | LOG_DEBUG3 = 4
 18 | 
 19 | BUFFER_INTERVAL = 0.2
 20 | 
 21 | def update():
 22 |     logger.update()
 23 | 
 24 | def fatal_exit(exitcode, *msg):
 25 |     logger.do_log(LOG_FATAL, os.path.basename(sys.argv[0]), ": fatal: ", *msg)
 26 |     exit(exitcode)
 27 | 
 28 | def fatal(*msg):
 29 |     fatal_exit(1, *msg)
 30 | 
 31 | def warn(*msg):
 32 |     logger.do_log(LOG_WARN, "warning: ", *msg)
 33 | 
 34 | def error(*msg):
 35 |     logger.do_log(LOG_ERROR, "error: ", *msg)
 36 | 
 37 | def info(*msg):
 38 |     logger.do_log(LOG_INFO, *msg)
 39 | 
 40 | def debug1(*msg):
 41 |     logger.do_log(LOG_DEBUG1, *msg)
 42 | 
 43 | def debug2(*msg):
 44 |     logger.do_log(LOG_DEBUG2, *msg)
 45 | 
 46 | def debug3(*msg):
 47 |     logger.do_log(LOG_DEBUG3, *msg)
 48 | 
 49 | 
 50 | class Logger(object):
 51 |     def __init__(self, loglevel=LOG_WARN, logfile=sys.stderr, colors='auto'):
 52 |         self.loglevel = loglevel
 53 |         self._file = logfile
 54 |         self.set_colors(colors)
 55 | 
 56 |     def set_colors(self, preference):
 57 |         if preference == 'always':
 58 |             colors = Colors()
 59 |         elif preference == 'auto':
 60 |             if self._file.isatty():
 61 |                 colors = Colors()
 62 |             else:
 63 |                 colors = NoColors()
 64 |         elif preference == 'never':
 65 |             colors = NoColors()
 66 |         else:
 67 |             raise ValueError
 68 |         self.colors = ColorSchemeDefault(colors)
 69 |         self._make_colormap()
 70 | 
 71 |     def _make_colormap(self):
 72 |         self._colormap = {
 73 |                 LOG_WARN  : self.colors.WARN,
 74 |                 LOG_ERROR : self.colors.ERROR,
 75 |                 LOG_FATAL : self.colors.ERROR,
 76 |                 LOG_DEBUG1: self.colors.DEBUG1,
 77 |                 LOG_DEBUG2: self.colors.DEBUG2,
 78 |                 LOG_DEBUG3: self.colors.DEBUG3
 79 |             }
 80 | 
 81 |     def _write_log(self, msg):
 82 |         self._file.write(msg)
 83 | 
 84 |     def _colorize_msg(self, level, *msg):
 85 |         try:
 86 |             return self.colors.wrap_list(self._colormap[level], list(msg))
 87 |         except KeyError:
 88 |             return msg
 89 | 
 90 |     def _compile_msg(self, *msg):
 91 |         l = list(map(str, msg))
 92 |         l.append("\n")
 93 |         return ''.join(l)
 94 | 
 95 |     def do_log(self, level, *msg):
 96 |         if self.loglevel < level and level > LOG_FATAL:
 97 |             return
 98 |         msg = self._colorize_msg(level, *msg)
 99 |         msg = self._compile_msg(*msg)
100 |         self._write_log(msg)
101 | 
102 |     def update(self):
103 |         pass
104 | 
105 |     def set_status_generator(self, generator, formatfunc):
106 |         pass
107 | 
108 |     def block_signals(self):
109 |         pass
110 | 
111 |     def unblock_signals(self):
112 |         pass
113 | 
114 | 
115 | received_sigwinch = False
116 | 
117 | def sigwinch_handler(signum, frame):
118 |     global received_sigwinch
119 |     received_sigwinch = True
120 | 
121 | def setup_signal_handling():
122 |     signal.signal(signal.SIGWINCH,sigwinch_handler)
123 |     signal.siginterrupt(signal.SIGWINCH, False)
124 | 
125 | def reset_signal_handling():
126 |     signal.signal(signal.SIGWINCH, signal.SIG_DFL)
127 | 
128 | class ProgressLineLogger(Logger):
129 |     def __init__(self, loglevel=LOG_WARN, logfile=sys.stderr, colors='auto'):
130 |         super(ProgressLineLogger,self).__init__(loglevel,logfile, colors)
131 |         self._generator = None
132 |         self._formatter = None
133 |         self._buffer = collections.deque()
134 |         self._flush_interval = 0.0
135 |         self._last_flush = 0
136 |         self._screen_width = 0
137 |         self._screen_height = 0
138 |         self._current_status = None
139 |         self._statuslines = None
140 |         self._enabled = False
141 | 
142 |     def from_logger(logger):
143 |         plogger = ProgressLineLogger(logger.loglevel, logger._file)
144 |         plogger.colors = logger.colors
145 |         plogger._colormap = logger._colormap
146 |         return plogger
147 | 
148 |     def enable(self):
149 |         self._last_flush = 0;
150 |         self._flush_interval = BUFFER_INTERVAL
151 |         setup_signal_handling()
152 |         self._determine_screen_size()
153 |         self._enabled = True
154 | 
155 |     def disable(self):
156 |         self._enabled = False
157 |         self.flush()
158 |         reset_signal_handling()
159 |         self._last_flush = 0
160 |         self._flush_interval = 0
161 | 
162 |     def block_signals(self):
163 |         if not self._enabled:
164 |             return
165 |         signal.signal(signal.SIGWINCH, signal.SIG_IGN)
166 | 
167 |     def unblock_signals(self):
168 |         if not self._enabled:
169 |             return
170 |         setup_signal_handling()
171 |         # in case a signal would have arrived in the meantime
172 |         self._determine_screen_size()
173 | 
174 | 
175 | 
176 |     def set_status_generator(self, generator, formatfunc):
177 |         self.flush()
178 |         self._generator, self._formatter = generator, formatfunc
179 |         if generator is not None:
180 |             self.enable()
181 |         else:
182 |             self.disable()
183 | 
184 |     def flush(self):
185 |         self._format_statuslines()
186 |         self._write_log(''.join(self._buffer))
187 |         self._buffer.clear()
188 |         self._file.flush()
189 |         self._last_flush = time.monotonic()
190 | 
191 |     def do_log(self, level, *msg):
192 |         if level == LOG_FATAL:
193 |             self.set_status_generator(None, None)
194 |             msg = self._colorize_msg(level, *msg)
195 |             self._buffer.append(self._compile_msg(*msg))
196 |             self.disable()
197 |             return
198 |         if self.loglevel >= level:
199 |             msg = self._colorize_msg(level, *msg)
200 |             self._buffer.append(self._compile_msg(*msg))
201 |             self.update(force=(level <= LOG_WARN))
202 |         elif level <= LOG_DEBUG1:
203 |             self.update(force=(level <= LOG_WARN))
204 | 
205 |     def update(self, force=False):
206 |         if self._generator is not None:
207 |             gen = self._generator
208 |             self._current_status = gen()
209 |         if not force and time.monotonic() - self._last_flush < self._flush_interval:
210 |             return
211 |         self.flush()
212 | 
213 |     def _format_statuslines(self):
214 |         if self._current_status is None or self._formatter is None:
215 |             self._statuslines = None
216 |             self._current_status = None
217 |             return
218 |         # new statusline
219 |         global received_sigwinch
220 |         if received_sigwinch:
221 |             received_sigwinch = False
222 |             self._determine_screen_size()
223 |         if self._statuslines is not None and self._file.isatty():
224 |             # clear old statuslines
225 |             self._buffer.appendleft(''.join(("\r\033[0K","\033[A\033[0K"*len(self._statuslines))))
226 |         formatfunc = self._formatter
227 |         self._statuslines = formatfunc(self._screen_width,
228 |                                        *(self._current_status))
229 |         self._current_status = None
230 |         self._buffer.extend(('\n'.join(self._statuslines), "\n"))
231 | 
232 | 
233 |     def _determine_screen_size(self):
234 |         if self._file.isatty():
235 |             buf = array.array('h', [0, 0, 0, 0])
236 |             res = fcntl.ioctl(self._file.fileno(), termios.TIOCGWINSZ, buf)
237 |             if res != 0:
238 |                 raise EnvironmentError("ioctl() failed")
239 |             self._screen_height = buf[0]
240 |             self._screen_width = buf[1]
241 |         else:
242 |             self._screen_height = 20
243 |             self._screen_width = 80
244 | 
245 | class NoColors:
246 |     RESET          = ''
247 |     RED            = ''
248 |     GREEN          = ''
249 |     YELLOW         = ''
250 |     BLUE           = ''
251 |     MAGENTA        = ''
252 |     CYAN           = ''
253 |     BRIGHT_RED     = ''
254 |     BRIGHT_GREEN   = ''
255 |     BRIGHT_YELLOW  = ''
256 |     BRIGHT_BLUE    = ''
257 |     BRIGHT_MAGENTA = ''
258 |     BRIGHT_CYAN    = ''
259 | 
260 |     def __init__(self):
261 |         pass
262 | 
263 |     def wrap(self, color, s):
264 |         return s
265 | 
266 |     def wrap_list(self, color, l):
267 |         return l
268 | 
269 | class Colors(NoColors):
270 |     RESET          = '\033[0m'
271 |     RED            = '\033[31m'
272 |     GREEN          = '\033[32m'
273 |     YELLOW         = '\033[33m'
274 |     BLUE           = '\033[34m'
275 |     MAGENTA        = '\033[35m'
276 |     CYAN           = '\033[36m'
277 |     BRIGHT_RED     = '\033[1;31m'
278 |     BRIGHT_GREEN   = '\033[1;32m'
279 |     BRIGHT_YELLOW  = '\033[1;33m'
280 |     BRIGHT_BLUE    = '\033[1;34m'
281 |     BRIGHT_MAGENTA = '\033[1;35m'
282 |     BRIGHT_CYAN    = '\033[1;36m'
283 | 
284 |     def __init__(self):
285 |         pass
286 | 
287 |     def wrap(self, color, s):
288 |         return ''.join((color, s, self.RESET))
289 | 
290 |     def wrap_list(self, color, l):
291 |         l.insert(0, color)
292 |         l.append(self.RESET)
293 |         return l
294 | 
295 | class ColorSchemeDefault:
296 |     def __init__(self, colors):
297 |         if colors is None:
298 |             colors = Colors()
299 |         self.WARN = colors.BRIGHT_YELLOW
300 |         self.ERROR = colors.BRIGHT_RED
301 |         self.DEBUG1 = colors.CYAN
302 |         self.DEBUG2 = colors.CYAN
303 |         self.DEBUG3 = colors.CYAN
304 |         self.PROGRESSBAR = colors.CYAN
305 |         self.PROGRESS = colors.BRIGHT_CYAN
306 |         self.RECORDS = colors.BRIGHT_GREEN
307 |         self.NUMBERS = colors.CYAN
308 |         self.ZONE = colors.BRIGHT_BLUE
309 |         self.DECO = colors.BRIGHT_MAGENTA
310 | 
311 |         self.RESET = colors.RESET
312 |         self.colors = colors
313 | 
314 |     def wrap(self, color, s):
315 |         return self.colors.wrap(color, s)
316 | 
317 |     def wrap_list(self, color, l):
318 |         return self.colors.wrap_list(color, l)
319 | 
320 |     def gradient(self, ratio):
321 |         if ratio < 0.33:
322 |             return self.colors.BRIGHT_CYAN
323 |         elif ratio < 0.66:
324 |             return self.colors.BRIGHT_GREEN
325 |         elif ratio < 1.0:
326 |             return self.colors.BRIGHT_YELLOW
327 |         else:
328 |             return self.colors.BRIGHT_RED
329 | 
330 | 


--------------------------------------------------------------------------------
/n3map/map.py:
--------------------------------------------------------------------------------
  1 | import getopt
  2 | import multiprocessing
  3 | import re
  4 | import sys
  5 | import os
  6 | import time
  7 | from datetime import timedelta
  8 | 
  9 | from . import log
 10 | from . import prehash
 11 | from . import queryprovider
 12 | from .query import query_ns_records
 13 | from . import rrfile
 14 | from .exception import N3MapError, FileParseError, HashLimitReached
 15 | from .nsec3walker import NSEC3Walker
 16 | from .predict import create_zone_predictor
 17 | from .nsecwalker import NSECWalkerN, NSECWalkerMixed, NSECWalkerA
 18 | 
 19 | import n3map.name
 20 | import n3map.walker
 21 | 
 22 | def _def_num_of_processes():
 23 |     try:
 24 |         ncpus = multiprocessing.cpu_count()
 25 |     except NotImplementedError:
 26 |         log.error("could not detect number of cpus.")
 27 |         ncpus = 1
 28 | 
 29 |     if ncpus > 1:
 30 |         return ncpus - 1
 31 |     return 1
 32 | 
 33 | def _compute_query_interval(n, unit):
 34 |     units = { 's': 1.0, 'm' : 60.0, 'h': 3600.0 }
 35 |     return units[unit]/n
 36 | 
 37 | def _query_interval(s):
 38 |     p = re.compile('^(([0-9]\.|[1-9][0-9]*[.]?)[0-9]*)/([smh])$')
 39 |     m = p.match(s)
 40 |     if m is None:
 41 |         raise ValueError
 42 |     try:
 43 |         return _compute_query_interval(n=float(m.group(1)), unit=m.group(3))
 44 |     except ZeroDivisionError:
 45 |         raise ValueError
 46 | 
 47 | 
 48 | def _human_number(s):
 49 |     units = {
 50 |             'K': 1000,
 51 |             'M': 1000000,
 52 |             'G': 1000000000,
 53 |             'T': 1000000000000,
 54 |             }
 55 |     m = re.fullmatch(r'(\d+)([a-zA-Z]?)', s)
 56 |     if m is None:
 57 |         raise ValueError
 58 |     n = int(m.group(1))
 59 |     unit = m.group(2)
 60 |     if unit != '':
 61 |         try:
 62 |             n *= units[unit.upper()]
 63 |         except KeyError:
 64 |             raise ValueError
 65 |     return n
 66 | 
 67 | def check_part_of_zone(rr, zone):
 68 |     if not rr.part_of_zone(zone):
 69 |         raise N3MapError(("not all read records are part of the specified zone"))
 70 | 
 71 | def get_nameservers(zone, ipproto='', ns_names=None):
 72 |     if ns_names is not None:
 73 |         return queryprovider.nameserver_from_text(ipproto, *ns_names)
 74 | 
 75 |     ns_names = query_ns_records(zone)
 76 |     nslist = queryprovider.nameserver_from_text(ipproto, *ns_names,
 77 |                                                 ignore_unresolved=True)
 78 |     for ns in nslist:
 79 |         log.info("using nameserver: ", str(ns))
 80 |     return nslist
 81 | 
 82 | def read_input_file(input_filename, cont, zone, zone_type):
 83 |     chain = None
 84 |     records_file = None
 85 |     label_counter = None
 86 |     try:
 87 |         records_file = rrfile.open_input_rrfile(input_filename)
 88 |     except FileNotFoundError as e:
 89 |         if cont:
 90 |             log.info('zone file {} does not exist yet, creating it'
 91 |                     .format(input_filename))
 92 |             return (None, None)
 93 |         else:
 94 |             log.fatal("unable to open input file: \n", str(e))
 95 |     try:
 96 |         chain = []
 97 |         if zone_type == 'nsec3':
 98 |             for rr in records_file.nsec3_reader():
 99 |                 check_part_of_zone(rr, zone)
100 |                 chain.append(rr)
101 |             label_counter = records_file.label_counter
102 |         elif zone_type == 'nsec':
103 |             for rr in records_file.nsec_reader():
104 |                 check_part_of_zone(rr, zone)
105 |                 chain.append(rr)
106 |     except IOError as e:
107 |         log.fatal("unable to read input file: \n", str(e))
108 |     except FileParseError as e:
109 |         log.fatal("unable to parse input file: \n", str(e))
110 |     finally:
111 |         if records_file is not None:
112 |             records_file.close()
113 |     if cont:
114 |         try:
115 |             records_file.into_backup()
116 |         except OSError as e:
117 |             log.fatal("failed to create backup file: \n", str(e))
118 |     return (chain, label_counter)
119 | 
120 | 
121 | 
122 | def n3map_main(argv):
123 |     log.logger = log.Logger()
124 |     try:
125 |         (options, ns_names, zone) = parse_arguments(argv)
126 |     except N3MapError as e:
127 |         log.fatal_exit(2, e)
128 | 
129 |     output_rrfile = None
130 |     chain = None
131 |     label_counter = None
132 |     walker = None
133 |     process_pool = None
134 |     hash_queues = None
135 |     if options['progress']:
136 |         log.logger = log.ProgressLineLogger.from_logger(log.logger)
137 | 
138 |     log.info("n3map {}: starting mapping of {}".format(
139 |         n3map.__version__, str(zone)))
140 | 
141 |     try:
142 |         nslist = get_nameservers(zone, options['ipproto'], ns_names)
143 |         stats = {}
144 |         options['timeout'] /= 1000.0
145 |         qprovider = queryprovider.QueryProvider(nslist,
146 |                 timeout=options['timeout'], max_retries=options['max_retries'],
147 |                 max_errors=options['max_errors'],
148 |                 query_interval = options['query_interval'], stats=stats)
149 | 
150 |         if options['soa_check']:
151 |             n3map.walker.check_soa(zone, qprovider)
152 | 
153 |         if options['dnskey_check']:
154 |             n3map.walker.check_dnskey(zone, qprovider)
155 | 
156 |         if options['zone_type'] == 'auto':
157 |             options['zone_type'] = n3map.walker.detect_dnssec_type(zone,
158 |                     qprovider, options['detection_attempts'])
159 |             if options['detect_only']:
160 |                 print("{}: {}".format(str(zone), options['zone_type']))
161 |                 return 0
162 | 
163 |         if options['zone_type'] == 'nsec3':
164 |             (hash_queues, process_pool) = prehash.create_prehash_pool(
165 |                 options['processes'], options['queue_element_size'],
166 |                 options['use_openssl'])
167 |             if options['predict']:
168 |                 proc,pipe = create_zone_predictor()
169 |                 predictor = (proc,pipe)
170 |             else:
171 |                 predictor = None
172 | 
173 | 
174 |         if options['continue'] is not None:
175 |             chain, label_counter = read_input_file(options['continue'], True,
176 |                     zone, options['zone_type'])
177 |             try:
178 |                 output_rrfile =  rrfile.open_output_rrfile(options['continue'])
179 |             except IOError as e:
180 |                 log.fatal("unable to open output file: ", str(e))
181 |         else:
182 |             if options['input'] is not None:
183 |                 chain, label_counter = read_input_file(options['input'], False,
184 |                         zone, options['zone_type'])
185 |             if options['output'] is not None:
186 |                 if options['output'] == '-':
187 |                     output_rrfile = rrfile.RRFileStream(sys.stdout)
188 |                 else:
189 |                     try:
190 |                         output_rrfile =  rrfile.open_output_rrfile(
191 |                                 options['output'])
192 |                     except IOError as e:
193 |                         log.fatal("unable to open output file: ", str(e))
194 | 
195 | 
196 | 
197 |         if options['zone_type'] == 'nsec3':
198 |             if output_rrfile is not None:
199 |                 output_rrfile.write_header(zone, "List of NSEC3 RRs")
200 |             if options['label_counter'] is not None:
201 |                 label_counter = options['label_counter']
202 |             walker = NSEC3Walker(zone,
203 |                                  qprovider,
204 |                                  hash_queues,
205 |                                  process_pool,
206 |                                  nsec3_records=[] if chain is None else chain,
207 |                                  ignore_overlapping=options['ignore_overlapping'],
208 |                                  label_counter=label_counter,
209 |                                  output_file=output_rrfile,
210 |                                  stats=stats,
211 |                                  predictor=predictor,
212 |                                  aggressive=options['aggressive'],
213 |                                  hashlimit=options['hashlimit']
214 |                                  )
215 | 
216 |         elif options['zone_type'] == 'nsec':
217 |             if output_rrfile is not None:
218 |                 output_rrfile.write_header(zone, "List of NSEC RRs")
219 | 
220 |             if options['query_mode'] == "mixed":
221 |                 walker = NSECWalkerMixed(zone,
222 |                                          qprovider,
223 |                                          options['query_chars'] == 'ldh',
224 |                                          nsec_chain=chain,
225 |                                          startname=options['start'],
226 |                                          endname=options['end'],
227 |                                          stats=stats,
228 |                                          output_file=output_rrfile,
229 |                                          never_prefix_label=options['no_prefix_labels'])
230 |             elif options['query_mode'] == "A":
231 |                 walker = NSECWalkerA(zone,
232 |                                      qprovider,
233 |                                      options['query_chars'] == 'ldh',
234 |                                      nsec_chain=chain,
235 |                                      startname=options['start'],
236 |                                      endname=options['end'],
237 |                                      stats=stats,
238 |                                      output_file=output_rrfile,
239 |                                      never_prefix_label=options['no_prefix_labels'])
240 |             else:
241 |                 walker = NSECWalkerN(zone,
242 |                                      qprovider,
243 |                                      nsec_chain=chain,
244 |                                      startname=options['start'],
245 |                                      endname=options['end'],
246 |                                      stats=stats,
247 |                                      output_file=output_rrfile)
248 |         finished = False
249 |         if walker is not None:
250 |             starttime = time.monotonic()
251 |             stopped_prematurely = False
252 |             try:
253 |                 walker.walk()
254 |             except HashLimitReached:
255 |                 stopped_prematurely = True
256 |             elapsed = timedelta(seconds=time.monotonic() - starttime)
257 |             if stopped_prematurely:
258 |                 log.info("stopped mapping of {0:s} after {1:s}: hashlimit reached"
259 |                          .format( str(zone), str(elapsed)))
260 |             else:
261 |                 log.info("finished mapping of {0:s} in {1:s}"
262 |                          .format( str(zone), str(elapsed)))
263 |             finished = True
264 | 
265 |         if output_rrfile is not None:
266 |             output_rrfile.write_stats(stats)
267 |             if finished and options['continue'] is not None:
268 |                 output_rrfile.unlink_backup()
269 | 
270 |     except N3MapError as e:
271 |         log.fatal(e)
272 |     except IOError as e:
273 |         log.fatal(str(e))
274 |     finally:
275 |         if output_rrfile is not None:
276 |             output_rrfile.close()
277 |     return 0
278 | 
279 | def default_options():
280 |     opts = {
281 |             'zone_type' : 'auto',
282 |             'output': None,
283 |             'input' : None,
284 |             'continue' : None,
285 |             'aggressive' : 0,
286 |             'ignore_overlapping' : False,
287 |             'query_mode' : 'mixed',
288 |             'query_chars' : 'binary',
289 |             'start' : None,
290 |             'end' : None,
291 |             'no_prefix_labels' : False,
292 |             'label_counter' : None,
293 |             'hashlimit' : 0,
294 |             'timeout' : 2500,
295 |             'max_retries' : 5,
296 |             'max_errors' : 1,
297 |             'query_interval' : None,
298 |             'detection_attempts' : 5,
299 |             'soa_check' : True,
300 |             'dnskey_check' : True,
301 |             'predict' : False,
302 |             'processes' : _def_num_of_processes(),
303 |             'progress' : True,
304 |             'queue_element_size' : 256,
305 |             'use_openssl' : True,
306 |             'ipproto' : '',
307 |             'detect_only' : False,
308 |             }
309 |     return opts
310 | 
311 | def invalid_argument(opt, arg):
312 |     log.fatal_exit(2, "invalid " + opt + " argnument `" + str(arg) + "'")
313 | 
314 | def parse_arguments(argv):
315 |     long_opts = [
316 |             'aggressive=',
317 |             'auto',
318 |             'binary',
319 |             'continue=',
320 |             'end=',
321 |             'help',
322 |             'ignore-overlapping',
323 |             'input=',
324 |             'label-counter=',
325 |             'hashlimit=',
326 |             'ldh',
327 |             'limit-rate=',
328 |             'max-retries=',
329 |             'max-errors=',
330 |             'mixed',
331 |             'nsec',
332 |             'nsec3',
333 |             'omit-soa-check',
334 |             'omit-dnskey-check',
335 |             'detection-attempts=',
336 |             'output=',
337 |             'predict',
338 |             'processes=',
339 |             'query-mode=',
340 |             'queue-element-size=',
341 |             'quiet',
342 |             'start=',
343 |             'no-prefix-labels',
344 |             'timeout=',
345 |             'no-openssl',
346 |             'verbose',
347 |             'color=',
348 |             'version',
349 |             'detect-only'
350 |     ]
351 |     options = default_options()
352 |     opts = '346AMNabc:e:f:hi:lm:no:pqs:v'
353 |     try:
354 |         opts, args = getopt.gnu_getopt(argv[1:], opts, long_opts)
355 |     except getopt.GetoptError as err:
356 |         log.fatal_exit(2, err, "\n", "Try `",
357 |                 str(os.path.basename(argv[0])),
358 |                 " --help' for more information.")
359 | 
360 |     for opt, arg in opts:
361 |         if opt in ('-h', '--help'):
362 |             usage(os.path.basename(argv[0]))
363 |             sys.exit(0)
364 | 
365 |         elif opt in ('-a' '--auto'):
366 |             options['zone_type'] = 'auto'
367 | 
368 |         elif opt in ('-n', '--nsec'):
369 |             options['zone_type'] = 'nsec'
370 | 
371 |         elif opt in ('-3', '--nsec3'):
372 |             options['zone_type'] = 'nsec3'
373 | 
374 |         elif opt in ('--detect-only'):
375 |             options['detect_only'] = True
376 | 
377 |         elif opt in ('-4',):
378 |             options['ipproto'] = 'ipv4'
379 | 
380 |         elif opt in ('-6',):
381 |             options['ipproto'] = 'ipv6'
382 | 
383 |         elif opt in ('-c', '--continue'):
384 |             options['continue'] = arg
385 | 
386 |         elif opt in ('-i', '--input'):
387 |             options['input'] = arg
388 | 
389 |         elif opt in ('-o', '--output'):
390 |             options['output'] = arg
391 | 
392 |         elif opt in ('--label-counter',):
393 |             try:
394 |                 options['label_counter'] = int(arg, 0)
395 |             except ValueError:
396 |                 invalid_argument(opt, arg)
397 |             if options['label_counter']  < 0:
398 |                 invalid_argument(opt, arg)
399 | 
400 |         elif opt in ('--hashlimit',):
401 |             try:
402 |                 options['hashlimit'] = _human_number(arg)
403 |             except ValueError:
404 |                 invalid_argument(opt, arg)
405 | 
406 |         elif opt in ('--ignore-overlapping',):
407 |             options['ignore_overlapping'] = True
408 | 
409 |         elif opt in ('-m', '--query-mode'):
410 |             if arg not in ('mixed', 'NSEC', 'A'):
411 |                 invalid_argument(opt, arg)
412 |             options['query_mode'] = arg
413 | 
414 |         elif opt in ('-M', '--mixed'):
415 |             options['query_mode']  = 'mixed'
416 | 
417 |         elif opt in ('-A',):
418 |             options['query_mode']  = 'A'
419 | 
420 |         elif opt in ('-N',):
421 |             options['query_mode']  = 'NSEC'
422 | 
423 |         elif opt in ('-l', '--ldh'):
424 |             options['query_chars'] = 'ldh'
425 | 
426 |         elif opt in ('-b', '--binary'):
427 |             options['query_chars'] = 'binary'
428 | 
429 |         elif opt in ('--no-prefix-labels',):
430 |             options['no_prefix_labels'] = True
431 | 
432 |         elif opt in ('-e', '--end'):
433 |             options['end'] = arg
434 | 
435 |         elif opt in ('--limit-rate',):
436 |             try:
437 |                 options['query_interval'] = _query_interval(arg)
438 |             except ValueError:
439 |                 invalid_argument(opt, arg)
440 | 
441 |         elif opt in ('--max-retries',):
442 |             try:
443 |                 options['max_retries'] = int(arg)
444 |             except ValueError:
445 |                 invalid_argument(opt, arg)
446 |             if options['max_retries'] < -1:
447 |                 invalid_argument(opt, arg)
448 | 
449 |         elif opt in ('--max-errors',):
450 |             try:
451 |                 options['max_errors'] = int(arg)
452 |             except ValueError:
453 |                 invalid_argument(opt, arg)
454 |             if options['max_errors'] < -1:
455 |                 invalid_argument(opt, arg)
456 | 
457 |         elif opt in ('--detection-attempts',):
458 |             try:
459 |                 options['detection_attempts'] = int(arg)
460 |             except ValueError:
461 |                 invalid_argument(opt, arg)
462 |             if options['detection_attempts'] < 0:
463 |                 invalid_argument(opt, arg)
464 | 
465 | 
466 |         elif opt in ('--omit-soa-check',):
467 |             options['soa_check'] = False
468 | 
469 |         elif opt in ('--omit-dnskey-check',):
470 |             options['dnskey_check'] = False
471 | 
472 |         elif opt in ('-f', '--aggressive',):
473 |             try:
474 |                 options['aggressive'] = int(arg)
475 |             except ValueError:
476 |                 invalid_argument(opt, arg)
477 |             if options['aggressive'] < 1:
478 |                 invalid_argument(opt, arg)
479 | 
480 |         elif opt in ('-p', '--predict',):
481 |             options['predict'] = True
482 | 
483 |         elif opt in ('--processes',):
484 |             try:
485 |                 options['processes'] = int(arg)
486 |             except ValueError:
487 |                 invalid_argument(opt, arg)
488 |             if options['processes'] < 1:
489 |                 invalid_argument(opt, arg)
490 | 
491 | 
492 |         elif opt in ('--queue-element-size',):
493 |             try:
494 |                 options['queue_element_size'] = int(arg)
495 |             except ValueError:
496 |                 invalid_argument(opt, arg)
497 |             if options['queue_element_size']  < 1:
498 |                 invalid_argument(opt, arg)
499 | 
500 |         elif opt in ('-q', '--quiet'):
501 |             options['progress'] = False
502 | 
503 |         elif opt in ('-s', '--start'):
504 |             options['start'] = arg
505 | 
506 |         elif opt in ('--timeout',):
507 |             try:
508 |                 options['timeout'] = int(arg)
509 |             except ValueError:
510 |                 invalid_argument(opt, arg)
511 |             if options['timeout']  < 1:
512 |                 invalid_argument(opt, arg)
513 | 
514 |         elif opt in ('--no-openssl',):
515 |             options['use_openssl'] = False
516 | 
517 |         elif opt in ('-v', '--verbose'):
518 |             log.logger.loglevel += 1
519 | 
520 |         elif opt in ('--color',):
521 |             try:
522 |                 log.logger.set_colors(arg)
523 |             except ValueError:
524 |                 invalid_argument(opt, arg)
525 | 
526 |         elif opt in ('--version'):
527 |             version()
528 |             sys.exit(0)
529 | 
530 |         else:
531 |             invalid_argument(opt, "")
532 | 
533 |     if len(args) < 1:
534 |         log.fatal_exit(2, 'missing arguments', "\n", "Try `",
535 |                 str(os.path.basename(argv[0])),
536 |                 " --help' for more information.")
537 |     else:
538 |         zone = n3map.name.fqdn_from_text(args[-1])
539 |         if len(args) >= 2:
540 |             ns_names = args[:-1]
541 |         else:
542 |             ns_names = None
543 | 
544 |     if options['continue'] is not None and (options['input'] is not None or
545 |             options['output'] is not None):
546 |         log.fatal_exit(2, 'Invalid arguments: use -c xor (-i or -o)')
547 | 
548 |     return (options, ns_names, zone)
549 | 
550 | def version():
551 |     sys.stdout.write("nsec3map " + n3map.__version__ + "\n")
552 | 
553 | 
554 | def usage(program_name):
555 |     def_opts = default_options()
556 |     sys.stdout.write(
557 |             'Usage: {0:s} [option]... [-o file] [nameserver[:port]]... zone'
558 |             .format(program_name))
559 |     sys.stdout.write(
560 | '''
561 | Enumerate a DNSSEC signed zone based on NSEC or NSEC3 resource records
562 | 
563 | Options:
564 |       --version              show program's version number and exit
565 |   -h, --help                 show this help message and exit
566 |   -v, --verbose              increase verbosity level (use multiple times for
567 |                                greater effect)
568 |       --color=WHEN           colorize output; WHEN can be 'auto' (default),
569 |                                'always' or 'never'.
570 | 
571 | Enumeration:
572 |   -a, --auto                 autodetect enumeration method (default)
573 |   -3, --nsec3                use NSEC3 enumeration
574 |   -n, --nsec                 use NSEC enumeration
575 |   -o, --output=FILE          write all records to FILE (use '-' for stdout)
576 |   -i, --input=FILE           read records from FILE and continue
577 |                                the enumeration.
578 |   -c, --continue=FILE        same as -i FILE -o FILE, but will preserve FILE as
579 |                                a backup file until the enumeration is finished.
580 |                                Will create FILE if it does not exist yet.
581 | 
582 | NSEC Options:
583 |   -m, --query-mode=MODE      sets the query mode. Possible values are
584 |                                'mixed', 'A', and 'NSEC' (default {qmode:s})
585 |   -M, --mixed                shortcut for --query-mode=mixed
586 |   -A                         shortcut for --query-mode=A
587 |   -N                         shortcut for --query-mode=NSEC
588 |   -b, --binary               use all possible binary values in queries (default)
589 |   -l, --ldh                  use only lowercase characters, digits and hyphen in
590 |                                queries
591 |   -s, --start=DOMAIN
592 |   -e, --end=DOMAIN           use DOMAIN as the enumeration start-/endpoint.
593 |                                DOMAIN is relative to the zone name.
594 | 
595 |       --no-prefix-labels     do not add leading labels ("\\x00.") to increment
596 |                                query names. Applies to 'A' query mode.
597 |                                This will result in incomplete enumeration for
598 |                                many zones, but is useful to avoid descending
599 |                                into subzones when querying non-authoritative
600 |                                namservers or to speed up the enumeration of TLDs.
601 | 
602 | NSEC3 Options:
603 |   -f, --aggressive=N         send up to N queries in parallel. This may speed
604 |                                up the enumeration significantly if the DNS
605 |                                server's round-trip time is high. However, it will
606 |                                also cause n3map to make more queries than usual
607 |                                because it cannot completely avoid queries which
608 |                                resolve to the same NSEC3 records.
609 |                                Use with caution.
610 |       --ignore-overlapping   ignore overlapping NSEC3 records. Useful when
611 |                                enumerating large zones that may change during
612 |                                enumeration.
613 |   -p, --predict              try to predict the size of the zone based on the
614 |                                records already received. Note that this option
615 |                                might slow down the enumeration process
616 |                                (experimental)
617 |       --processes=N          defines the number of pre-hashing processes.
618 |                                Default is 1 or the number of CPUs - 1 on
619 |                                multiprocessor systems ({processes:d} on this system)
620 |       --hashlimit=N[K|M|G|T]
621 |                              stop the enumeration after checking N hashes, even
622 |                                if it is not finished. Default = 0 (unlimited).
623 | 
624 | Advanced NSEC3 Options:
625 |   Use with caution.
626 |       --label-counter=N      set the initial label counter
627 |       --queue-element-size=N set the queue elment size. (default {queue_element_sz:d})
628 |       --no-openssl           do not use OpenSSL for hashing (slower, especially
629 |                               at high iteration counts)
630 | 
631 | General Options:
632 |   -q, --quiet                do not display progress information during enumeration
633 |       --limit-rate=N{{/s|/m|/h}}
634 |                              limit the query rate (default = unlimited)
635 |       --max-retries=N        limit the maximum number of retries when a DNS query
636 |                                times out. Defaults to {max_retries:d}.
637 |                                N=-1 means no limit.
638 |       --max-errors=N         limit the maximum number of consecutive
639 |                                errors/wrongful responses a DNS server may
640 |                                return. Defaults to {max_errors:d}.
641 |                                N=-1 means no limit (use with extreme caution).
642 |       --timeout=N            timeout to wait for a server response,
643 |                                in miliseconds (default {timeout:d})
644 |       --detection-attempts=N limit the maximum number of zone type (NSEC/NSEC3)
645 |                                detection attempts. N=0 specifies no limit.
646 |                                (default {detection_attempts:d})
647 |       --detect-only          detect and print zone type only, don't enumerate
648 |                                the zone.
649 |       --omit-soa-check       don't check the SOA record of the zone
650 |                                before starting enumeration (use with caution).
651 |       --omit-dnskey-check    don't check the DNSKEY record of the zone
652 |                                before starting enumeration (use with caution).
653 |       -4                     Use IPv4 only.
654 |       -6                     Use IPv6 only.
655 | '''.format(qmode=def_opts['query_mode'], processes=def_opts['processes'],
656 |         queue_element_sz=def_opts['queue_element_size'],
657 |         timeout=def_opts['timeout'], max_retries=def_opts['max_retries'],
658 |         max_errors=def_opts['max_errors'],
659 |         detection_attempts=def_opts['detection_attempts'])
660 |     )
661 | 
662 | def main():
663 |     try:
664 |         sys.exit(n3map_main(sys.argv))
665 |     except KeyboardInterrupt:
666 |         sys.stderr.write("\nreceived SIGINT, terminating\n")
667 |         sys.exit(3)
668 | 
669 | 


--------------------------------------------------------------------------------
/n3map/name.py:
--------------------------------------------------------------------------------
  1 | import string
  2 | import struct
  3 | import functools
  4 | 
  5 | from . import vis
  6 | from .exception import (
  7 |         InvalidDomainNameError,
  8 |         MaxDomainNameLengthError,
  9 |         MaxLabelLengthError,
 10 |         MaxLabelValueError
 11 |     )
 12 | 
 13 | # see RFC1035, section 2.3.4 "Size limits" for details
 14 | MAX_LABEL = 63
 15 | # for wire format:
 16 | MAX_DOMAINNAME = 255
 17 | 
 18 | range_ld = b"0123456789abcdefghijklmnopqrstuvwxyz"
 19 | range_ldh = b"-0123456789abcdefghijklmnopqrstuvwxyz"
 20 | 
 21 | def hex_label(l):
 22 |     return b"%x" % l
 23 | 
 24 | #def binary_label(l: int):
 25 | #    return l.to_bytes((l.bit_length() + 7) // 8, 'big')
 26 | 
 27 | ## XXX: much slower than hex_label
 28 | #def b32_label(l: int):
 29 | #     return base64.b32encode(l.to_bytes((l.bit_length() + 7) // 8,
 30 | #         'big')).rstrip(b'=')
 31 | 
 32 | def label_generator(label_fun, init=0):
 33 |     l = init
 34 |     while True:
 35 |         lblstr = label_fun(l)
 36 |         yield (Label(lblstr), l)
 37 |         l += 1
 38 | 
 39 | def _split_domainname_str(s):
 40 |     if s == b'.':
 41 |         return (b"",)
 42 |     else:
 43 |         return s.split(b'.')
 44 | 
 45 | 
 46 | def unvis_domainname(s):
 47 |     return DomainName(*[Label(vis.strunvis(l)) for l in
 48 |         _split_domainname_str(s)])
 49 | 
 50 | def fqdn_from_text(s):
 51 |     if not s.endswith('.'):
 52 |         s = s + '.'
 53 |     return domainname_from_text(s)
 54 | 
 55 | def domainname_from_text(s):
 56 |     try:
 57 |         bstr = s.encode('ascii')
 58 |         return DomainName(*list(map(Label, _split_domainname_str(bstr))))
 59 |     except UnicodeError:
 60 |         raise InvalidDomainNameError('invalid encoding')
 61 | 
 62 | def domainname_from_wire(ws):
 63 |     wire_bytes = []
 64 |     wire_bytes[:] = reversed(struct.unpack('B'*len(ws), ws))
 65 |     labels = []
 66 |     try:
 67 |         while True:
 68 |             n = wire_bytes.pop()
 69 |             lbl = []
 70 |             for i in range(n):
 71 |                 try:
 72 |                     lbl.append(wire_bytes.pop())
 73 |                 except IndexError:
 74 |                     raise InvalidDomainNameError('invalid wire format')
 75 |             labels.append(lbl)
 76 |     except IndexError:
 77 |         return DomainName(*[Label(struct.pack('B'*len(lbl), *lbl)) for lbl in
 78 |             labels])
 79 | 
 80 | def _label_ldh():
 81 |     return Label(range_ld[0:1])
 82 | 
 83 | def _label_binary():
 84 |     return Label(b"\x00")
 85 | 
 86 | @functools.total_ordering
 87 | class Label(object):
 88 |     def __init__(self, labelstr):
 89 |         if len(labelstr) > MAX_LABEL:
 90 |             raise MaxLabelLengthError
 91 |         self.label = labelstr
 92 |         self._canonicalize()
 93 | 
 94 |     def forward_next(self, ldh, extend):
 95 |         if ldh:
 96 |             return self.forward_next_ldh(extend)
 97 |         else:
 98 |             return self.forward_next_binary(extend)
 99 | 
100 |     def _extend_labelstr_binary(self, labelstr):
101 |         return labelstr + b'\x00'
102 | 
103 |     def forward_next_binary(self, extend):
104 |         if extend:
105 |             try:
106 |                 return Label(self._extend_labelstr_binary(self.label))
107 |             except MaxLabelLengthError:
108 |                 pass
109 |         return Label(self._increase_labelstr_binary(self.label))
110 | 
111 | 
112 |     def _increase_labelstr_binary(self, labelstr):
113 |         if self.has_max_value(False):
114 |             raise MaxLabelValueError
115 |         s = []
116 |         increased = False
117 |         for i, c in enumerate(reversed(struct.unpack('B'*len(labelstr),
118 |             labelstr))):
119 |             if not increased:
120 |                 if c == 0xff:
121 |                     if i == len(labelstr)-1:
122 |                         # end of string and cannot increase more
123 |                         raise MaxLabelValueError
124 |                     c = 0
125 |                     s.append(c)
126 |                 else:
127 |                     c += 1
128 |                     s.append(c)
129 |                     increased = True
130 |             else:
131 |                 s.append(c)
132 |         return struct.pack('B'*len(s), *reversed(s))
133 | 
134 |     def _extend_labelstr_ldh(self, labelstr):
135 |         return labelstr + range_ld[0:1]
136 | 
137 |     def forward_next_ldh(self, extend):
138 |         if extend:
139 |             try:
140 |                 return Label(self._extend_labelstr_ldh(self.label))
141 |             except MaxLabelLengthError:
142 |                 pass
143 |         return Label(self._increase_labelstr_ldh(self.label))
144 | 
145 |     def _range_next(self, rng, c):
146 |         for rc in rng:
147 |             if c < rc:
148 |                 return rc
149 |         return -1
150 | 
151 |     def has_max_value(self, ldh):
152 |         if ldh:
153 |             for i, c in enumerate(self.label):
154 |                 if i == 0 or i == len(self.label) - 1:
155 |                     if c != range_ld[-1]:
156 |                         return False
157 |                 else:
158 |                     if c != range_ldh[-1]:
159 |                         return False
160 |             return True
161 | 
162 |         else:
163 |             for c in self.label:
164 |                 if c != 0xff:
165 |                     return False
166 |             return True
167 | 
168 | 
169 |     def _increase_labelstr_ldh(self, labelstr):
170 |         if self.has_max_value(True):
171 |             raise MaxLabelValueError
172 |         s = []
173 |         increased = False
174 |         for i, c in enumerate(reversed(struct.unpack('B'*len(labelstr),
175 |             labelstr))):
176 |             if not increased:
177 |                 if i == 0 or i == len(labelstr)-1:
178 |                     # at beginning or end of string
179 |                     inc = self._range_next(range_ld, c)
180 |                     if inc == -1:
181 |                         # end of range
182 |                         if i == len(labelstr)-1:
183 |                             raise MaxLabelValueError
184 |                         inc = range_ld[0]
185 |                     else:
186 |                         increased = True
187 |                     s.append(inc)
188 |                 else:
189 |                     # in the middle
190 |                     inc = self._range_next(range_ldh, c)
191 |                     if inc == -1:
192 |                         inc = range_ldh[0]
193 |                     else:
194 |                         increased = True
195 |                     s.append(inc)
196 |             else:
197 |                 s.append(c)
198 | 
199 |         return struct.pack('B'*len(s), *reversed(s))
200 | 
201 |     def wire_length(self):
202 |         return 1 + len(self.label)
203 | 
204 |     def _canonicalize(self):
205 |         # don't use locale-aware lowercase function, we only want
206 |         # to convert the ASCII characters
207 |         # since label is a bytes object, this will only convert ASCII characters
208 |         self.label = self.label.lower()
209 | 
210 |     def to_wire(self):
211 |         # see RFC1035, section 3.1 "Name space definitions" for more info
212 |         return bytes([len(self.label)]) + self.label
213 | 
214 |     def __lt__(self, other):
215 |         return self.label < other.label
216 | 
217 |     def __eq__(self, other):
218 |         return self.label == other.label
219 | 
220 |     def __str__(self):
221 |         return vis.strvis(self.label).decode("ascii")
222 | 
223 | 
224 | @functools.total_ordering
225 | class DomainName(object):
226 |     def __init__(self, *labels):
227 |         if len(labels) == 0:
228 |             raise InvalidDomainNameError('no label specified')
229 |         self.labels = list(labels)
230 |         if self.wire_length() > MAX_DOMAINNAME:
231 |             raise MaxDomainNameLengthError
232 | 
233 |     def wire_length(self):
234 |         return sum([l.wire_length() for l in self.labels])
235 | 
236 |     def next_label_add(self, ldh):
237 |         lbls = self.labels[:]
238 |         if ldh:
239 |             return DomainName(_label_ldh(), *lbls)
240 |         else:
241 |             return DomainName(_label_binary(), *lbls)
242 | 
243 |     def next_extend_increase(self, ldh):
244 |         lbls = self.labels[:]
245 |         increased = False
246 |         newlabels = []
247 |         extend = False
248 |         if MAX_DOMAINNAME > self.wire_length() + 1:
249 |             extend = True
250 |         for label in lbls:
251 |             if not increased:
252 |                 try:
253 |                     newlabels.append(label.forward_next(ldh, extend))
254 |                     increased = True
255 |                 except (MaxLabelLengthError, MaxLabelValueError):
256 |                     newlabels.append(label)
257 |             else:
258 |                 newlabels.append(label)
259 |         if not increased:
260 |             raise MaxDomainNameLengthError(('cannot increase domain name'))
261 |         return DomainName(*newlabels)
262 | 
263 |     def covered_by(self, owner, next_owner):
264 |         if owner >= next_owner:
265 |             # last NSEC[3] record
266 |             return (self >= owner or self <= next_owner)
267 |         return (self >= owner and self <= next_owner)
268 | 
269 |     def covered_by_exclusive(self, owner, next_owner):
270 |         if owner >= next_owner:
271 |             return (self > owner or self < next_owner)
272 |         return (self > owner and self < next_owner)
273 | 
274 |     def num_labels(self):
275 |         return len(self.labels)
276 | 
277 |     # this really checks if self is below (or equal) to zone
278 |     def part_of_zone(self, zone):
279 |         if len(self.labels) >= len(zone.labels):
280 |             for l in zip(reversed(zone.labels), reversed(self.labels)):
281 |                 if l[0] != l[1]:
282 |                     return False
283 |             return True
284 |         return False
285 | 
286 |     def split(self, position):
287 |         first_labels = []
288 |         second_labels = []
289 |         i = 0
290 |         labels = first_labels
291 |         for l in self.labels:
292 |             if i == position:
293 |                 labels = second_labels
294 |             labels.append(l)
295 |             i += 1
296 |         return (DomainName(*first_labels), DomainName(*second_labels))
297 | 
298 |     def __lt__(self, other):
299 |         s = self.labels[:]
300 |         o = other.labels[:]
301 |         s.reverse()
302 |         o.reverse()
303 |         for i in range(0, max((len(s), len(o)))):
304 |             if i >= len(s):
305 |                 return True
306 |             if i >= len(o):
307 |                 return False
308 |             if s[i] < o[i]:
309 |                 return True
310 |             elif s[i] > o[i]:
311 |                 return False
312 |         return False
313 | 
314 |     def __eq__(self, other):
315 |         s = self.labels[:]
316 |         o = other.labels[:]
317 |         s.reverse()
318 |         o.reverse()
319 |         for i in range(0, max((len(s), len(o)))):
320 |             if i >= len(s):
321 |                 return False
322 |             if i >= len(o):
323 |                 return False
324 |             if s[i] != o[i]:
325 |                 return False
326 |         return True
327 | 
328 |     def is_root(self):
329 |         return (len(self.labels) == 1 and self.labels[0].label == b"")
330 | 
331 |     def to_wire(self):
332 |         # see RFC1035, section 3.1 "Name space definitions" for more info
333 |         wirelabels = bytearray()
334 |         for label in self.labels:
335 |             wirelabels += label.to_wire()
336 |         return bytes(wirelabels)
337 | 
338 |     def __str__(self):
339 |         if self.is_root():
340 |             return '.'
341 |         else:
342 |             return '.'.join(str(l) for l in self.labels)
343 | 
344 | 


--------------------------------------------------------------------------------
/n3map/nsec3chain.py:
--------------------------------------------------------------------------------
 1 | from . import log
 2 | from . import util
 3 | from .exception import ZoneChangedError
 4 | from .tree.nsec3tree import NSEC3Tree, OverLapError
 5 | from .rrtypes.nsec3 import SHA1_MAX
 6 | 
 7 | class NSEC3Chain(object):
 8 |     def __init__(self, iterable=None, ignore_overlapping=False):
 9 |         self.tree = NSEC3Tree(hash_max=SHA1_MAX)
10 |         self.salt = None
11 |         self.iterations = None
12 |         self.zone = None
13 |         self.tree.ignore_overlapping = ignore_overlapping
14 |         if iterable is not None:
15 |             for nsec3 in iterable:
16 |                 self.insert(nsec3)
17 | 
18 |     def _sortedvalues(self):
19 |         values = []
20 |         self.tree.inorder(lambda n: values.append(n.value))
21 |         return values
22 | 
23 |     def _check_salt(self, nsec3):
24 |         if self.salt is None:
25 |             self.salt = nsec3.salt
26 |             log.debug2("salt = 0x", self.salt.hex())
27 |         elif self.salt != nsec3.salt:
28 |             raise ZoneChangedError("NSEC3 salt changed")
29 |         else:
30 |             nsec3.salt = self.salt
31 | 
32 |     def _check_iterations(self, nsec3):
33 |         if self.iterations is None:
34 |             self.iterations = nsec3.iterations
35 |             log.debug2("number of iterations = ", self.iterations)
36 |         elif self.iterations != nsec3.iterations:
37 |             raise ZoneChangedError("NSEC3 number of iterations changed")
38 | 
39 |     def _check_zone(self, nsec3):
40 |         if self.zone is None:
41 |             self.zone = nsec3.zone
42 |         elif self.zone != nsec3.zone:
43 |                 raise ZoneChangedError("NSEC3 zone name changed")
44 |         else:
45 |             nsec3.zone = self.zone
46 | 
47 |     def insert(self, nsec3):
48 |         """Inserts an NSEC3 record into the tree.
49 | 
50 |         Returns True if the record didn't already exist in the tree, False
51 |         otherwise
52 |         """
53 |         self._check_zone(nsec3)
54 |         self._check_salt(nsec3)
55 |         self._check_iterations(nsec3)
56 | 
57 |         key = nsec3.hashed_owner
58 |         int_end = nsec3.next_hashed_owner
59 |         try:
60 |             new, was_updated = self.tree.insert(key, None, int_end)
61 |         except OverLapError:
62 |             raise ZoneChangedError("NSEC3 record overlaps with " +
63 |                     "another NSEC3 record")
64 |         return (not was_updated)
65 | 
66 |     def find_hash(self, h):
67 |         n = self.tree.find(h)
68 |         if n is None:
69 |             return None
70 |         else:
71 |             return n.value
72 | 
73 |     def covers(self, nsec3_hash):
74 |         return (self.tree.find_interval(nsec3_hash) is not None)
75 | 
76 |     def covers_zone(self):
77 |         return (self.tree.hash_max <= self.tree.covered_distance)
78 | 
79 |     def coverage(self):
80 |         return float(self.tree.covered_distance)/float(self.tree.hash_max)
81 | 
82 |     def size(self):
83 |         return self.tree.size()
84 | 
85 |     def get_list(self):
86 |         return self._sortedvalues()
87 | 
88 | 


--------------------------------------------------------------------------------
/n3map/nsec3hash.c:
--------------------------------------------------------------------------------
  1 | #define PY_SSIZE_T_CLEAN
  2 | #include <Python.h>
  3 | #include <openssl/evp.h>
  4 | 
  5 | struct hash_ctx {
  6 | 	int iterations;
  7 | 	Py_ssize_t salt_length;
  8 | 	const unsigned char *salt;
  9 | };
 10 | 
 11 | int compute_hash(const unsigned char *dn, unsigned int dn_length,
 12 | 		struct hash_ctx *ctx, unsigned char *result,
 13 | 		unsigned int *presult_len);
 14 | 
 15 | PyMODINIT_FUNC PyInit_nsec3hash(void);
 16 | static PyObject *py_compute_hash(PyObject *self, PyObject *args);
 17 | 
 18 | static PyMethodDef nsec3_methods[] = {
 19 | 	{"compute_hash", py_compute_hash, METH_VARARGS,
 20 | 		"compute an NSEC3 hash"},
 21 | 	{NULL, NULL, 0, NULL}
 22 | };
 23 | 
 24 | static struct PyModuleDef nsec3hash_module = {
 25 | 	PyModuleDef_HEAD_INIT,
 26 | 	"nsec3hash",
 27 | 	NULL,
 28 | 	-1,
 29 | 	nsec3_methods,
 30 | };
 31 | 
 32 | static PyObject *nsec3hash_error;
 33 | 
 34 | PyMODINIT_FUNC PyInit_nsec3hash(void)
 35 | {
 36 | 	PyObject *m;
 37 | 	m = PyModule_Create(&nsec3hash_module);
 38 | 	if (m == NULL) {
 39 | 		return NULL;
 40 | 	}
 41 | 
 42 | 	nsec3hash_error = PyErr_NewException("nsec3hash.error", NULL, NULL);
 43 | 	Py_XINCREF(nsec3hash_error);
 44 | 	if (PyModule_AddObject(m, "error", nsec3hash_error) < 0) {
 45 | 		Py_XDECREF(nsec3hash_error);
 46 | 		Py_CLEAR(nsec3hash_error);
 47 | 		Py_DECREF(m);
 48 | 		return NULL;
 49 | 	}
 50 | 
 51 | 	return m;
 52 | }
 53 | 
 54 | int compute_hash(const unsigned char *dn, unsigned int dn_length,
 55 | 		struct hash_ctx *ctx, unsigned char *result,
 56 | 		unsigned int *presult_len)
 57 | {
 58 | 	int i = 0;
 59 | 	int ret = -1;
 60 | 	EVP_MD_CTX *mdctx;
 61 | 
 62 | 	if ((mdctx = EVP_MD_CTX_new()) == NULL)
 63 | 		goto allocerror;
 64 | 	if (1 != EVP_DigestInit_ex(mdctx, EVP_sha1(), NULL))
 65 | 		goto error;
 66 | 	if (1 != EVP_DigestUpdate(mdctx, dn, dn_length))
 67 | 		goto error;
 68 | 	if (1 != EVP_DigestUpdate(mdctx, ctx->salt, ctx->salt_length))
 69 | 		goto error;
 70 | 	if (1 != EVP_DigestFinal_ex(mdctx, result, presult_len))
 71 | 		goto error;
 72 | 
 73 | 	while (i++ < ctx->iterations) {
 74 | 		if (1 != EVP_DigestInit_ex2(mdctx, NULL, NULL))
 75 | 			goto error;
 76 | 		if (1 != EVP_DigestUpdate(mdctx, result, *presult_len))
 77 | 			goto error;
 78 | 		if (1 != EVP_DigestUpdate(mdctx, ctx->salt, ctx->salt_length))
 79 | 			goto error;
 80 | 		if (1 != EVP_DigestFinal_ex(mdctx, result, presult_len))
 81 | 			goto error;
 82 | 	}
 83 | 
 84 | 	ret = 0;
 85 | 
 86 | error:
 87 | 	EVP_MD_CTX_free(mdctx);
 88 | allocerror:
 89 | 	return ret;
 90 | 
 91 | }
 92 | 
 93 | static PyObject *py_compute_hash(PyObject *self, PyObject *args)
 94 | {
 95 | 	struct hash_ctx ctx;
 96 | 	const unsigned char *dn;
 97 | 	Py_ssize_t dn_length;
 98 | 	unsigned char result[EVP_MAX_MD_SIZE];
 99 | 	unsigned int result_len;
100 | 
101 | 	/* dn, salt, iterations, result */
102 | 	if (!PyArg_ParseTuple(args, "y#y#i", &dn,
103 | 				&dn_length,
104 | 				&ctx.salt,
105 | 				&ctx.salt_length,
106 | 				&ctx.iterations))
107 | 		return NULL;
108 | 	if (-1 == compute_hash(dn, dn_length, &ctx, result, &result_len)) {
109 | 		PyErr_SetString(nsec3hash_error, "compute_hash() failed");
110 | 		return NULL;
111 | 	}
112 | 	return Py_BuildValue("y#", result, result_len);
113 | }
114 | 


--------------------------------------------------------------------------------
/n3map/nsec3lookup.py:
--------------------------------------------------------------------------------
  1 | import sys
  2 | import os
  3 | import getopt
  4 | 
  5 | from . import log
  6 | from . import rrfile
  7 | from . import rrtypes
  8 | from .exception import N3MapError, ZoneChangedError
  9 | import n3map.name
 10 | 
 11 | stats = {'queries': 0,
 12 |          'found' : 0}
 13 | 
 14 | def usage(argv):
 15 |     sys.stderr.write("usage: " + os.path.basename(argv[0]) + " file [-o outfile] [-z zone] [-v]\n")
 16 |     sys.exit(2)
 17 | 
 18 | def lookup_nsec3(nsec3_chain, salt, iterations,  zone, line, out):
 19 |     line = line.rstrip()
 20 |     if zone is None:
 21 |         dn = n3map.name.fqdn_from_text(line)
 22 |     else:
 23 |         if line == "":
 24 |             dn = zone
 25 |         else:
 26 |             owner = n3map.name.domainname_from_text(line)
 27 |             dn = n3map.name.DomainName(*(owner.labels + zone.labels))
 28 |     stats['queries'] += 1
 29 |     try:
 30 |         rr =  nsec3_chain[rrtypes.nsec3.compute_hash(dn, salt, iterations)]
 31 |     except KeyError:
 32 |         return;
 33 |     out.write(str(dn) + ": " + str(rr) + "\n")
 34 |     stats['found'] += 1
 35 | 
 36 | 
 37 | def nsec3lookup_main(argv):
 38 |     log.logger = log.Logger()
 39 |     out = None
 40 |     zone = None
 41 |     try:
 42 |         nsec3_chain = {}
 43 |         try:
 44 |             opts, args = getopt.gnu_getopt(argv[1:], "z:o:v")
 45 |         except getopt.GetoptError as err:
 46 |             usage(argv)
 47 |         for opt, arg in opts:
 48 |             if opt == '-z':
 49 |                 zone = n3map.name.fqdn_from_text(arg)
 50 |             if opt == '-o':
 51 |                 out = open(arg, "w")
 52 |             if opt == '-v':
 53 |                 log.logger.loglevel += 1
 54 | 
 55 |         if out is None:
 56 |             out = sys.stdout
 57 | 
 58 |         if len(args) < 1:
 59 |             usage(argv)
 60 | 
 61 |         records_file = rrfile.open_input_rrfile(args[0])
 62 |         salt = None
 63 |         iterations = None
 64 |         for nsec3 in records_file.nsec3_reader():
 65 |             if salt == None or iterations == None:
 66 |                 salt = nsec3.salt
 67 |                 iterations = nsec3.iterations
 68 |             elif salt != nsec3.salt or iterations != nsec3.iterations:
 69 |                 raise ZoneChangedError("zone salt or iterations not unique!")
 70 |             nsec3_chain[nsec3.hashed_owner] = nsec3;
 71 |         records_file.close()
 72 |         log.info("read {0:d} records. ready for input!".format(len(nsec3_chain)))
 73 | 
 74 |         if len(nsec3_chain) == 0:
 75 |             return 0
 76 |         if sys.stdin.isatty():
 77 |             try:
 78 |                 while True:
 79 |                     line = input()
 80 |                     lookup_nsec3(nsec3_chain, salt, iterations, zone, line, out)
 81 |             except (EOFError) as e:
 82 |                 pass
 83 |         else:
 84 |             for line in sys.stdin:
 85 |                 lookup_nsec3(nsec3_chain, salt, iterations, zone, line, out)
 86 | 
 87 |         log.info( "queries total = {0:d}\nhits = {1:d}".format(
 88 |             stats['queries'], stats['found']))
 89 | 
 90 |     except (IOError, N3MapError) as e:
 91 |         log.fatal(e)
 92 |     finally:
 93 |         if out is not None:
 94 |             out.close()
 95 | 
 96 | def main():
 97 |     try:
 98 |         sys.exit(nsec3lookup_main(sys.argv))
 99 |     except KeyboardInterrupt:
100 |         sys.stderr.write("\nreceived SIGINT, terminating\n")
101 |         sys.exit(3)
102 | 
103 | 


--------------------------------------------------------------------------------
/n3map/nsec3walker.py:
--------------------------------------------------------------------------------
  1 | import itertools
  2 | import secrets
  3 | 
  4 | from . import log
  5 | from . import name
  6 | from . import prehash
  7 | from . import util
  8 | from . import walker
  9 | 
 10 | from .queryprovider import create_aggressive_qp
 11 | 
 12 | from .statusline import format_statusline_nsec3
 13 | 
 14 | from .exception import N3MapError, NSEC3WalkError, HashLimitReached
 15 | from .nsec3chain import NSEC3Chain
 16 | 
 17 | 
 18 | class NSEC3Walker(walker.Walker):
 19 |     def __init__(self, zone, queryprovider, hash_queues, prehash_pool,
 20 |             nsec3_records, ignore_overlapping=False, label_counter=None,
 21 |             output_file=None, stats=None, predictor=None, aggressive=0,
 22 |             hashlimit=0):
 23 |         super(NSEC3Walker, self).__init__(zone, queryprovider, output_file, stats)
 24 |         self.stats['tested_hashes'] = 0
 25 |         self.hashlimit = hashlimit
 26 | 
 27 |         self._prediction_current = None
 28 |         if predictor is not None:
 29 |             self._predictor_proc,self._predictor_pipe = predictor
 30 |         else:
 31 |             self._predictor_proc = None
 32 | 
 33 |         self._write_chain(nsec3_records)
 34 |         self.nsec3_chain = NSEC3Chain(ignore_overlapping=ignore_overlapping)
 35 |         self._update_predictor_state()
 36 |         for rr in nsec3_records:
 37 |             self.nsec3_chain.insert(rr)
 38 |             self._update_predictor_state()
 39 | 
 40 |         self._prehash_processes = prehash_pool
 41 | 
 42 |         if label_counter is not None:
 43 |             log.debug2("setting initial label counter to 0x{0:x}".format(
 44 |                         label_counter))
 45 |             self._label_counter_init = label_counter
 46 |         elif len(nsec3_records) > 0:
 47 |             self._label_counter_init = secrets.randbits(60)
 48 |             log.warn("could not restore label counter from input file\n",
 49 |                     "picking an outrageous value at random instead: 0x{:x}"
 50 |                     .format(self._label_counter_init))
 51 |         else:
 52 |             self._label_counter_init = 0
 53 | 
 54 |         self._label_counter_state = 0
 55 |         self._hash_queues = itertools.cycle(hash_queues)
 56 |         self._reset_prehashing()
 57 |         self._aggressive = aggressive
 58 | 
 59 |     def _process_query_result(self, query_dn, res, ns):
 60 |         recv_nsec3 = res.find_NSEC3()
 61 |         if len(recv_nsec3) == 0:
 62 |             if res.status() == "NOERROR":
 63 |                 log.info("hit an existing owner name: ", str(query_dn))
 64 |                 ns.reset_errors()
 65 |                 return
 66 |             elif res.status() == 'NXDOMAIN':
 67 |                 log.error('no NSEC3 RR received\n',
 68 |                         "Maybe the zone doesn't support DNSSEC or uses NSEC RRs")
 69 |                 self.queryprovider.add_ns_error(ns)
 70 |                 return
 71 |             else:
 72 |                 log.error('unexpected response status: ', res.status(),
 73 |                         ' from ', str(ns))
 74 |                 self.queryprovider.add_ns_error(ns)
 75 |                 return
 76 |         ns.reset_errors()
 77 |         if not self._insert_records(recv_nsec3):
 78 |             log.warn("did not receive any new NSEC3 records for query: ",
 79 |                      str(query_dn))
 80 | 
 81 |     def _insert_records(self, recv_rr):
 82 |         got_new = False
 83 |         # TODO: check if records cover query name
 84 |         for rr in recv_rr:
 85 |             log.debug3('received NSEC3 RR: ', str(rr))
 86 |         for rr in recv_rr:
 87 |             if not rr.part_of_zone(self.zone):
 88 |                 log.warn('NSEC3 RR not part of zone: ', str(rr))
 89 |                 continue
 90 | 
 91 |             # check if the record is minimally-covering
 92 |             #  ref 'NSEC3 White Lies':
 93 |             #  https://tools.ietf.org/html/rfc7129#appendix-B
 94 |             if rr.distance_covered() == 2:
 95 |                 raise NSEC3WalkError('Received minimally-covering NSEC3 record\n',
 96 |                              'This zone likely uses "NSEC3 White Lies" to prevent zone enumeration\n',
 97 |                              '(See https://tools.ietf.org/html/rfc7129#appendix-B)')
 98 |             was_new = self.nsec3_chain.insert(rr)
 99 |             if was_new:
100 |                 got_new = True
101 |                 log.debug1("discovered: ", str(rr.owner), " ",
102 |                         ' '.join(rr.types))
103 |                 self._write_record(rr)
104 |                 self._update_predictor_state()
105 |         return got_new
106 | 
107 |     def _map_aggressive(self):
108 |         queries = {}
109 |         max_queries = self._aggressive
110 |         oldqp = self.queryprovider
111 |         self.queryprovider = create_aggressive_qp(self.queryprovider,
112 |                                                   max_queries)
113 |         try:
114 |             while not self.nsec3_chain.covers_zone():
115 |                 num_queries = len(queries)
116 |                 query_dn,dn_hash = self._find_uncovered_dn(num_queries > 0)
117 |                 results = self.queryprovider.collectresponses(
118 |                         block=(num_queries >= max_queries))
119 |                 for qid, (res, ns) in results:
120 |                     self._process_query_result(queries.pop(qid),res, ns)
121 |                 if query_dn is None or self.nsec3_chain.covers(dn_hash):
122 |                     continue
123 |                 queries[self.queryprovider.query_ff(query_dn, rrtype='A')] = query_dn
124 |         finally:
125 |             self.queryprovider.stop()
126 |             self.queryprovider = oldqp
127 | 
128 |     def _map_normal(self):
129 |         while not self.nsec3_chain.covers_zone():
130 |             query_dn,dn_hash = self._find_uncovered_dn()
131 |             result, ns = self.queryprovider.query(query_dn, rrtype='A')
132 |             self._process_query_result(query_dn, result, ns)
133 | 
134 |     def _map_zone(self):
135 |         generator = name.label_generator(name.hex_label, self._label_counter_init)
136 |         while self.nsec3_chain.size() == 0:
137 |             query_dn = name.DomainName(next(generator)[0], *self.zone.labels)
138 |             res, ns = self.queryprovider.query(query_dn, rrtype='A')
139 |             self._process_query_result(query_dn, res, ns)
140 |             self._label_counter_init += 1
141 |         self._start_prehashing()
142 |         if self._aggressive > 0:
143 |             self._map_aggressive()
144 |         else:
145 |             self._map_normal()
146 | 
147 |         self._write_number_of_records(self.nsec3_chain.size())
148 |         self._stop_prehashing()
149 |         self._stop_predictor()
150 | 
151 | 
152 |     def walk(self):
153 |         log.info("starting NSEC3 enumeration...")
154 |         if self.hashlimit > 0:
155 |             log.info("will stop after checking ~{} hashes"
156 |                      .format(self.hashlimit))
157 |         self._set_status_generator()
158 |         try:
159 |             self._map_zone()
160 |         except (KeyboardInterrupt, N3MapError) as e:
161 |             if self._output_file is not None:
162 |                 self._output_file.write_label_counter(self._label_counter_state)
163 |             self._stop_prehashing()
164 |             self._stop_predictor()
165 |             raise e
166 |         finally:
167 |             log.update()
168 |             log.logger.set_status_generator(None, None)
169 | 
170 |         return self.nsec3_chain
171 | 
172 | 
173 |     def _find_uncovered_dn(self, break_early=False):
174 |         is_covered = self.nsec3_chain.covers
175 |         while True:
176 |             for ptlabel,dn_hash in self._prehash_iter:
177 |                 if not is_covered(dn_hash):
178 |                     dn = name.DomainName(name.Label(ptlabel), *self.zone.labels)
179 |                     owner_b32 = util.base32_ext_hex_encode( dn_hash).lower()
180 |                     hashed_dn = name.DomainName( name.Label(owner_b32), *self.zone.labels)
181 |                     log.debug3('found uncovered dn: ', str(dn), '; hashed: ', str(hashed_dn))
182 |                     return dn,dn_hash
183 | 
184 |             self.stats['tested_hashes'] += len(self._prehash_list)
185 |             if (self.hashlimit > 0 and
186 |                     self.stats['tested_hashes'] >= self.hashlimit):
187 |                 raise HashLimitReached
188 |             hashes, label_counter_state = next(self._hash_queues).recv()
189 |             if self._label_counter_state < label_counter_state:
190 |                 self._label_counter_state = label_counter_state
191 |             self._prehash_list = hashes
192 |             self._prehash_iter = iter(hashes)
193 |             log.update()
194 |             if break_early:
195 |                 return None,None
196 | 
197 | 
198 |     def _start_prehashing(self):
199 |         for pipe, proc in self._prehash_processes:
200 |             pipe.send((self._label_counter_init, self.zone,
201 |                 self.nsec3_chain.salt, self.nsec3_chain.iterations))
202 |         self._prehash_started = True
203 | 
204 |     def _reset_prehashing(self):
205 |         self._prehash_list = []
206 |         self._prehash_iter = iter(self._prehash_list)
207 |         self._prehash_started = False
208 | 
209 |     def _stop_prehashing(self):
210 |         for pipe, proc in self._prehash_processes:
211 |             proc.terminate()
212 |         self._reset_prehashing()
213 | 
214 |     def _stop_predictor(self):
215 |         if self._predictor_proc is not None:
216 |             self._predictor_proc.terminate()
217 | 
218 |     def _update_predictor_state(self):
219 |         if self._predictor_proc is not None:
220 |             self._predictor_pipe.send((self.nsec3_chain.coverage(),
221 |                                        self.nsec3_chain.size()))
222 |             if self._predictor_pipe.poll():
223 |                 self._prediction_current = self._predictor_pipe.recv()
224 | 
225 |     def _set_status_generator(self):
226 |         def status_generator():
227 |             return (str(self.zone),
228 |                     self.stats['queries'],
229 |                     self.nsec3_chain.size(),
230 |                     self.stats['tested_hashes'],
231 |                     self.nsec3_chain.coverage(),
232 |                     self.queryprovider.query_rate(),
233 |                     self._prediction_current
234 |                 )
235 |         log.logger.set_status_generator(status_generator, format_statusline_nsec3)
236 | 
237 | 
238 | 


--------------------------------------------------------------------------------
/n3map/nsecwalker.py:
--------------------------------------------------------------------------------
  1 | import itertools
  2 | import enum
  3 | 
  4 | from . import log
  5 | from . import name
  6 | from . import walker
  7 | 
  8 | from .exception import N3MapError
  9 | 
 10 | from .statusline import format_statusline_nsec
 11 | 
 12 | from .exception import (
 13 |         MaxDomainNameLengthError,
 14 |         MaxDomainNameLengthError,
 15 |         NSECWalkError
 16 |     )
 17 | 
 18 | 
 19 | class ResultStatus(enum.Enum):
 20 |     OK        = enum.auto()
 21 |     ERROR     = enum.auto()
 22 |     SUBZONE   = enum.auto()
 23 |     HITOWNER  = enum.auto()
 24 | 
 25 | class NSECResult:
 26 |     def __init__(self, walk_zone, query_dn, query_type, queryresult, ns):
 27 |         self.walk_zone = walk_zone
 28 |         self.query_dn = query_dn
 29 |         self.query_type = query_type
 30 |         self.queryresult = queryresult
 31 |         self.ns = ns
 32 | 
 33 |     def log_NSEC_rrs(self):
 34 |         for nsec in self.all_NSEC_rrs():
 35 |             log.debug3('received NSEC RR: ' + str(nsec))
 36 |             if not nsec.part_of_zone(self.walk_zone):
 37 |                 log.warn("received invalid NSEC RR, not part of zone: ",
 38 |                          str(nsec))
 39 | 
 40 |     def _find_RRSIG_signer(self, owner, type_covered):
 41 |         signer = self.queryresult.find_RRSIG_signer(owner, type_covered, False)
 42 |         if signer is not None:
 43 |             return signer
 44 |         return self.queryresult.find_RRSIG_signer(owner, type_covered, True)
 45 | 
 46 |     def _RRSIG_signer_matches_zone(self, name, rrtype):
 47 |         signer = self._find_RRSIG_signer(name, rrtype)
 48 |         return signer is not None and signer == self.walk_zone
 49 | 
 50 |     def all_NSEC_rrs(self):
 51 |         return self.queryresult.all_NSEC_rrs()
 52 | 
 53 |     def num_NSEC_rrs(self):
 54 |         return sum(1 for _ in self.all_NSEC_rrs())
 55 | 
 56 |     def find_covering_nsec(self, check_signer=True, inclusive=True):
 57 |         covering_nsec = None
 58 |         for nsec in self.all_NSEC_rrs():
 59 |             if not nsec.part_of_zone(self.walk_zone):
 60 |                 continue
 61 | 
 62 |             if check_signer and not self._RRSIG_signer_matches_zone(
 63 |                     nsec.owner, 'NSEC'):
 64 |                 continue
 65 | 
 66 |             if ((inclusive and nsec.covers(self.query_dn)) or
 67 |                     (not inclusive and nsec.covers_exclusive(self.query_dn))
 68 |                     or nsec.next_owner == self.walk_zone):
 69 |                 covering_nsec = nsec
 70 |                 break
 71 |         return covering_nsec
 72 | 
 73 |     def status(self):
 74 |         return self.queryresult.status()
 75 | 
 76 |     def _detect_subdomain_soa(self):
 77 |         soa_owner = self.queryresult.find_SOA(in_answer=False)
 78 |         if (soa_owner is not None and soa_owner != self.walk_zone
 79 |                 and soa_owner.part_of_zone(self.walk_zone)):
 80 |             log.debug1("subdomain SOA RR received: ", str(soa_owner))
 81 |             return soa_owner
 82 |         return None
 83 | 
 84 |     def _detect_subdomain_ns(self):
 85 |         ns_owner = self.queryresult.find_NS(in_answer=False)
 86 |         if (ns_owner is not None and ns_owner != self.walk_zone
 87 |                 and ns_owner.part_of_zone(self.walk_zone)):
 88 |             log.debug1("subdomain NS RR received: ", str(ns_owner))
 89 |             return ns_owner
 90 |         return None
 91 | 
 92 |     def _detect_subdomain_auth(self):
 93 |         # check for NS or SOA records in authority
 94 |         ns_owner = self._detect_subdomain_ns()
 95 |         if ns_owner is not None:
 96 |             log.warn("walked into a sub-zone at ", str(self.query_dn),
 97 |                      " (subdomain NS received)")
 98 |             return ns_owner
 99 |         soa_owner = self._detect_subdomain_soa()
100 |         if soa_owner is not None:
101 |             log.warn("walked into a sub-zone at ", str(self.query_dn),
102 |                      " (subdomain SOA received)")
103 |             return soa_owner
104 |         return None
105 | 
106 | 
107 |     def _extract_from_NSEC_query(self):
108 |         nsec = self.find_covering_nsec()
109 |         if nsec is not None:
110 |             return (ResultStatus.OK, nsec, None)
111 | 
112 |         nsec = self.find_covering_nsec(check_signer=False)
113 |         if nsec is not None:
114 |             # got NSEC record, but RRSIG signer doesn't match zone
115 |             log.warn("walked into a sub-zone at ", str(self.query_dn),
116 |                      " (RRSIG signer for NSEC RR does not match zone)")
117 |             return (ResultStatus.SUBZONE, nsec,
118 |                     self._find_RRSIG_signer(nsec.owner, 'NSEC'))
119 | 
120 |         # check for NS or SOA records in authority section
121 |         sub_owner = self._detect_subdomain_auth()
122 |         if sub_owner is not None:
123 |             return (ResultStatus.SUBZONE, None, sub_owner)
124 | 
125 |         log.error("no covering NSEC RR received for domain name ",
126 |                 str(self.query_dn))
127 |         return (ResultStatus.ERROR, None, None)
128 | 
129 |     def _extract_from_A_query(self):
130 |         if self.status() == 'NXDOMAIN':
131 |             nsec = self.find_covering_nsec(inclusive=False)
132 |             if nsec is not None:
133 |                 return (ResultStatus.OK, nsec, None)
134 | 
135 |             nsec = self.find_covering_nsec(check_signer=False, inclusive=False)
136 |             if nsec is not None:
137 |                 # got NSEC record, but RRSIG signer doesn't match zone
138 |                 log.warn("walked into a sub-zone at ", str(self.query_dn),
139 |                          " (RRSIG signer for NSEC RR does not match zone)")
140 |                 return (ResultStatus.SUBZONE, nsec,
141 |                         self._find_RRSIG_signer(nsec.owner, 'NSEC'))
142 | 
143 |             # NXDOMAIN but no NSEC
144 | 
145 |             # check for NS or SOA records in authority section
146 |             sub_owner = self._detect_subdomain_auth()
147 |             if sub_owner is not None:
148 |                 return (ResultStatus.SUBZONE, None, sub_owner)
149 | 
150 |             log.error("no covering NSEC RR received in NXDOMAIN response for ",
151 |                       str(self.query_dn))
152 |             return (ResultStatus.ERROR, None, None)
153 | 
154 |         elif self.status() == 'NOERROR':
155 |             if self.queryresult.answer_length() > 0:
156 |                 log.warn("hit an existing owner name: ", str(self.query_dn))
157 |                 signer = self._find_RRSIG_signer(self.query_dn, self.query_type)
158 |                 if signer is None:
159 |                     log.warn("walked into a sub-zone at ", str(self.query_dn),
160 |                              " (no RRSIG found)")
161 |                     return (ResultStatus.SUBZONE, None, None)
162 |                 if signer != self.walk_zone:
163 |                     log.warn("walked into a sub-zone at ", str(self.query_dn),
164 |                              " (RRSIG signer does not match zone)")
165 |                     return (ResultStatus.SUBZONE, None, signer)
166 |                 # part of this zone
167 | 
168 |                 # check for NSEC records anyway. This can happen e.g. if the
169 |                 # owner name we hit was actually a wildcard
170 |                 # FIXME: wildcards could probably be handled more explicitly
171 |                 nsec = self.find_covering_nsec(inclusive=False)
172 |                 if nsec is not None:
173 |                     return (ResultStatus.OK, nsec, None)
174 | 
175 |                 return (ResultStatus.HITOWNER, None, None)
176 |             # this happens e.g. when the query name with added label
177 |             # (usually \x00) in front is part of a zone delegated to a
178 |             # (possibly different) nameserver
179 | 
180 |             # check for NS or SOA records in authority section
181 |             # this check is just to provide better feedback,
182 |             # we'll treat this as a sub-zone in any case
183 |             sub_owner = self._detect_subdomain_auth()
184 |             if sub_owner is not None:
185 |                 return (ResultStatus.SUBZONE, None, sub_owner)
186 | 
187 |             log.warn("got NOERROR response but no RRs for owner: ",
188 |                      str(self.query_dn), ", looks like a sub-zone")
189 |             return (ResultStatus.SUBZONE, None, None)
190 | 
191 |         # this should never happen as anything other than 'NXDOMAIN' or
192 |         # 'NOERROR' already causes an error in queryprovider
193 |         log.error('unexpected response status: ', str(self.status()))
194 |         return (ResultStatus.ERROR, None, None)
195 | 
196 |     def extract(self):
197 |         if self.query_type == 'NSEC':
198 |             return self._extract_from_NSEC_query()
199 |         # non-NSEC (A) query
200 |         return self._extract_from_A_query()
201 | 
202 | class NSECWalker(walker.Walker):
203 |     def __init__(self, zone, queryprovider, nsec_chain=None, startname=None,
204 |             endname=None, output_file=None, stats=None):
205 |         super(NSECWalker, self).__init__(zone, queryprovider, output_file,
206 |                 stats)
207 |         if nsec_chain is not None:
208 |             self.nsec_chain = list(sorted(nsec_chain, key=lambda x: x.owner))
209 |             self._write_chain(self.nsec_chain)
210 |         else:
211 |             self.nsec_chain = []
212 |         self.start, self.end = self._get_start_end(startname, endname)
213 | 
214 |     def _query(self, query_dn, rrtype='A'):
215 |         if not query_dn.part_of_zone(self.zone):
216 |             raise NSECWalkError('query_dn not part of zone!')
217 |         result, ns = self.queryprovider.query(query_dn, rrtype)
218 |         nresult = NSECResult(self.zone, query_dn, rrtype, result, ns)
219 |         nresult.log_NSEC_rrs()
220 |         return nresult
221 | 
222 | 
223 |     def walk(self):
224 |         self._set_status_generator()
225 |         try:
226 |             nsec_chain= self._walk_zone()
227 |             self._write_number_of_records(len(nsec_chain))
228 |             return nsec_chain
229 |         except (KeyboardInterrupt, N3MapError) as e:
230 |             raise e
231 |         finally:
232 |             log.logger.set_status_generator(None,None)
233 | 
234 |     def _append_covering_record(self, covering_nsec):
235 |         log.debug2('covering NSEC RR found: ', str(covering_nsec))
236 | 
237 |         self._write_record(covering_nsec)
238 | 
239 |         if (covering_nsec.owner > covering_nsec.next_owner and
240 |                 covering_nsec.next_owner != self.zone):
241 |             raise NSECWalkError('NSEC owner > next_owner, ',
242 |                     'but next_owner != zone')
243 | 
244 |         self.nsec_chain.append(covering_nsec)
245 |         log.debug1('discovered owner: ', str(covering_nsec.owner),
246 |                 "\t", ' '.join(covering_nsec.types))
247 |         log.update()
248 | 
249 | 
250 |     def _no_NSEC_error(self, ns):
251 |         return ("no NSEC RR received\n" +
252 |                "Maybe the zone doesn't support DNSSEC or uses NSEC3 RRs\n")
253 | 
254 |     def _walk_zone(self):
255 |         raise NotImplementedError
256 | 
257 |     def _finished(self, dname):
258 |         return (((dname is not None and dname == self.zone) or (self.end is not
259 |             None and dname >= self.end)) and len(self.nsec_chain) > 0)
260 | 
261 | 
262 |     def _get_start(self, startname):
263 |         if len(self.nsec_chain) > 0:
264 |             return self.nsec_chain[-1].next_owner
265 | 
266 |         if startname is None:
267 |             return self.zone
268 |         else:
269 |             return name.DomainName(
270 |                     *(name.domainname_from_text(startname).labels +
271 |                         self.zone.labels))
272 | 
273 |     def _get_end(self, endname):
274 |         if endname is None:
275 |             end = None
276 |         else:
277 |             end = name.DomainName(
278 |                     *(name.domainname_from_text(endname).labels +
279 |                         self.zone.labels))
280 |         return end
281 | 
282 |     def _get_start_end(self, startname, endname):
283 |         start = self._get_start(startname)
284 |         end = self._get_end(endname)
285 |         if end is not None:
286 |             if start >= end:
287 |                 raise NSECWalkError("invalid start / endpoint specified")
288 | 
289 |         return (start, end)
290 | 
291 |     def _set_status_generator(self):
292 |         def status_generator():
293 |             return (str(self.zone),
294 |                     self.stats['queries'],
295 |                     len(self.nsec_chain),
296 |                     self.queryprovider.query_rate()
297 |                 )
298 |         log.logger.set_status_generator(status_generator, format_statusline_nsec)
299 | 
300 | 
301 | class NSECWalkerN(NSECWalker):
302 |     def __init__(self, zone, queryprovider, nsec_chain=None, startname=None,
303 |             endname=None, output_file=None, stats=None):
304 |         super(NSECWalkerN, self).__init__(zone, queryprovider, nsec_chain,
305 |                 startname, endname, output_file, stats)
306 | 
307 |     def walk(self):
308 |         log.info("starting enumeration in NSEC query mode...")
309 |         return super(NSECWalkerN,self).walk()
310 | 
311 |     def _walk_zone(self):
312 |         dname = self.start
313 |         covering_nsec = None
314 |         while not self._finished(dname):
315 |             nresult = self._query(dname, rrtype='NSEC')
316 |             (status, covering_nsec, subzone) = nresult.extract()
317 |             if status == ResultStatus.ERROR:
318 |                 if nresult.num_NSEC_rrs() == 0:
319 |                     log.error(self._no_NSEC_error(nresult.ns))
320 |                 self.queryprovider.add_ns_error(nresult.ns)
321 |                 continue
322 |             elif status == ResultStatus.SUBZONE:
323 |                 if covering_nsec is not None:
324 |                     # we write this record down anyway
325 |                     self._append_covering_record(covering_nsec)
326 |                 raise NSECWalkError('walked into subzone at: ', str(dname),
327 |                         "\ndon't know how to continue enumeration.\n",
328 |                         "Try using 'mixed' or 'A' query mode instead.")
329 |             elif status == ResultStatus.OK:
330 |                 nresult.ns.reset_errors()
331 |             else:
332 |                 # in case we ever extend ResultStatus
333 |                 raise N3MapError(
334 |                         "Unexpected ResultStatus. This should never happen")
335 | 
336 |             # status == OK:
337 |             self._append_covering_record(covering_nsec)
338 |             log.debug2("next in chain: ", str(covering_nsec.next_owner))
339 |             dname = covering_nsec.next_owner
340 | 
341 |         return self.nsec_chain
342 | 
343 |     def _no_NSEC_error(self, ns):
344 |         return (super()._no_NSEC_error(ns) +
345 |                 "or the server {} does not allow NSEC queries.\n".format(ns) +
346 |                 "Perhaps try using --query-mode=A")
347 | 
348 | class NSECWalkerA(NSECWalker):
349 |     def __init__(self, zone, queryprovider, ldh = False, nsec_chain=None,
350 |             startname=None, endname=None, output_file=None, stats=None,
351 |                  never_prefix_label=False):
352 |         super(NSECWalkerA, self).__init__(zone, queryprovider, nsec_chain,
353 |                 startname, endname, output_file, stats)
354 |         self.ldh = ldh
355 |         self._never_prefix_label = never_prefix_label
356 | 
357 |     def walk(self):
358 |         log.info("starting enumeration in A query mode...")
359 |         return super(NSECWalkerA,self).walk()
360 | 
361 |     def _increase_dn_next_step(self, dname):
362 |         if self._never_prefix_label:
363 |             return dname
364 |         return self._next_dn_extend_increase(dname)
365 | 
366 | 
367 |     def _skip_subzone(self, dname, query_dn, subzone):
368 |         if dname == self.zone:
369 |             log.warn("trying to skip sub-zone ", str(query_dn))
370 |             return self._increase_dn_next_step(query_dn)
371 | 
372 |         if (subzone is not None
373 |                 and subzone.num_labels() <= dname.num_labels()):
374 |             # if we know the subzone, we can move on from there
375 |             log.debug1("learned sub-zone from response: ",
376 |                        str(subzone))
377 |             log.warn("trying to skip confirmed sub-zone ", str(subzone))
378 |             return self._increase_dn_next_step(subzone)
379 | 
380 |         if dname.num_labels() > self.zone.num_labels() + 1:
381 |             (_, dname) = dname.split(1)
382 |             log.warn("could not learn sub-zone name from response,",
383 |                      " skipping ", str(dname))
384 |             return self._increase_dn_next_step(dname)
385 | 
386 |         log.warn("trying to skip sub-zone ", str(dname))
387 |         return self._increase_dn_next_step(dname)
388 | 
389 | 
390 |     def _extract_next_NSEC_a(self, dname):
391 |         while not self._finished(dname):
392 |             if self._never_prefix_label and dname != self.zone:
393 |                 query_dn = self._next_dn_extend_increase(dname)
394 |             else:
395 |                 query_dn = self._next_dn_label_add(dname)
396 |             nresult = self._query(query_dn, rrtype='A')
397 |             (status, covering_nsec, subzone) = nresult.extract()
398 |             if status == ResultStatus.ERROR:
399 |                 if query_dn.num_labels() == self.zone.num_labels() + 1:
400 |                     # this query_dn *has* to be part of the zone and yet we got
401 |                     # nothing: some server error
402 |                     if nresult.num_NSEC_rrs() == 0:
403 |                         log.error(self._no_NSEC_error(nresult.ns))
404 |                     self.queryprovider.add_ns_error(nresult.ns)
405 |                     continue
406 | 
407 |                 dname = self._skip_subzone(dname, query_dn, None)
408 |                 continue
409 |             elif status == ResultStatus.SUBZONE:
410 |                 nresult.ns.reset_errors()
411 |                 if covering_nsec is not None:
412 |                     # we write this record down anyway
413 |                     self._append_covering_record(covering_nsec)
414 |                 dname = self._skip_subzone(dname, query_dn, subzone)
415 |                 continue
416 |             elif status == ResultStatus.HITOWNER:
417 |                 # hit an existing name, but it is part of this zone
418 |                 nresult.ns.reset_errors()
419 |                 # add or increase label in nextg iteration
420 |                 dname = query_dn
421 |                 continue
422 |             elif status == ResultStatus.OK:
423 |                 nresult.ns.reset_errors()
424 |             else:
425 |                 # in case we ever extend ResultStatus
426 |                 raise N3MapError(
427 |                         "Unexpected ResultStatus. This should never happen")
428 | 
429 |             # status == OK:
430 |             # at this point we have our next record
431 |             return (covering_nsec, dname)
432 |         return (None, dname)
433 | 
434 |     def _walk_zone(self):
435 |         dname = self.start
436 |         covering_nsec = None
437 |         while not self._finished(dname):
438 |             covering_nsec, dname = self._extract_next_NSEC_a(dname)
439 |             if covering_nsec is None:
440 |                 # only happens when self._finished(dname) == True
441 |                 break
442 | 
443 |             self._append_covering_record(covering_nsec)
444 |             log.debug2("next in chain: ", str(covering_nsec.next_owner))
445 |             dname = covering_nsec.next_owner
446 | 
447 |         return self.nsec_chain
448 | 
449 |     def _next_dn_label_add(self, dname):
450 |         try:
451 |             query_dn = dname.next_label_add(self.ldh)
452 |         except MaxDomainNameLengthError:
453 |             query_dn = self._next_dn_extend_increase(dname)
454 | 
455 |         self._check_query_dn(query_dn)
456 |         return query_dn
457 | 
458 |     def _next_dn_extend_increase(self, dname):
459 |         try:
460 |             query_dn = dname.next_extend_increase(self.ldh)
461 |         except MaxDomainNameLengthError as e:
462 |             raise NSECWalkError(str(e))
463 |         self._check_query_dn(query_dn)
464 |         return query_dn
465 | 
466 |     def _check_query_dn(self, query_dn):
467 |         if not query_dn.part_of_zone(self.zone):
468 |             raise NSECWalkError('unable to increase ' +
469 |                     'domain name any more.')
470 | 
471 | class NSECWalkerMixed(NSECWalkerA):
472 | 
473 |     def walk(self):
474 |         log.info("starting enumeration in mixed query mode...")
475 |         return NSECWalker.walk(self)
476 | 
477 |     def _walk_zone(self):
478 |         dname = self.start
479 |         covering_nsec = None
480 |         while not self._finished(dname):
481 |             nresult = self._query(dname, rrtype='NSEC')
482 |             (status, covering_nsec, subzone) = nresult.extract()
483 |             if status == ResultStatus.ERROR:
484 |                 if nresult.num_NSEC_rrs() == 0:
485 |                     log.error(self._no_NSEC_error(nresult.ns))
486 |                 self.queryprovider.add_ns_error(nresult.ns)
487 |                 continue
488 |             elif status == ResultStatus.SUBZONE:
489 |                 if covering_nsec is not None:
490 |                     # we write this record down anyway
491 |                     self._append_covering_record(covering_nsec)
492 |                 nresult.ns.reset_errors()
493 |                 # try to skip subzone using 'A' queries
494 |                 log.warn("trying to skip sub-zone at ", str(dname),
495 |                          " using 'A' queries")
496 |                 if dname != self.zone and not self._never_prefix_label:
497 |                     dname = self._next_dn_extend_increase(dname)
498 |                 (covering_nsec, dname) = self._extract_next_NSEC_a(dname)
499 |                 if covering_nsec is None:
500 |                     # finished
501 |                     break
502 |                 log.debug2("continuing in mixed mode...")
503 |             elif status == ResultStatus.OK:
504 |                 nresult.ns.reset_errors()
505 |             else:
506 |                 # in case we ever extend ResultStatus
507 |                 raise N3MapError(
508 |                         "Unexpected ResultStatus. This should never happen")
509 | 
510 |             # got our next record
511 | 
512 |             self._append_covering_record(covering_nsec)
513 |             log.debug2("next in chain: ", str(covering_nsec.next_owner))
514 |             dname = covering_nsec.next_owner
515 | 
516 |         return self.nsec_chain
517 | 


--------------------------------------------------------------------------------
/n3map/predict.py:
--------------------------------------------------------------------------------
  1 | import os
  2 | import gc
  3 | import sys
  4 | import multiprocessing
  5 | import signal
  6 | import math
  7 | 
  8 | from . import log
  9 | 
 10 | from .exception import N3MapError
 11 | 
 12 | HAS_NUMPY = False
 13 | try:
 14 |     import numpy as np
 15 |     HAS_NUMPY = True
 16 | except ImportError:
 17 |     pass
 18 | HAS_SCIPY = False
 19 | try:
 20 |     from scipy.optimize import leastsq
 21 |     HAS_SCIPY = True
 22 | except ImportError:
 23 |     pass
 24 | 
 25 | 
 26 | def np_func(p,x):
 27 |     a,b = p
 28 |     return b - np.sqrt(np.exp(a)*(np.ones(len(x))-x))
 29 | 
 30 | def np_dfunc(p,x,y):
 31 |     a,b = p
 32 |     return np.array([-0.5 * np.sqrt(-np.exp(a)*(x-np.ones(len(x)))),
 33 |             np.ones(len(x))])
 34 | 
 35 | def np_residuals(p,x,y):
 36 |     return np_func(p,x) - y
 37 | 
 38 | def compute_fit(params,xdata,ydata):
 39 |     res = [0,0]
 40 |     try:
 41 |         args = (xdata, ydata)
 42 |         res = leastsq(np_residuals,params, Dfun=np_dfunc,col_deriv=True, args=args)
 43 |     except (ValueError,OverflowError,ZeroDivisionError):
 44 |         pass
 45 |     return res[0]
 46 | 
 47 | def sample(data, n):
 48 |     length = float(len(data))
 49 |     return [data[i] for i in [int(math.ceil(j * length / n)) for j in range(n)]]
 50 | 
 51 | 
 52 | def create_zone_predictor():
 53 |     if not HAS_NUMPY:
 54 |         raise N3MapError("failed to start predictor: could not import numpy")
 55 |     if not HAS_SCIPY:
 56 |         raise N3MapError("failed to start predictor: could not import scipy")
 57 |     par,chld = multiprocessing.Pipe(True)
 58 |     proc = PredictorProcess(chld)
 59 |     proc.start()
 60 |     return proc,par
 61 | 
 62 | class PredictorProcess(multiprocessing.Process):
 63 |     def __init__ (self, pipe):
 64 |         multiprocessing.Process.__init__(self)
 65 |         self.daemon = True
 66 |         self.pipe = pipe
 67 |         self._coverage_data = []
 68 | 
 69 |     def run(self):
 70 |         try:
 71 |             signal.signal(signal.SIGINT, signal.SIG_IGN)
 72 |             # sometimes scipy spills warnings to stderr
 73 |             # redirect stdout,stderr to /dev/null
 74 |             nullfd = os.open(os.devnull,os.O_RDWR)
 75 |             os.dup2(nullfd,sys.stdout.fileno())
 76 |             os.dup2(nullfd,sys.stderr.fileno())
 77 | 
 78 |             os.nice(15)
 79 |             gc.collect()
 80 |             log.logger = None
 81 |             repredict_threshold = 20
 82 |             while True:
 83 |                 cov,rec = self.pipe.recv()
 84 |                 self._coverage_data.append((cov,rec))
 85 |                 for i in range(repredict_threshold):
 86 |                     if not self.pipe.poll():
 87 |                         break;
 88 |                     cov,rec = self.pipe.recv()
 89 |                     self._coverage_data.append((cov,rec))
 90 |                 size = self._predict_zone_size()
 91 |                 self.pipe.send(int(size))
 92 |         except EOFError:
 93 |             sys.exit(0)
 94 |         except KeyboardInterrupt:
 95 |             sys.exit(3)
 96 | 
 97 |     def _predict_zone_size(self):
 98 |         npts = len(self._coverage_data)
 99 |         if npts <= 1:
100 |             return 1e8
101 | 
102 |         sample_sz = 5
103 |         if npts < sample_sz:
104 |             sample_sz = npts
105 | 
106 |         subset = sample(self._coverage_data,sample_sz-1)
107 |         subset.append(self._coverage_data[-1])
108 | 
109 |         xdata,ydata = list(zip(*subset))
110 |         lastcov = xdata[-1]
111 |         if lastcov < 1e-8:
112 |             lastcov = 1e-8
113 |         binit = (1/lastcov*ydata[-1])
114 |         ainit = 2.0*math.log(binit)
115 |         a,b = compute_fit([ainit,binit],xdata,ydata)
116 |         current_records = self._coverage_data[-1][1]
117 |         return b if b >= current_records else current_records
118 | 


--------------------------------------------------------------------------------
/n3map/prehash.py:
--------------------------------------------------------------------------------
  1 | import gc
  2 | import multiprocessing
  3 | import os
  4 | import sys
  5 | 
  6 | from . import log
  7 | from . import rrtypes
  8 | from . import name
  9 | from .name import DomainName,Label
 10 | 
 11 | 
 12 | HAS_NSEC3HASH = False
 13 | try:
 14 |     from . import nsec3hash
 15 |     HAS_NSEC3HASH = True
 16 | except ImportError:
 17 |     pass
 18 | 
 19 | def _process_label_generator(label_fun, gap, process_id, num_processes, init=0):
 20 |     start = l = int(process_id*gap+init)
 21 |     end = start + gap
 22 |     while True:
 23 |         if l >= end:
 24 |             start += int(num_processes*gap)
 25 |             end = start + gap
 26 |             l = start
 27 |         lblstr = label_fun(l)
 28 |         yield (lblstr, l)
 29 |         l += 1
 30 | 
 31 | def create_prehash_pool(num_processes, element_size,
 32 |         use_cext):
 33 |     processes = []
 34 |     hash_queues = []
 35 |     for i in range(num_processes):
 36 |         par,chld = multiprocessing.Pipe(True)
 37 |         p = PreHashProcess(chld, element_size, i, name.hex_label,
 38 |                 num_processes, use_cext)
 39 |         p.start()
 40 |         processes.append((par,p))
 41 |         hash_queues.append(par)
 42 | 
 43 |     return hash_queues, processes
 44 | 
 45 | 
 46 | class PreHashProcess(multiprocessing.Process):
 47 |     def __init__ (self, pipe, element_size,
 48 |             process_id, label_fun, num_processes,  use_cext):
 49 |         multiprocessing.Process.__init__(self)
 50 |         # Kills this Process when parent exits
 51 |         self.daemon = True
 52 | 
 53 |         self.pipe = pipe
 54 |         self.id = process_id
 55 |         self.element_size = element_size
 56 |         self.use_cext = use_cext
 57 |         self.label_fun = label_fun
 58 |         self.num_processes = num_processes
 59 | 
 60 |         if self.use_cext and not HAS_NSEC3HASH:
 61 |             log.error("failed to import nsec3hash module, ",
 62 |                       "falling back to Python-based hashing\n",
 63 |                       "use --no-openssl to avoid printing this error")
 64 |             self.use_cext = False
 65 | 
 66 |         self.zone = None
 67 |         self.generator = None
 68 |         self.salt = None
 69 |         self.iterations = None
 70 | 
 71 |     def run(self):
 72 |         try:
 73 |             os.nice(15)
 74 |             gc.collect()
 75 |             log.logger = None
 76 |             (label_counter_init,  self.zone, self.salt,
 77 |                     self.iterations) = self.pipe.recv()
 78 |             self.generator = _process_label_generator(label_fun =
 79 |                     self.label_fun, gap = 1024, process_id = self.id,
 80 |                     num_processes = self.num_processes,
 81 |                     init = label_counter_init)
 82 |             if self.use_cext:
 83 |                 self._precompute_hashes(self._hash_cext)
 84 |             else:
 85 |                 self._precompute_hashes(self._hash)
 86 |         except KeyboardInterrupt:
 87 |             sys.exit(3)
 88 | 
 89 | 
 90 |     def _hash(self, dn):
 91 |         return rrtypes.nsec3.compute_hash(dn, self.salt,
 92 |                 self.iterations)
 93 | 
 94 |     def _hash_cext(self, dn):
 95 |         return nsec3hash.compute_hash(dn.to_wire(), self.salt,
 96 |                 self.iterations)
 97 | 
 98 |     def _precompute_hashes(self, hash_func):
 99 |         counter_state = 0
100 |         element_size = self.element_size
101 |         generator = self.generator
102 |         while True:
103 |             element = []
104 |             for i in range(element_size):
105 |                 ptlabel, counter_state = next(generator)
106 |                 dn = DomainName(Label(ptlabel), *self.zone.labels)
107 |                 hashed_owner =  hash_func(dn)
108 |                 element.append((ptlabel,hashed_owner))
109 | 
110 |             self.pipe.send((element, counter_state))
111 | 
112 | 


--------------------------------------------------------------------------------
/n3map/query.py:
--------------------------------------------------------------------------------
  1 | import struct
  2 | import itertools
  3 | 
  4 | import dns.resolver
  5 | import dns.exception
  6 | import dns.message
  7 | import dns.name
  8 | import dns.query
  9 | import dns.rcode
 10 | import dns.rdataclass
 11 | import dns.rdatatype
 12 | import dns.flags
 13 | 
 14 | from . import name
 15 | from .rrtypes import nsec3
 16 | from .rrtypes import nsec
 17 | from . import rrtypes
 18 | from . import exception
 19 | from . import log
 20 | 
 21 | def _rrtypes_from_window_list(window_list):
 22 |     # see RFC 3845, section 2.1.2 "The List of Type Bit Map(s) Field"
 23 |     types = []
 24 |     for win_nr, bitmap in window_list:
 25 |         offset = win_nr * 256
 26 |         octet_counter = 0
 27 |         for b in struct.unpack('B'*len(bitmap), bitmap):
 28 |             bitmask = 0x80
 29 |             bit_counter = 0
 30 |             while bitmask:
 31 |                 if b & bitmask:
 32 |                     types.append(offset + octet_counter*8 + bit_counter)
 33 |                 bit_counter += 1
 34 |                 bitmask >>= 1
 35 |             octet_counter += 1
 36 | 
 37 |     return types
 38 | 
 39 | def _rrtypes_to_text(types):
 40 |     # TODO: exception
 41 |     types_text = []
 42 |     for r in types:
 43 |         types_text.append(dns.rdatatype.to_text(r))
 44 | 
 45 |     return types_text
 46 | 
 47 | 
 48 | class DNSPythonResult(object):
 49 |     def __init__(self, dnspython_result):
 50 |         self._result = dnspython_result
 51 | 
 52 |     def status(self):
 53 |         return dns.rcode.to_text(self._result.rcode())
 54 | 
 55 |     def find_SOA(self, in_answer=True):
 56 |         for r in self._result.answer if in_answer else self._result.authority:
 57 |             if (r.rdclass == dns.rdataclass.IN and
 58 |                     r.rdtype == dns.rdatatype.SOA):
 59 |                 return name.domainname_from_wire(
 60 |                         r.name.to_wire(file=None, compress=None, origin=None))
 61 |         return None
 62 | 
 63 |     def find_NS(self, in_answer=True):
 64 |         for r in self._result.answer if in_answer else self._result.authority:
 65 |             if (r.rdclass == dns.rdataclass.IN and
 66 |                     r.rdtype == dns.rdatatype.NS):
 67 |                 return name.domainname_from_wire(
 68 |                         r.name.to_wire(file=None, compress=None, origin=None))
 69 |         return None
 70 | 
 71 |     def find_DNSKEY(self):
 72 |         for r in self._result.answer:
 73 |             if (r.rdclass == dns.rdataclass.IN and
 74 |                     r.rdtype == dns.rdatatype.DNSKEY):
 75 |                 return name.domainname_from_wire(
 76 |                         r.name.to_wire(file=None, compress=None, origin=None))
 77 |         return None
 78 | 
 79 |     def answer_length(self):
 80 |         return len(self._result.answer)
 81 | 
 82 |     def find_RRSIG_signer(self, owner, type_covered, in_answer=True):
 83 |         type_covered = dns.rdatatype.from_text(type_covered)
 84 |         for r in self._result.answer if in_answer else self._result.authority:
 85 |             if (r.rdclass == dns.rdataclass.IN
 86 |                     and r.rdtype == dns.rdatatype.RRSIG
 87 |                     and r[0].type_covered == type_covered
 88 |                     and owner ==  name.domainname_from_wire(
 89 |                         r.name.to_wire(file=None, compress=None, origin=None))
 90 |                    ):
 91 |                 return name.domainname_from_wire(r[0].signer.to_wire(file=None,
 92 |                                                                   compress=None,
 93 |                                                                   origin=None))
 94 |         return None
 95 | 
 96 | 
 97 |     def find_NSEC(self, in_answer=False):
 98 |         nsec = []
 99 |         for r in self._result.authority if not in_answer else self._result.answer:
100 |             if (r.rdclass == dns.rdataclass.IN and
101 |                     r.rdtype == dns.rdatatype.NSEC):
102 |                 types = _rrtypes_from_window_list(r[0].windows)
103 |                 nsec.append(rrtypes.nsec.NSEC(
104 |                         name.domainname_from_wire(
105 |                             r.name.to_wire(file=None, compress=None,
106 |                                 origin=None)),
107 |                         r.ttl,
108 |                         'IN',
109 |                         name.domainname_from_wire(
110 |                             r[0].next.to_wire(file=None, compress=None,
111 |                                 origin=None)),
112 |                         _rrtypes_to_text(types)
113 |                         ))
114 |         return nsec
115 | 
116 |     def all_NSEC_rrs(self):
117 |         return itertools.chain(self.find_NSEC(in_answer=False),
118 |                                self.find_NSEC(in_answer=True))
119 | 
120 | 
121 |     def find_NSEC3(self):
122 |         nsec3 = []
123 |         for r in self._result.authority:
124 |             if (r.rdclass == dns.rdataclass.IN and
125 |                     r.rdtype == dns.rdatatype.NSEC3):
126 |                 types = _rrtypes_from_window_list(r[0].windows)
127 |                 nsec3.append(rrtypes.nsec3.NSEC3(
128 |                     name.domainname_from_wire(r.name.to_wire(file=None,
129 |                         compress=None, origin=None)),
130 |                     r.ttl,
131 |                     'IN',
132 |                     r[0].algorithm,
133 |                     r[0].flags,
134 |                     r[0].iterations,
135 |                     r[0].salt,
136 |                     r[0].next,
137 |                     _rrtypes_to_text(types)))
138 |         return nsec3
139 | 
140 | def dnspython_query(dname, ns_ip, ns_port, rrtype, timeout):
141 |     # XXX:
142 |     qname = dns.name.from_wire(dname.to_wire(), 0)[0]
143 | 
144 |     q = dns.message.make_query(qname,
145 |                                rrtype,
146 |                                want_dnssec=True,
147 |                                payload = 4096)
148 |     r = dns.query.udp(q, ns_ip, port=ns_port, timeout=timeout,
149 |             ignore_unexpected=True)
150 |     if r.flags & dns.flags.TC:
151 |         r = dns.query.tcp(q, ns_ip, port=ns_port, timeout=timeout)
152 | 
153 |     return DNSPythonResult(r)
154 | 
155 | 
156 | def query(dname, ns, rrtype, timeout):
157 |     try:
158 |         res = dnspython_query(dname, ns.ip_str(), ns.port, rrtype, timeout)
159 |     except dns.exception.Timeout:
160 |         return exception.TimeOutError()
161 |     except dns.query.BadResponse:
162 |         return exception.QueryError()
163 |     if res.status() != 'NOERROR' and res.status() != 'NXDOMAIN':
164 |         return exception.UnexpectedResponseStatus(res.status())
165 |     return res
166 | 
167 | def query_ns_records(zone):
168 |     try:
169 |         log.info("looking up nameservers for zone ", str(zone))
170 |         zname = dns.name.from_wire(zone.to_wire(),0)[0]
171 |         ans = dns.resolver.query(zname, 'NS')
172 |         return set([rd.to_text() for rd in ans])
173 |     except dns.resolver.NXDOMAIN as e:
174 |         raise exception.N3MapError('failed to resolve nameservers for zone: NXDOMAIN')
175 |     except dns.exception.DNSException as e:
176 |         raise exception.N3MapError('failed to resolve nameservers for zone')
177 | 
178 | 
179 | 
180 | 


--------------------------------------------------------------------------------
/n3map/queryprovider.py:
--------------------------------------------------------------------------------
  1 | import collections
  2 | import socket
  3 | import time
  4 | import itertools
  5 | import threading
  6 | import queue
  7 | import re
  8 | import ipaddress
  9 | 
 10 | from . import vis
 11 | from .util import printsafe
 12 | from . import query
 13 | from . import log
 14 | from .exception import (
 15 |         N3MapError,
 16 |         InvalidPortError,
 17 |         InvalidAddressError,
 18 |         NameResolutionError,
 19 |         QueryError,
 20 |         TimeOutError,
 21 |         MaxRetriesError,
 22 |         MaxNsErrors,
 23 |         UnexpectedResponseStatus,
 24 |     )
 25 | 
 26 | 
 27 | DEFAULT_PORT = 53
 28 | QR_MEASUREMENTS = 256
 29 | 
 30 | 
 31 | class QueryProvider(object):
 32 |     def __init__(self,
 33 |                  ns_list,
 34 |                  timeout,
 35 |                  max_retries,
 36 |                  max_errors = 1,
 37 |                  stats=None,
 38 |                  query_interval=None):
 39 |         self.ns_list = ns_list
 40 |         self.next_ns_idx = 0
 41 |         self.timeout = timeout
 42 |         self.max_retries = max_retries
 43 |         self.max_errors = max_errors
 44 |         self.query_interval = query_interval
 45 |         self._last_query_time = None
 46 | 
 47 |         self.stats = stats if stats is not None else {}
 48 |         self.stats['queries'] = 0
 49 |         self._qr_measurements = collections.deque(maxlen=QR_MEASUREMENTS)
 50 | 
 51 |     def _ns_cycle(self, step=1):
 52 |         self.next_ns_idx = (self.next_ns_idx + step) % len(self.ns_list)
 53 | 
 54 |     def _next_ns(self):
 55 |         ns = self.ns_list[self.next_ns_idx]
 56 |         self._ns_cycle()
 57 |         return ns
 58 | 
 59 |     def _remove_ns(self, ns):
 60 |         try:
 61 |             ns_idx = self.ns_list.index(ns)
 62 |         except ValueError:
 63 |             # may have been already removed
 64 |             return
 65 |         removed_ns = self.ns_list.pop(ns_idx)
 66 | 
 67 |         log.warn("removed misbehaving/unresponsive nameserver ", str(removed_ns))
 68 | 
 69 |         if len(self.ns_list) == 0:
 70 |             self._ns_cycle = 0
 71 |             raise N3MapError("ran out of working nameservers!")
 72 | 
 73 |         # ensure the correct server is next in line:
 74 |         if ns_idx < self.next_ns_idx:
 75 |             self._ns_cycle(-1)
 76 |         else:
 77 |             self._ns_cycle(0)
 78 | 
 79 | 
 80 |         if self.query_interval is not None:
 81 |             # make sure we reduce the query rate such that each server receives
 82 |             # the same q/s as before
 83 |             single_server_interval = self.query_interval * (len(self.ns_list)+1)
 84 |             self.query_interval = single_server_interval/len(self.ns_list)
 85 |             log.warn("reducing query rate to avoid increasing the load ",
 86 |                     "on remaining servers")
 87 | 
 88 |     def add_ns_error(self, ns):
 89 |         try:
 90 |             ns.add_error(self.max_errors)
 91 |         except MaxNsErrors:
 92 |             self._remove_ns(ns)
 93 | 
 94 |     def add_ns_timeout(self, ns):
 95 |         try:
 96 |             ns.add_timeouterror(self.max_retries)
 97 |         except MaxRetriesError:
 98 |             self._remove_ns(ns)
 99 | 
100 |     def _query_timing(self, query_dn, rrtype, ns):
101 |         self._wait_query_interval()
102 |         self._qr_measurements.append(time.monotonic())
103 |         return ns
104 | 
105 |     def _sendquery(self, query_dn, ns, rrtype):
106 |         # XXX
107 |         # need to block signals because dnspython doesn't handle EINTR
108 |         # correctly
109 |         log.logger.block_signals()
110 |         try:
111 |             self.stats['queries'] += 1
112 |             log.debug2('query: ', query_dn, '; ns = ', ns, '; rrtype = ', rrtype)
113 |             return query.query(query_dn, ns, rrtype, self.timeout)
114 |         finally:
115 |             log.logger.unblock_signals()
116 | 
117 | 
118 |     def query(self, query_dn, rrtype='A'):
119 |         ns = self._next_ns()
120 |         self._query_timing(query_dn, rrtype, ns)
121 |         while True:
122 |             res = self._sendquery(query_dn, ns, rrtype)
123 |             if not isinstance(res, N3MapError):
124 |                 ns.retries = 0
125 |                 # don't know yet if we can reset the error counter, caller
126 |                 # decides
127 |                 return (res, ns)
128 |             if isinstance(res, TimeOutError):
129 |                 self.add_ns_timeout(ns)
130 |                 ns = self._next_ns()
131 |                 continue
132 |             if isinstance(res, QueryError) or isinstance(res,
133 |                     UnexpectedResponseStatus):
134 |                 log.error("{} from server {}".format(res, ns))
135 |                 self.add_ns_error(ns)
136 |                 ns = self._next_ns()
137 |                 continue
138 | 
139 | 
140 |     def query_rate(self):
141 |         t = time.monotonic()
142 |         # discard any data older than 2 seconds:
143 |         while (len(self._qr_measurements) > 0 and
144 |                 self._qr_measurements[0] + 2 < t):
145 |             self._qr_measurements.popleft()
146 |         if len(self._qr_measurements) < 2:
147 |             return 0.0
148 |         else:
149 |             interval = t - self._qr_measurements[0]
150 |             return len(self._qr_measurements)/interval
151 | 
152 |     def _wait_query_interval(self):
153 |         if (self.query_interval is not None and
154 |                 self._last_query_time is not None):
155 | 
156 |             # the loop is needed because time.sleep()
157 |             # may be interrupted by a signal
158 |             while True:
159 |                 diff = time.monotonic() - self._last_query_time
160 |                 if diff < 0 or diff >= self.query_interval:
161 |                     break
162 |                 time.sleep(self.query_interval - diff)
163 | 
164 |         self._last_query_time = time.monotonic()
165 | 
166 | 
167 | class Query(object):
168 |     def __init__(self, id, query_dn, ns, rrtype, timeout):
169 |         self.id = id
170 |         self.query_dn = query_dn
171 |         self.ns = ns
172 |         self.rrtype = rrtype
173 |         self.timeout = timeout
174 | 
175 | def create_aggressive_qp(queryprovider, num_threads):
176 |     return AggressiveQueryProvider(queryprovider.ns_list,
177 |                                    queryprovider.timeout,
178 |                                    queryprovider.max_retries,
179 |                                    queryprovider.max_errors,
180 |                                    queryprovider.stats,
181 |                                    queryprovider.query_interval,
182 |                                    num_threads)
183 | 
184 | class AggressiveQueryProvider(QueryProvider):
185 |     def __init__(self,
186 |                  ns_list,
187 |                  timeout,
188 |                  max_retries,
189 |                  max_errors,
190 |                  stats=None,
191 |                  query_interval=None,
192 |                  num_threads=1):
193 |         super(AggressiveQueryProvider,self).__init__(
194 |                  ns_list,
195 |                  timeout,
196 |                  max_retries,
197 |                  max_errors,
198 |                  stats,
199 |                  query_interval)
200 |         self._current_queryid = 0
201 |         self._active_queries = {}
202 |         self._results = {}
203 |         self._query_queue = queue.Queue()
204 |         self._result_queue = queue.Queue()
205 |         self._querythreads = []
206 |         self._start_query_threads(num_threads)
207 | 
208 |     def _start_query_threads(self,num=1):
209 |         for i in range(num):
210 |             qt = QueryThread(self._query_queue, self._result_queue)
211 |             self._querythreads.append(qt)
212 |             qt.start()
213 | 
214 |     def stop(self):
215 |         for i in range(len(self._querythreads)):
216 |             self._query_queue.put(None)
217 |         for qt in self._querythreads:
218 |             qt.join()
219 | 
220 |     def _gen_query_id(self):
221 |         self._current_queryid += 1
222 |         return self._current_queryid
223 | 
224 |     def _sendquery(self, query):
225 |         self.stats['queries'] += 1
226 |         log.debug2('query: ', query.query_dn, '; ns = ', query.ns, '; rrtype = ', query.rrtype)
227 |         self._active_queries[query.id] = query
228 |         self._query_queue.put(query)
229 |         return query.id
230 | 
231 |     def _checkresult(self, qid, res):
232 |         q = self._active_queries[qid]
233 |         if not isinstance(res, N3MapError):
234 |             q.ns.retries = 0
235 |             self._results[qid] = (res, q.ns)
236 |             del self._active_queries[qid]
237 |             return
238 |         try:
239 |             raise res
240 |         except TimeOutError:
241 |             try:
242 |                 self.add_ns_timeout(q.ns)
243 |             except N3MapError as e:
244 |                 # happens when we run out of servers
245 |                 del self._active_queries[qid]
246 |                 raise e
247 |             q.ns = self._next_ns()
248 |             self._sendquery(q)
249 |         except (QueryError, UnexpectedResponseStatus) as e:
250 |             log.error("{} from server {}".format(e, q.ns))
251 |             try:
252 |                 self.add_ns_error(q.ns)
253 |             except N3MapError as e:
254 |                 # happens when we run out of servers
255 |                 del self._active_queries[qid]
256 |                 raise e
257 |             q.ns = self._next_ns()
258 |             self._sendquery(q)
259 | 
260 | 
261 |     def _collectresponses(self, block=True):
262 |         if block:
263 |             self._checkresult(*self._result_queue.get(True))
264 |         has_responses = True
265 |         while has_responses:
266 |             try:
267 |                 self._checkresult(*self._result_queue.get(False))
268 |             except queue.Empty:
269 |                 has_responses = False
270 | 
271 |     def collectresponses(self, block=True):
272 |         self._collectresponses(block)
273 |         res = list(self._results.items())
274 |         self._results.clear()
275 |         return res
276 | 
277 | 
278 |     def query_ff(self, query_dn, rrtype='A'):
279 |         ns = self._next_ns()
280 |         self._query_timing(query_dn, rrtype, ns)
281 |         return self._sendquery(Query(self._gen_query_id(), query_dn, ns, rrtype, self.timeout))
282 | 
283 | 
284 |     def query(self, query_dn, rrtype='A'):
285 |         qid = self.query_ff(query_dn, rrtype)
286 |         while True:
287 |             self._collectresponses(block=True)
288 |             res = self._results.pop(qid,None)
289 |             if res is not None:
290 |                 return res
291 | 
292 | 
293 | class QueryThread(threading.Thread):
294 |     def __init__(self, query_queue, result_queue):
295 |         super(QueryThread, self).__init__()
296 |         self.daemon = True
297 |         self._query_queue = query_queue
298 |         self._result_queue = result_queue
299 | 
300 |     def run(self):
301 |         query_queue = self._query_queue
302 |         result_queue = self._result_queue
303 |         while True:
304 |             q = query_queue.get()
305 |             if q is None:
306 |                 return
307 |             result_queue.put((q.id,query.query(q.query_dn, q.ns, q.rrtype, q.timeout)))
308 | 
309 | 
310 | 
311 | class NameServer(object):
312 |     def __init__(self, ip, port, name):
313 |         if port < 0 or port > 65535:
314 |             raise InvalidPortError(str(port))
315 |         self.ip = ip
316 |         self.port = port
317 |         self.name = vis.strvis(name.encode()).decode()
318 |         self.retries = 0
319 |         self.errors = 0
320 | 
321 |     def add_timeouterror(self, max_retries):
322 |         if max_retries != -1:
323 |             self.retries += 1
324 |             retries_left = max_retries - self.retries
325 |             log.warn("timeout reached when waiting for response from ", str(self),
326 |                     ", ", str(max(0,retries_left)), " retries left")
327 |             if retries_left <= 0:
328 |                 raise MaxRetriesError('no response from server: ' + str(self))
329 |         else:
330 |             log.debug2("timeout reached when waiting for response from ", str(self))
331 | 
332 |     def add_error(self, max_errors):
333 |         self.errors += 1
334 |         if max_errors != -1:
335 |             errors_left = max_errors - self.errors
336 |             log.warn(str(max(0,errors_left)), " errors left for ", str(self))
337 |             if errors_left <= 0:
338 |                 raise MaxNsErrors()
339 |         else:
340 |             log.debug2(str(self), " had ", str(self.errors), " error(s)")
341 | 
342 |     def reset_errors(self):
343 |         self.errors = 0
344 | 
345 |     def ip_str(self):
346 |         return str(self.ip)
347 | 
348 |     def __str__(self):
349 |         try:
350 |             ipaddress.ip_address(self.name)
351 |             name = ''
352 |         except ValueError:
353 |             name = ' ({})'.format(self.name)
354 | 
355 |         if self.port == DEFAULT_PORT:
356 |             return '{}{}'.format(self.ip, name)
357 |         elif self.ip.version == 6:
358 |             return '[{}]:{}{}'.format(self.ip, self.port, name)
359 |         return '{}:{}{}'.format(self.ip, self.port, name)
360 | 
361 | 
362 | def _resolve(host, port, protofamily=''):
363 |     try:
364 |         if protofamily == 'ipv4':
365 |             family = socket.AF_INET
366 |         elif protofamily == 'ipv6':
367 |             family = socket.AF_INET6
368 |         else:
369 |             family = 0
370 |         for info in socket.getaddrinfo(host, port, family,
371 |                 socket.SOCK_DGRAM, socket.IPPROTO_UDP):
372 |             if info[0] == socket.AF_INET and (
373 |                     protofamily == '' or protofamily == 'ipv4'):
374 |                 return ipaddress.ip_address(info[4][0])
375 |             elif info[0] == socket.AF_INET6 and (
376 |                     protofamily == '' or protofamily == 'ipv6'):
377 |                 return ipaddress.ip_address(info[4][0])
378 |         raise NameResolutionError("no suitable address found for host '{}'"
379 |                 .format(printsafe(host)))
380 |     except socket.gaierror as e:
381 |         raise NameResolutionError("could not resolve host '" +
382 |                 str(printsafe(host)) + "': " + str(e))
383 | 
384 | def port_from_s(s):
385 |     try:
386 |         p = int(s)
387 |     except ValueError:
388 |         raise InvalidPortError(str(v))
389 | 
390 |     if p < 0 or p > 65535:
391 |         raise InvalidPortError(str(p))
392 |     return p
393 | 
394 | 
395 | def ip6_from_s(s):
396 |     try:
397 |         return ipaddress.IPv6Address(s)
398 |     except ipaddress.AddressValueError as e:
399 |         raise InvalidAddressError(str(e))
400 | 
401 | pat_ipv6_hostp = re.compile(r'\[([:0-9a-fA-F]+)\]:([0-9]+)')
402 | pat_ipv6_host = re.compile(r'([:0-9a-fA-F]+)')
403 | pat_hostp = re.compile(r'(.*):([0-9]+)')
404 | 
405 | def host_port_from_s(s):
406 |     m = pat_ipv6_hostp.fullmatch(s)
407 |     if m is not None:
408 |         ip = m.group(1)
409 |         port = m.group(2)
410 |         return (str(ip6_from_s(ip)), port_from_s(port))
411 | 
412 |     m = pat_ipv6_host.fullmatch(s)
413 |     if m is not None:
414 |         ip = m.group(1)
415 |         return (str(ip6_from_s(ip)), DEFAULT_PORT)
416 | 
417 |     m = pat_hostp.fullmatch(s)
418 |     if m is not None:
419 |         host = m.group(1)
420 |         port = m.group(2)
421 |         return (host, port_from_s(port))
422 | 
423 |     return (s, DEFAULT_PORT)
424 | 
425 | 
426 | def nameserver_from_text(protofamily, *hosts, ignore_unresolved=False):
427 |     lst = []
428 |     ns_dict = {}
429 |     for s in hosts:
430 |         host, port = host_port_from_s(s)
431 |         try:
432 |             ip = _resolve(host, port, protofamily)
433 |         except NameResolutionError as e:
434 |             estr = "failed to resolve nameserver: {}".format(str(e))
435 |             if ignore_unresolved:
436 |                 log.warn(estr)
437 |                 continue
438 |             raise N3MapError(estr)
439 | 
440 |         ns = NameServer(ip, port, host)
441 |         if (ip, port) in ns_dict:
442 |             original = ns_dict[(ip, port)]
443 |             if host != original[0]:
444 |                 log.warn("nameserver {} is a duplicate of {}, ignoring it"
445 |                         .format(printsafe(s), str(original[1])))
446 |             continue
447 |         ns_dict[(ip, port)] = (host, ns)
448 |         lst.append(ns)
449 |     if len(lst) == 0:
450 |         raise N3MapError("no nameservers found!")
451 |     return lst
452 | 
453 | 
454 | 


--------------------------------------------------------------------------------
/n3map/rrfile.py:
--------------------------------------------------------------------------------
  1 | import re
  2 | import gzip
  3 | import os
  4 | 
  5 | from . import log
  6 | from .rrtypes import nsec
  7 | from .rrtypes import nsec3
  8 | from . import rrtypes
  9 | from .exception import (
 10 |         FileParseError,
 11 |         MaxDomainNameLengthError,
 12 |         MaxLabelLengthError,
 13 |         NSECError,
 14 |         NSEC3Error,
 15 |         ParseError
 16 |     )
 17 | 
 18 | _comment_pattern = r'^\s*([;#].*)?$'
 19 | 
 20 | def _open(filename, mode):
 21 |     if filename.endswith(".gz"):
 22 |         return gzip.open(filename, mode + 't', encoding="utf-8")
 23 |     return open(filename, mode, encoding="utf-8")
 24 | 
 25 | def open_output_rrfile(filename):
 26 |     return RRFile(_open(filename, "w+"), filename)
 27 | 
 28 | def open_input_rrfile(filename):
 29 |     return RRFile(_open(filename, "r"), filename)
 30 | 
 31 | class RRFileStream(object):
 32 |     def __init__(self, f):
 33 |         self.f = f
 34 |         self.label_counter = None
 35 | 
 36 |     def fsync(self):
 37 |         pass
 38 | 
 39 |     def seek(self, offset):
 40 |         pass
 41 | 
 42 |     def close(self):
 43 |         if self.f is not None:
 44 |             if self.f.writable():
 45 |                 # ensure data is written to disk before we try to delete the
 46 |                 # backup file
 47 |                 self.f.flush()
 48 |                 self.fsync()
 49 |             self.f.close()
 50 |             self.f = None
 51 | 
 52 |     def write_header(self, zone, title):
 53 |         self.f.write(';' *  80 + '\n')
 54 |         zonestr = " zone: " + str(zone)
 55 |         self.f.write(';' + zonestr.center(79).rstrip() + '\n')
 56 |         self.f.write(';' + title.center(79).rstrip() + '\n')
 57 |         self.f.write(';' * 80 + '\n')
 58 | 
 59 |     def write_number_of_rrs(self, n):
 60 |         self.f.write("; number of records = " + str(n) + "\n")
 61 | 
 62 |     def write_stats(self, stats):
 63 |         self.f.write("\n;; statistics\n")
 64 |         for k, v in stats.items():
 65 |             self.f.write("; " + str(k) + " = " + str(v) + '\n')
 66 | 
 67 |     def write_record(self, rr):
 68 |         self.f.write(str(rr) + '\n')
 69 | 
 70 |     def _desc_filename(self):
 71 |         return self.f.name
 72 | 
 73 |     def nsec_reader(self):
 74 |         log.info("reading NSEC RRs from ", str(self.f.name))
 75 |         self.seek(0)
 76 |         p_ignore = re.compile(_comment_pattern)
 77 |         nsec_parse = rrtypes.nsec.parser()
 78 |         for i, line in enumerate(self.f):
 79 |             i += 1
 80 |             if p_ignore.match(line):
 81 |                 continue
 82 |             try:
 83 |                 nsec = nsec_parse(line)
 84 |                 if nsec is None:
 85 |                     raise FileParseError(self._desc_filename(), i,
 86 |                             "invalid file format")
 87 |                 yield nsec
 88 |             except ParseError:
 89 |                 raise FileParseError(self._desc_filename(), i,
 90 |                         "could not parse NSEC record")
 91 |             except (NSECError,
 92 |                     MaxDomainNameLengthError,
 93 |                     MaxLabelLengthError) as e:
 94 |                 raise FileParseError(self._desc_filename(), i,
 95 |                         "invalid NSEC record:\n" + str(e))
 96 | 
 97 |     def nsec3_reader(self):
 98 |         log.info("reading NSEC3 RRs from ", str(self.f.name))
 99 |         self.seek(0)
100 |         p_counter = re.compile("^;;;; label_counter\s*=\s*0x([0-9a-fA-F]+)")
101 |         p_ignore = re.compile(_comment_pattern)
102 |         nsec3_parse = rrtypes.nsec3.parser()
103 |         for i, line in enumerate(self.f, start=1):
104 |             m_counter = p_counter.match(line)
105 |             if m_counter is not None:
106 |                 try:
107 |                     self.label_counter = int(m_counter.group(1), 16)
108 |                 except ValueError:
109 |                     raise FileParseError(self._desc_filename(), i,
110 |                             "cannot parse label counter value")
111 |                 continue
112 |             elif p_ignore.match(line):
113 |                 continue
114 |             try:
115 |                 nsec3 = nsec3_parse(line)
116 |                 if nsec3 is None:
117 |                     raise FileParseError(self._desc_filename(), i,
118 |                             "invalid file format")
119 |                 yield nsec3
120 |             except ParseError:
121 |                 raise FileParseError(self._desc_filename(), i,
122 |                         "could not parse NSEC3 record")
123 |             except (NSEC3Error,
124 |                     MaxDomainNameLengthError,
125 |                     MaxLabelLengthError) as e:
126 |                 raise FileParseError(self._desc_filename(), i,
127 |                         "invalid NSEC3 record:\n" + str(e))
128 | 
129 |     def write_label_counter(self, label_counter):
130 |         self.f.write(";;;; label_counter = 0x{0:x}\n".format(label_counter))
131 | 
132 | 
133 | class RRFile(RRFileStream):
134 |     def __init__(self, f, fname):
135 |         super().__init__(f)
136 |         self.filename = fname
137 | 
138 |     def fsync(self):
139 |         os.fsync(self.f.fileno())
140 | 
141 |     def seek(self, offset):
142 |         self.f.seek(offset)
143 | 
144 |     def _backup_filename(self):
145 |         return self.filename + '~'
146 | 
147 |     def unlink_backup(self):
148 |         try:
149 |             os.unlink(self._backup_filename())
150 |         except OSError as e:
151 |             log.debug2("failed to unlink backup file: \n", str(e))
152 | 
153 |     def into_backup(self):
154 |         os.replace(self.filename, self._backup_filename())
155 | 
156 | 
157 | 
158 | def nsec_from_file(filename):
159 |     """Read NSEC records from a file"""
160 |     rrf = None
161 |     try:
162 |         rrf = open_input_rrfile(filename)
163 |         return list(rrf.nsec_reader())
164 |     finally:
165 |         if rrf is not None:
166 |             rrf.close()
167 | 
168 | def nsec3_from_file(filename):
169 |     """Read NSEC3 records from a file"""
170 |     rrf = None
171 |     try:
172 |         rrf = open_input_rrfile(filename)
173 |         return list(rrf.nsec3_reader())
174 |     finally:
175 |         if rrf is not None:
176 |             rrf.close()
177 | 
178 | 
179 | 
180 | 
181 | 


--------------------------------------------------------------------------------
/n3map/rrtypes/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/anonion0/nsec3map/d145b134742e88dc74579e2252db827a321316c7/n3map/rrtypes/__init__.py


--------------------------------------------------------------------------------
/n3map/rrtypes/nsec.py:
--------------------------------------------------------------------------------
 1 | import re
 2 | 
 3 | from . import rr
 4 | from .. import name
 5 | from .. import vis
 6 | from ..exception import NSECError, ParseError
 7 | 
 8 | class NSEC(rr.RR):
 9 |     def __init__(self, owner, ttl, cls, next_owner, types):
10 |         super(NSEC, self).__init__(owner, ttl, cls)
11 |         self.next_owner = next_owner
12 |         self.types = types
13 | 
14 |     def covers(self, dname):
15 |         return dname.covered_by(self.owner, self.next_owner)
16 | 
17 |     def covers_exclusive(self, dname):
18 |         return dname.covered_by_exclusive(self.owner, self.next_owner)
19 | 
20 |     def part_of_zone(self, zone):
21 |         return (self.owner.part_of_zone(zone) and
22 |                 self.next_owner.part_of_zone(zone))
23 | 
24 |     def __str__(self):
25 |         return '\t'.join((super(NSEC, self).__str__(), "NSEC", str(self.next_owner), ' '.join(self.types)))
26 | 
27 | def parser():
28 |     p_nsec = re.compile(r'^NSEC\s+(([a-zA-Z0-9\\_*-]+\.|\.)+)((\s+[A-Z0-9]+)*)\s*$')
29 |     rr_parse = rr.parser()
30 |     def nsec_from_text(s):
31 |         try:
32 |             res = rr_parse(s)
33 |             if res is None:
34 |                 return None
35 |             owner, ttl, cls, rest = res
36 |             m = p_nsec.match(rest)
37 |             if m is None:
38 |                 return None
39 |             next_owner = name.unvis_domainname(m.group(1).encode("ascii"))
40 |             types = m.group(3).strip()
41 |             if not types.isprintable():
42 |                 raise ValueError
43 |             types = types.split(' ')
44 |         except ValueError:
45 |                 raise ParseError
46 |         return NSEC(owner, ttl, cls, next_owner, types)
47 |     return nsec_from_text
48 | 


--------------------------------------------------------------------------------
/n3map/rrtypes/nsec3.py:
--------------------------------------------------------------------------------
  1 | import re
  2 | import hashlib
  3 | 
  4 | from . import rr
  5 | from .. import name
  6 | from .. import vis
  7 | from .. import util
  8 | from ..exception import (
  9 |         NSEC3Error,
 10 |         InvalidDomainNameError,
 11 |         ParseError
 12 |     )
 13 | 
 14 | SHA1 = 1
 15 | SHA1_LENGTH = 20
 16 | SHA1_MAX = 2**160-1
 17 | 
 18 | def distance_covered(hashed_owner, next_hashed_owner):
 19 |     if hashed_owner == next_hashed_owner:
 20 |         # empty zone case
 21 |         return SHA1_MAX
 22 |     return abs(int.from_bytes(next_hashed_owner, "big") -
 23 |                 int.from_bytes(hashed_owner, "big"))
 24 | 
 25 | def covered_by_nsec3_interval(nsec3_hash, hashed_owner, next_hashed_owner):
 26 |     if hashed_owner >= next_hashed_owner:
 27 |         # this is the last NSEC3 record in a chain
 28 |         # this will also catch the empty zone case in which
 29 |         # there is a single record with hashed_owner == next_hashed_owner
 30 |         return (nsec3_hash >= hashed_owner or nsec3_hash <= next_hashed_owner)
 31 |     return (nsec3_hash >= hashed_owner and nsec3_hash <= next_hashed_owner)
 32 | 
 33 | class NSEC3(rr.RR):
 34 |     def __init__(self, hashed_owner, ttl, cls, algorithm, flags, iterations,
 35 |             salt, next_hashed_owner, types):
 36 |         super(NSEC3, self).__init__(hashed_owner, ttl, cls)
 37 |         self.algorithm = algorithm
 38 |         self.flags = flags
 39 |         self.iterations = iterations
 40 |         self.salt = salt
 41 |         self.next_hashed_owner = next_hashed_owner
 42 |         self.types = types
 43 | 
 44 |     @property
 45 |     def owner(self):
 46 |         return self.hashed_owner_dn()
 47 | 
 48 |     @owner.setter
 49 |     def owner(self, hashed_owner):
 50 |         try:
 51 |             hash_dn, zone = hashed_owner.split(1)
 52 |             hashed_owner = util.base32_ext_hex_decode(hash_dn.labels[0].label)
 53 |             if len(hashed_owner) != SHA1_LENGTH:
 54 |                 raise NSEC3Error('NSEC3 RR: invalid hashed_owner length')
 55 |             self.hashed_owner = hashed_owner
 56 |             self.zone = zone
 57 |         except (InvalidDomainNameError, TypeError, IndexError):
 58 |             raise NSEC3Error("NSEC3 RR: could not decode hashed owner name")
 59 | 
 60 |     @property
 61 |     def algorithm(self):
 62 |         return self._algorithm
 63 | 
 64 |     @algorithm.setter
 65 |     def algorithm(self, algorithm):
 66 |         if not (algorithm & SHA1):
 67 |             raise NSEC3Error('NSEC3 RR: unknown hash function')
 68 |         self._algorithm = algorithm
 69 | 
 70 |     @property
 71 |     def next_hashed_owner(self):
 72 |         return self._next_hashed_owner
 73 | 
 74 |     @next_hashed_owner.setter
 75 |     def next_hashed_owner(self, v):
 76 |         if len(v) != SHA1_LENGTH:
 77 |             raise NSEC3Error('NSEC3 RR: invalid next_hashed_owner length')
 78 |         self._next_hashed_owner = v
 79 | 
 80 |     @property
 81 |     def iterations(self):
 82 |         return self._iterations
 83 | 
 84 |     @iterations.setter
 85 |     def iterations(self, v):
 86 |         if v < 0 or v > 2500:
 87 |             raise NSEC3Error("NSEC3 RR: invalid number of iterations")
 88 |         self._iterations = v
 89 | 
 90 |     def part_of_zone(self, zone):
 91 |         return (zone == self.zone)
 92 | 
 93 |     def hashed_owner_dn(self):
 94 |         hashed_owner = util.base32_ext_hex_encode(self.hashed_owner).lower()
 95 |         return name.DomainName(name.Label(hashed_owner),
 96 |                                       *self.zone.labels)
 97 | 
 98 |     def next_hashed_owner_dn(self):
 99 |         next_hashed_owner = util.base32_ext_hex_encode(self.next_hashed_owner).lower()
100 |         return name.DomainName(name.Label(next_hashed_owner),
101 |                                       *self.zone.labels)
102 | 
103 |     def covers_hash(self, nsec3_hash):
104 |         return covered_by_nsec3_interval(nsec3_hash, self.hashed_owner, self.next_hashed_owner)
105 | 
106 |     def __str__(self):
107 |         return '\t'.join((super(NSEC3, self).__str__(),
108 |             ' '.join(("NSEC3",
109 |                       str(self.algorithm),
110 |                       str(self.flags),
111 |                       str(self.iterations),
112 |                       (self.salt.hex() if len(self.salt) > 0 else '-'),
113 |                       util.base32_ext_hex_encode(self.next_hashed_owner).lower()
114 |                       .decode())),
115 |             ' '.join(self.types)))
116 | 
117 |     def distance_covered(self):
118 |         return distance_covered(self.hashed_owner, self.next_hashed_owner)
119 | 
120 | 
121 | 
122 | def compute_hash(owner_name, salt, iterations, algorithm=SHA1):
123 |     # see RFC5155 for details
124 |     if not (algorithm & SHA1):
125 |         raise NSEC3Error('unknown hash function')
126 |     x = owner_name.to_wire()
127 |     i = 0
128 |     while True:
129 |         h = hashlib.sha1()
130 |         h.update(x)
131 |         h.update(salt)
132 |         x = h.digest()
133 |         i += 1
134 |         if i > iterations:
135 |             break
136 |     return h.digest()
137 | 
138 | def parser():
139 |     p_nsec3 = re.compile(r'^NSEC3\s+([0-9]|[1-9][0-9]*)\s+([0-9]|[1-9][0-9]*)\s+([0-9]|[1-9][0-9]*)\s+([a-fA-F0-9]+|\-)\s+([a-vA-V0-9]+)((\s+[A-Z0-9]+)*)\s*$')
140 |     rr_parse = rr.parser()
141 |     def nsec3_from_text(s):
142 |         try:
143 |             res = rr_parse(s)
144 |             if res is None:
145 |                 return None
146 |             owner, ttl, cls, rest = res
147 |             m = p_nsec3.match(rest)
148 |             if m is None:
149 |                 return None
150 |             algorithm = int(m.group(1))
151 |             flags = int(m.group(2))
152 |             iterations = int(m.group(3))
153 |             salt  = m.group(4)
154 |             if salt == '-':
155 |                 salt = b""
156 |             else:
157 |                 salt = bytes.fromhex(m.group(4))
158 |             next_hashed_owner = util.base32_ext_hex_decode(m.group(5))
159 |             types = m.group(6).strip()
160 |             if not types.isprintable():
161 |                 raise ValueError
162 |             types = types.split(' ')
163 |         except (TypeError, ValueError):
164 |             raise ParseError
165 |         return NSEC3(owner, ttl, cls, algorithm, flags, iterations, salt, next_hashed_owner, types)
166 |     return nsec3_from_text
167 | 
168 | 


--------------------------------------------------------------------------------
/n3map/rrtypes/rr.py:
--------------------------------------------------------------------------------
 1 | import re
 2 | 
 3 | from .. import name
 4 | from ..exception import ParseError
 5 | 
 6 | class RR(object):
 7 |     """General resource record"""
 8 |     def __init__(self, owner, ttl, cls):
 9 |         self.owner = owner
10 |         self.ttl = ttl
11 |         self.cls = cls
12 | 
13 |     def __str__(self):
14 |         return '\t'.join((str(self.owner), str(self.ttl), self.cls))
15 | 
16 | 
17 | def parser():
18 |     """Returns a parser for a general resource record"""
19 |     p = re.compile(r'^(([a-zA-Z0-9\\_*-]+\.)+|\.)\s+([0-9]|[1-9][0-9]*)\s+IN\s+(.*)$')
20 |     def rr_from_text(s):
21 |         m = p.match(s)
22 |         try:
23 |             if m is None:
24 |                 return None
25 |             else:
26 |                 # owner, tt, class, rest
27 |                 owner = name.unvis_domainname(m.group(1).encode("ascii"))
28 |                 return owner, int(m.group(3)), 'IN', m.group(4)
29 |         except ValueError:
30 |             raise ParseError
31 |     return rr_from_text
32 | 


--------------------------------------------------------------------------------
/n3map/statusline.py:
--------------------------------------------------------------------------------
  1 | import math
  2 | import re
  3 | 
  4 | from . import log
  5 | 
  6 | class ColorCode:
  7 |     def __init__(self, ccode):
  8 |         self.ccode = ccode
  9 | 
 10 |     def __str__(self):
 11 |         return self.ccode
 12 | 
 13 |     def __len__(self):
 14 |         return 0
 15 | 
 16 | def printlen(l):
 17 |     return sum(len(x) for x in l)
 18 | 
 19 | def truncate_line(l, width, cs):
 20 |     length = 0
 21 |     newlist = []
 22 |     for i,element in enumerate(l):
 23 |         newlength = length + len(element)
 24 |         if newlength > width:
 25 |             newlist.append(element[:width-newlength])
 26 |             newlist.append(ColorCode(cs.RESET))
 27 |             break
 28 |         newlist.append(element)
 29 |         length = newlength
 30 |     return newlist
 31 | 
 32 | def assemble_line(l):
 33 |     return ''.join(str(element) for element in l)
 34 | 
 35 | def compose_leftright(cs, leftlabels, leftvalues, rightlabels, rightvalues):
 36 |     leftline = []
 37 |     for lbl,val in zip(leftlabels, leftvalues):
 38 |         leftline += [*lbl , " = ", *val, '; ']
 39 | 
 40 |     rightline = [' ']
 41 |     for i,(lbl,val) in enumerate(zip(rightlabels, rightvalues)):
 42 |         if i > 0:
 43 |             rightline.append('; ')
 44 |         rightline += [*lbl , " = ", *val]
 45 |     rightline += [
 46 |             ColorCode(cs.DECO),
 47 |             ' ;;',
 48 |             ColorCode(cs.RESET),
 49 |             ]
 50 |     return leftline,rightline
 51 | 
 52 | 
 53 | def format_statusline_nsec3(width,
 54 |                 zone,
 55 |                 queries,
 56 |                 records,
 57 |                 hashes,
 58 |                 coverage,
 59 |                 queryrate,
 60 |                 prediction
 61 |             ):
 62 |     cs = log.logger.colors
 63 |     # first line ======
 64 |     lines = []
 65 |     left = [
 66 |             ColorCode(cs.DECO),
 67 |             ";;",
 68 |             ColorCode(cs.RESET),
 69 |             " mapping ",
 70 |             ColorCode(cs.ZONE), str(zone), ColorCode(cs.RESET),
 71 |             " ",
 72 |             ]
 73 |     right = [
 74 |             ColorCode(cs.DECO),
 75 |             " ;;",
 76 |             ColorCode(cs.RESET),
 77 |             ]
 78 |     pad = width - printlen(left) - printlen(right)
 79 |     if prediction is not None and pad >= 10:
 80 |         if prediction < records:
 81 |             prediction = records
 82 |         ratio = records/float(prediction) if prediction > 0 else 0
 83 |         percentage = [
 84 |                 ColorCode(cs.PROGRESS),
 85 |                 "{0:3d}% ".format(int(ratio*100)),
 86 |                 ColorCode(cs.RESET),
 87 |                 ]
 88 |         proglen = pad-printlen(percentage)-2
 89 |         filllen = int(math.ceil(ratio*proglen))
 90 |         progress = [
 91 |             "[",
 92 |             ColorCode(cs.PROGRESSBAR),
 93 |             "{0:s}{1:s}".format("="*filllen," "*(proglen-filllen)),
 94 |             ColorCode(cs.RESET),
 95 |             "]"
 96 |             ]
 97 |         right =  percentage + progress + right
 98 |     elif pad > 0:
 99 |         right = ['.' * pad] + right
100 |     lines.append(left + right)
101 | 
102 |     # second line =======
103 |     leftlabels =  [['records'],['queries'],['hashes']]
104 |     leftshortlabels =  [['r'],['q'],['h']]
105 |     leftvalues = [
106 |             [ColorCode(cs.RECORDS), "{0:3d}".format(records),
107 |                 ColorCode(cs.RESET)],
108 |             [ColorCode(cs.NUMBERS), "{0:3d}".format(queries),
109 |                 ColorCode(cs.RESET)],
110 |             [ColorCode(cs.NUMBERS), "{0:3d}".format(hashes),
111 |                 ColorCode(cs.RESET)],
112 |             ]
113 |     if prediction is not None:
114 |         leftlabels.append("predicted zone size")
115 |         leftshortlabels.append("pred")
116 |         leftvalues += [
117 |                 [ColorCode(cs.NUMBERS), "{0:3d}".format(prediction),
118 |                     ColorCode(cs.RESET)],
119 |                 ]
120 |     rightlabels = [['q/s'], ['coverage']]
121 |     rightshortlabels = [['q/s'], ['c']]
122 |     rightvalues = [
123 |             [ColorCode(cs.gradient(round(queryrate)/100.0)),
124 |                 "{0:.0f}".format(queryrate),
125 |                 ColorCode(cs.RESET)
126 |                 ],
127 |             [ColorCode(cs.NUMBERS), "{0:11.6%}".format(coverage),
128 |                 ColorCode(cs.RESET)],
129 |             ]
130 |     left,right = compose_leftright(cs, leftlabels, leftvalues,
131 |                                    rightlabels, rightvalues)
132 |     leftprefix = [ ColorCode(cs.DECO), ';; ', ColorCode(cs.RESET), ]
133 |     pad = width - printlen(leftprefix) - printlen(left) - printlen(right)
134 |     if pad < 0:
135 |         left,right = compose_leftright(cs, leftshortlabels, leftvalues,
136 |                                        rightshortlabels, rightvalues)
137 |         pad = width - printlen(leftprefix) - printlen(left) - printlen(right)
138 |     if pad < 0:
139 |         pad = 0
140 | 
141 |     lines.append([*leftprefix, *left, pad * '.', *right])
142 | 
143 |     return [assemble_line(truncate_line(l, width, cs)) for l in lines]
144 | 
145 | def format_statusline_nsec(width,
146 |                 zone,
147 |                 queries,
148 |                 records,
149 |                 queryrate
150 |             ):
151 | 
152 |     cs = log.logger.colors
153 |     mappinglabel = [
154 |             ColorCode(cs.DECO),
155 |             ";;",
156 |             ColorCode(cs.RESET),
157 |             " walking ",
158 |             ColorCode(cs.ZONE), str(zone), ColorCode(cs.RESET),
159 |             ": ",
160 |             ]
161 |     leftlabels =  [['records'],['queries']]
162 |     leftvalues = [
163 |             [ColorCode(cs.RECORDS), "{0:3d}".format(records),
164 |                 ColorCode(cs.RESET)],
165 |             [ColorCode(cs.NUMBERS), "{0:3d}".format(queries),
166 |                 ColorCode(cs.RESET)],
167 |             ]
168 |     rightlabels = [['q/s']]
169 |     rightvalues = [
170 |             [
171 |                 ColorCode(cs.gradient(round(queryrate)/100.0)),
172 |                 "{0:.0f}".format(queryrate),
173 |                 ColorCode(cs.RESET),
174 |             ]
175 |             ]
176 |     left,right = compose_leftright(cs, leftlabels, leftvalues,
177 |                                    rightlabels, rightvalues)
178 |     left = mappinglabel + left
179 |     pad = width - printlen(left) - printlen(right)
180 |     if pad < 0:
181 |         pad = 0
182 |     line = left + [pad * '.'] + right
183 |     return [assemble_line(truncate_line(line, width, cs))]
184 | 
185 | 


--------------------------------------------------------------------------------
/n3map/tree/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/anonion0/nsec3map/d145b134742e88dc74579e2252db827a321316c7/n3map/tree/__init__.py


--------------------------------------------------------------------------------
/n3map/tree/bstree.py:
--------------------------------------------------------------------------------
 1 | class BSTreeNode(object):
 2 |     """Abstract implementation of a binary search tree node."""
 3 | 
 4 |     def __init__(self, k, v, nil=None):
 5 |         self.key =  k
 6 |         self.value = v
 7 | 
 8 |         self.left = nil
 9 |         self.right = nil
10 |         self.parent = nil
11 | 
12 | class BSTree(object):
13 |     """Abstract implementation of a binary search tree."""
14 | 
15 |     def __init__(self, node_type=BSTreeNode):
16 |         self.node_type = node_type
17 |         self.nil = self.node_type(k=None, v=None)
18 |         self.root = self.nil
19 |         self.root.parent = self.nil
20 | 
21 |     def contains(self, k):
22 |         return self.find(k) is not None
23 | 
24 | 
25 |     def find(self, k):
26 |         """Finds the node with key k. Returns None if k is not found.
27 | 
28 |         Time complexity: O(lg n) (balanced)"""
29 |         x = self.root
30 |         while x is not self.nil and k != x.key:
31 |             if k < x.key:
32 |                 x = x.left
33 |             else:
34 |                 x = x.right
35 |         return x if x is not self.nil else None
36 | 
37 | 
38 |     def inorder(self, f):
39 |         """Does an inorder traversal and calls f(x) for every node x.
40 | 
41 |         Time complexity: O(n)
42 |         """
43 |         return self._inorder_recurse(self.root, f)
44 | 
45 |     def _inorder_recurse(self, x, f):
46 |         if x is self.nil:
47 |             return
48 |         self._inorder_recurse(x.left, f)
49 |         f(x)
50 |         self._inorder_recurse(x.right, f)
51 | 
52 |     def minimum(self, x=None):
53 |         """Finds the node with the minimal key
54 | 
55 |         Returns nil if tree is empty
56 |         Time complexity: O(lg n) (balanced)"""
57 |         if x is None:
58 |             x = self.root
59 | 
60 |         while x.left is not self.nil:
61 |             x = x.left
62 |         return x if x is not self.nil else None
63 | 
64 |     def maximum(self, x=None):
65 |         """Finds the node with the maximum key
66 | 
67 |         Time complexity: O(lg n) (balanced)"""
68 |         if x is None:
69 |             x = self.root
70 | 
71 |         while x.right is not self.nil:
72 |             x = x.right
73 |         return x if x is not self.nil else None
74 | 
75 |     def successor(self, x):
76 |         """Finds the successor of node x in sorted order
77 | 
78 |         Time complexity: O(lg n) (balanced)"""
79 |         if x.right is not self.nil:
80 |             return self.minimum(x.right)
81 |         y = x.parent
82 |         while y is not self.nil and x is y.right:
83 |             x = y
84 |             y = y.parent
85 |         return y if y is not self.nil else None
86 | 
87 |     def predecessor(self, x):
88 |         """Finds the predecessor of node x in sorted order
89 | 
90 |         Time complexity: O(lg n) (balanced)"""
91 |         if x.left is not self.nil:
92 |             return self.maximum(x.left)
93 |         y = x.parent
94 |         while y is not self.nil and x is y.left:
95 |             x = y
96 |             y = y.parent
97 |         return y if y is not self.nil else None
98 | 
99 | 


--------------------------------------------------------------------------------
/n3map/tree/nsec3tree.py:
--------------------------------------------------------------------------------
  1 | from . import rbtree
  2 | from .. import log
  3 | from ..exception import N3MapError
  4 | 
  5 | class NSEC3TreeNode(rbtree.RBTreeNode):
  6 |     def __init__(self, k, v, int_end=None, nil=None):
  7 |         super(NSEC3TreeNode, self).__init__(k, v, nil)
  8 |         self.int_end = int_end
  9 | 
 10 |     def covers(self, k):
 11 |         if self.is_only():
 12 |             return True
 13 |         if self.is_last():
 14 |             return (k >= self.key or k <= self.int_end)
 15 |         return k >= self.key and k <= self.int_end
 16 | 
 17 |     def covered_distance(self, hash_max):
 18 |         l1 = int.from_bytes(self.key, "big")
 19 |         l2 = int.from_bytes(self.int_end, "big")
 20 |         if self.is_only():
 21 |             return hash_max
 22 |         if self.is_last():
 23 |             return hash_max - l1 + l2
 24 |         return l2-l1
 25 | 
 26 | 
 27 |     def is_last(self):
 28 |         return (self.key >= self.int_end)
 29 | 
 30 |     def is_only(self):
 31 |         # 1 record covers entire zone
 32 |         return (self.key == self.int_end)
 33 | 
 34 | class NSEC3Tree(rbtree.RBTree):
 35 |     def __init__(self, hash_max, node_type=NSEC3TreeNode):
 36 |         super(NSEC3Tree, self).__init__(node_type)
 37 |         self.last = None
 38 |         self.hash_max = hash_max
 39 |         self.covered_distance = int(0)
 40 |         self.ignore_overlapping = False
 41 | 
 42 |     def find_interval(self, k):
 43 |         """Finds the node n for which n.key <= k <= n.int_end
 44 | 
 45 |         Time complexity: O(lg n) (balanced)"""
 46 | 
 47 |         x = self.root
 48 |         p = self.nil
 49 | 
 50 |         if self.last is not None and self.last.covers(k):
 51 |             return self.last
 52 | 
 53 |         while x is not self.nil and k != x.key:
 54 |             p = x
 55 |             if k < x.key:
 56 |                 x = x.left
 57 |             else:
 58 |                 x = x.right
 59 | 
 60 |         if x is self.nil and p is not self.nil:
 61 |             if p.covers(k):
 62 |                 x = p
 63 |             elif p.key > k:
 64 |                 y = self.predecessor(p)
 65 |                 if y is not None:
 66 |                     if y.covers(k):
 67 |                         x = y
 68 | 
 69 |         return x if x is not self.nil else None
 70 | 
 71 |     def update(self, x, new):
 72 |         if x.int_end != new.int_end:
 73 |             # same hashed owner name, but interval changed
 74 |             log.warn("next hashed owner changed for existing NSEC3 record\n",
 75 |                             "zone may have changed")
 76 |             self.covered_distance += new.covered_distance(self.hash_max)
 77 |             self.covered_distance -= x.covered_distance(self.hash_max)
 78 |         x.value = new.value
 79 |         x.int_end = new.int_end
 80 | 
 81 |     def _check_overlap(self, node):
 82 |         if self.ignore_overlapping:
 83 |             return
 84 |         pre = self.predecessor(node)
 85 |         if pre is not None:
 86 |             if pre.int_end > node.key:
 87 |                 raise OverLapError
 88 |         suc = self.successor(node)
 89 |         if suc is not None:
 90 |             if node.int_end > suc.key:
 91 |                 raise OverLapError
 92 | 
 93 |     def insert(self, k, v, int_end):
 94 |         was_updated = False
 95 |         new = self.node_type(k=k, v=v, int_end=int_end)
 96 |         inserted = super(NSEC3Tree, self).insert_node(new)
 97 |         if new is inserted:
 98 |             # node didn't already exist
 99 |             self.covered_distance += new.covered_distance(self.hash_max)
100 |         else:
101 |             was_updated = True
102 |         new = inserted
103 | 
104 |         if self.last is None and new.is_last():
105 |             self.last = new
106 | 
107 |         self._check_overlap(new)
108 |         return (new, was_updated)
109 | 
110 |     def delete(self, node):
111 |         deleted = super(NSEC3Tree, self).delete(node)
112 |         if self.last is deleted:
113 |             self.last = None
114 | 
115 |         self.covered_distance -= deleted.covered_distance(self.hash_max)
116 | 
117 |         return deleted
118 | 
119 | class OverLapError(N3MapError):
120 |     pass
121 | 


--------------------------------------------------------------------------------
/n3map/tree/rbtree.py:
--------------------------------------------------------------------------------
  1 | from . import bstree
  2 | 
  3 | RED = 0
  4 | BLACK = 1
  5 | 
  6 | class RBTreeNode(bstree.BSTreeNode):
  7 |     """A node of a Red-Black Tree"""
  8 | 
  9 |     def __init__(self, k, v, nil=None):
 10 |         super(RBTreeNode, self).__init__(k, v, nil)
 11 |         self.color = BLACK
 12 |         self.size = 0
 13 | 
 14 |     def update_size(self):
 15 |         """Update the size attribute using the size attribute of left and right childs.
 16 | 
 17 |         Time complexity: O(1)"""
 18 |         self.size = 1 + self.left.size + self.right.size
 19 | 
 20 | class RBTree(bstree.BSTree):
 21 |     """A Red-Black Binary Search Tree"""
 22 |     def __init__(self, node_type=RBTreeNode):
 23 |         super(RBTree, self).__init__(node_type)
 24 |         self.nil.color = BLACK
 25 | 
 26 |     def _left_rotate(self, x):
 27 |         """Perform a left rotation around node x
 28 | 
 29 |         Time complexity: O(1)"""
 30 |         y = x.right
 31 |         x.right = y.left
 32 |         if y.left is not self.nil:
 33 |             y.left.parent = x
 34 |         y.parent = x.parent
 35 |         if x.parent is self.nil:
 36 |             self.root = y
 37 |         elif x is x.parent.left:
 38 |             x.parent.left = y
 39 |         else:
 40 |             x.parent.right = y
 41 |         y.left = x
 42 |         x.parent = y
 43 |         y.size = x.size
 44 |         x.update_size()
 45 | 
 46 |     def _right_rotate(self, x):
 47 |         """Perform a right rotation around node x
 48 | 
 49 |         Time complexity: O(1)"""
 50 |         y = x.left
 51 |         x.left = y.right
 52 |         if y.right is not self.nil:
 53 |             y.right.parent = x
 54 |         y.parent = x.parent
 55 |         if x.parent is self.nil:
 56 |             self.root = y
 57 |         elif x is x.parent.right:
 58 |             x.parent.right = y
 59 |         else:
 60 |             x.parent.left = y
 61 |         y.right = x
 62 |         x.parent = y
 63 |         y.size = x.size
 64 |         x.update_size()
 65 | 
 66 |     def _insert_fixup(self, new):
 67 |         """Restore Red-Black properties of the tree after node insertion.
 68 | 
 69 |         Time complexity: O(lg n)"""
 70 |         while new.parent.color == RED:
 71 |             if new.parent is new.parent.parent.left:
 72 |                 y = new.parent.parent.right
 73 |                 if y.color == RED:
 74 |                     new.parent.color = BLACK
 75 |                     y.color = BLACK
 76 |                     new.parent.parent.color = RED
 77 |                     new = new.parent.parent
 78 |                 else:
 79 |                     if new is new.parent.right:
 80 |                         new = new.parent
 81 |                         self._left_rotate(new)
 82 |                     new.parent.color = BLACK
 83 |                     new.parent.parent.color = RED
 84 |                     self._right_rotate(new.parent.parent)
 85 |             else:
 86 |                 y = new.parent.parent.left
 87 |                 if y.color == RED:
 88 |                     new.parent.color = BLACK
 89 |                     y.color = BLACK
 90 |                     new.parent.parent.color = RED
 91 |                     new = new.parent.parent
 92 |                 else:
 93 |                     if new is new.parent.left:
 94 |                         new = new.parent
 95 |                         self._right_rotate(new)
 96 |                     new.parent.color = BLACK
 97 |                     new.parent.parent.color = RED
 98 |                     self._left_rotate(new.parent.parent)
 99 |         self.root.color = BLACK
100 | 
101 |     def _transplant(self, old, new):
102 |         """Replace subtree rooted at node old with the subtree rooted at node new
103 | 
104 |         Time complexity: O(1)"""
105 |         if old.parent is self.nil:
106 |             self.root = new
107 |         elif old is old.parent.left:
108 |             old.parent.left = new
109 |         else:
110 |             old.parent.right = new
111 |         new.parent = old.parent
112 | 
113 |     def _delete_fixup(self, x):
114 |         """Restore Red-Black properties of the tree after node deletion.
115 | 
116 |         Time complexity: O(lg n)"""
117 |         while x is not self.root and x.color == BLACK:
118 |             if x is x.parent.left:
119 |                 w = x.parent.right
120 |                 if w.color == RED:
121 |                     w.color = BLACK
122 |                     x.parent.color = RED
123 |                     self._left_rotate(x.parent)
124 |                     w = x.parent.right
125 |                 if w.left.color == BLACK and w.right.color == BLACK:
126 |                     w.color = RED
127 |                     x = x.parent
128 |                 else:
129 |                     if w.right.color == BLACK:
130 |                         w.left.color = BLACK
131 |                         w.color = RED
132 |                         self._right_rotate(w)
133 |                         w = x.parent.right
134 |                     w.color = x.parent.color
135 |                     x.parent.color = BLACK
136 |                     w.right.color = BLACK
137 |                     self._left_rotate(x.parent)
138 |                     x = self.root
139 |             else:
140 |                 w = x.parent.left
141 |                 if w.color == RED:
142 |                     w.color = BLACK
143 |                     x.parent.color = RED
144 |                     self._right_rotate(x.parent)
145 |                     w = x.parent.left
146 |                 if w.right.color == BLACK and w.left.color == BLACK:
147 |                     w.color = RED
148 |                     x = x.parent
149 |                 else:
150 |                     if w.left.color == BLACK:
151 |                         w.right.color = BLACK
152 |                         w.color = RED
153 |                         self._left_rotate(w)
154 |                         w = x.parent.left
155 |                     w.color = x.parent.color
156 |                     x.parent.color = BLACK
157 |                     w.left.color = BLACK
158 |                     self._right_rotate(x.parent)
159 |                     x = self.root
160 |         x.color = BLACK
161 | 
162 |     def _update_size(self, node):
163 |         """Updates the size attribute on all nodes from node to the root.
164 | 
165 |         Time complexity: O(lg n)"""
166 |         while node is not self.nil:
167 |             node.update_size()
168 |             node = node.parent
169 | 
170 |     def deletekey(self, k):
171 |         node = self.find(k)
172 |         if node is not None:
173 |             node = self.delete(node)
174 |         return node
175 | 
176 |     def delete(self, node):
177 |         """Delete node from the tree, preserving all red-black properties.
178 | 
179 |         Returns the deleted node.
180 |         Time complexity: O(lg n)"""
181 |         y = node
182 |         y_orig_color = y.color
183 |         if node.left is self.nil:
184 |             x = node.right
185 |             sz_update = node.parent
186 |             self._transplant(node, node.right)
187 |             self._update_size(sz_update)
188 |         elif node.right is self.nil:
189 |             x = node.left
190 |             sz_update = node.parent
191 |             self._transplant(node, node.left)
192 |             self._update_size(sz_update)
193 |         else:
194 |             y = self.minimum(node.right)
195 |             y_orig_color = y.color
196 |             x = y.right
197 |             if y.parent is node:
198 |                 sz_update = y
199 |                 x.parent = y
200 |             else:
201 |                 sz_update = y.parent
202 |                 self._transplant(y, y.right)
203 |                 y.right = node.right
204 |                 y.right.parent = y
205 |             self._transplant(node, y)
206 |             y.left = node.left
207 |             y.left.parent = y
208 |             y.color = node.color
209 |             self._update_size(sz_update)
210 | 
211 | 
212 |         if y_orig_color == BLACK:
213 |             self._delete_fixup(x)
214 |         return node
215 | 
216 |     def update(self, x, new):
217 |         """Set change value(s) of node x to those from node new
218 | 
219 |         Time Complexity: O(1)
220 |         """
221 |         x.value = new.value
222 | 
223 | 
224 |     def insert_node(self, new):
225 |         """Insert a new node with distinct key k and value v into the tree, preserving all red-black properties.
226 | 
227 |         Returns the newly inserted/updated node
228 |         Time complexity: O(lg n)"""
229 |         y = self.nil
230 |         x = self.root
231 |         while x is not self.nil:
232 |             y = x
233 |             if new.key < x.key:
234 |                 x = x.left
235 |             elif new.key > x.key:
236 |                 x = x.right
237 |             else:
238 |                 # key is already in tree
239 |                 self.update(x, new)
240 |                 return x
241 | 
242 |         new.parent = y
243 |         if y is self.nil:
244 |             self.root = new
245 |         elif new.key < y.key:
246 |             y.left = new
247 |         else:
248 |             y.right = new
249 |         new.left = self.nil
250 |         new.right = self.nil
251 |         new.color = RED
252 |         new.size = 1
253 |         self._update_size(new)
254 |         self._insert_fixup(new)
255 |         return new
256 | 
257 | 
258 |     def size(self):
259 |         """Returns the number of nodes stored in the tree.
260 | 
261 |         Time complexity: O(1)"""
262 |         return self.root.size
263 | 


--------------------------------------------------------------------------------
/n3map/util.py:
--------------------------------------------------------------------------------
 1 | import string
 2 | import struct
 3 | import base64
 4 | 
 5 | # see RFC4648 for details
 6 | # TODO: python3.10 has support for this included: base64.b32hexencode()
 7 | b32_to_b32_ext_hex = bytes.maketrans(b"ABCDEFGHIJKLMNOPQRSTUVWXYZ234567",
 8 |                                      b"0123456789ABCDEFGHIJKLMNOPQRSTUV")
 9 | b32_ext_hex_to_b32 = bytes.maketrans(b"0123456789ABCDEFGHIJKLMNOPQRSTUV",
10 |                                      b"ABCDEFGHIJKLMNOPQRSTUVWXYZ234567")
11 | def base32_ext_hex_encode(s):
12 |     return base64.b32encode(s).translate(b32_to_b32_ext_hex)
13 | 
14 | def base32_ext_hex_decode(s):
15 |     return base64.b32decode(s.upper().translate(b32_ext_hex_to_b32))
16 | 
17 | def printsafe(s):
18 |     return ''.join(map(lambda c: c if c.isprintable() else '\uFFFD', s))
19 | 
20 | 
21 | 


--------------------------------------------------------------------------------
/n3map/vis.py:
--------------------------------------------------------------------------------
 1 | 
 2 | import string
 3 | import struct
 4 | 
 5 | hex_chars = b"0123456789abcdefABCDEF"
 6 | 
 7 | ascii_printable = string.printable.encode("ascii")
 8 | 
 9 | def vis(char):
10 |     """ Returns True if a character is safe to print
11 | 
12 |     char:    the character to test
13 | 
14 |     """
15 |     return char in ascii_printable
16 | 
17 | def strvis(s):
18 |     """Encode a string so it is safe to print on a tty
19 | 
20 |     s:    the string to encode
21 | 
22 |     """
23 |     enc_str = []
24 |     chars = struct.unpack('B' * len(s), s)
25 |     for c in chars:
26 |         if vis(c):
27 |             enc_str.append(struct.pack('B', c))
28 |             if c == struct.unpack('B', b'\\')[0]:
29 |                 enc_str.append(b'\\')
30 |         else:
31 |             enc_str.append(b"\\x%02x" % c)
32 |     return b''.join(enc_str)
33 | 
34 | 
35 | def strunvis(s):
36 |     """Decode a strvis-encoded string
37 | 
38 |     s:    the string to decode
39 | 
40 |     """
41 |     i = 0
42 |     d_s = []
43 |     push = None
44 |     while i < len(s):
45 |         if push is not None:
46 |             push = push + bytes([s[i]])
47 |             if push == b'\\\\':
48 |                 d_s.append(b'\\')
49 |                 push = None
50 |             elif len(push) == 4:
51 |                 if push[:2] == b'\\x' and all([c in hex_chars for c in
52 |                     push[2:]]):
53 |                     d_s.append(bytes([int(push[2:], 16)]))
54 |                     push = None
55 |                 else:
56 |                     raise ValueError
57 |         elif bytes([s[i]]) == b'\\':
58 |             push = b'\\'
59 |         else:
60 |             d_s.append(bytes([s[i]]))
61 |         i += 1
62 | 
63 |     if push is not None:
64 |         raise ValueError
65 | 
66 |     return b''.join(d_s)
67 | 
68 | 


--------------------------------------------------------------------------------
/n3map/walker.py:
--------------------------------------------------------------------------------
 1 | from . import log
 2 | from . import name
 3 | from .exception import N3MapError
 4 | 
 5 | import secrets
 6 | 
 7 | def detect_dnssec_type(zone, queryprovider, attempts=5):
 8 |     log.info("detecting zone type...")
 9 |     i = 0
10 |     while attempts == 0 or i < attempts:
11 |         label_gen = name.label_generator(name.hex_label,
12 |                                          init=secrets.randbits(30 +
13 |                                              secrets.randbelow(31)))
14 |         dname = name.DomainName(next(label_gen)[0], *zone.labels)
15 |         result, _ = queryprovider.query(dname, rrtype='A')
16 | 
17 |         # check for NSEC/3 records even if we got a NOERROR response
18 |         # to try and avoid loops when the zone contains a wildcard domain
19 |         if len(result.find_NSEC()) > 0:
20 |             log.info("zone uses NSEC records")
21 |             return 'nsec'
22 |         elif len(result.find_NSEC3()) > 0:
23 |             log.info("zone uses NSEC3 records")
24 |             return 'nsec3'
25 | 
26 |         if result.status() == "NXDOMAIN":
27 |             raise N3MapError("zone doesn't seem to be DNSSEC-enabled")
28 |         elif result.status() != "NOERROR":
29 |             raise N3MapError("unexpected response status: ", result.status())
30 | 
31 |         # result.status() == "NOERROR":
32 |         log.info("hit an existing owner name")
33 |         i += 1
34 |     raise N3MapError("failed to detect zone type after {0:d} attempt(s), terminating.".format(attempts))
35 | 
36 | def check_dnskey(zone, queryprovider):
37 |     log.info('checking DNSKEY...')
38 |     res, _ = queryprovider.query(zone, rrtype='DNSKEY')
39 |     dnskey_owner = res.find_DNSKEY()
40 |     if dnskey_owner is None:
41 |         raise N3MapError("no DNSKEY RR found at ", zone,
42 |                 "\nZone is not DNSSEC-enabled.")
43 |     if dnskey_owner != zone:
44 |         raise N3MapError("invalid DNSKEY RR received. Aborting")
45 | 
46 | def check_soa(zone, queryprovider):
47 |     log.info('checking SOA...')
48 |     res, _ = queryprovider.query(zone, rrtype='SOA')
49 |     soa_owner = res.find_SOA()
50 |     if soa_owner is None:
51 |         raise N3MapError("no SOA RR found at ", zone,
52 |                 "\nZone name may be incorrect.")
53 |     if soa_owner != zone:
54 |         raise N3MapError("invalid SOA RR received. Aborting")
55 | 
56 | class Walker(object):
57 |     def __init__(self,
58 |                  zone,
59 |                  queryprovider,
60 |                  output_file=None,
61 |                  stats=None):
62 |         self.zone = zone
63 |         self.queryprovider = queryprovider
64 |         self.stats = stats if stats is not None else {}
65 |         self._output_file = output_file
66 | 
67 |     def _write_chain(self, chain):
68 |         for record in chain:
69 |             self._write_record(record)
70 | 
71 |     def _write_record(self, record):
72 |         if self._output_file is not None:
73 |             self._output_file.write_record(record)
74 | 
75 |     def _write_number_of_records(self, num):
76 |         if self._output_file is not None:
77 |             self._output_file.write_number_of_rrs(num)
78 | 


--------------------------------------------------------------------------------
/nsec3-lookup.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env python3
2 | 
3 | import n3map.nsec3lookup
4 | 
5 | if __name__ == '__main__':
6 |     n3map.nsec3lookup.main()
7 | 
8 | 


--------------------------------------------------------------------------------
/nsec3_gen_fmt_plug.c:
--------------------------------------------------------------------------------
  1 | /*
  2 |  * DNSSEC NSEC3 hash cracker. Developed as part of the nsec3map DNS zone
  3 |  * enumerator project (https://github.com/anonion0/nsec3map).
  4 |  *
  5 |  * This software is Copyright (c) 2016 Ralf Sager <nsec3map at 3fnc.org>,
  6 |  * and it is hereby released to the general public under the following terms:
  7 |  * Redistribution and use in source and binary forms, with or without
  8 |  * modification, are permitted.
  9 |  *
 10 |  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
 11 |  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 12 |  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 13 |  * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
 14 |  * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
 15 |  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
 16 |  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
 17 |  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
 18 |  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 19 |  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 20 |  * POSSIBILITY OF SUCH DAMAGE.
 21 |  *
 22 |  * Some of this code was inspired by sha1_gen_fmt_plug.c by Solar Designer
 23 |  */
 24 | 
 25 | #if FMT_EXTERNS_H
 26 | extern struct fmt_main fmt_nsec3_gen;
 27 | #elif FMT_REGISTERS_H
 28 | john_register_one(&fmt_nsec3_gen);
 29 | #else
 30 | 
 31 | #include <ctype.h>
 32 | #include <string.h>
 33 | #include <stdint.h>
 34 | #include "sha.h"
 35 | 
 36 | #include "arch.h"
 37 | #include "params.h"
 38 | #include "common.h"
 39 | #include "formats.h"
 40 | 
 41 | #define FORMAT_LABEL			"nsec3"
 42 | #define FORMAT_NAME			"DNSSEC NSEC3"
 43 | #define ALGORITHM_NAME			"32/" ARCH_BITS_STR
 44 | 
 45 | #define BENCHMARK_COMMENT		""
 46 | #define BENCHMARK_LENGTH		0
 47 | 
 48 | #define PLAINTEXT_LENGTH		125
 49 | 
 50 | #define MIN_KEYS_PER_CRYPT		1
 51 | #define MAX_KEYS_PER_CRYPT		1
 52 | 
 53 | #define BINARY_SIZE			20
 54 | #define BINARY_ALIGN			4
 55 | #define N3_MAX_SALT_SIZE		255
 56 | #define N3_MAX_ZONE_SIZE		255
 57 | 
 58 | 
 59 | #define HASH_LENGTH			20
 60 | #define SALT_SIZE			sizeof(struct salt_t)
 61 | #define SALT_ALIGN			1
 62 | 
 63 | struct salt_t {
 64 | 	uint16_t iterations;
 65 | 	size_t salt_length;
 66 | 	size_t zone_length;
 67 | 	unsigned char salt[N3_MAX_SALT_SIZE];
 68 | 	unsigned char zone_wf[N3_MAX_ZONE_SIZE];
 69 | };
 70 | 
 71 | static struct fmt_tests tests[] = {
 72 | 	{ "$NSEC3$100$4141414141414141$8c2d583acbe22616c69bb457e0c2111ced0a6e77$example.com.", "www" },
 73 | 	{ "$NSEC3$100$42424242$8fb38d13720815ed5b5fcefd973e0d7c3906ab02$example.com.", "mx" },
 74 | 	{ NULL }
 75 | };
 76 | 
 77 | 
 78 | static struct salt_t saved_salt;
 79 | /* length of the saved label, without the length field */
 80 | static int saved_key_length;
 81 | static unsigned char saved_key[PLAINTEXT_LENGTH + 1];
 82 | static unsigned char saved_wf_label[PLAINTEXT_LENGTH + 2];
 83 | 
 84 | static SHA_CTX sha_ctx;
 85 | static ARCH_WORD_32 crypt_out[5];
 86 | 
 87 | static void convert_label_wf(void)
 88 | {
 89 | 	int last_dot = saved_key_length - 1;
 90 | 	int i;
 91 | 	unsigned char *out = saved_wf_label;
 92 | 	if (saved_key_length == 0)
 93 | 		return;
 94 | 	++out;
 95 | 	for (i = last_dot ; i >= 0; ) {
 96 | 		if (saved_key[i] == '.') {
 97 | 			out[i] = (unsigned char)(last_dot - i); 
 98 | 			last_dot = --i;
 99 | 		} else {
100 | 			out[i] = tolower(saved_key[i]);
101 | 			--i;
102 | 		}
103 | 	}
104 | 	*(--out) = (unsigned char)(last_dot - i);
105 | }
106 | 
107 | static size_t parse_zone(char *zone, unsigned char *zone_wf_out)
108 | {
109 | 	char *lbl_end, *lbl_start;
110 | 	unsigned int lbl_len;
111 | 	unsigned int index = 0;
112 | 	unsigned int zone_len = strlen(zone);
113 | 
114 | 	/* TODO: unvis */
115 | 	if (zone_len == 0) {
116 | 		return 0;
117 | 	} else if (zone_len > N3_MAX_ZONE_SIZE) {
118 | 		return 0;
119 | 	} 
120 | 
121 | 	lbl_end = strchr(zone, '.');
122 | 	lbl_start = zone;
123 | 	while (lbl_end != NULL) {
124 | 		lbl_len = lbl_end - lbl_start;
125 | 		zone_wf_out[index] = (unsigned char) lbl_len;
126 | 		if (lbl_len > 0) {
127 | 			memcpy(&zone_wf_out[++index], lbl_start, lbl_len);
128 | 		}
129 | 		index += lbl_len;
130 | 		lbl_start = lbl_end+1;
131 | 		if (lbl_start - zone == zone_len) {
132 | 			zone_wf_out[index] = 0;
133 | 			break;
134 | 		} else {
135 | 			lbl_end = strchr(lbl_start, '.');
136 | 		}
137 | 	}
138 | 	if (lbl_end == NULL)
139 | 		return 0;
140 | 	return index + 1;
141 | }
142 | 
143 | 
144 | /* format:
145 |  * $NSEC3$iter$salt$hash$zone
146 |  */
147 | 
148 | static int valid(char *ciphertext, struct fmt_main *pFmt)
149 | {
150 | 	char *p, *q;
151 | 	int i;
152 | 	unsigned char zone[N3_MAX_ZONE_SIZE];
153 | 	unsigned int iter;
154 | 
155 | 	if (strncmp(ciphertext, "$NSEC3$", 7))
156 | 		return 0;
157 | 	p = ciphertext;
158 | 	for (i = 0; i < 4; ++i) {
159 | 		p = strchr(p, '$');
160 | 		if (p == NULL || *(++p) == 0)
161 | 			return 0;
162 | 		switch (i) {
163 | 		case 0:
164 | 			continue;
165 | 		case 1:
166 | 			/* iterations */
167 | 			iter = atoi(p);
168 | 			if (iter < 0 || iter > UINT16_MAX)
169 | 				return 0;
170 | 			break;
171 | 		case 2:
172 | 			/*  salt */
173 | 			q = p;
174 | 			while (atoi16[ARCH_INDEX(*q)] != 0x7F)
175 | 				++q;
176 | 			if (*q != '$' || q-p > N3_MAX_SALT_SIZE*2 || (q-p) % 2)
177 | 				return 0;
178 | 			break;
179 | 		case 3:
180 | 			/* hash */
181 | 			q = p;
182 | 			while (atoi16[ARCH_INDEX(*q)] != 0x7F)
183 | 				++q;
184 | 			if (*q != '$' || q-p > HASH_LENGTH*2)
185 | 				return 0;
186 | 			p = q+1;
187 | 			break;
188 | 		}
189 | 	}
190 | 	/* zone */
191 | 	if (*p== 0)
192 | 		return 0;
193 | 	if (parse_zone(p, zone) == 0) {
194 | 		return 0;
195 | 	}
196 | 	return 1;
197 | }
198 | 
199 | static void *get_binary(char *ciphertext)
200 | {
201 | 	static unsigned char out[BINARY_SIZE];
202 | 	char *p;
203 | 	int i;
204 | 
205 | 	p = ciphertext;
206 | 	for (i = 0; i < 4; ++i) {
207 | 		p = strchr(p, '$') + 1;
208 | 	}
209 | 
210 | 	for (i = 0; i < sizeof(out); ++i) {
211 | 		out[i] = (atoi16[ARCH_INDEX(*p)] << 4) |
212 | 			atoi16[ARCH_INDEX(p[1])];
213 | 		p += 2;
214 | 	}
215 | 	return out;
216 | }
217 | 
218 | static void *salt(char *ciphertext)
219 | {
220 | 	static struct salt_t out;
221 | 	unsigned int salt_length;
222 | 	int i;
223 | 	char *p, *q;
224 | 
225 | 	memset(&out, 0, sizeof(out));
226 | 	p = ciphertext;
227 | 	for (i = 0; i < 2; ++i) 
228 | 		p = strchr(p, '$') + 1;
229 | 	out.iterations = (uint16_t) atoi(p);
230 | 
231 | 	p = strchr(p, '$') + 1;
232 | 	q = strchr(p, '$'); 
233 | 	salt_length = q-p;
234 | 	for (i = 0; i < salt_length; i += 2) {
235 | 		out.salt[i/2] = (atoi16[ARCH_INDEX(*p)] << 4 |
236 | 				atoi16[ARCH_INDEX(p[1])]);
237 | 		p += 2;
238 | 	}
239 | 	out.salt_length = (unsigned char)((salt_length)/2);
240 | 
241 | 	p = strchr(q+1, '$') + 1;
242 | 	out.zone_length =  parse_zone(p, out.zone_wf);
243 | 
244 | 	return &out;
245 | }
246 | 
247 | 
248 | static int salt_hash(void *salt)
249 | {
250 | 	unsigned int hash = 0;
251 | 	int i;
252 | 	for (i = 0; i < SALT_SIZE; ++i) {
253 | 		hash <<= 1;
254 | 		hash += (unsigned char)((unsigned char *)salt)[i];
255 | 		if (hash >> 10) {
256 | 			hash ^= hash >> 10;
257 | 			hash &= 0x3FF;
258 | 		}
259 | 	}
260 | 	hash ^= hash >> 10;
261 | 	hash &= 0x3FF;
262 | 
263 | 	return hash;
264 | }
265 | 
266 | static void set_salt(void *salt)
267 | {
268 | 	memcpy(&saved_salt, salt, SALT_SIZE);
269 | }
270 | 
271 | static void set_key(char *key, int index)
272 | {
273 | 	saved_key_length = strlen(key);
274 | 	if (saved_key_length > PLAINTEXT_LENGTH)
275 | 		saved_key_length = PLAINTEXT_LENGTH;
276 | 	memcpy(saved_key, key, saved_key_length);
277 | 	convert_label_wf();
278 | }
279 | 
280 | static  char *get_key(int index)
281 | {
282 | 	saved_key[saved_key_length] = 0;
283 | 	return (char *) saved_key;
284 | }
285 | 
286 | static int crypt_all(int *pcount, struct db_salt *salt)
287 | {
288 | 	int count = *pcount;
289 | 
290 | 	register int i = 0;
291 | 	register uint16_t iterations = saved_salt.iterations;
292 | 	register size_t salt_length =  saved_salt.salt_length;
293 | 
294 | 	SHA1_Init(&sha_ctx);
295 | 	if (saved_key_length > 0)
296 | 		SHA1_Update(&sha_ctx, saved_wf_label, saved_key_length+1);
297 | 	SHA1_Update(&sha_ctx, saved_salt.zone_wf, saved_salt.zone_length);
298 | 	SHA1_Update(&sha_ctx, saved_salt.salt, salt_length);
299 | 	SHA1_Final((unsigned char *)crypt_out, &sha_ctx);
300 | 	while (i++ < iterations) {
301 | 		SHA1_Init(&sha_ctx);
302 | 		SHA1_Update(&sha_ctx, crypt_out, BINARY_SIZE);
303 | 		SHA1_Update(&sha_ctx, saved_salt.salt, salt_length);
304 | 		SHA1_Final((unsigned char *)crypt_out, &sha_ctx);
305 | 	}
306 | 
307 | 	return count;
308 | }
309 | 
310 | static int cmp_all(void *binary, int count)
311 | {
312 | 	return !memcmp(binary, crypt_out, BINARY_SIZE);
313 | }
314 | 
315 | static int cmp_exact(char *source, int index) { return 1; }
316 | 
317 | static int get_hash_0(int index) { return crypt_out[0] & 0xF; } 
318 | static int get_hash_1(int index) { return crypt_out[0] & 0xFF; } 
319 | static int get_hash_2(int index) { return crypt_out[0] & 0xFFF; } 
320 | static int get_hash_3(int index) { return crypt_out[0] & 0xFFFF; } 
321 | static int get_hash_4(int index) { return crypt_out[0] & 0xFFFFF; }
322 | static int get_hash_5(int index) { return crypt_out[0] & 0xFFFFFF; }
323 | static int get_hash_6(int index) { return crypt_out[0] & 0x7FFFFFF; }
324 | 
325 | struct fmt_main fmt_nsec3_gen = {
326 | 	{
327 | 		FORMAT_LABEL,
328 | 		FORMAT_NAME,
329 | 		ALGORITHM_NAME,
330 | 		BENCHMARK_COMMENT,
331 | 		BENCHMARK_LENGTH,
332 | 		PLAINTEXT_LENGTH,
333 | 		BINARY_SIZE,
334 | 		BINARY_ALIGN,
335 | 		SALT_SIZE,
336 | 		SALT_ALIGN,
337 | 		MIN_KEYS_PER_CRYPT,
338 | 		MAX_KEYS_PER_CRYPT,
339 | 		0,
340 | #if FMT_MAIN_VERSION > 11
341 | 		{ NULL },
342 | #endif
343 | 		tests
344 | 	}, {
345 | 		fmt_default_init,
346 | 		fmt_default_done,
347 | 		fmt_default_reset,
348 | 		fmt_default_prepare,
349 | 		valid,
350 | 		fmt_default_split,
351 | 		get_binary,
352 | 		salt,
353 | #if FMT_MAIN_VERSION > 11
354 | 		{ NULL },
355 | #endif
356 | 		fmt_default_source,
357 | 		{
358 | 			fmt_default_binary_hash_0,
359 | 			fmt_default_binary_hash_1,
360 | 			fmt_default_binary_hash_2,
361 | 			fmt_default_binary_hash_3,
362 | 			fmt_default_binary_hash_4,
363 | 			fmt_default_binary_hash_5,
364 | 			fmt_default_binary_hash_6
365 | 		},
366 | 		salt_hash,
367 | 		set_salt,
368 | 		set_key,
369 | 		get_key,
370 | 		fmt_default_clear_keys,
371 | 		crypt_all,
372 | 		{
373 | 			get_hash_0,
374 | 			get_hash_1,
375 | 			get_hash_2,
376 | 			get_hash_3,
377 | 			get_hash_4,
378 | 			get_hash_5,
379 | 			get_hash_6
380 | 		},
381 | 		cmp_all,
382 | 		cmp_all,
383 | 		cmp_exact
384 | 	}
385 | };
386 | #endif  /* plugin */
387 | 


--------------------------------------------------------------------------------
/pyproject.toml:
--------------------------------------------------------------------------------
 1 | [build-system]
 2 | requires = ["setuptools"]
 3 | build-backend = "setuptools.build_meta"
 4 | 
 5 | [project]
 6 | name = "n3map"
 7 | description = "Enumerate DNS zones based on DNSSEC records"
 8 | dynamic = ["version"]
 9 | authors = [
10 | 	{ name = "Ralf Sager", email = "nsec3map@3fnc.org" },
11 | ]
12 | readme = "README.md"
13 | requires-python = ">=3.9"
14 | license = { file = "COPYING" }
15 | dependencies = [
16 | 	"dnspython",
17 | ]
18 | 
19 | keywords = ["security",  "network", "cryptography",
20 | 	 "dns", "dnssec", "nsec", "nsec3", "scanner"]
21 | classifiers = [
22 |     "Development Status :: 5 - Production/Stable",
23 |     "Environment :: Console",
24 |     "Intended Audience :: Developers",
25 |     "Intended Audience :: Information Technology",
26 |     "Intended Audience :: Science/Research",
27 |     "Intended Audience :: System Administrators",
28 |     "Intended Audience :: Telecommunications Industry",
29 |     "License :: OSI Approved :: GNU General Public License v3 (GPLv3)",
30 |     "Operating System :: POSIX",
31 |     "Programming Language :: C",
32 |     "Programming Language :: Python :: 3",
33 |     "Topic :: Security",
34 |     "Topic :: Security :: Cryptography",
35 |     "Topic :: System :: Networking",
36 |     "Topic :: Internet :: Name Service (DNS)",
37 |     "Topic :: Internet",
38 | ]
39 | 
40 | [project.optional-dependencies]
41 | predict = [ "numpy", "scipy" ]
42 | 
43 | [project.scripts]
44 | n3map = 'n3map.map:main'
45 | n3map-johnify = 'n3map.johnify:main'
46 | n3map-hashcatify = 'n3map.hashcatify:main'
47 | n3map-nsec3-lookup = 'n3map.nsec3lookup:main'
48 | 
49 | [project.urls]
50 | "Homepage" = "https://github.com/anonion0/nsec3map"
51 | 
52 | [tool.setuptools.dynamic]
53 | version = {attr = "n3map.__version__"}
54 | 
55 | [tool.setuptools]
56 | data-files = { 'share/man/man1' = [
57 |                 'doc/n3map.1',
58 |                 'doc/n3map-nsec3-lookup.1',
59 |                 'doc/n3map-johnify.1',
60 |                 'doc/n3map-hashcatify.1',
61 |                 ] }
62 | 
63 | 


--------------------------------------------------------------------------------
/screenshot.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/anonion0/nsec3map/d145b134742e88dc74579e2252db827a321316c7/screenshot.png


--------------------------------------------------------------------------------
/setup.py:
--------------------------------------------------------------------------------
 1 | from setuptools import setup, Extension
 2 | 
 3 | 
 4 | setup(
 5 |     ext_modules = [
 6 |         Extension(
 7 |             name = "n3map.nsec3hash",
 8 |             sources = ["n3map/nsec3hash.c"],
 9 |             libraries = ["crypto"],
10 |             extra_compile_args = ["-O3"],
11 |             ),
12 |         ],
13 | )
14 | 


--------------------------------------------------------------------------------