├── .gitignore ├── .travis.yml ├── LICENSE ├── README.md ├── TODO ├── build.sbt ├── project ├── assembly.sbt ├── build.properties ├── plugins.sbt └── project │ └── target │ └── config-classes │ ├── $8ef092c91510c23606ca.cache │ └── $c840bd9eec2738c3fcb5.cache └── src ├── main ├── resources │ └── banner.txt └── scala │ └── com │ └── gilazaria │ └── subsearch │ ├── SubSearch.scala │ ├── connection │ ├── DNSLookup.scala │ ├── DNSLookupImpl.scala │ ├── LookupFactory.scala │ ├── LookupFactoryImpl.scala │ ├── ZoneTransferInFactory.scala │ └── ZoneTransferInFactoryImpl.scala │ ├── controller │ └── Controller.scala │ ├── core │ ├── ArgumentParser.scala │ ├── AuthoritativeScanner.scala │ ├── ZoneTransferScanner.scala │ └── subdomainscanner │ │ ├── Dispatcher.scala │ │ ├── DispatcherQueue.scala │ │ ├── Listener.scala │ │ ├── Message.scala │ │ ├── PauseHandler.scala │ │ ├── Scanner.scala │ │ └── SubdomainScanner.scala │ ├── discovery │ ├── DNSDumpsterScanner.scala │ ├── Scanner.scala │ └── VirusTotalScanner.scala │ ├── model │ ├── Record.scala │ └── RecordType.scala │ ├── output │ ├── CLIOutput.scala │ ├── CSVOutput.scala │ ├── Logger.scala │ ├── LoggerImpl.scala │ ├── Output.scala │ └── StandardOutput.scala │ └── utils │ ├── FileUtils.scala │ ├── HostnameUtils.scala │ ├── IPUtils.scala │ ├── IterableUtils.scala │ ├── MathUtils.scala │ ├── TimeUtils.scala │ └── TimeoutScheduler.scala └── test ├── resources └── DNSDumpster │ ├── DNSDumpsterExample.html │ └── ExampledomainExample.html └── scala └── com └── gilazaria └── subsearch ├── connection ├── DNSLookupImplSpec.scala └── LookupTestUtils.scala ├── core ├── AuthoritativeScannerSpec.scala └── ZoneTransferScannerSpec.scala ├── discovery ├── DNSDumpsterScannerSpec.scala └── VirusTotalScannerSpec.scala └── utils ├── HostnameUtilsSpec.scala ├── IPUtilsSpec.scala ├── MathUtilsSpec.scala └── TimeUtilsSpec.scala /.gitignore: -------------------------------------------------------------------------------- 1 | /target/ 2 | /project/target/ 3 | *.class 4 | *.iml 5 | /lib_managed/ 6 | /src_managed/ 7 | *.ipr 8 | *.iws 9 | .idea 10 | out 11 | .DS_Store -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: scala 2 | 3 | scala: 4 | - 2.11.8 5 | 6 | jdk: 7 | - oraclejdk8 8 | 9 | script: "sbt clean coverage test" 10 | after_success: "sbt coverageReport coveralls" -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | GNU GENERAL PUBLIC LICENSE 2 | Version 2, June 1991 3 | 4 | Copyright (C) 1989, 1991 Free Software Foundation, Inc., 5 | 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA 6 | Everyone is permitted to copy and distribute verbatim copies 7 | of this license document, but changing it is not allowed. 8 | 9 | Preamble 10 | 11 | The licenses for most software are designed to take away your 12 | freedom to share and change it. By contrast, the GNU General Public 13 | License is intended to guarantee your freedom to share and change free 14 | software--to make sure the software is free for all its users. This 15 | General Public License applies to most of the Free Software 16 | Foundation's software and to any other program whose authors commit to 17 | using it. (Some other Free Software Foundation software is covered by 18 | the GNU Lesser General Public License instead.) You can apply it to 19 | your programs, too. 20 | 21 | When we speak of free software, we are referring to freedom, not 22 | price. Our General Public Licenses are designed to make sure that you 23 | have the freedom to distribute copies of free software (and charge for 24 | this service if you wish), that you receive source code or can get it 25 | if you want it, that you can change the software or use pieces of it 26 | in new free programs; and that you know you can do these things. 27 | 28 | To protect your rights, we need to make restrictions that forbid 29 | anyone to deny you these rights or to ask you to surrender the rights. 30 | These restrictions translate to certain responsibilities for you if you 31 | distribute copies of the software, or if you modify it. 32 | 33 | For example, if you distribute copies of such a program, whether 34 | gratis or for a fee, you must give the recipients all the rights that 35 | you have. You must make sure that they, too, receive or can get the 36 | source code. And you must show them these terms so they know their 37 | rights. 38 | 39 | We protect your rights with two steps: (1) copyright the software, and 40 | (2) offer you this license which gives you legal permission to copy, 41 | distribute and/or modify the software. 42 | 43 | Also, for each author's protection and ours, we want to make certain 44 | that everyone understands that there is no warranty for this free 45 | software. If the software is modified by someone else and passed on, we 46 | want its recipients to know that what they have is not the original, so 47 | that any problems introduced by others will not reflect on the original 48 | authors' reputations. 49 | 50 | Finally, any free program is threatened constantly by software 51 | patents. We wish to avoid the danger that redistributors of a free 52 | program will individually obtain patent licenses, in effect making the 53 | program proprietary. To prevent this, we have made it clear that any 54 | patent must be licensed for everyone's free use or not licensed at all. 55 | 56 | The precise terms and conditions for copying, distribution and 57 | modification follow. 58 | 59 | GNU GENERAL PUBLIC LICENSE 60 | TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION 61 | 62 | 0. This License applies to any program or other work which contains 63 | a notice placed by the copyright holder saying it may be distributed 64 | under the terms of this General Public License. The "Program", below, 65 | refers to any such program or work, and a "work based on the Program" 66 | means either the Program or any derivative work under copyright law: 67 | that is to say, a work containing the Program or a portion of it, 68 | either verbatim or with modifications and/or translated into another 69 | language. (Hereinafter, translation is included without limitation in 70 | the term "modification".) Each licensee is addressed as "you". 71 | 72 | Activities other than copying, distribution and modification are not 73 | covered by this License; they are outside its scope. The act of 74 | running the Program is not restricted, and the output from the Program 75 | is covered only if its contents constitute a work based on the 76 | Program (independent of having been made by running the Program). 77 | Whether that is true depends on what the Program does. 78 | 79 | 1. You may copy and distribute verbatim copies of the Program's 80 | source code as you receive it, in any medium, provided that you 81 | conspicuously and appropriately publish on each copy an appropriate 82 | copyright notice and disclaimer of warranty; keep intact all the 83 | notices that refer to this License and to the absence of any warranty; 84 | and give any other recipients of the Program a copy of this License 85 | along with the Program. 86 | 87 | You may charge a fee for the physical act of transferring a copy, and 88 | you may at your option offer warranty protection in exchange for a fee. 89 | 90 | 2. You may modify your copy or copies of the Program or any portion 91 | of it, thus forming a work based on the Program, and copy and 92 | distribute such modifications or work under the terms of Section 1 93 | above, provided that you also meet all of these conditions: 94 | 95 | a) You must cause the modified files to carry prominent notices 96 | stating that you changed the files and the date of any change. 97 | 98 | b) You must cause any work that you distribute or publish, that in 99 | whole or in part contains or is derived from the Program or any 100 | part thereof, to be licensed as a whole at no charge to all third 101 | parties under the terms of this License. 102 | 103 | c) If the modified program normally reads commands interactively 104 | when run, you must cause it, when started running for such 105 | interactive use in the most ordinary way, to print or display an 106 | announcement including an appropriate copyright notice and a 107 | notice that there is no warranty (or else, saying that you provide 108 | a warranty) and that users may redistribute the program under 109 | these conditions, and telling the user how to view a copy of this 110 | License. (Exception: if the Program itself is interactive but 111 | does not normally print such an announcement, your work based on 112 | the Program is not required to print an announcement.) 113 | 114 | These requirements apply to the modified work as a whole. If 115 | identifiable sections of that work are not derived from the Program, 116 | and can be reasonably considered independent and separate works in 117 | themselves, then this License, and its terms, do not apply to those 118 | sections when you distribute them as separate works. But when you 119 | distribute the same sections as part of a whole which is a work based 120 | on the Program, the distribution of the whole must be on the terms of 121 | this License, whose permissions for other licensees extend to the 122 | entire whole, and thus to each and every part regardless of who wrote it. 123 | 124 | Thus, it is not the intent of this section to claim rights or contest 125 | your rights to work written entirely by you; rather, the intent is to 126 | exercise the right to control the distribution of derivative or 127 | collective works based on the Program. 128 | 129 | In addition, mere aggregation of another work not based on the Program 130 | with the Program (or with a work based on the Program) on a volume of 131 | a storage or distribution medium does not bring the other work under 132 | the scope of this License. 133 | 134 | 3. You may copy and distribute the Program (or a work based on it, 135 | under Section 2) in object code or executable form under the terms of 136 | Sections 1 and 2 above provided that you also do one of the following: 137 | 138 | a) Accompany it with the complete corresponding machine-readable 139 | source code, which must be distributed under the terms of Sections 140 | 1 and 2 above on a medium customarily used for software interchange; or, 141 | 142 | b) Accompany it with a written offer, valid for at least three 143 | years, to give any third party, for a charge no more than your 144 | cost of physically performing source distribution, a complete 145 | machine-readable copy of the corresponding source code, to be 146 | distributed under the terms of Sections 1 and 2 above on a medium 147 | customarily used for software interchange; or, 148 | 149 | c) Accompany it with the information you received as to the offer 150 | to distribute corresponding source code. (This alternative is 151 | allowed only for noncommercial distribution and only if you 152 | received the program in object code or executable form with such 153 | an offer, in accord with Subsection b above.) 154 | 155 | The source code for a work means the preferred form of the work for 156 | making modifications to it. For an executable work, complete source 157 | code means all the source code for all modules it contains, plus any 158 | associated interface definition files, plus the scripts used to 159 | control compilation and installation of the executable. However, as a 160 | special exception, the source code distributed need not include 161 | anything that is normally distributed (in either source or binary 162 | form) with the major components (compiler, kernel, and so on) of the 163 | operating system on which the executable runs, unless that component 164 | itself accompanies the executable. 165 | 166 | If distribution of executable or object code is made by offering 167 | access to copy from a designated place, then offering equivalent 168 | access to copy the source code from the same place counts as 169 | distribution of the source code, even though third parties are not 170 | compelled to copy the source along with the object code. 171 | 172 | 4. You may not copy, modify, sublicense, or distribute the Program 173 | except as expressly provided under this License. Any attempt 174 | otherwise to copy, modify, sublicense or distribute the Program is 175 | void, and will automatically terminate your rights under this License. 176 | However, parties who have received copies, or rights, from you under 177 | this License will not have their licenses terminated so long as such 178 | parties remain in full compliance. 179 | 180 | 5. You are not required to accept this License, since you have not 181 | signed it. However, nothing else grants you permission to modify or 182 | distribute the Program or its derivative works. These actions are 183 | prohibited by law if you do not accept this License. Therefore, by 184 | modifying or distributing the Program (or any work based on the 185 | Program), you indicate your acceptance of this License to do so, and 186 | all its terms and conditions for copying, distributing or modifying 187 | the Program or works based on it. 188 | 189 | 6. Each time you redistribute the Program (or any work based on the 190 | Program), the recipient automatically receives a license from the 191 | original licensor to copy, distribute or modify the Program subject to 192 | these terms and conditions. You may not impose any further 193 | restrictions on the recipients' exercise of the rights granted herein. 194 | You are not responsible for enforcing compliance by third parties to 195 | this License. 196 | 197 | 7. If, as a consequence of a court judgment or allegation of patent 198 | infringement or for any other reason (not limited to patent issues), 199 | conditions are imposed on you (whether by court order, agreement or 200 | otherwise) that contradict the conditions of this License, they do not 201 | excuse you from the conditions of this License. If you cannot 202 | distribute so as to satisfy simultaneously your obligations under this 203 | License and any other pertinent obligations, then as a consequence you 204 | may not distribute the Program at all. For example, if a patent 205 | license would not permit royalty-free redistribution of the Program by 206 | all those who receive copies directly or indirectly through you, then 207 | the only way you could satisfy both it and this License would be to 208 | refrain entirely from distribution of the Program. 209 | 210 | If any portion of this section is held invalid or unenforceable under 211 | any particular circumstance, the balance of the section is intended to 212 | apply and the section as a whole is intended to apply in other 213 | circumstances. 214 | 215 | It is not the purpose of this section to induce you to infringe any 216 | patents or other property right claims or to contest validity of any 217 | such claims; this section has the sole purpose of protecting the 218 | integrity of the free software distribution system, which is 219 | implemented by public license practices. Many people have made 220 | generous contributions to the wide range of software distributed 221 | through that system in reliance on consistent application of that 222 | system; it is up to the author/donor to decide if he or she is willing 223 | to distribute software through any other system and a licensee cannot 224 | impose that choice. 225 | 226 | This section is intended to make thoroughly clear what is believed to 227 | be a consequence of the rest of this License. 228 | 229 | 8. If the distribution and/or use of the Program is restricted in 230 | certain countries either by patents or by copyrighted interfaces, the 231 | original copyright holder who places the Program under this License 232 | may add an explicit geographical distribution limitation excluding 233 | those countries, so that distribution is permitted only in or among 234 | countries not thus excluded. In such case, this License incorporates 235 | the limitation as if written in the body of this License. 236 | 237 | 9. The Free Software Foundation may publish revised and/or new versions 238 | of the General Public License from time to time. Such new versions will 239 | be similar in spirit to the present version, but may differ in detail to 240 | address new problems or concerns. 241 | 242 | Each version is given a distinguishing version number. If the Program 243 | specifies a version number of this License which applies to it and "any 244 | later version", you have the option of following the terms and conditions 245 | either of that version or of any later version published by the Free 246 | Software Foundation. If the Program does not specify a version number of 247 | this License, you may choose any version ever published by the Free Software 248 | Foundation. 249 | 250 | 10. If you wish to incorporate parts of the Program into other free 251 | programs whose distribution conditions are different, write to the author 252 | to ask for permission. For software which is copyrighted by the Free 253 | Software Foundation, write to the Free Software Foundation; we sometimes 254 | make exceptions for this. Our decision will be guided by the two goals 255 | of preserving the free status of all derivatives of our free software and 256 | of promoting the sharing and reuse of software generally. 257 | 258 | NO WARRANTY 259 | 260 | 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY 261 | FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN 262 | OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES 263 | PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED 264 | OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 265 | MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS 266 | TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE 267 | PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, 268 | REPAIR OR CORRECTION. 269 | 270 | 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING 271 | WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR 272 | REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, 273 | INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING 274 | OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED 275 | TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY 276 | YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER 277 | PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE 278 | POSSIBILITY OF SUCH DAMAGES. 279 | 280 | END OF TERMS AND CONDITIONS 281 | 282 | How to Apply These Terms to Your New Programs 283 | 284 | If you develop a new program, and you want it to be of the greatest 285 | possible use to the public, the best way to achieve this is to make it 286 | free software which everyone can redistribute and change under these terms. 287 | 288 | To do so, attach the following notices to the program. It is safest 289 | to attach them to the start of each source file to most effectively 290 | convey the exclusion of warranty; and each file should have at least 291 | the "copyright" line and a pointer to where the full notice is found. 292 | 293 | 294 | Copyright (C) 295 | 296 | This program is free software; you can redistribute it and/or modify 297 | it under the terms of the GNU General Public License as published by 298 | the Free Software Foundation; either version 2 of the License, or 299 | (at your option) any later version. 300 | 301 | This program is distributed in the hope that it will be useful, 302 | but WITHOUT ANY WARRANTY; without even the implied warranty of 303 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 304 | GNU General Public License for more details. 305 | 306 | You should have received a copy of the GNU General Public License along 307 | with this program; if not, write to the Free Software Foundation, Inc., 308 | 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 309 | 310 | Also add information on how to contact you by electronic and paper mail. 311 | 312 | If the program is interactive, make it output a short notice like this 313 | when it starts in an interactive mode: 314 | 315 | Gnomovision version 69, Copyright (C) year name of author 316 | Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. 317 | This is free software, and you are welcome to redistribute it 318 | under certain conditions; type `show c' for details. 319 | 320 | The hypothetical commands `show w' and `show c' should show the appropriate 321 | parts of the General Public License. Of course, the commands you use may 322 | be called something other than `show w' and `show c'; they could even be 323 | mouse-clicks or menu items--whatever suits your program. 324 | 325 | You should also get your employer (if you work as a programmer) or your 326 | school, if any, to sign a "copyright disclaimer" for the program, if 327 | necessary. Here is a sample; alter the names: 328 | 329 | Yoyodyne, Inc., hereby disclaims all copyright interest in the program 330 | `Gnomovision' (which makes passes at compilers) written by James Hacker. 331 | 332 | , 1 April 1989 333 | Ty Coon, President of Vice 334 | 335 | This General Public License does not permit incorporating your program into 336 | proprietary programs. If your program is a subroutine library, you may 337 | consider it more useful to permit linking proprietary applications with the 338 | library. If this is what you want to do, use the GNU Lesser General 339 | Public License instead of this License. 340 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # subsearch 2 | subsearch is a command line tool designed to discover subdomain names. It is aimed at penetration testers and bug 3 | bounty hunters and has been built with a focus on speed, stealth and reporting. 4 | 5 | The current release is version 0.2.0 and was published on 14/4/2016. 6 | 7 | ## Features 8 | 9 | - Scan a single hostname or a list of hostnames 10 | - Takes as arguments a comma separated list of DNS resolvers, and/or a file containing newline delimited list of resolvers 11 | - Recursive scanning: If a CNAME, MX, NS or SRV record is discovered, the any subdomains will be added to a priority list 12 | of subdomains to scan for 13 | - Support for additional scanners: 14 | - Attempt a Zone Transfer on the hostname's authoritative name servers 15 | - Retrieve seen subdomains from Virus Total 16 | - Retrieve seen subdomains from DNS Dumpster 17 | - Different levels of verbosity 18 | - Multiple real-time reporting capabilities 19 | - Supports the use of massive wordlists 20 | 21 | ## Requirements 22 | 23 | subsearch is built in scala using the Java 8 SDK. 24 | 25 | ## How to use 26 | 27 | A compiled copy of the latest version can be downloaded from the [releases page](https://github.com/gavia/subsearch/releases). 28 | Alternatively you can clone the latest commit and compile it by executing `sbt assembly` in the root folder. The compiled 29 | jar will be located in `target/scala-2.11/`. 30 | 31 | To show the below help text, execute `java -jar --help`. 32 | 33 | ``` 34 | subsearch 0.2.0 35 | Usage: subsearch [options] 36 | 37 | Options: 38 | --help 39 | Prints this usage text. 40 | 41 | Mandatory: 42 | -h HOSTNAME | --hostname HOSTNAME 43 | The hostname(s) to scan. Enter more than one by separating with a comma. 44 | and/or 45 | -H HOSTLIST | --hostlist HOSTLIST 46 | A file containing a newline delimited list of hostnames to scan. 47 | -w WORDLIST | --wordlist WORDLIST 48 | A newline delimited list of subdomain names. 49 | -r RESOLVERS | --resolvers RESOLVERS 50 | The name server(s) to scan with. Enter more than one by separating with a comma. 51 | and/or 52 | -R RESOLVERSLIST | --resolverslist RESOLVERSLIST 53 | A file containing a newline delimited list of name servers to scan with. 54 | 55 | General Settings: 56 | -a | --auth-resolvers 57 | Include the hostname's authoritative name servers in the list of resolvers. Defaults to false. 58 | -c | --concurrent-resolver-requests 59 | Allow for more than one request to each resolver at the same time. If true, it can result in being blacklisted or rate limited by some resolvers. Defaults to false. 60 | -t THREADCOUNT | --threads THREADCOUNT 61 | The number of concurrent threads whilst scanning. Defaults to 10. 62 | -v | --verbose 63 | Show more extended command line output such as the addresses that A, AAAA and CNAME records point to. Defaults to false. 64 | --comprehensive 65 | Runs all additional scanners. 66 | 67 | Additional Scanners: 68 | --dns-dumpster 69 | Attempts to lookup possible records from dnsdumpster.com 70 | --virus-total 71 | Attempts to lookup possible records from virustotal.com 72 | -z | --zone-transfer 73 | Attempts a zone transfer against the host's authoritative name servers. 74 | 75 | Reporting: 76 | --report-csv OUTPUTFILE 77 | Outputs a CSV report of discovered subdomains including timestamp, subdomain, record type and record data. 78 | --report-stdout OUTPUTFILE 79 | Outputs standard out to a file. 80 | 81 | ``` 82 | 83 | Subdomain and resolvers lists are not bundled with this tool as excellent resources already exist in other locations. 84 | For those that do not already possess these resources, [fuzzdb](https://github.com/fuzzdb-project/fuzzdb) and 85 | [subbrute](https://github.com/TheRook/subbrute) are good places to start. 86 | 87 | ## Issues 88 | 89 | If you have any problems or questions please open an issue and I'll try to help. If you're going to submit a bug, please 90 | provide steps to reproduce the issue and a copy of the program output. 91 | 92 | ## New features 93 | 94 | Contributions and suggestions are welcome! I'm going to continue to update this tool with new features, but if you have 95 | an idea for some great functionality then open an issue, or alternatively give it a go yourself and create a pull request. 96 | For a list of things that are already on my roadmap, checkout the `TODO` file. 97 | 98 | ## Changelog 99 | 100 | - 0.2.0 - 14/4/2016 101 | - Added Virus Total and DNS Dumpster as additional scanners 102 | - Zone Transfer is now off by default 103 | - Added an option to report standard out to file 104 | - Numerous bug fixes 105 | - Began writing tests 106 | - Removed check to see if domain is "valid" 107 | - 0.1.1 - 14/3/2016 108 | - subsearch can now handle massive wordlists, wordlists aren't loaded into memory in one go 109 | - resolver timeouts increased from 5, 10 and 15 seconds to 10, 20 and 30 seconds 110 | - other minor bug fixes 111 | - 0.1.0 - 14/3/2016 - Initial release 112 | 113 | ## License 114 | 115 | This tool is released under the GNU General Public License, version 2. A copy of this license can be found in the `LICENSE` 116 | file. 117 | 118 | The design for the user interface has been inspired by the fantastic tool [dirsearch](https://github.com/maurosoria/dirsearch) 119 | by Mauro Soria. Anything that could be considered a direct copy of his tool is Copyright (C) Mauro Soria. 120 | 121 | All other work is Copyright (C) Gil Azaria. 122 | -------------------------------------------------------------------------------- /TODO: -------------------------------------------------------------------------------- 1 | TODO: Change execution context to be imported from an object 2 | TODO: Prevent Listener from printing messages after pausing 3 | TODO: Implement third party services as scan options (e.g. autogeneration of short letter and number combinations, http://bgp.he.net lookup) 4 | TODO: Read file resolvers in to memory as they're used 5 | TODO: Create a list of found subdomains so that they are not rescanned 6 | TODO: Read in previously found subdomains csv file 7 | TODO: Write more unit tests 8 | TODO: Implement stop asking for ANY detection, e.g: 9 | 10 | ; <<>> DiG 9.8.3-P1 <<>> ANY some.domain.net 11 | ;; global options: +cmd 12 | ;; Got answer: 13 | ;; ->>HEADER<<- opcode: QUERY, status: NOERROR, id: 12941 14 | ;; flags: qr rd ra; QUERY: 1, ANSWER: 1, AUTHORITY: 0, ADDITIONAL: 0 15 | 16 | ;; QUESTION SECTION: 17 | some.domain.net. IN ANY 18 | 19 | ;; ANSWER SECTION: 20 | some.domain.net. 5 IN HINFO "Please stop asking for ANY" "See draft-ietf-dnsop-refuse-any" 21 | 22 | TODO: Implement RRL prevention in the form of multiple resolver lookups per subdomain 23 | TODO: Implement multiple verbosity options. 1st shows all records, 2nd also says when blacklisting, 3rd also says when times out fails -------------------------------------------------------------------------------- /build.sbt: -------------------------------------------------------------------------------- 1 | name := "subsearch" 2 | 3 | assemblyJarName in assembly := s"subsearch-0.2.0.jar" 4 | 5 | scalaVersion := "2.11.8" 6 | 7 | scalacOptions ++= Seq("-unchecked", "-deprecation") 8 | 9 | resolvers += Resolver.sonatypeRepo("public") 10 | 11 | libraryDependencies += "org.scalatest" % "scalatest_2.11" % "2.2.1" % "test" 12 | libraryDependencies += "org.scalamock" %% "scalamock-scalatest-support" % "3.2.2" % "test" 13 | libraryDependencies += "net.databinder.dispatch" %% "dispatch-core" % "0.11.2" 14 | libraryDependencies += "org.slf4j" % "slf4j-simple" % "1.6.6" 15 | libraryDependencies += "com.github.scopt" %% "scopt" % "3.4.0" 16 | libraryDependencies += "pl.project13.scala" %% "rainbow" % "0.2" 17 | libraryDependencies += "dnsjava" % "dnsjava" % "2.1.7" 18 | libraryDependencies += "com.typesafe.akka" %% "akka-actor" % "2.4.1" 19 | libraryDependencies += "org.scala-lang.modules" % "scala-jline" % "2.12.1" 20 | libraryDependencies += "net.ruippeixotog" %% "scala-scraper" % "1.0.0" -------------------------------------------------------------------------------- /project/assembly.sbt: -------------------------------------------------------------------------------- 1 | addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "0.13.0") -------------------------------------------------------------------------------- /project/build.properties: -------------------------------------------------------------------------------- 1 | sbt.version = 0.13.8 -------------------------------------------------------------------------------- /project/plugins.sbt: -------------------------------------------------------------------------------- 1 | logLevel := Level.Info 2 | 3 | addSbtPlugin("org.scoverage" % "sbt-scoverage" % "1.3.5") 4 | addSbtPlugin("org.scoverage" % "sbt-coveralls" % "1.1.0") -------------------------------------------------------------------------------- /project/project/target/config-classes/$8ef092c91510c23606ca.cache: -------------------------------------------------------------------------------- 1 | sbt.internals.DslEntry -------------------------------------------------------------------------------- /project/project/target/config-classes/$c840bd9eec2738c3fcb5.cache: -------------------------------------------------------------------------------- 1 | sbt.internals.DslEntry -------------------------------------------------------------------------------- /src/main/resources/banner.txt: -------------------------------------------------------------------------------- 1 | _ _ 2 | ___ _ _ | |__ ___ ___ _ _ ___ ___ | |_ vVERSION 3 | / __|| | | || _ \ / __| / _ \ / _` ||` __|/ __|| __ \ 4 | \__ \| |_| || |_) |\__ \( __/( (_| || | | (__ | | | | 5 | /___/ \__,_||_,__/ /___/ \___| \__,_||_| \___||_| |_| 6 | -------------------------------------------------------------------------------- /src/main/scala/com/gilazaria/subsearch/SubSearch.scala: -------------------------------------------------------------------------------- 1 | package com.gilazaria.subsearch 2 | 3 | import com.gilazaria.subsearch.controller.Controller 4 | import com.gilazaria.subsearch.core.{ArgumentParser, Arguments} 5 | import com.gilazaria.subsearch.output.{Logger, LoggerImpl} 6 | 7 | class SubSearch(args: Array[String]) { 8 | val arguments: Arguments = ArgumentParser.parseArguments(args) 9 | val logger: Logger = LoggerImpl.create(arguments.extendedOutput, arguments.csvReportFile, arguments.stdoutReportFile) 10 | val controller: Controller = Controller.create(arguments, logger) 11 | } 12 | 13 | object SubSearch { 14 | def main(args: Array[String]): Unit = 15 | new SubSearch(args) 16 | 17 | val version = "0.2.0" 18 | } -------------------------------------------------------------------------------- /src/main/scala/com/gilazaria/subsearch/connection/DNSLookup.scala: -------------------------------------------------------------------------------- 1 | package com.gilazaria.subsearch.connection 2 | 3 | import com.gilazaria.subsearch.model.{Record, RecordType} 4 | 5 | import scala.collection.SortedSet 6 | import scala.util.Try 7 | 8 | trait DNSLookup { 9 | def performQueryOfTypeANY(hostname: String, resolver: String): Try[SortedSet[Record]] = 10 | performQueryOfType(hostname, resolver, RecordType.ANY) 11 | 12 | def performQueryOfType(hostname: String, resolver: String, recordType: RecordType): Try[SortedSet[Record]] 13 | } 14 | -------------------------------------------------------------------------------- /src/main/scala/com/gilazaria/subsearch/connection/DNSLookupImpl.scala: -------------------------------------------------------------------------------- 1 | package com.gilazaria.subsearch.connection 2 | 3 | import com.gilazaria.subsearch.model.{Record, RecordType} 4 | import com.gilazaria.subsearch.utils.IterableUtils._ 5 | import org.xbill.DNS._ 6 | 7 | import scala.annotation.tailrec 8 | import scala.util.Try 9 | import scala.collection.JavaConverters._ 10 | import scala.collection.SortedSet 11 | 12 | class DNSLookupImpl extends DNSLookup { 13 | import DNSLookupImpl.{HostNotFoundException, ServerFailureException} 14 | 15 | override def performQueryOfType(hostname: String, resolver: String, recordType: RecordType): Try[SortedSet[Record]] = 16 | if (recordType == RecordType.AXFR) 17 | Try(performZoneTransfer(hostname, resolver)) 18 | else 19 | Try(performLookup(hostname, resolver, recordType)) 20 | 21 | private[connection] def performZoneTransfer(hostname: String, resolver: String): SortedSet[Record] = { 22 | val transfer: ZoneTransferInFactory = ZoneTransferInFactoryImpl.newAXFR(new Name(hostname), resolver, null) 23 | val xbillRecords: Set[org.xbill.DNS.Record] = xbillRecordsFromTransfer(transfer).getOrElse(Set.empty) 24 | 25 | recordsFromXbillRecords(xbillRecords) 26 | .filter(dnsRecord => !dnsRecord.name.startsWith(hostname)) 27 | } 28 | 29 | private[connection] def xbillRecordsFromTransfer(transfer: ZoneTransferInFactory): Try[Set[org.xbill.DNS.Record]] = { 30 | Try { 31 | Option(transfer.run()) 32 | .map(_.asScala.toSet) 33 | .getOrElse(Set.empty) 34 | .asInstanceOf[Set[org.xbill.DNS.Record]] 35 | } 36 | } 37 | 38 | private[connection] def performLookup(hostname: String, resolver: String, recordType: RecordType): SortedSet[Record] = { 39 | val lookup: LookupFactory = new LookupFactoryImpl(hostname, recordType.intValue) 40 | lookup.setResolver(new SimpleResolver(resolver)) 41 | query(hostname, resolver, lookup) 42 | } 43 | 44 | @tailrec 45 | private[connection] final def query(hostname: String, resolver: String, lookup: LookupFactory, attempt: Int = 1): SortedSet[Record] = { 46 | lookup.run() 47 | 48 | lookup.getResult match { 49 | case Lookup.SUCCESSFUL => 50 | val xbillRecords = 51 | Option(lookup.getAnswers) 52 | .map(_.toSet) 53 | .getOrElse(Set.empty) 54 | 55 | recordsFromXbillRecords(xbillRecords) 56 | 57 | case Lookup.HOST_NOT_FOUND => 58 | throw new HostNotFoundException(s"The hostname $hostname was not found.") 59 | 60 | case Lookup.UNRECOVERABLE => 61 | throw new ServerFailureException(s"There was a data or server error with the resolver $resolver.") 62 | 63 | case Lookup.TYPE_NOT_FOUND => 64 | SortedSet.empty 65 | 66 | case Lookup.TRY_AGAIN => 67 | if (attempt >= 3) SortedSet.empty 68 | else query(hostname, resolver, lookup, attempt + 1) 69 | } 70 | } 71 | 72 | private[connection] def recordsFromXbillRecords(xbillRecords: Set[org.xbill.DNS.Record]): SortedSet[Record] = 73 | xbillRecords 74 | .map(xbillRecord => Record(xbillRecord.getName.toString, RecordType.fromInt(xbillRecord.getType), xbillRecord.rdataToString)) 75 | .toSortedSet 76 | } 77 | 78 | object DNSLookupImpl { 79 | def create(): DNSLookup = 80 | new DNSLookupImpl() 81 | 82 | case class HostNotFoundException(msg: String) extends Exception(msg) 83 | case class ServerFailureException(msg: String) extends Exception(msg) 84 | } -------------------------------------------------------------------------------- /src/main/scala/com/gilazaria/subsearch/connection/LookupFactory.scala: -------------------------------------------------------------------------------- 1 | package com.gilazaria.subsearch.connection 2 | 3 | import org.xbill.DNS.{Record, SimpleResolver} 4 | 5 | trait LookupFactory { 6 | def run(): Array[Record] 7 | def setResolver(resolver: SimpleResolver): Unit 8 | def getResult: Int 9 | def getAnswers: Array[Record] 10 | } 11 | -------------------------------------------------------------------------------- /src/main/scala/com/gilazaria/subsearch/connection/LookupFactoryImpl.scala: -------------------------------------------------------------------------------- 1 | package com.gilazaria.subsearch.connection 2 | 3 | import org.xbill.DNS.{Lookup, Record, SimpleResolver} 4 | 5 | class LookupFactoryImpl(hostname: String, recordType: Int) extends LookupFactory { 6 | private val lookup: Lookup = new Lookup(hostname, recordType) 7 | 8 | def run(): Array[Record] = lookup.run() 9 | def setResolver(resolver: SimpleResolver): Unit = lookup.setResolver(resolver) 10 | def getResult: Int = lookup.getResult 11 | def getAnswers: Array[Record] = lookup.getAnswers 12 | } 13 | -------------------------------------------------------------------------------- /src/main/scala/com/gilazaria/subsearch/connection/ZoneTransferInFactory.scala: -------------------------------------------------------------------------------- 1 | package com.gilazaria.subsearch.connection 2 | 3 | trait ZoneTransferInFactory { 4 | def run(): java.util.List[_] 5 | } 6 | -------------------------------------------------------------------------------- /src/main/scala/com/gilazaria/subsearch/connection/ZoneTransferInFactoryImpl.scala: -------------------------------------------------------------------------------- 1 | package com.gilazaria.subsearch.connection 2 | 3 | import org.xbill.DNS._ 4 | 5 | class ZoneTransferInFactoryImpl(private val zoneTransferIn: ZoneTransferIn) extends ZoneTransferInFactory { 6 | def run(): java.util.List[_] = zoneTransferIn.run() 7 | } 8 | 9 | object ZoneTransferInFactoryImpl { 10 | def newAXFR(zone: Name, host: String, key: TSIG): ZoneTransferInFactory = 11 | new ZoneTransferInFactoryImpl(ZoneTransferIn.newAXFR(zone, host, key)) 12 | } 13 | -------------------------------------------------------------------------------- /src/main/scala/com/gilazaria/subsearch/controller/Controller.scala: -------------------------------------------------------------------------------- 1 | package com.gilazaria.subsearch.controller 2 | 3 | import java.util.concurrent.Executors 4 | 5 | import com.gilazaria.subsearch.SubSearch 6 | import com.gilazaria.subsearch.core.{Arguments, AuthoritativeScanner, ZoneTransferScanner} 7 | import com.gilazaria.subsearch.core.subdomainscanner.{SubdomainScanner, SubdomainScannerArguments} 8 | import com.gilazaria.subsearch.discovery.{DNSDumpsterScanner, Scanner, VirusTotalScanner} 9 | import com.gilazaria.subsearch.output.Logger 10 | import com.gilazaria.subsearch.utils.{FileUtils, TimeUtils} 11 | 12 | import scala.concurrent.{Await, ExecutionContext, Future} 13 | import scala.concurrent.ExecutionContext.Implicits.global 14 | import scala.util.Try 15 | 16 | object Controller { 17 | def create(arguments: Arguments, logger: Logger) = 18 | new Controller(arguments, logger) 19 | } 20 | 21 | class Controller(private val arguments: Arguments, private val logger: Logger) { 22 | initialise() 23 | 24 | def initialise() = { 25 | printHeader() 26 | printConfig() 27 | 28 | arguments.hostnames.foreach { 29 | hostname => Await.result(runScanForHostname(hostname), TimeUtils.awaitDuration) 30 | } 31 | 32 | exitGracefully() 33 | } 34 | 35 | def printHeader() = { 36 | val header: String = 37 | FileUtils 38 | .getResourceSource("banner.txt") 39 | .replaceFirst("VERSION", SubSearch.version) 40 | 41 | logger.logHeader(header) 42 | } 43 | 44 | def printConfig() = { 45 | val wordlistSize = arguments.wordlist.get.numberOfLines 46 | val resolversSize = arguments.resolvers.size 47 | 48 | logger.logConfig(arguments.threads, wordlistSize, resolversSize) 49 | } 50 | 51 | def runScanForHostname(hostname: String): Future[Unit] = { 52 | logger.logTarget(hostname) 53 | runScanners(hostname) 54 | } 55 | 56 | private def runScanners(hostname: String): Future[Unit] = { 57 | val executorService = Executors.newFixedThreadPool(arguments.threads) 58 | implicit val ec: ExecutionContext = ExecutionContext.fromExecutorService(executorService) 59 | 60 | val authoritativeNameServers: Set[String] = retrieveAuthoritativeNameServers(hostname) 61 | 62 | val zoneTransferSubdomains: Set[String] = performZoneTransferScan(hostname, authoritativeNameServers) 63 | 64 | val discoveredSubdomains: Set[String] = performAdditionalScans(hostname) 65 | 66 | val resolvers = 67 | if (arguments.includeAuthoritativeNameServersWithResolvers) (arguments.resolvers ++ authoritativeNameServers).distinct 68 | else arguments.resolvers 69 | 70 | if (arguments.includeAuthoritativeNameServersWithResolvers) 71 | logger.logAddingAuthNameServersToResolvers(resolvers.size) 72 | 73 | val subdomainScannerArguments = SubdomainScannerArguments(hostname, arguments.wordlist.get, zoneTransferSubdomains.toList, discoveredSubdomains.toList, resolvers, arguments.threads, arguments.concurrentResolverRequests) 74 | 75 | SubdomainScanner.performScan(subdomainScannerArguments, logger) 76 | } 77 | 78 | private def retrieveAuthoritativeNameServers(hostname: String)(implicit ec: ExecutionContext): Set[String] = { 79 | val scanner = AuthoritativeScanner.create(logger) 80 | val lookup = scanner.performLookupOnHostname(hostname, arguments.resolvers.head) 81 | Await.result(lookup, TimeUtils.awaitDuration) 82 | } 83 | 84 | private def performAdditionalScans(hostname: String): Set[String] = { 85 | val scanners: Set[Scanner] = 86 | Set( 87 | DNSDumpsterScanner.conditionallyCreate(arguments.performDNSDumpsterScan), 88 | VirusTotalScanner.conditionallyCreate(arguments.performVirusTotalScan) 89 | ).flatten 90 | 91 | if (scanners.isEmpty) return Set.empty 92 | 93 | logger.logAdditionalScansStarted() 94 | 95 | val futures: Set[Future[Set[String]]] = 96 | scanners.map { 97 | scanner: Scanner => 98 | val subdomains: Future[Set[String]] = scanner 99 | .scan(hostname) 100 | .andThen { 101 | case subdomains: Try[Set[String]] => 102 | if (subdomains.isSuccess) logger.logAdditionalScannerFoundSubdomains(scanner.name, subdomains.get) 103 | } 104 | .recover { 105 | case cause: Throwable => 106 | logger.logAdditionalScannerError(scanner.name, cause.getMessage) 107 | Set.empty 108 | } 109 | subdomains 110 | } 111 | 112 | val future: Future[Set[String]] = 113 | Future 114 | .sequence(futures) 115 | .map(_.flatten) 116 | .andThen { case _ => logger.logAdditionalScansCompleted() } 117 | 118 | Await.result(future, TimeUtils.awaitDuration) 119 | } 120 | 121 | private def performZoneTransferScan(hostname: String, resolvers: Set[String]): Set[String] = 122 | if (arguments.performZoneTransfer) 123 | Await.result( 124 | ZoneTransferScanner 125 | .create(logger) 126 | .performLookup(hostname, resolvers), 127 | TimeUtils.awaitDuration 128 | ) 129 | else 130 | Set.empty 131 | 132 | def exitGracefully() = 133 | logger.completedLoggingFuture.andThen { case _ => System.exit(0) } 134 | } 135 | -------------------------------------------------------------------------------- /src/main/scala/com/gilazaria/subsearch/core/ArgumentParser.scala: -------------------------------------------------------------------------------- 1 | package com.gilazaria.subsearch.core 2 | 3 | import com.gilazaria.subsearch.SubSearch 4 | import scopt.OptionParser 5 | import com.gilazaria.subsearch.utils.{File, HostnameUtils, IPUtils} 6 | 7 | case class Arguments(hostnames: List[String] = List.empty, 8 | wordlist: Option[File] = None, 9 | resolvers: List[String] = List.empty, 10 | includeAuthoritativeNameServersWithResolvers: Boolean = false, 11 | concurrentResolverRequests: Boolean = false, 12 | extendedOutput: Boolean = false, 13 | threads: Int = 10, 14 | performZoneTransfer: Boolean = false, 15 | performDNSDumpsterScan: Boolean = false, 16 | performVirusTotalScan: Boolean = false, 17 | csvReportFile: Option[File] = None, 18 | stdoutReportFile: Option[File] = None) 19 | 20 | private class ArgumentParser(private val args: Array[String]) { 21 | 22 | private val parser = new OptionParser[Arguments]("subsearch") { 23 | head("subsearch", SubSearch.version) 24 | 25 | note("Options:") 26 | 27 | help("help") 28 | .text("Prints this usage text.") 29 | 30 | note("") 31 | note("Mandatory:") 32 | 33 | opt[String]('h', "hostname") 34 | .valueName("HOSTNAME") 35 | .text("The hostname(s) to scan. Enter more than one by separating with a comma.") 36 | .action { 37 | (argument, config) => 38 | val hostnames = argument.split(",").toList.map(HostnameUtils.normalise) 39 | hostnames.foreach(verifyHostname) 40 | config.copy(hostnames = hostnames) 41 | } 42 | note("and/or") 43 | opt[String]('H', "hostlist") 44 | .valueName("HOSTLIST") 45 | .text("A file containing a newline delimited list of hostnames to scan.") 46 | .action { 47 | (argument, config) => 48 | val file: File = File.fromFilename(argument) 49 | verifyFile(file, "hostlist") 50 | val hostnames = file.getLines.map(HostnameUtils.normalise) 51 | hostnames.foreach(verifyHostname) 52 | config.copy(hostnames = (config.hostnames ++ hostnames).distinct) 53 | } 54 | 55 | opt[String]('w', "wordlist") 56 | .required() 57 | .valueName("WORDLIST") 58 | .text("A newline delimited list of subdomain names.") 59 | .action { 60 | (argument, config) => 61 | val file: File = File.fromFilename(argument) 62 | verifyFile(file, "wordlist") 63 | config.copy(wordlist = Option(file)) 64 | } 65 | 66 | opt[String]('r', "resolvers") 67 | .valueName("RESOLVERS") 68 | .text("The name server(s) to scan with. Enter more than one by separating with a comma.") 69 | .action { 70 | (argument, config) => 71 | val resolvers = argument.split(",").toList.map(IPUtils.normalise) 72 | resolvers.foreach(verifyResolver) 73 | config.copy(resolvers = resolvers) 74 | } 75 | note("and/or") 76 | opt[String]('R', "resolverslist") 77 | .valueName("RESOLVERSLIST") 78 | .text("A file containing a newline delimited list of name servers to scan with.") 79 | .action { 80 | (argument, config) => 81 | val file: File = File.fromFilename(argument) 82 | verifyFile(file, "resolvers list") 83 | val resolvers = file.getLines.map(IPUtils.normalise) 84 | resolvers.foreach(verifyResolver) 85 | config.copy(resolvers = (config.resolvers ++ resolvers).distinct) 86 | } 87 | 88 | note("") 89 | note("General Settings:") 90 | 91 | opt[Unit]('a', "auth-resolvers") 92 | .text("Include the hostname's authoritative name servers in the list of resolvers. Defaults to false.") 93 | .action { 94 | (argument, config) => 95 | config.copy(includeAuthoritativeNameServersWithResolvers = true) 96 | } 97 | 98 | opt[Unit]('c', "concurrent-resolver-requests") 99 | .text("Allow for more than one request to each resolver at the same time. If true, it can result in being blacklisted or rate limited by some resolvers. Defaults to false.") 100 | .action { 101 | (argument, config) => 102 | config.copy(concurrentResolverRequests = true) 103 | } 104 | 105 | opt[Int]('t', "threads") 106 | .valueName("THREADCOUNT") 107 | .text("The number of concurrent threads whilst scanning. Defaults to 10.") 108 | .action { 109 | (threads, config) => 110 | verifyThreads(threads) 111 | config.copy(threads = threads) 112 | } 113 | 114 | opt[Unit]('v', "verbose") 115 | .text("Show more extended command line output such as the addresses that A, AAAA and CNAME records point to. Defaults to false.") 116 | .action { 117 | (argument, config) => 118 | config.copy(extendedOutput = true) 119 | } 120 | 121 | opt[Unit]("comprehensive") 122 | .text("Runs all additional scanners.") 123 | .action { 124 | (argument, config) => 125 | config.copy( 126 | performZoneTransfer = true, 127 | performDNSDumpsterScan = true, 128 | performVirusTotalScan = true 129 | ) 130 | } 131 | 132 | note("") 133 | note("Additional Scanners:") 134 | 135 | opt[Unit]("dns-dumpster") 136 | .text("Attempts to lookup possible records from dnsdumpster.com") 137 | .action { 138 | (argument, config) => 139 | config.copy(performDNSDumpsterScan = true) 140 | } 141 | 142 | opt[Unit]("virus-total") 143 | .text("Attempts to lookup possible records from virustotal.com") 144 | .action { 145 | (argument, config) => 146 | config.copy(performVirusTotalScan = true) 147 | } 148 | 149 | opt[Unit]('z', "zone-transfer") 150 | .text("Attempts a zone transfer against the host's authoritative name servers.") 151 | .action { 152 | (argument, config) => 153 | config.copy(performZoneTransfer = true) 154 | } 155 | 156 | note("") 157 | note("Reporting:") 158 | 159 | opt[String]("report-csv") 160 | .valueName("OUTPUTFILE") 161 | .text("Outputs a CSV report of discovered subdomains including timestamp, subdomain, record type and record data.") 162 | .action { 163 | (argument, config) => 164 | val file: File = File.fromFilename(argument) 165 | if (!file.isWriteable) 166 | printErrorThenExit("The output file is not writeble.") 167 | config.copy(csvReportFile = Some(file)) 168 | } 169 | 170 | opt[String]("report-stdout") 171 | .valueName("OUTPUTFILE") 172 | .text("Outputs standard out to a file.") 173 | .action { 174 | (argument, config) => 175 | val file: File = File.fromFilename(argument) 176 | if (!file.isWriteable) 177 | printErrorThenExit("The output file is not writeble.") 178 | config.copy(stdoutReportFile = Some(file)) 179 | } 180 | } 181 | 182 | def verifyHostname(hostname: String) = 183 | if (!HostnameUtils.isValidDomain(hostname)) 184 | printErrorThenExit("The hostname '$hostname' is invalid.") 185 | 186 | def verifyResolver(resolver: String) = 187 | if (!IPUtils.isValidIPv4(resolver)) 188 | printErrorThenExit("The resolver '$resolver' is not a valid IPv4 address.") 189 | 190 | def verifyFile(file: File, description : String) = { 191 | if (!file.exists) 192 | printErrorThenExit("The " + description + "file does not exist.") 193 | else if (!file.isFile) 194 | printErrorThenExit("The " + description + "file is invalid.") 195 | else if (!file.isReadable) 196 | printErrorThenExit("The " + description + "file cannot be read.") 197 | } 198 | 199 | def verifyThreads(threads: Int) = { 200 | if (threads < 1) 201 | printErrorThenExit("Threads must be a positive integer.") 202 | } 203 | 204 | def parse(): Arguments = 205 | parser.parse(args, Arguments()) match { 206 | case Some(arguments) => 207 | postVerifyArguments(arguments) 208 | arguments 209 | case None => 210 | System.exit(1) 211 | Arguments() 212 | } 213 | 214 | private def printErrorThenExit(message: String) = { 215 | println("Error: " + message) 216 | System.exit(1) 217 | } 218 | 219 | private def postVerifyArguments(arguments: Arguments) = { 220 | if (arguments.hostnames.isEmpty) 221 | printErrorThenExit("At least one hostname must be specified.") 222 | if (arguments.resolvers.isEmpty) 223 | printErrorThenExit("At least one resolver must be specified.") 224 | } 225 | 226 | } 227 | 228 | object ArgumentParser { 229 | def parseArguments(args: Array[String]): Arguments = { 230 | val parser = new ArgumentParser(args) 231 | parser.parse() 232 | } 233 | 234 | } -------------------------------------------------------------------------------- /src/main/scala/com/gilazaria/subsearch/core/AuthoritativeScanner.scala: -------------------------------------------------------------------------------- 1 | package com.gilazaria.subsearch.core 2 | 3 | import com.gilazaria.subsearch.output.Logger 4 | import com.gilazaria.subsearch.connection.{DNSLookupImpl, DNSLookup} 5 | import com.gilazaria.subsearch.model.{Record, RecordType} 6 | 7 | import scala.concurrent.{ExecutionContext, Future} 8 | import scala.collection.SortedSet 9 | 10 | class AuthoritativeScanner private[core] (private val logger: Logger, private val lookup: DNSLookup)(implicit ec: ExecutionContext) { 11 | def performLookupOnHostname(hostname: String, resolver: String): Future[Set[String]] = { 12 | logger.logAuthoritativeScanStarted() 13 | 14 | dataFromQuery(hostname, resolver, RecordType.NS) 15 | .flatMap(nameservers => ipsForNameServers(nameservers, resolver)) 16 | .map(printAuthoritativeNameServers) 17 | } 18 | 19 | private[core] def ipsForNameServers(nameServers: Set[String], resolver: String): Future[Set[String]] = { 20 | Future 21 | .sequence(nameServers.map(ns => dataFromQuery(ns, resolver, RecordType.A))) 22 | .map(_.flatten) 23 | } 24 | 25 | private[core] def dataFromQuery(hostname: String, resolver: String, recordType: RecordType): Future[Set[String]] = { 26 | Future { 27 | lookup 28 | .performQueryOfType(hostname, resolver, recordType) 29 | .getOrElse(SortedSet.empty[Record]) 30 | .map(_.data) 31 | .toSet 32 | } 33 | } 34 | 35 | private[core] def printAuthoritativeNameServers(nameServers: Set[String]) = { 36 | nameServers.foreach(logger.logAuthoritativeNameServer) 37 | logger.logAuthoritativeScanCompleted() 38 | nameServers 39 | } 40 | } 41 | 42 | object AuthoritativeScanner { 43 | def create(logger: Logger)(implicit ec: ExecutionContext): AuthoritativeScanner = 44 | new AuthoritativeScanner(logger, DNSLookupImpl.create()) 45 | } -------------------------------------------------------------------------------- /src/main/scala/com/gilazaria/subsearch/core/ZoneTransferScanner.scala: -------------------------------------------------------------------------------- 1 | package com.gilazaria.subsearch.core 2 | 3 | import com.gilazaria.subsearch.connection.{DNSLookupImpl, DNSLookup} 4 | import com.gilazaria.subsearch.model.{Record, RecordType} 5 | import com.gilazaria.subsearch.output.Logger 6 | 7 | import scala.concurrent.{ExecutionContext, Future} 8 | import scala.util.Try 9 | import scala.collection.SortedSet 10 | 11 | class ZoneTransferScanner private[core] (private val logger: Logger, private val lookup: DNSLookup)(implicit ec: ExecutionContext) { 12 | def performLookup(hostname: String, resolvers: Set[String]): Future[Set[String]] = { 13 | logger.logStartedZoneTransfer() 14 | 15 | zoneTransfersForHostnameAndResolvers(hostname, resolvers) 16 | .map(records => recordsEndingWithHostname(hostname, records)) 17 | .map(printFoundRecords) 18 | .map(namesFromRecords) 19 | } 20 | 21 | private[core] def zoneTransfersForHostnameAndResolvers(hostname: String, resolvers: Set[String]): Future[SortedSet[Record]] = 22 | Future 23 | .sequence(resolvers.map(resolver => zoneTransferForHostnameAndResolver(hostname, resolver))) 24 | .map(flattenRecords) 25 | 26 | private[core] def zoneTransferForHostnameAndResolver(hostname: String, resolver: String): Future[SortedSet[Record]] = { 27 | val lookupFut: Future[SortedSet[Record]] = 28 | Future { 29 | lookup 30 | .performQueryOfType(hostname, resolver, RecordType.AXFR) 31 | .getOrElse(SortedSet.empty[Record]) 32 | } 33 | 34 | lookupFut 35 | .andThen { 36 | case records: Try[SortedSet[Record]] => 37 | if (records.getOrElse(SortedSet.empty[Record]).nonEmpty) 38 | logger.logNameServerVulnerableToZoneTransfer(resolver) 39 | } 40 | } 41 | 42 | private[core] def flattenRecords(set: Set[SortedSet[Record]]): SortedSet[Record] = 43 | if (set.isEmpty) SortedSet.empty[Record] 44 | else set.reduce(_ ++ _) 45 | 46 | private[core] def recordsEndingWithHostname(hostname: String, records: SortedSet[Record]): SortedSet[Record] = 47 | records.filter(_.name.endsWith(hostname)) 48 | 49 | private[core] def printFoundRecords(records: SortedSet[Record]): SortedSet[Record] = { 50 | if (records.isEmpty) 51 | logger.logNameServersNotVulnerableToZoneTransfer() 52 | else 53 | logger.logRecords(records) 54 | 55 | logger.logZoneTransferCompleted() 56 | 57 | records 58 | } 59 | 60 | private[core] def namesFromRecords(records: SortedSet[Record]): Set[String] = 61 | records.map(_.name).toSet 62 | 63 | } 64 | 65 | object ZoneTransferScanner { 66 | def create(logger: Logger)(implicit ec: ExecutionContext): ZoneTransferScanner = 67 | new ZoneTransferScanner(logger, DNSLookupImpl.create()) 68 | } -------------------------------------------------------------------------------- /src/main/scala/com/gilazaria/subsearch/core/subdomainscanner/Dispatcher.scala: -------------------------------------------------------------------------------- 1 | package com.gilazaria.subsearch.core.subdomainscanner 2 | 3 | import akka.actor._ 4 | import com.gilazaria.subsearch.core.subdomainscanner.DispatcherMessage._ 5 | import com.gilazaria.subsearch.core.subdomainscanner.ListenerMessage._ 6 | import com.gilazaria.subsearch.core.subdomainscanner.ScannerMessage.{Scan, ScanAvailable} 7 | 8 | import scala.concurrent.ExecutionContext 9 | 10 | object Dispatcher { 11 | def props(arguments: SubdomainScannerArguments, listener: ActorRef)(implicit ec: ExecutionContext): Props = 12 | Props(new Dispatcher(arguments, listener)) 13 | 14 | private def createScanners(context: ActorContext, listener: ActorRef, threads: Int, hostname: String)(implicit ec: ExecutionContext): Set[ActorRef] = 15 | Vector.fill(threads) { 16 | val scanner = context.actorOf(Scanner.props(listener, hostname)) 17 | context.watch(scanner) 18 | scanner 19 | }.toSet 20 | } 21 | 22 | class Dispatcher(arguments: SubdomainScannerArguments, listener: ActorRef)(implicit ec: ExecutionContext) extends Actor { 23 | var master: Option[ActorRef] = None 24 | 25 | var pauseScanning = false 26 | var numberOfPausedScanners = 0 27 | var whoToNotifyAboutPaused: Option[ActorRef] = None 28 | 29 | val dispatcherQueue: DispatcherQueue = DispatcherQueue.create(arguments.hostname, arguments.wordlist, arguments.omitSubdomains, arguments.prioritySubdomains, arguments.resolvers, arguments.concurrentResolverRequests) 30 | 31 | var scansSoFar: Int = 0 32 | 33 | var currentlyScanning: Set[String] = Set.empty 34 | 35 | var scannerRefs: Set[ActorRef] = Dispatcher.createScanners(context, listener, arguments.threads, arguments.hostname) 36 | scannerRefs.foreach(_ ! ScanAvailable) 37 | 38 | def receive = { 39 | case ResumeScanning => 40 | scanningHasResumed() 41 | scannerRefs.foreach(_ ! ScanAvailable) 42 | listener ! ResumedScanning 43 | 44 | case PauseScanning => 45 | scanningHasPaused() 46 | whoToNotifyAboutPaused = Some(sender) 47 | listener ! PausingScanning 48 | 49 | case CompletedScan(subdomain, resolver) => 50 | subdomainHasBeenScanned(subdomain) 51 | dispatcherQueue.recycleResolver(resolver) 52 | scannerIsAvailableToScan(sender) 53 | 54 | case FailedScan(subdomain, resolver) => 55 | dispatcherQueue.requeueSubdomain(subdomain) 56 | dispatcherQueue.blacklistResolver(resolver) 57 | listener ! BlacklistedResolver(resolver) 58 | scannerIsAvailableToScan(sender) 59 | 60 | case AvailableForScan => 61 | scannerIsAvailableToScan(sender) 62 | 63 | case NotifyOnCompletion => 64 | master = Some(sender) 65 | 66 | case PriorityScanSubdomain(subdomain: String) => 67 | dispatcherQueue.enqueuePrioritySubdomain(subdomain) 68 | 69 | case Terminated(scanner) => 70 | scannerHasTerminated(scanner) 71 | 72 | if (scanningHasNotBeenPaused && allScannersHaveTerminated) { 73 | if (allSubdomainsHaveBeenScanned) { 74 | listener ! TaskCompleted(master) 75 | } else if (dispatcherQueue.isOutOfResolvers) { 76 | // All resolvers are dead. Scan must terminate 77 | listener ! TaskFailed(master) 78 | } else { 79 | // Add any missed subdomains back to the queue 80 | currentlyScanning.foreach(_ => dispatcherQueue.requeueSubdomain(_)) 81 | currentlyScanning = Set.empty 82 | 83 | // Start scanning again. 84 | val numberOfScannersToCreate: Int = 85 | Array(dispatcherQueue.remainingNumberOfSubdomains, 86 | dispatcherQueue.remainingNumberOfResolvers, 87 | arguments.threads).min 88 | 89 | scannerRefs = Dispatcher.createScanners(context, listener, numberOfScannersToCreate, arguments.hostname) 90 | scannerRefs.foreach(_ ! ScanAvailable) 91 | } 92 | } 93 | } 94 | 95 | def scannerIsAvailableToScan(scanner: ActorRef) = { 96 | if (scanningHasBeenPaused) { 97 | // Don't send anything to the scanner, consider it paused 98 | aScannerHasBeenPaused() 99 | } 100 | else if (!dispatcherQueue.isOutOfResolvers) { 101 | val resolver = dispatcherQueue.dequeueResolver() 102 | val subdomainOpt: Option[String] = dispatcherQueue.dequeueSubdomain() 103 | 104 | if (subdomainOpt.isDefined) { 105 | val subdomain = subdomainOpt.get 106 | 107 | scanningSubdomain(subdomain) 108 | scanner ! Scan(subdomain, resolver, 1) 109 | scansSoFar += 1 110 | listener ! LastScan(subdomain, scansSoFar, dispatcherQueue.totalNumberOfSubdomains) 111 | } else { 112 | // There aren't any subdomains for this scanner. Stop this scanner from working 113 | terminateScanner(scanner) 114 | } 115 | } else { 116 | // We don't have enough resolvers to go around. Stop this scanner from working 117 | listener ! NotEnoughResolvers 118 | terminateScanner(scanner) 119 | } 120 | } 121 | 122 | // Keeping track of subdomains currently being scanned 123 | def subdomainHasBeenScanned(subdomain: String) = currentlyScanning = currentlyScanning.diff(Set(subdomain)) 124 | def scanningSubdomain(subdomain: String) = currentlyScanning = currentlyScanning ++ Set(subdomain) 125 | def allSubdomainsHaveBeenScanned = dispatcherQueue.isOutOfSubdomains && currentlyScanning.isEmpty 126 | 127 | // Keeping track of scanning and whether it's been paused 128 | def scanningHasBeenPaused: Boolean = pauseScanning 129 | def scanningHasNotBeenPaused: Boolean = !pauseScanning 130 | def aScannerHasBeenPaused() = { 131 | numberOfPausedScanners += 1 132 | 133 | if (allScannersHaveBeenPaused && whoToNotifyAboutPaused.isDefined) 134 | whoToNotifyAboutPaused.get ! true 135 | } 136 | def allScannersHaveBeenPaused: Boolean = numberOfPausedScanners == scannerRefs.size 137 | def scanningHasPaused() = pauseScanning = true 138 | def scanningHasResumed() = { 139 | pauseScanning = false 140 | numberOfPausedScanners = 0 141 | } 142 | 143 | // Keeping track of scanner references 144 | def scannerHasTerminated(scanner: ActorRef) = scannerRefs = scannerRefs.diff(Set(scanner)) 145 | def terminateScanner(scanner: ActorRef) = context.stop(scanner) 146 | def allScannersHaveTerminated = scannerRefs.isEmpty 147 | } -------------------------------------------------------------------------------- /src/main/scala/com/gilazaria/subsearch/core/subdomainscanner/DispatcherQueue.scala: -------------------------------------------------------------------------------- 1 | package com.gilazaria.subsearch.core.subdomainscanner 2 | 3 | import com.gilazaria.subsearch.utils.{HostnameUtils, File} 4 | 5 | import scala.collection.mutable 6 | import scala.util.Random 7 | 8 | object DispatcherQueue { 9 | def create(hostname: String, 10 | wordlist: File, 11 | omitSubdomains: List[String], 12 | prioritySubdomains: List[String], 13 | resolvers: List[String], 14 | concurrentResolvers: Boolean): DispatcherQueue = 15 | new DispatcherQueue(hostname, wordlist, omitSubdomains, prioritySubdomains, resolvers, concurrentResolvers) 16 | } 17 | 18 | class DispatcherQueue(private val hostname: String, 19 | private val wordlist: File, 20 | private val omitSubdomains: List[String], 21 | private val prioritySubdomains: List[String], 22 | private val resolvers: List[String], 23 | private val concurrentResolvers: Boolean) { 24 | 25 | private var totalNumberOfScans: Int = wordlist.numberOfLines 26 | private var scannedSoFar: Int = 0 27 | 28 | private var allSeenSubdomains: Set[String] = omitSubdomains.toSet 29 | 30 | private val subdomainsIterator: Iterator[String] = wordlist.linesIterator 31 | private val prioritySubdomainsQueue: mutable.Queue[String] = mutable.Queue() ++= prioritySubdomains 32 | private val resolversQueue: mutable.Queue[String] = mutable.Queue() ++= Random.shuffle(resolvers).toSet 33 | 34 | private var blacklistedResolvers: List[String] = List.empty 35 | 36 | def remainingNumberOfSubdomains: Int = prioritySubdomainsQueue.size 37 | def remainingNumberOfResolvers: Int = resolversQueue.size 38 | def isOutOfSubdomains: Boolean = prioritySubdomainsQueue.isEmpty && !subdomainsIterator.hasNext 39 | def isOutOfResolvers: Boolean = resolversQueue.isEmpty || blacklistedResolvers.size == resolvers.size 40 | def requeueSubdomain(subdomain: String) = prioritySubdomainsQueue.enqueue(subdomain) 41 | 42 | def recycleResolver(resolver: String) = 43 | if (!concurrentResolvers && !blacklistedResolvers.contains(resolver)) resolversQueue.enqueue(resolver) 44 | 45 | def dequeueResolver(): String = 46 | if (!concurrentResolvers) resolversQueue.dequeue() 47 | else resolvers.diff(blacklistedResolvers)(Random.nextInt(resolvers.diff(blacklistedResolvers).size)) 48 | 49 | /** 50 | * Getting a new subdomain does NOT add it to the allSeenSubdomains set, as this will quickly 51 | * take too much memory. We're instead making an assumption that the subdomains word list does not 52 | * contain repeats. 53 | */ 54 | 55 | def dequeueSubdomain(): Option[String] = { 56 | scannedSoFar += 1 57 | 58 | if (prioritySubdomainsQueue.nonEmpty) Option(prioritySubdomainsQueue.dequeue()) 59 | else nextSubdomainsIterator() 60 | } 61 | 62 | private def nextSubdomainsIterator(): Option[String] = { 63 | if (subdomainsIterator.hasNext) { 64 | val subdomainPart = HostnameUtils.normalise(subdomainsIterator.next) 65 | val subdomainPartIsValid = HostnameUtils.isValidSubdomainPart(subdomainPart) 66 | val fullyQualifiedSubdomain = HostnameUtils.ensureSubdomainEndsWithHostname(subdomainPart, hostname) 67 | 68 | if (subdomainPartIsValid && !allSeenSubdomains.contains(fullyQualifiedSubdomain)) { 69 | Option(fullyQualifiedSubdomain) 70 | } 71 | else { 72 | scannedSoFar += 1 73 | nextSubdomainsIterator() 74 | } 75 | } else { 76 | None 77 | } 78 | } 79 | 80 | def enqueuePrioritySubdomain(subdomain: String) = 81 | if (!allSeenSubdomains.contains(subdomain)) { 82 | prioritySubdomainsQueue.enqueue(subdomain) 83 | allSeenSubdomains = allSeenSubdomains ++ Set(subdomain) 84 | totalNumberOfScans += 1 85 | } 86 | 87 | def totalNumberOfSubdomains: Int = 88 | totalNumberOfScans 89 | 90 | def blacklistResolver(resolver: String) = 91 | blacklistedResolvers = (blacklistedResolvers ++ List(resolver)).distinct 92 | } 93 | -------------------------------------------------------------------------------- /src/main/scala/com/gilazaria/subsearch/core/subdomainscanner/Listener.scala: -------------------------------------------------------------------------------- 1 | package com.gilazaria.subsearch.core.subdomainscanner 2 | 3 | import com.gilazaria.subsearch.output.Logger 4 | import com.gilazaria.subsearch.core.subdomainscanner.ListenerMessage._ 5 | 6 | import akka.actor.{ActorRef, Actor, Props} 7 | 8 | object Listener { 9 | def props(logger: Logger) = Props(new Listener(logger)) 10 | } 11 | 12 | class Listener(logger: Logger) extends Actor { 13 | def receive = { 14 | case FoundSubdomain(subdomain, records) => 15 | logger.logRecordsDuringScan(records) 16 | 17 | case LastScan(subdomain, requestsSoFar, totalRequests) => 18 | logger.logLastRequest(subdomain, requestsSoFar, totalRequests) 19 | 20 | case PausingScanning => 21 | logger.logPausingThreads() 22 | 23 | case NotEnoughResolvers => 24 | logger.logNotEnoughResolvers() 25 | 26 | case TaskCompleted(master: Option[ActorRef]) => 27 | logger.logTaskCompleted() 28 | if (master.isDefined) master.get ! None 29 | context.system.terminate() 30 | 31 | case TaskFailed(master: Option[ActorRef]) => 32 | logger.logTaskFailed() 33 | if (master.isDefined) master.get ! None 34 | context.system.terminate() 35 | 36 | case ScanTimeout(subdomain, resolver, attempt) => 37 | val duration = 38 | if (attempt == 2) "20 seconds" 39 | else "30 seconds" 40 | logger.logTimedOutScan(subdomain, resolver, duration) 41 | 42 | case BlacklistedResolver(resolver) => 43 | logger.logBlacklistedResolver(resolver) 44 | } 45 | } -------------------------------------------------------------------------------- /src/main/scala/com/gilazaria/subsearch/core/subdomainscanner/Message.scala: -------------------------------------------------------------------------------- 1 | package com.gilazaria.subsearch.core.subdomainscanner 2 | 3 | import akka.actor.ActorRef 4 | import com.gilazaria.subsearch.model.Record 5 | 6 | import scala.collection.SortedSet 7 | import scala.util.Try 8 | 9 | trait Message 10 | 11 | object DispatcherMessage extends Message { 12 | case object ResumeScanning 13 | case object PauseScanning 14 | case object NotifyOnCompletion 15 | case object AvailableForScan 16 | 17 | case class PriorityScanSubdomain(subdomain: String) 18 | case class CompletedScan(subdomain: String, resolver: String) 19 | case class FailedScan(subdomain: String, resolver: String) 20 | } 21 | 22 | object ScannerMessage extends Message { 23 | case object ScanAvailable 24 | 25 | case class Scan(subdomain: String, resolver: String, attempt: Int) 26 | case class ScanComplete(records: Try[SortedSet[Record]], subdomain: String, resolver: String) 27 | case class ScanFailed(cause: Throwable, subdomain: String, resolver: String, attempt: Int) 28 | } 29 | 30 | object ListenerMessage extends Message { 31 | case object PausingScanning 32 | case object ResumedScanning 33 | case object NotEnoughResolvers 34 | 35 | case class FoundSubdomain(subdomain: String, records: SortedSet[Record]) 36 | case class LastScan(subdomain: String, requestsSoFar: Int, totalRequests: Int) 37 | case class TaskCompleted(master: Option[ActorRef]) 38 | case class TaskFailed(master: Option[ActorRef]) 39 | case class ScanTimeout(subdomain: String, resolver: String, attempt: Int) 40 | case class BlacklistedResolver(resolver: String) 41 | } -------------------------------------------------------------------------------- /src/main/scala/com/gilazaria/subsearch/core/subdomainscanner/PauseHandler.scala: -------------------------------------------------------------------------------- 1 | package com.gilazaria.subsearch.core.subdomainscanner 2 | 3 | import com.gilazaria.subsearch.core.subdomainscanner.DispatcherMessage.{PauseScanning, ResumeScanning} 4 | import com.gilazaria.subsearch.output.Logger 5 | import com.gilazaria.subsearch.utils.TimeUtils 6 | import akka.actor.ActorRef 7 | import akka.pattern.ask 8 | import scala.concurrent.ExecutionContext.Implicits.global 9 | import sun.misc.{Signal, SignalHandler} 10 | 11 | import scala.concurrent.Await 12 | 13 | /** 14 | * I'll admit, this isn't exactly kosher. Apparently this can't be done in Java 9. That should be fun. 15 | * How to do this in Java was found here: http://twit88.com/blog/2008/02/06/java-signal-handling/ 16 | */ 17 | 18 | object PauseHandler { 19 | def create(dispatcher: ActorRef, logger: Logger): PauseHandler = 20 | new PauseHandler(List("INT"), dispatcher, logger) 21 | 22 | case class InterruptException(msg: String) extends Exception(msg) 23 | case class ContinueException(msg: String) extends Exception(msg) 24 | } 25 | 26 | class PauseHandler(signalNames: List[String], dispatcher: ActorRef, logger: Logger) extends SignalHandler { 27 | import PauseHandler.{InterruptException, ContinueException} 28 | 29 | private val signalMap = signalNames.map(name => (name, Signal.handle(new Signal(name), this))).toMap 30 | 31 | private var pausingCalled: Boolean = false 32 | 33 | override def handle(signal: Signal) = { 34 | if (pausingCalled) 35 | forceExit() 36 | else 37 | pausingCalled = true 38 | 39 | implicit val timeout = TimeUtils.akkaAskTimeout 40 | Await.result(dispatcher ? PauseScanning, TimeUtils.awaitDuration) 41 | 42 | try { 43 | while (true) { 44 | logger.logPauseOptions() 45 | 46 | val option: String = System.console.readLine().toLowerCase 47 | 48 | if (option == "e") 49 | throw new InterruptException("Exited the program.") 50 | else if (option == "c") 51 | throw new ContinueException("Continuing the scan.") 52 | else 53 | logger.logInvalidPauseOption() 54 | } 55 | } catch { 56 | case InterruptException(msg) => 57 | exit() 58 | case ContinueException(msg) => 59 | resume() 60 | } 61 | } 62 | 63 | private def forceExit() = { 64 | logger.logScanForceCancelled() 65 | System.exit(0) 66 | } 67 | 68 | private def exit() = { 69 | logger.logScanCancelled() 70 | logger.completedLoggingFuture.andThen { case _ => System.exit(0) } 71 | } 72 | 73 | private def resume() = { 74 | dispatcher ! ResumeScanning 75 | pausingCalled = false 76 | } 77 | } -------------------------------------------------------------------------------- /src/main/scala/com/gilazaria/subsearch/core/subdomainscanner/Scanner.scala: -------------------------------------------------------------------------------- 1 | package com.gilazaria.subsearch.core.subdomainscanner 2 | 3 | import com.gilazaria.subsearch.connection.{DNSLookupImpl, DNSLookup} 4 | import com.gilazaria.subsearch.core.subdomainscanner.ScannerMessage._ 5 | import com.gilazaria.subsearch.core.subdomainscanner.DispatcherMessage.{AvailableForScan, CompletedScan, FailedScan, PriorityScanSubdomain} 6 | import com.gilazaria.subsearch.core.subdomainscanner.ListenerMessage.{FoundSubdomain, ScanTimeout} 7 | import com.gilazaria.subsearch.utils.TimeoutFuture._ 8 | import akka.actor.{Actor, Props, ActorRef} 9 | import scala.concurrent.{ExecutionContext, Future} 10 | import scala.concurrent.duration._ 11 | 12 | object Scanner { 13 | def props(listener: ActorRef, hostname: String)(implicit ec: ExecutionContext): Props = 14 | Props(new Scanner(listener, hostname)) 15 | } 16 | 17 | class Scanner(listener: ActorRef, hostname: String)(implicit ec: ExecutionContext) extends Actor { 18 | private val lookup: DNSLookup = DNSLookupImpl.create() 19 | 20 | override def postRestart(reason: Throwable) = { 21 | preStart 22 | // Reporting for duty after restart 23 | context.parent ! AvailableForScan 24 | } 25 | 26 | def receive = { 27 | case ScanAvailable => 28 | // Notified about available work by parent (Subdomain Dispatcher) 29 | context.parent ! AvailableForScan 30 | 31 | case Scan(subdomain, resolver, attempt) => 32 | val timeout = 33 | if (attempt == 1) 10.seconds 34 | else if (attempt == 2) 20.seconds 35 | else 30.seconds 36 | 37 | Future(lookup.performQueryOfTypeANY(subdomain, resolver)) 38 | .withTimeout(timeout) 39 | .map(records => self ! ScanComplete(records, subdomain, resolver)) 40 | .recover { case cause => self ! ScanFailed(cause, subdomain, resolver, attempt+1) } 41 | 42 | case ScanComplete(recordsAttempt, subdomain, resolver) => 43 | if (recordsAttempt.isSuccess) { 44 | val records = recordsAttempt.get 45 | 46 | if (records.nonEmpty) 47 | listener ! FoundSubdomain(subdomain, records) 48 | 49 | records 50 | .filter(_.recordType.isOneOf("CNAME", "SRV", "MX")) 51 | .filter(record => record.data.endsWith(hostname)) 52 | .map(_.data) 53 | .foreach((subdomain: String) => context.parent ! PriorityScanSubdomain(subdomain)) 54 | } else { 55 | // Do nothing. This indicates that the DNSLookup class tried three times to lookup the subdomain. 56 | // For the moment, we aren't going to try again and will mark this subdomain scan as completed. 57 | } 58 | 59 | context.parent ! CompletedScan(subdomain, resolver) 60 | 61 | case ScanFailed(cause, subdomain, resolver, attempt) => 62 | if (attempt < 4) { 63 | listener ! ScanTimeout(subdomain, resolver, attempt) 64 | self ! Scan(subdomain, resolver, attempt) 65 | } 66 | else { 67 | context.parent ! FailedScan(subdomain, resolver) 68 | } 69 | } 70 | } -------------------------------------------------------------------------------- /src/main/scala/com/gilazaria/subsearch/core/subdomainscanner/SubdomainScanner.scala: -------------------------------------------------------------------------------- 1 | package com.gilazaria.subsearch.core.subdomainscanner 2 | 3 | import com.gilazaria.subsearch.core.subdomainscanner.DispatcherMessage.NotifyOnCompletion 4 | import com.gilazaria.subsearch.output.Logger 5 | import com.gilazaria.subsearch.utils.{File, TimeUtils} 6 | 7 | import akka.actor.ActorSystem 8 | import akka.pattern.ask 9 | import scala.concurrent.{ExecutionContext, Future} 10 | 11 | case class SubdomainScannerArguments(hostname: String, 12 | wordlist: File, 13 | omitSubdomains: List[String], 14 | prioritySubdomains: List[String], 15 | resolvers: List[String], 16 | threads: Int, 17 | concurrentResolverRequests: Boolean) 18 | 19 | object SubdomainScanner { 20 | def performScan(arguments: SubdomainScannerArguments, logger: Logger)(implicit ec: ExecutionContext): Future[Unit] = { 21 | logger.logStartedSubdomainSearch() 22 | val scanner: SubdomainScanner = new SubdomainScanner(arguments, logger) 23 | scanner.future.map(_ => None) 24 | } 25 | } 26 | 27 | class SubdomainScanner(arguments: SubdomainScannerArguments, logger: Logger)(implicit ec: ExecutionContext) { 28 | val system = ActorSystem("SubdomainScanner") 29 | val listener = system.actorOf(Listener.props(logger), "listener") 30 | val dispatcher = system.actorOf(Dispatcher.props(arguments, listener), "dispatcher") 31 | 32 | implicit val timeout = TimeUtils.akkaAskTimeout 33 | val future: Future[Any] = dispatcher ? NotifyOnCompletion 34 | 35 | private val pauseHandler = PauseHandler.create(dispatcher, logger) 36 | } -------------------------------------------------------------------------------- /src/main/scala/com/gilazaria/subsearch/discovery/DNSDumpsterScanner.scala: -------------------------------------------------------------------------------- 1 | package com.gilazaria.subsearch.discovery 2 | 3 | import com.ning.http.client.cookie.Cookie 4 | import dispatch._ 5 | import net.ruippeixotog.scalascraper.browser.{Browser, JsoupBrowser} 6 | import net.ruippeixotog.scalascraper.dsl.DSL.Extract._ 7 | import net.ruippeixotog.scalascraper.dsl.DSL._ 8 | import net.ruippeixotog.scalascraper.model.Element 9 | 10 | import scala.concurrent.{ExecutionContext, Future} 11 | 12 | class DNSDumpsterScanner private[discovery] (private val browser: Browser = JsoupBrowser(), 13 | private val http: HttpExecutor = Http) 14 | (implicit ec: ExecutionContext) 15 | extends Scanner { 16 | override val name: String = "DNS Dumpster Scanner" 17 | 18 | override def scan(hostname: String): Future[Set[String]] = 19 | retrieveCSRFToken() 20 | .flatMap(token => retrieveHTMLWithTokenForHostname(token, hostname)) 21 | .map(html => extractSubdomains(html, hostname)) 22 | 23 | private[discovery] val dnsDumpsterURL: String = "https://dnsdumpster.com/" 24 | 25 | private[discovery] def retrieveCSRFToken(): Future[String] = 26 | Future(browser.get(dnsDumpsterURL)) 27 | .map { 28 | document => 29 | val tokenInputList: List[Element] = 30 | (document.body >> elementList("input")) 31 | .filter(_.attr("name") == "csrfmiddlewaretoken") 32 | 33 | if (tokenInputList.isEmpty) throw new DNSDumpsterScanner.TokenNotFoundException("No CSRF token was found.") 34 | else tokenInputList.head.attr("value") 35 | } 36 | 37 | private[discovery] def retrieveHTMLWithTokenForHostname(token: String, hostname: String): Future[String] = { 38 | val request: Req = 39 | url(dnsDumpsterURL) 40 | .POST 41 | .setHeader("referer", dnsDumpsterURL) 42 | .setHeader("origin", dnsDumpsterURL) 43 | .addCookie(new Cookie("csrftoken", token, token, "dnsdumpster.com", "/", -1, -1, false, false)) 44 | .addParameter("csrfmiddlewaretoken", token) 45 | .addParameter("targetip", hostname) 46 | 47 | http(request).map(_.getResponseBody) 48 | } 49 | 50 | private[discovery] def extractSubdomains(html: String, hostname: String): Set[String] = 51 | (browser.parseString(html).body >> elementList(".col-md-4")) 52 | .map(e => e.innerHtml.split("
").head) 53 | .filter(subdomain => subdomain.endsWith(hostname) && subdomain != hostname) 54 | .toSet 55 | } 56 | 57 | object DNSDumpsterScanner { 58 | def conditionallyCreate(create: Boolean)(implicit ec: ExecutionContext): Option[DNSDumpsterScanner] = 59 | if (create) Some(DNSDumpsterScanner.create()) 60 | else None 61 | 62 | def create()(implicit ec: ExecutionContext): DNSDumpsterScanner = 63 | new DNSDumpsterScanner() 64 | 65 | case class TokenNotFoundException(msg: String) extends Exception(msg) 66 | } -------------------------------------------------------------------------------- /src/main/scala/com/gilazaria/subsearch/discovery/Scanner.scala: -------------------------------------------------------------------------------- 1 | package com.gilazaria.subsearch.discovery 2 | 3 | import scala.concurrent.Future 4 | 5 | trait Scanner { 6 | def scan(hostname: String): Future[Set[String]] 7 | val name: String 8 | } 9 | -------------------------------------------------------------------------------- /src/main/scala/com/gilazaria/subsearch/discovery/VirusTotalScanner.scala: -------------------------------------------------------------------------------- 1 | package com.gilazaria.subsearch.discovery 2 | 3 | import dispatch.{Http, HttpExecutor, Req, url} 4 | import net.ruippeixotog.scalascraper.browser.{Browser, JsoupBrowser} 5 | import net.ruippeixotog.scalascraper.dsl.DSL.Extract._ 6 | import net.ruippeixotog.scalascraper.dsl.DSL._ 7 | 8 | import scala.concurrent.{ExecutionContext, Future} 9 | 10 | class VirusTotalScanner private[discovery] (private val browser: Browser = JsoupBrowser(), 11 | private val http: HttpExecutor = Http) 12 | (implicit ec: ExecutionContext) 13 | extends Scanner { 14 | override val name: String = "Virus Total Scanner" 15 | 16 | override def scan(hostname: String): Future[Set[String]] = { 17 | retrieveHTML(hostname) 18 | .map(html => extractSubdomains(html, hostname)) 19 | } 20 | 21 | private[discovery] def extractSubdomains(html: String, hostname: String): Set[String] = 22 | (browser.parseString(html).body >> elementList("a")) 23 | .map(e => e.innerHtml) 24 | .filter(subdomain => subdomain.endsWith(hostname) && subdomain != hostname) 25 | .toSet 26 | 27 | private[discovery] def retrieveHTML(hostname: String): Future[String] = { 28 | val request: Req = 29 | url(s"https://www.virustotal.com/en-gb/domain/$hostname/information/") 30 | .GET 31 | .setHeader("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.10; rv:45.0) Gecko/20100101 Firefox/45.0") 32 | .setHeader("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8") 33 | .setHeader("Accept-Language", "en-US,en;q=0.5") 34 | 35 | http(request).map(_.getResponseBody) 36 | } 37 | } 38 | 39 | object VirusTotalScanner { 40 | def conditionallyCreate(create: Boolean)(implicit ec: ExecutionContext): Option[VirusTotalScanner] = 41 | if (create) Some(VirusTotalScanner.create()) 42 | else None 43 | 44 | def create()(implicit ec: ExecutionContext): VirusTotalScanner = 45 | new VirusTotalScanner() 46 | } -------------------------------------------------------------------------------- /src/main/scala/com/gilazaria/subsearch/model/Record.scala: -------------------------------------------------------------------------------- 1 | package com.gilazaria.subsearch.model 2 | 3 | import com.gilazaria.subsearch.utils.HostnameUtils 4 | 5 | class Record private(val name: String, val recordType: RecordType, val data: String) extends Ordered[Record] { 6 | override def compare(that: Record): Int = { 7 | val compareName = compareString(this.name, that.name) 8 | lazy val compareType = recordType.compare(that.recordType) 9 | lazy val compareData = compareString(this.data, that.data) 10 | 11 | if (compareName != 0) compareName 12 | else if (compareType != 0) compareType 13 | else compareData 14 | } 15 | 16 | private def compareString(a: String, b: String): Int = 17 | if (a == b) 0 18 | else if (a > b) 1 19 | else -1 20 | } 21 | 22 | object Record { 23 | def apply(name: String, recordType: RecordType, data: String): Record = { 24 | val transformedName = HostnameUtils.normalise(name) 25 | 26 | val transformedData = 27 | if (recordType.isOneOf("CNAME", "NS")) HostnameUtils.normalise(data) 28 | else if (recordType.stringValue == "SRV") HostnameUtils.normalise(data.split(" ")(3)) 29 | else if (recordType.stringValue == "MX") HostnameUtils.normalise(data.split(" ")(1)) 30 | else data 31 | 32 | new Record(transformedName, recordType, transformedData) 33 | } 34 | 35 | def unapply(record: Record): Option[(String, RecordType, String)] = 36 | Some((record.name, record.recordType, record.data)) 37 | } 38 | -------------------------------------------------------------------------------- /src/main/scala/com/gilazaria/subsearch/model/RecordType.scala: -------------------------------------------------------------------------------- 1 | package com.gilazaria.subsearch.model 2 | 3 | import org.xbill.DNS.Type 4 | 5 | case class RecordType(stringValue: String) extends Ordered[RecordType] { 6 | override def toString = stringValue 7 | lazy val intValue: Int = Type.value(stringValue) 8 | 9 | def isOneOf(types: String*) = 10 | types.contains(stringValue) 11 | 12 | override def compare(that: RecordType): Int = 13 | if (this.stringValue == that.stringValue) 0 14 | else if (this.stringValue > that.stringValue) 1 15 | else -1 16 | } 17 | 18 | object RecordType { 19 | lazy val A = RecordType("A") 20 | lazy val AAAA = RecordType("AAAA") 21 | lazy val ANY = RecordType("ANY") 22 | lazy val AXFR = RecordType("AXFR") 23 | lazy val CNAME = RecordType("CNAME") 24 | lazy val MX = RecordType("MX") 25 | lazy val NS = RecordType("NS") 26 | lazy val SOA = RecordType("SOA") 27 | lazy val RRSIG = RecordType("RRSIG") 28 | lazy val HINFO = RecordType("HINFO") 29 | lazy val DNSKEY = RecordType("DNSKEY") 30 | lazy val NSEC = RecordType("NSEC") 31 | lazy val PTR = RecordType("PTR") 32 | lazy val SRV = RecordType("SRV") 33 | 34 | private lazy val types: Set[RecordType] = Set(A, AAAA, ANY, AXFR, CNAME, MX, NS, SOA, RRSIG, HINFO, DNSKEY, NSEC, PTR, SRV) 35 | 36 | def fromInt(int: Int): RecordType = { 37 | val matchingTypes: Set[RecordType] = types.filter(_.intValue == int) 38 | 39 | if (matchingTypes.isEmpty) RecordType(Type.string(int)) 40 | else matchingTypes.head 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /src/main/scala/com/gilazaria/subsearch/output/CLIOutput.scala: -------------------------------------------------------------------------------- 1 | package com.gilazaria.subsearch.output 2 | 3 | import java.io.PrintStream 4 | 5 | import com.gilazaria.subsearch.model.{Record, RecordType} 6 | import pl.project13.scala.rainbow.Rainbow._ 7 | 8 | import scala.tools.jline.TerminalFactory 9 | import scala.collection.SortedSet 10 | 11 | class CLIOutput(private val printStream: PrintStream, private val verbose: Boolean) extends StandardOutput(None, verbose) { 12 | override def print(string: String) = printStream.print(string) 13 | private def eraseln() = print("\u001b[1K\u001b[0G") 14 | 15 | override def printSuccessWithoutTime(string: String) = println(string.green) 16 | override def printStatusWithoutTime(string: String) = println(string.yellow.bold) 17 | override def printInfoWithoutTime(string: String) = println(string.blue) 18 | override def printErrorWithoutTime(string: String) = println(string.onRed.white.bold) 19 | 20 | override def printStatusDuringScan(string: String) = { 21 | eraseln() 22 | super.printStatusDuringScan(string) 23 | printLastRequest() 24 | } 25 | 26 | override def printInfoDuringScan(string: String) = { 27 | eraseln() 28 | super.printInfoDuringScan(string) 29 | printLastRequest() 30 | } 31 | 32 | override def printHeader(header: String) = { 33 | println(header.magenta) 34 | println() 35 | } 36 | 37 | override def printConfig(config: List[(String, String)], separator: String) = { 38 | val string: String = 39 | config 40 | .map((tuple: (String, String)) => tuple._1.yellow + tuple._2.green) 41 | .mkString(separator.magenta) 42 | 43 | println(string) 44 | println() 45 | } 46 | 47 | override def printTarget(text: String, hostname: String) = { 48 | println(s"${text.yellow}${hostname.green}".bold) 49 | println() 50 | } 51 | 52 | override def printTaskCompleted(text: String) = { 53 | eraseln() 54 | super.printTaskCompleted(text) 55 | } 56 | 57 | override def printTaskFailed(text: String) = { 58 | eraseln() 59 | super.printTaskFailed(text) 60 | } 61 | 62 | override def printPausingThreads(text: String) = { 63 | eraseln() 64 | printStatusWithoutTime(text) 65 | } 66 | 67 | override def printPauseOptions(text: String) = { 68 | print(text) 69 | } 70 | 71 | override def printInvalidPauseOptions(text: String) = { 72 | println(text) 73 | } 74 | private var lastRequest: String = "" 75 | override def printLastRequest(text: String) = { 76 | lastRequest = text 77 | eraseln() 78 | printLastRequest() 79 | } 80 | 81 | lazy val terminal = TerminalFactory.create() 82 | override def printLastRequest() = { 83 | val terminalWidth: Int = terminal.getWidth 84 | val text = if (lastRequest.length <= terminalWidth) lastRequest 85 | else lastRequest.substring(0, terminalWidth) 86 | 87 | print(lastRequest) 88 | } 89 | 90 | override def printRecordsDuringScan(records: SortedSet[Record]) = { 91 | eraseln() 92 | super.printRecordsDuringScan(records) 93 | } 94 | 95 | override protected def formatRecordTypeAndSubdomainForPrinting(recordType: RecordType, subdomain: String): String = 96 | super.formatRecordTypeAndSubdomainForPrinting(recordType,subdomain).green 97 | } 98 | 99 | object CLIOutput { 100 | def create(verbose: Boolean): CLIOutput = create(System.out, verbose) 101 | private[this] def create(printStream: PrintStream, verbose: Boolean): CLIOutput = new CLIOutput(printStream, verbose) 102 | } -------------------------------------------------------------------------------- /src/main/scala/com/gilazaria/subsearch/output/CSVOutput.scala: -------------------------------------------------------------------------------- 1 | package com.gilazaria.subsearch.output 2 | 3 | import com.gilazaria.subsearch.model.Record 4 | import com.gilazaria.subsearch.utils.{File, TimeUtils} 5 | 6 | import scala.collection.SortedSet 7 | import scala.concurrent.Future 8 | import scala.concurrent.ExecutionContext.Implicits.global 9 | 10 | class CSVOutput(private val file: Option[File]) extends Output { 11 | override def print(string: String): Unit = {} 12 | 13 | /** 14 | * Using a future and chaining it means that writing to file will happen on a different thread to printing to CLI 15 | */ 16 | 17 | private var saveToFileFuture: Future[Unit] = Future(Unit) 18 | override def printRecords(records: SortedSet[Record]) = { 19 | if (file.isDefined) { 20 | saveToFileFuture = saveToFileFuture.map { 21 | _ => 22 | val lines = records.map(record => s"${TimeUtils.timestampNow},${record.name},${record.recordType},${record.data}") 23 | file.get.write(lines.mkString("\n") + "\n") 24 | } 25 | } 26 | } 27 | 28 | override def writingToFileFuture: Future[Unit] = { 29 | saveToFileFuture 30 | } 31 | } 32 | 33 | object CSVOutput { 34 | def create(fileOption: Option[File]): Option[CSVOutput] = 35 | if (fileOption.isDefined) Some(new CSVOutput(fileOption)) 36 | else None 37 | } -------------------------------------------------------------------------------- /src/main/scala/com/gilazaria/subsearch/output/Logger.scala: -------------------------------------------------------------------------------- 1 | package com.gilazaria.subsearch.output 2 | 3 | import com.gilazaria.subsearch.model.Record 4 | 5 | import scala.collection.SortedSet 6 | import scala.concurrent.Future 7 | 8 | trait Logger { 9 | // Controller 10 | def logHeader(header: String) 11 | def logConfig(threads: Int, wordlistSize: Int, resolverslistSize: Int) 12 | def logTarget(hostname: String) 13 | 14 | // Authoritative Scanner 15 | def logAuthoritativeScanStarted() 16 | def logAuthoritativeNameServer(nameServer: String) 17 | def logAuthoritativeScanCompleted() 18 | def logAddingAuthNameServersToResolvers(totalResolversSize: Int) 19 | 20 | // Zone Transfer Scanner 21 | def logStartedZoneTransfer() 22 | def logNameServersNotVulnerableToZoneTransfer() 23 | def logNameServerVulnerableToZoneTransfer(nameServer: String) 24 | def logZoneTransferCompleted() 25 | 26 | // Additional Scanners 27 | def logAdditionalScansStarted() 28 | def logAdditionalScannerError(scannerName: String, msg: String) 29 | def logAdditionalScannerFoundSubdomains(scannerName: String, subdomains: Set[String]) 30 | def logAdditionalScansCompleted() 31 | 32 | // Subdomain Bruteforce Scanner 33 | def logStartedSubdomainSearch() 34 | def logTaskCompleted() 35 | def logTaskFailed() 36 | def logPausingThreads() 37 | def logPauseOptions() 38 | def logInvalidPauseOption() 39 | def logNotEnoughResolvers() 40 | def logTimedOutScan(subdomain: String, resolver: String, duration: String) 41 | def logBlacklistedResolver(resolver: String) 42 | def logScanCancelled() 43 | def logScanForceCancelled() 44 | def logLastRequest(subdomain: String, numberOfRequestsSoFar: Int, totalNumberOfSubdomains: Int) 45 | 46 | // Records 47 | def logRecords(records: SortedSet[Record]) 48 | def logRecordsDuringScan(records: SortedSet[Record]) 49 | 50 | // Utility 51 | def completedLoggingFuture: Future[Unit] 52 | } 53 | -------------------------------------------------------------------------------- /src/main/scala/com/gilazaria/subsearch/output/LoggerImpl.scala: -------------------------------------------------------------------------------- 1 | package com.gilazaria.subsearch.output 2 | 3 | import java.util.concurrent.Executors 4 | 5 | import com.gilazaria.subsearch.model.Record 6 | import com.gilazaria.subsearch.utils.File 7 | import com.gilazaria.subsearch.utils.MathUtils.percentage 8 | 9 | import scala.concurrent.{ExecutionContext, Future} 10 | import scala.collection.SortedSet 11 | 12 | class LoggerImpl(private val verbose: Boolean, csvReportFile: Option[File], stdoutReportFile: Option[File])(implicit ec: ExecutionContext) extends Logger { 13 | private val outputs: Set[Output] = 14 | Set( 15 | Some(CLIOutput.create(verbose)), 16 | StandardOutput.create(stdoutReportFile, verbose), 17 | CSVOutput.create(csvReportFile) 18 | ).flatten 19 | 20 | /** Controller **/ 21 | 22 | def logHeader(header: String) = 23 | outputs.foreach(_.printHeader(header)) 24 | 25 | def logConfig(threads: Int, wordlistSize: Int, resolverslistSize: Int) = { 26 | val config = List(("Threads: ", threads.toString), 27 | ("Wordlist size: ", wordlistSize.toString), 28 | ("Number of resolvers: ", resolverslistSize.toString)) 29 | val separator = " | " 30 | 31 | outputs.foreach(_.printConfig(config, separator)) 32 | } 33 | 34 | def logTarget(hostname: String) = 35 | outputs.foreach(_.printTarget("Target: ", hostname)) 36 | 37 | /** Authoritative Scanner **/ 38 | 39 | def logAuthoritativeScanStarted() = 40 | outputs.foreach(_.printStatus("Identifying authoritative name servers:")) 41 | 42 | def logAuthoritativeNameServer(nameServer: String) = 43 | outputs.foreach(_.printSuccess(nameServer)) 44 | 45 | def logAuthoritativeScanCompleted() = 46 | outputs.foreach(_.println()) 47 | 48 | def logAddingAuthNameServersToResolvers(totalResolversSize: Int) = 49 | outputs.foreach { 50 | output => 51 | output.printStatus(s"Adding authoritative name servers to list of resolvers with a total of $totalResolversSize") 52 | output.println() 53 | } 54 | 55 | /** Zone Transfer Scanner **/ 56 | 57 | def logStartedZoneTransfer() = 58 | outputs.foreach(_.printStatus("Attempting zone transfer:")) 59 | 60 | def logNameServersNotVulnerableToZoneTransfer() = 61 | outputs.foreach(_.printInfo("Name servers aren't vulnerable to zone transfer")) 62 | 63 | def logNameServerVulnerableToZoneTransfer(nameServer: String) = 64 | outputs.foreach(_.printSuccess(s"$nameServer vulnerable to zone transfer!")) 65 | 66 | def logZoneTransferCompleted() = 67 | outputs.foreach(_.println()) 68 | 69 | /** Additional Scanners **/ 70 | 71 | override def logAdditionalScansStarted() = 72 | outputs.foreach(_.printStatus("Launching additional scanners:")) 73 | 74 | override def logAdditionalScannerError(scannerName: String, msg: String) = 75 | outputs.foreach(_.printInfo(s"$scannerName - Error: $msg")) 76 | 77 | override def logAdditionalScannerFoundSubdomains(scannerName: String, subdomains: Set[String]) = 78 | outputs.foreach(_.printSuccess(s"$scannerName found ${subdomains.size} possible subdomains.")) 79 | 80 | override def logAdditionalScansCompleted() = 81 | outputs.foreach(_.println()) 82 | 83 | /** Subdomain Bruteforce Scanner **/ 84 | 85 | def logStartedSubdomainSearch() = 86 | outputs.foreach(_.printStatus("Starting subdomain search:")) 87 | 88 | def logTaskCompleted() = 89 | outputs.foreach(_.printTaskCompleted("Task Completed")) 90 | 91 | def logTaskFailed() = 92 | outputs.foreach(_.printTaskFailed("Scan aborted as all resolvers are dead.")) 93 | 94 | def logPausingThreads() = 95 | outputs.foreach(_.printPausingThreads("CTRL+C detected: Pausing threads, please wait...")) 96 | 97 | def logPauseOptions() = 98 | outputs.foreach(_.printPauseOptions("[e]xit / [c]ontinue: ")) 99 | 100 | def logInvalidPauseOption() = 101 | outputs.foreach(_.printInvalidPauseOptions("")) 102 | 103 | def logNotEnoughResolvers() = 104 | outputs.foreach(_.printInfoDuringScan("There aren't enough resolvers for each thread. Reducing thread count by 1.")) 105 | 106 | def logTimedOutScan(subdomain: String, resolver: String, duration: String) = 107 | outputs.foreach(_.printInfoDuringScan(s"Lookup of $subdomain using $resolver timed out. Increasing timeout to $duration.")) 108 | 109 | def logBlacklistedResolver(resolver: String) = 110 | outputs.foreach(_.printInfoDuringScan(s"Lookup using $resolver timed out three times. Blacklisting resolver.")) 111 | 112 | def logScanCancelled() = 113 | outputs.foreach { 114 | output => 115 | output.println() 116 | output.println() 117 | output.printErrorWithoutTime("Cancelled by the user") 118 | } 119 | 120 | def logScanForceCancelled() = { 121 | logScanCancelled() 122 | outputs.foreach { 123 | output => 124 | if (!completedLoggingFuture.isCompleted && (csvReportFile.isDefined || stdoutReportFile.isDefined)) 125 | output.printErrorWithoutTime("WARNING: Reports may not be complete due to unexpected exit.") 126 | } 127 | } 128 | 129 | def logLastRequest(subdomain: String, numberOfRequestsSoFar: Int, totalNumberOfSubdomains: Int) = { 130 | val progress: Float = percentage(numberOfRequestsSoFar, totalNumberOfSubdomains) 131 | outputs.foreach(_.printLastRequest(f"$progress%.2f" + s"% - Last request to: $subdomain")) 132 | } 133 | 134 | /** Records **/ 135 | 136 | def logRecords(records: SortedSet[Record]) = { 137 | val newRecords = filterOutSeenAndInvalidRecords(records) 138 | saveNewRecords(newRecords) 139 | 140 | outputs.foreach(_.printRecords(newRecords)) 141 | } 142 | 143 | def logRecordsDuringScan(records: SortedSet[Record]) = { 144 | val newRecords = filterOutSeenAndInvalidRecords(records) 145 | saveNewRecords(newRecords) 146 | 147 | outputs.foreach(_.printRecordsDuringScan(newRecords)) 148 | } 149 | 150 | /** Utility **/ 151 | 152 | def completedLoggingFuture: Future[Unit] = { 153 | Future.sequence(outputs.map(_.writingToFileFuture)).map(_ => Unit) 154 | } 155 | 156 | /** Internal **/ 157 | 158 | private var allSeenRecords: SortedSet[Record] = SortedSet.empty 159 | private def filterOutSeenAndInvalidRecords(records: SortedSet[Record]): SortedSet[Record] = 160 | records 161 | .filter(!_.recordType.isOneOf("NSEC", "RRSIG", "SOA")) 162 | .diff(allSeenRecords) 163 | 164 | private def saveNewRecords(records: SortedSet[Record]) = 165 | allSeenRecords = allSeenRecords ++ records 166 | } 167 | 168 | object LoggerImpl { 169 | def create(extendedOutput: Boolean, csvReportFile: Option[File], stdoutReportFile: Option[File]): Logger = { 170 | val executorService = Executors.newFixedThreadPool(1) 171 | implicit val ec: ExecutionContext = ExecutionContext.fromExecutorService(executorService) 172 | 173 | new LoggerImpl(extendedOutput, csvReportFile, stdoutReportFile)(ec) 174 | } 175 | } -------------------------------------------------------------------------------- /src/main/scala/com/gilazaria/subsearch/output/Output.scala: -------------------------------------------------------------------------------- 1 | package com.gilazaria.subsearch.output 2 | 3 | import com.gilazaria.subsearch.model.Record 4 | import com.gilazaria.subsearch.utils.TimeUtils 5 | 6 | import scala.collection.SortedSet 7 | import scala.concurrent.Future 8 | import scala.concurrent.ExecutionContext.Implicits.global 9 | 10 | trait Output { 11 | def print(string: String) 12 | def println(): Unit = println("") 13 | def println(string: String): Unit = print(s"$string\n") 14 | 15 | def printSuccess(string: String) = printSuccessWithoutTime(prependTime(string)) 16 | def printStatus(string: String) = printStatusWithoutTime(prependTime(string)) 17 | def printInfo(string: String) = printInfoWithoutTime(prependTime(string)) 18 | def printError(string: String) = printErrorWithoutTime(prependTime(string)) 19 | 20 | def printSuccessWithoutTime(string: String) = println(string) 21 | def printStatusWithoutTime(string: String) = println(string) 22 | def printInfoWithoutTime(string: String) = println(string) 23 | def printErrorWithoutTime(string: String) = println(string) 24 | 25 | def printSuccessDuringScan(string: String) = printSuccess(string) 26 | def printStatusDuringScan(string: String) = printStatus(string) 27 | def printInfoDuringScan(string: String) = printInfo(string) 28 | def printErrorDuringScan(string: String) = printError(string) 29 | 30 | // Utility 31 | 32 | final def prependTime(string: String): String = 33 | s"${TimeUtils.currentTimePretty} $string" 34 | 35 | def writingToFileFuture: Future[Unit] = Future(Unit) 36 | 37 | // Application Specific 38 | 39 | def printHeader(header: String) = { 40 | println(header) 41 | println() 42 | } 43 | 44 | def printConfig(config: List[(String, String)], separator: String) = { 45 | val string: String = 46 | config 47 | .map((tuple: (String, String)) => tuple._1 + tuple._2) 48 | .mkString(separator) 49 | 50 | println(string) 51 | println() 52 | } 53 | 54 | def printTarget(text: String, hostname: String) = { 55 | println(s"$text$hostname") 56 | println() 57 | } 58 | 59 | def printTaskCompleted(text: String) = { 60 | println() 61 | printStatusWithoutTime(text) 62 | println() 63 | } 64 | 65 | def printTaskFailed(text: String) = { 66 | println() 67 | printErrorWithoutTime(text) 68 | println() 69 | } 70 | 71 | def printPausingThreads(text: String) = {} 72 | 73 | def printPauseOptions(text: String) = {} 74 | 75 | def printInvalidPauseOptions(text: String) = {} 76 | 77 | def printLastRequest(text: String) = {} 78 | 79 | def printLastRequest() = {} 80 | 81 | def printRecords(records: SortedSet[Record]) 82 | 83 | def printRecordsDuringScan(records: SortedSet[Record]) = printRecords(records) 84 | } 85 | -------------------------------------------------------------------------------- /src/main/scala/com/gilazaria/subsearch/output/StandardOutput.scala: -------------------------------------------------------------------------------- 1 | package com.gilazaria.subsearch.output 2 | 3 | import com.gilazaria.subsearch.model.{Record, RecordType} 4 | import com.gilazaria.subsearch.utils.File 5 | 6 | import scala.collection.SortedSet 7 | import scala.concurrent.Future 8 | import scala.concurrent.ExecutionContext.Implicits.global 9 | 10 | class StandardOutput(private val file: Option[File], private val verbose: Boolean) extends Output { 11 | private var saveToFileFuture: Future[Unit] = Future(Unit) 12 | 13 | override def print(string: String): Unit = { 14 | if (file.isDefined) { 15 | saveToFileFuture = saveToFileFuture.map { 16 | _ => file.get.write(string) 17 | } 18 | } 19 | } 20 | 21 | override def writingToFileFuture: Future[Unit] = { 22 | saveToFileFuture 23 | } 24 | 25 | override def printRecords(records: SortedSet[Record]) = { 26 | if (verbose) printRecordsVerbose(records) 27 | else printRecordsNormal(records) 28 | } 29 | 30 | protected def printRecordsVerbose(records: SortedSet[Record]) = { 31 | val lines: List[String] = 32 | records 33 | .map(_.name) 34 | .toList 35 | .flatMap { 36 | subdomain => 37 | val subdomainRecords: SortedSet[Record] = records.filter(_.name == subdomain) 38 | val recordTypes: SortedSet[RecordType] = subdomainRecords.map(_.recordType) 39 | 40 | recordTypes.flatMap { 41 | recordType => 42 | subdomainRecords.filter(_.recordType == recordType).map { 43 | case Record(_, _, data) => 44 | val msg = formatRecordTypeAndSubdomainForPrinting(recordType, subdomain) 45 | 46 | if (recordType.isOneOf("A", "AAAA", "CNAME", "NS", "SRV")) 47 | s"$msg -> $data" 48 | else if (recordType.stringValue == "MX") 49 | s"$msg @@ $data" 50 | else 51 | s"$msg -- $data" 52 | } 53 | } 54 | } 55 | 56 | if (lines.nonEmpty) 57 | println(lines.mkString("\n")) 58 | } 59 | 60 | protected def formatRecordTypeAndSubdomainForPrinting(recordType: RecordType, subdomain: String): String = 61 | prependTime(f"${recordType.toString}%-7s: $subdomain") 62 | 63 | protected def printRecordsNormal(records: SortedSet[Record]) = { 64 | val lines: List[String] = 65 | records 66 | .map(_.name) 67 | .toList 68 | .map(subdomain => (subdomain, records.filter(_.name == subdomain).map(_.recordType))) 69 | .map((data: (String, SortedSet[RecordType])) => s"${data._2.mkString(", ")}: ${data._1}") 70 | 71 | if (lines.nonEmpty) 72 | printSuccess(lines.mkString("\n")) 73 | } 74 | } 75 | 76 | object StandardOutput { 77 | def create(fileOption: Option[File], verbose: Boolean): Option[StandardOutput] = 78 | if (fileOption.isDefined) Some(new StandardOutput(fileOption, verbose)) 79 | else None 80 | } 81 | -------------------------------------------------------------------------------- /src/main/scala/com/gilazaria/subsearch/utils/FileUtils.scala: -------------------------------------------------------------------------------- 1 | package com.gilazaria.subsearch.utils 2 | 3 | import java.nio.charset.StandardCharsets 4 | import java.nio.file._ 5 | import scala.collection.JavaConverters._ 6 | import scala.io.Source 7 | import scala.util.Try 8 | 9 | class File(val path: Path) { 10 | def exists: Boolean = 11 | FileUtils.exists(path) 12 | 13 | def isFile: Boolean = 14 | FileUtils.isFile(path) 15 | 16 | def isReadable: Boolean = 17 | FileUtils.isReadable(path) 18 | 19 | def linesIterator: Iterator[String] = 20 | FileUtils.linesIterator(path) 21 | 22 | lazy val getLines: List[String] = 23 | FileUtils.getLines(path) 24 | 25 | lazy val numberOfLines: Int = 26 | FileUtils.numberOfLines(path) 27 | 28 | lazy val getSource: String = 29 | getLines.mkString("\n") 30 | 31 | def isWriteable: Boolean = 32 | FileUtils.isWriteable(path) 33 | 34 | def write(data: String) = 35 | FileUtils.writeToPath(data, path) 36 | } 37 | 38 | object File { 39 | def fromFilename(filename: String): File = 40 | new File(Paths.get(filename)) 41 | } 42 | 43 | object FileUtils { 44 | def exists(path: Path): Boolean = 45 | Files.exists(path) 46 | 47 | def isFile(path: Path): Boolean = 48 | Files.isRegularFile(path) 49 | 50 | def isReadable(path: Path): Boolean = 51 | Files.isReadable(path) 52 | 53 | def isWriteable(path: Path): Boolean = { 54 | if (!FileUtils.exists(path)) { 55 | val createFileAttempt = 56 | Try { 57 | Files.createFile(path) 58 | Files.delete(path) 59 | } 60 | 61 | createFileAttempt.isSuccess 62 | } else { 63 | Files.isWritable(path) 64 | } 65 | } 66 | 67 | def writeToPath(data: String, path: Path) = 68 | Files.write(path, data.getBytes(StandardCharsets.UTF_8), StandardOpenOption.CREATE, StandardOpenOption.APPEND) 69 | 70 | def linesIterator(path: Path): Iterator[String] = 71 | io.Source.fromFile(path.toUri).getLines 72 | 73 | def getLines(path: Path): List[String] = 74 | Files.readAllLines(path).asScala.toList 75 | 76 | def numberOfLines(path: Path): Int = 77 | io.Source.fromFile(path.toUri).getLines.size 78 | 79 | /** 80 | * This is a hacky way of getting the entire source of a resource 81 | * into a String object. Thanks to: 82 | * https://stackoverflow.com/questions/15110315/read-property-file-under-classpath-using-scala 83 | * and 84 | * https://stackoverflow.com/questions/18923864/read-all-lines-of-bufferedreader-in-scala-into-a-string 85 | */ 86 | def getResourceSource(filename: String): String = { 87 | val reader = 88 | Source 89 | .fromInputStream(getClass.getResourceAsStream("/" + filename)) 90 | .bufferedReader() 91 | 92 | Stream 93 | .continually(reader.readLine()) 94 | .takeWhile(_ != null) 95 | .mkString("\n") 96 | } 97 | } 98 | -------------------------------------------------------------------------------- /src/main/scala/com/gilazaria/subsearch/utils/HostnameUtils.scala: -------------------------------------------------------------------------------- 1 | package com.gilazaria.subsearch.utils 2 | 3 | import com.gilazaria.subsearch.model.Record 4 | 5 | object HostnameUtils { 6 | def normalise(hostname: String): String = 7 | hostname 8 | .trim // Trim leading/trailing whitespaces 9 | .stripPrefix(".").stripSuffix(".").trim // Trim leading/trailing dots 10 | .toLowerCase 11 | 12 | private val validDomainRegexPattern = "^[a-z0-9-_]+[.]([a-z0-9-_]+[.])*[a-z0-9-_]+$".r.pattern 13 | def isValidDomain(hostname: String): Boolean = 14 | validDomainRegexPattern.matcher(hostname).matches 15 | 16 | private val validSubdomainPartRegexPattern = "^[a-z0-9-_]+([.][a-z0-9-_]+)*$".r.pattern 17 | def isValidSubdomainPart(part: String): Boolean = 18 | validSubdomainPartRegexPattern.matcher(part).matches 19 | 20 | def ensureSubdomainEndsWithHostname(subdomain: String, hostname: String): String = 21 | if (subdomain.endsWith(hostname)) subdomain 22 | else s"$subdomain.$hostname" 23 | } 24 | -------------------------------------------------------------------------------- /src/main/scala/com/gilazaria/subsearch/utils/IPUtils.scala: -------------------------------------------------------------------------------- 1 | package com.gilazaria.subsearch.utils 2 | 3 | import scala.util.Try 4 | 5 | object IPUtils { 6 | def normalise(ip: String): String = 7 | ip.trim 8 | 9 | def isValidIPv4(ip: String): Boolean = { 10 | val parts = ip.split("\\.", -1) 11 | 12 | parts.length == 4 && 13 | parts 14 | .filter(string => Try(string.toInt).isSuccess) // It's an integer 15 | .filter(num => num.toInt != 0 || (num.toInt == 0 && num == "0")) // 0 is 0 and not 00 or 000 or ... 16 | .filter(num => !(num.toInt != 0 && num.startsWith("0"))) // 1 is not 01 17 | .map(_.toInt) 18 | .count(num => 0 <= num && num <= 255) == 4 19 | 20 | 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /src/main/scala/com/gilazaria/subsearch/utils/IterableUtils.scala: -------------------------------------------------------------------------------- 1 | package com.gilazaria.subsearch.utils 2 | 3 | import com.gilazaria.subsearch.model.Record 4 | 5 | import scala.collection.SortedSet 6 | 7 | object IterableUtils { 8 | implicit class IterableImprovements(iterable: Iterable[Record]) { 9 | def toSortedSet: SortedSet[Record] = { 10 | SortedSet[Record]() ++ iterable 11 | } 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /src/main/scala/com/gilazaria/subsearch/utils/MathUtils.scala: -------------------------------------------------------------------------------- 1 | package com.gilazaria.subsearch.utils 2 | 3 | object MathUtils { 4 | def percentage(a: Int, b: Int): Float = 5 | if (b == 0) throw new IllegalArgumentException("The second argument cannot be zero.") 6 | else a.toFloat / b.toFloat * 100 7 | } 8 | -------------------------------------------------------------------------------- /src/main/scala/com/gilazaria/subsearch/utils/TimeUtils.scala: -------------------------------------------------------------------------------- 1 | package com.gilazaria.subsearch.utils 2 | 3 | import java.time._ 4 | import akka.util.Timeout 5 | import scala.concurrent.duration._ 6 | 7 | object TimeUtils { 8 | def currentTimePretty: String = { 9 | val localTime = LocalTime.now() 10 | val hour = f"${localTime.getHour}%02d" 11 | val minute = f"${localTime.getMinute}%02d" 12 | val second = f"${localTime.getSecond}%02d" 13 | 14 | s"[$hour:$minute:$second]" 15 | } 16 | 17 | def akkaAskTimeout: Timeout = 18 | Timeout(FiniteDuration(21474835, "seconds")) 19 | 20 | def awaitDuration: FiniteDuration = 21 | 365.days 22 | 23 | def timestampNow: String = 24 | (System.currentTimeMillis / 1000).toString 25 | } 26 | -------------------------------------------------------------------------------- /src/main/scala/com/gilazaria/subsearch/utils/TimeoutScheduler.scala: -------------------------------------------------------------------------------- 1 | package com.gilazaria.subsearch.utils 2 | 3 | import java.util.concurrent.TimeUnit 4 | import org.jboss.netty.util.{Timeout, TimerTask, HashedWheelTimer} 5 | 6 | import scala.concurrent.{TimeoutException, Promise, Future} 7 | import scala.concurrent.duration.Duration 8 | 9 | /** 10 | * Taken from the excellent https://stackoverflow.com/questions/16304471/scala-futures-built-in-timeout 11 | */ 12 | 13 | object TimeoutScheduler { 14 | val timer = new HashedWheelTimer(10, TimeUnit.MILLISECONDS) 15 | 16 | def scheduleTimeout(promise:Promise[_], after:Duration) = { 17 | timer.newTimeout(new TimerTask { 18 | def run(timeout:Timeout){ 19 | promise.failure(new TimeoutException("Operation timed out after " + after.toMillis + " millis")) 20 | } 21 | }, after.toNanos, TimeUnit.NANOSECONDS) 22 | } 23 | } 24 | 25 | object TimeoutFuture { 26 | implicit class FutureWithTimeout[T](f: Future[T]) { 27 | import scala.concurrent.ExecutionContext 28 | 29 | def withTimeout(after: Duration)(implicit ec: ExecutionContext) = { 30 | val prom = Promise[T]() 31 | val timeout = TimeoutScheduler.scheduleTimeout(prom, after) 32 | val combinedFut = Future.firstCompletedOf(List(f, prom.future)) 33 | f onComplete { case result => timeout.cancel() } 34 | combinedFut 35 | } 36 | } 37 | } -------------------------------------------------------------------------------- /src/test/resources/DNSDumpster/DNSDumpsterExample.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | DNSdumpster.com - dns recon and research, find and lookup dns records 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 |
24 |
25 | 26 |
27 |
28 |
29 |
30 |

dns recon & research, find & lookup dns records

31 |

32 |

33 |
34 |
35 |
36 | 37 |
38 |
39 |
40 |
41 |
Loading...
42 |
43 |
44 |
45 |

46 | 47 | 48 | 49 |

Showing results for dnsdumpster.com

50 |
51 |
52 |
53 |
54 |

Hosting (IP block owners)

55 |
56 |
57 |

GeoIP of Host Locations

58 |
59 |
60 |
61 | 62 |

DNS Servers

63 |
64 | 65 | 66 | 71 | 72 | 77 | 78 | 83 | 84 | 89 | 90 | 95 | 96 |
freedns4.registrar-servers.com.
67 | 68 | 69 | 70 | 		62.210.149.102
freedns4.registrar-servers.com		AS12876 ONLINE S.A.S.
France
freedns5.registrar-servers.com.
73 | 74 | 75 | 76 | 		192.99.40.34
freedns5.registrar-servers.com		AS16276 OVH SAS
Canada
freedns1.registrar-servers.com.
79 | 80 | 81 | 82 | 		208.64.122.242
freedns1.registrar-servers.com		AS32421 Black Lotus Communications
United States
freedns3.registrar-servers.com.
85 | 86 | 87 | 88 | 		62.210.149.103
freedns3.registrar-servers.com		AS12876 ONLINE S.A.S.
France
freedns2.registrar-servers.com.
91 | 92 | 93 | 94 | 		72.20.38.137
freedns2.registrar-servers.com		AS25761 Staminus Communications
United States
97 |
98 | 99 |

MX Records ** This is where email for the domain goes...

100 |
101 | 102 | 103 |
104 |
105 | 106 |

TXT Records ** Find more hosts in Sender Policy Framework (SPF) configurations

107 |
108 | 109 | 110 |
111 |
112 | 113 | 114 |

Host Records (A) ** this data may not be current as it uses a static database (updated monthly)

115 |
116 | 117 | 118 | 124 | 125 |
dnsdumpster.com
119 | 120 | 121 | nginx/1.4.6 (Ubuntu) 122 | 123 | 		104.131.171.250
AS393406 Digital Ocean, Inc.
United States
126 |
127 | 128 | 129 |
130 |

Mapping the domain ** click for full size image

131 |

132 | 133 |

134 |
135 | 136 | 137 | 138 |

DNSdumpster.com is a FREE domain research tool that can discover hosts related to a domain. Finding visible hosts from the attackers perspective is an important part of the security assessment process.

139 | 140 |
141 | 142 |

this is a project

143 | 144 |

145 | 146 | 147 | 148 | 149 | 150 | 158 | 159 | 160 | 161 | 162 | 163 |
164 |
165 | 166 |
167 | 168 |
169 | 170 | 171 |

Map an organizations attack surface with a virtual dumpster dive* of the DNS records associated with the target organization.

172 |

*DUMPSTER DIVING: The practice of sifting refuse from an office or technical installation to extract confidential data, especially security-compromising information.
Dictionary.com

173 |

174 |
175 |
176 | 177 |
178 | 179 |
180 | 181 | 182 |

More than a simple DNS lookup this tool will discover those hard to find sub-domains and web hosts. No brute force of common sub-domains is undertaken as is common practice for many DNS recon tools. The search relies on data from search engines and the excellent scans.io research project.

183 |

184 |
185 |
186 | 187 | 188 | 189 |
190 | 191 |
192 | 193 | 194 |

Receive updates by following @hackertarget on Twitter
or subscribe to the low volume mailing list.

195 | 196 |

this is a project

197 |
198 | 199 |
200 |
201 | 202 | 204 | 205 | 206 | 207 | 208 | 209 | 210 | 211 | 212 | 213 | 214 | 215 | 216 | 217 | 263 | 264 | 265 | 295 | 296 | 332 | 333 | 334 | 344 | 354 | 355 | 356 | -------------------------------------------------------------------------------- /src/test/scala/com/gilazaria/subsearch/connection/DNSLookupImplSpec.scala: -------------------------------------------------------------------------------- 1 | package com.gilazaria.subsearch.connection 2 | 3 | import com.gilazaria.subsearch.connection.DNSLookupImpl.{HostNotFoundException, ServerFailureException} 4 | import com.gilazaria.subsearch.model.{Record, RecordType} 5 | import com.gilazaria.subsearch.connection.LookupTestUtils._ 6 | import org.scalatest.FlatSpec 7 | import org.scalamock.scalatest.MockFactory 8 | import org.xbill.DNS.{Lookup, ZoneTransferException} 9 | import org.xbill.DNS 10 | import scala.collection.JavaConverters._ 11 | 12 | import scala.collection.SortedSet 13 | import scala.util.Try 14 | 15 | class DNSLookupImplSpec extends FlatSpec with MockFactory { 16 | val lookup: DNSLookupImpl = new DNSLookupImpl() 17 | 18 | behavior of "xbillRecordsFromTransfer" 19 | 20 | val mockZoneTransferIn: ZoneTransferInFactory = mock[ZoneTransferInFactory] 21 | 22 | it should "return the correct records on a successful transfer with records" in { 23 | val hostname = "zonetransfer.me." 24 | val siphostname = s"_sip._tcp.$hostname" 25 | val arpahostname = s"157.177.147.217.IN-ADDR.ARPA.$hostname" 26 | val wwwhostname = s"www.$hostname" 27 | 28 | val expectedRecords: Set[DNS.Record] = 29 | Set( 30 | createSOARecord(hostname, "nsztm1.digi.ninja. robin.digi.ninja. 2014101603 172800 900 1209600 3600"), 31 | createRRSIGRecord(hostname, "SOA 8 2 7200 20160330133700 20160229123700 44244 zonetransfer.me. GzQojkYAP8zuTOB9UAx66mTDiEGJ26hVIIP2ifk2DpbQLrEAPg4M77i4 M0yFWHpNfMJIuuJ8nMxQgFVCU3yTOeT/EMbN98FYC8lVYwEZeWHtbMmS 88jVlF+cOz2WarjCdyV0+UJCTdGtBJriIczC52EXKkw2RCkv3gtdKKVa fBE="), 32 | createNSRecord(hostname, "nsztm1.digi.ninja."), 33 | createNSRecord(hostname, "nsztm2.digi.ninja."), 34 | createRRSIGRecord(hostname, "NS 8 2 7200 20160330133700 20160229123700 44244 zonetransfer.me. TyFngBk2PMWxgJc6RtgCE/RhE0kqeWfwhYSBxFxezupFLeiDjHeVXo+S WZxP54Xvwfk7jlFClNZ9lRNkL5qHyxRElhlH1JJI1hjvod0fycqLqCnx XIqkOzUCkm2Mxr8OcGf2jVNDUcLPDO5XjHgOXCK9tRbVVKIpB92f4Qal ulw="), 35 | createARecord(hostname, "217.147.177.157"), 36 | createRRSIGRecord(hostname, "A 8 2 7200 20160330133700 20160229123700 44244 zonetransfer.me. unoMaEPiyoAr0yAWg/coPbAFNznaAlUJW3/QrvJleer50VvGLW/cK+VE DcZLfCu6paQhgJHVddG4p145vVQe3QRvp7EJpUh+SU7dX0I3gngmOa4H k190S4utcXY5FhaN7xBKHVWBlavQaSHTg61g/iuLSB0lS1gp/DAMUpC+ WzE="), 37 | createHINFORecord(hostname, "\"Casio fx-700G\"", "\"Windows XP\""), 38 | createRRSIGRecord(hostname, "HINFO 8 2 300 20160330133700 20160229123700 44244 zonetransfer.me. Xebvrpv8nCGn/+iHqok1rcItTPqcskV6jpJ1pCo4WYbnqByLultzygWx JlyVzz+wJHEqRQYDjqGblOdyUgKn2FFnqb1O92kKghcHHvoMEh+Jf5i7 0trtucpRs3AtlneLj2vauOCIEdbjma4IxgdwPahKIhgtgWcUInVFh3Rr SwM="), 39 | createNSECRecord(hostname, "_sip._tcp.zonetransfer.me. A NS SOA HINFO RRSIG NSEC DNSKEY"), 40 | createRRSIGRecord(hostname, "NSEC 8 2 3600 20160330133700 20160229123700 44244 zonetransfer.me. MFHBRiIpk14ys3NIb81eCIl8rWULNDAokMeVUlXpwO9OF8jSuFOgPoHy zdAd5NdKe5O/QtuBFizLBTPoR41G5PzQwOZf3cwLEr+35cGsvujwu7Yz H2YU+bkNR/ZvOtYX8hMlM2WxivGyEM+ebZNVn0XcrOI3Kpi31VQ3SGTS Gcs="), 41 | createDNSKEYRecord(hostname, "256 3 8 AwEAAdXeqhjJnKHczUifC98Nz2xg2xM6DXe0JEE3dNenGNyoByUrxG2m QoZy78l2Pshjc/gKwcTjawHKeI8/aO16xP9nG4pgKyybLPJN9XvJRV5t XFok48g2DPbeu0/TPNoVYALrMeruqxFGv/a/RijzKxrdE3Eqle1906zg nXf43hkL"), 42 | createDNSKEYRecord(hostname, "256 3 8 AwEAAdd50fHa9wVKIf9/gpOhFx3NBq32WGq/SykRNUwSMnPU3OOkRzUT KEL+yxdoARvL/pbR+5pK3WAOVzsLKdy3+xeLqfILdgY1P0msjVXXNN2o mBV+Hs2Lip7qS4RkYJHg+Oq8RWixpeCSAJ1/DkSLE20HYYiq27a07pNE oc+OfNef"), 43 | createDNSKEYRecord(hostname, "257 3 8 AwEAAeFLORXsixEH2ftm5oo4VQI66D6zMBreinysTdx5jOVgdAtOa3I6 L3ye7lXZFJbdnPZw0w0df2NvWbIHDzICQxbJjXTUkWuN0wuXm9f6BFP8 VcNa8nPm14F2+6ue+cMxyMOXZW/4vt2DV5hsH+lNWQxGVeTFdCifUlYt sPloxspYETT5vhquKj/VNariRecX0zTzyM67F+grCR+cUo75CnZaRbVZ G7gIBQ783tVO5tFQCUJjqw3zO1f6eQp/xk4rNorCK7BDqitgUbpdPuFh CqmcKVYsFrEViNbaI/Nk9lJHH+nQaYBSZx+cWrTLPZtkNp3ahqY3gLwA 0RUgfYZCWnU="), 44 | createRRSIGRecord(hostname, "DNSKEY 8 2 300 20160330133700 20160229123700 44244 zonetransfer.me. LWKb719rS66s4b3Th1hCqN1dYsKM2hNTlSoHht7KSngj+zPbXckRQmEQ Dczk0DQO71mnxr8/V3iIHUwlEz54oVJZe/3VvNByL3Hdz88vJfXVjDl2 iPSZV9H+NMvocoWSa+63PinRXdfaSpt5wfd8PtRrO0cy9AZQfZc5Nvzx bvM="), 45 | createRRSIGRecord(hostname, "DNSKEY 8 2 300 20160330133700 20160229123700 47530 zonetransfer.me. zrmeIf1WdtuwrZgeJRgnP6SOLC+MXiw4UKUEvdA8bhdw1uHEAV8onMWO xQgHWOXV3/npzGhwTcLMoO6NqrV8dZyjL7hgo9PafECNWRIJmtEVOLhd uCiqd5fWFWabiqMr5fHkYIDocnjoz5iyau2ARhIun4U/sygKOPkVTAzP UhvywrNAp3vS+pSCTpPhybXuyU46dIKkZ0XAAwprfR4zT1utsSeafsK8 LGmf4RkmKZXO8nXiz2LKcBbrHv1JM9OvuNTYukI15RSyMDFRhfRCn+ZC UG12rJvWOg5zViaIK/phI5nNd281ghWvFs6Xb9/MJ3f3hWopDOk/ALel brPQ7g=="), 46 | createSRVRecord(siphostname, "0 0 5060 www.zonetransfer.me."), 47 | createRRSIGRecord(siphostname, "SRV 8 4 14000 20160330133700 20160229123700 44244 zonetransfer.me. trlrzT1EGDJ+I0GwCcCS1FU9HDiR+y9NOV3g+tyygeOUQDIE4Yr/UTvT e/UEjSEbluMOwAAPuMIFV9amchsbTBLIzxtVsZgPscJMB8vIA4UVSFOq NsuPbFd1uBsPJ9gJYjf7/L8hnZ3r6KDXxdOzHWhs/RhMKV9/pV+UMBzT fm0="), 48 | createNSECRecord(siphostname, "157.177.147.217.IN-ADDR.ARPA.zonetransfer.me. SRV RRSIG NSEC"), 49 | createRRSIGRecord(siphostname, "NSEC 8 4 3600 20160330133700 20160229123700 44244 zonetransfer.me. zxoMlL21uMJDNv54lggFbbp13q001YAqXobmIGBKyJR7fjD5vLNs54s7 nOLbTMee9rjzz6fMLUrNscPKh+Io9K8vSLhGK/SQbip5d/uHkITj0OW1 10QZ800ZbM5apzxVEg60expe7PmcbxlunEqGLr9+aO377DSADkDagXhB oQA="), 50 | createPTRRecord(arpahostname, "www.zonetransfer.me."), 51 | createRRSIGRecord(arpahostname, "PTR 8 8 7200 20160330133700 20160229123700 44244 zonetransfer.me. NLvoxfZKnVcap87fQs52MaUVX3Yi0pyQCrPJDkNNDIWvB78NHy6eNSZt 4cIb/oUgQW8Gw6/soXrMXBwJcG6gaVNXPEph1v8Yiobm3fyetGgeCSB8 rnhQwfCltiW1zCpIvkDZKIy7FgTLOSr+RjnPZA/cMu4c9QsJLgYXyytL hWs="), 52 | createNSECRecord(arpahostname, "asfdbauthdns.zonetransfer.me. PTR RRSIG NSEC"), 53 | createRRSIGRecord(arpahostname, "NSEC 8 8 3600 20160330133700 20160229123700 44244 zonetransfer.me. jU2Gm0tnY2m+JF8q2ca7U03I6HWSsCzGGQoYEzla+7QebAs9ho66PpAQ RdmPKX5KNIYZJSop+492a/1CBT/WR0hqUAgVS/UOuutCyrHqpeVWnl7x tx5CujVs1+Mn48RlGBzanTDzhRr44h/4nNJtZM3/OTWBkpnpakBWn/W6 EdI="), 54 | createARecord(wwwhostname, "217.147.177.157"), 55 | createRRSIGRecord(wwwhostname, "A 8 3 7200 20160330133700 20160229123700 44244 zonetransfer.me. tVFIQ2rklljhe64SM8JAn740XSk+MIEXuJJQ9u6Qzmti058hA0hEzdrz 7nAt/LmMcC9RCFTAOl8CSewVExhHgiwu1sO5i69jpI0G87eecAYnGnJe Cd9pf0Wfvm+3ucIxOKm59YmHXOF/48dJlyoKnXNEg6nCBzoW9aq3OreO 16s="), 56 | createNSECRecord(wwwhostname, "xss.zonetransfer.me. A RRSIG NSEC"), 57 | createRRSIGRecord(wwwhostname, "NSEC 8 3 3600 20160330133700 20160229123700 44244 zonetransfer.me. 0xCqc6tWcT11ACD24Ap68hc7HRyAcCf7MrkDqe2HyYMGuGS9YSwosiF3 QzffhuY5qagIFbpI3f7xVGxykngThTk37/JO2SrfI7Z5kvqLHdEd6GD9 sogsLqTfHE9UToOYYfuasO+IsJLyPALh89yk3bY+NipvpEPngSnxN6eh Ikc="), 58 | createSOARecord(hostname, "nsztm1.digi.ninja. robin.digi.ninja. 2014101603 172800 900 1209600 3600") 59 | ) 60 | 61 | (mockZoneTransferIn.run _).expects().returns(expectedRecords.toList.asJava) 62 | 63 | val actual: Try[Set[DNS.Record]] = lookup.xbillRecordsFromTransfer(mockZoneTransferIn) 64 | 65 | assert(actual.isSuccess) 66 | assert(expectedRecords == actual.get) 67 | } 68 | 69 | it should "return no records on a successful transfer with no records" in { 70 | (mockZoneTransferIn.run _).expects().returns(null) 71 | 72 | val expectedRecords: Set[DNS.Record] = Set.empty 73 | val actual: Try[Set[DNS.Record]] = lookup.xbillRecordsFromTransfer(mockZoneTransferIn) 74 | 75 | assert(actual.isSuccess) 76 | assert(expectedRecords == actual.get) 77 | } 78 | 79 | it should "return a failed try on a failed transfer" in { 80 | (mockZoneTransferIn.run _).expects().throws(new ZoneTransferException("Test message.")) 81 | 82 | val expectedException = intercept[ZoneTransferException] { 83 | lookup.xbillRecordsFromTransfer(mockZoneTransferIn).get 84 | } 85 | 86 | assert(expectedException.getMessage == "Test message.") 87 | } 88 | 89 | behavior of "query" 90 | 91 | val mockLookup: LookupFactory = mock[LookupFactory] 92 | 93 | it should "return the correct records on a successful lookup with records" in { 94 | val xbillRecords: Array[DNS.Record] = 95 | Array( 96 | createARecord("domain.com.", "10.10.10.10"), 97 | createNSRecord("domain.com.", "ns1.domain.com."), 98 | createNSRecord("domain.com.", "ns2.domain.com."), 99 | createSOARecord("domain.com.", "ns1.domain.com. root.domain.com. 118863930 900 900 1800 60") 100 | ) 101 | 102 | val records: Set[Record] = 103 | Set( 104 | Record("domain.com.", RecordType.A, "10.10.10.10"), 105 | Record("domain.com.", RecordType.NS, "ns1.domain.com."), 106 | Record("domain.com.", RecordType.NS, "ns2.domain.com."), 107 | Record("domain.com.", RecordType.SOA, "ns1.domain.com. root.domain.com. 118863930 900 900 1800 60") 108 | ) 109 | 110 | (mockLookup.run _).expects 111 | (mockLookup.getResult _ ).expects().returns(Lookup.SUCCESSFUL) 112 | (mockLookup.getAnswers _).expects().returns(xbillRecords) 113 | 114 | val expectedRecords = records 115 | val actualRecords = lookup.query("", "", mockLookup) 116 | 117 | assert(expectedRecords == actualRecords) 118 | } 119 | 120 | it should "return no records on a successful lookup with no records" in { 121 | (mockLookup.run _).expects() 122 | (mockLookup.getResult _).expects().returns(Lookup.SUCCESSFUL) 123 | (mockLookup.getAnswers _).expects().returns(null) 124 | 125 | val expectedRecords = SortedSet.empty[Record] 126 | val actualRecords = lookup.query("", "", mockLookup) 127 | 128 | assert(expectedRecords == actualRecords) 129 | } 130 | 131 | it should "attempt another a query if the first attempt fails" in { 132 | (mockLookup.run _).expects() 133 | (mockLookup.getResult _).expects().returns(Lookup.TRY_AGAIN) 134 | (mockLookup.run _).expects() 135 | (mockLookup.getResult _ ).expects().returns(Lookup.SUCCESSFUL) 136 | (mockLookup.getAnswers _).expects().returns(null) 137 | 138 | val expectedRecords = SortedSet.empty[Record] 139 | val actualRecords = lookup.query("", "", mockLookup) 140 | 141 | assert(expectedRecords == actualRecords) 142 | } 143 | 144 | it should "attempt another a query if the first and second attempt fails" in { 145 | (mockLookup.run _).expects() 146 | (mockLookup.getResult _).expects().returns(Lookup.TRY_AGAIN) 147 | (mockLookup.run _).expects() 148 | (mockLookup.getResult _).expects().returns(Lookup.TRY_AGAIN) 149 | (mockLookup.run _).expects() 150 | (mockLookup.getResult _ ).expects().returns(Lookup.SUCCESSFUL) 151 | (mockLookup.getAnswers _).expects().returns(null) 152 | 153 | val expectedRecords = SortedSet.empty[Record] 154 | val actualRecords = lookup.query("", "", mockLookup) 155 | 156 | assert(expectedRecords == actualRecords) 157 | 158 | } 159 | 160 | it should "return an empty set if the third attempt fails" in { 161 | (mockLookup.run _).expects() 162 | (mockLookup.getResult _).expects().returns(Lookup.TRY_AGAIN) 163 | (mockLookup.run _).expects() 164 | (mockLookup.getResult _).expects().returns(Lookup.TRY_AGAIN) 165 | (mockLookup.run _).expects() 166 | (mockLookup.getResult _ ).expects().returns(Lookup.TRY_AGAIN) 167 | 168 | val expectedRecords = SortedSet.empty[Record] 169 | val actualRecords = lookup.query("", "", mockLookup) 170 | 171 | assert(expectedRecords == actualRecords) 172 | 173 | } 174 | 175 | it should "throw an exception if the host is not found" in { 176 | val hostname = "domain.com" 177 | 178 | (mockLookup.run _).expects() 179 | (mockLookup.getResult _).expects().returns(Lookup.HOST_NOT_FOUND) 180 | 181 | val expectedException = intercept[HostNotFoundException] { 182 | lookup.query(hostname, "", mockLookup) 183 | } 184 | 185 | assert(expectedException.getMessage == s"The hostname $hostname was not found.") 186 | } 187 | 188 | it should "throw an exception if the lookup is unrecoverable" in { 189 | val resolver = "10.10.10.10" 190 | 191 | (mockLookup.run _).expects() 192 | (mockLookup.getResult _).expects().returns(Lookup.UNRECOVERABLE) 193 | 194 | val expectedException = intercept[ServerFailureException] { 195 | lookup.query("", resolver, mockLookup) 196 | } 197 | 198 | assert(expectedException.getMessage == s"There was a data or server error with the resolver $resolver.") 199 | } 200 | 201 | it should "return an empty set if the record type is not found" in { 202 | (mockLookup.run _).expects() 203 | (mockLookup.getResult _).expects().returns(Lookup.TYPE_NOT_FOUND) 204 | 205 | val expectedRecords = SortedSet.empty[Record] 206 | val actualRecords = lookup.query("", "", mockLookup) 207 | 208 | assert(expectedRecords == actualRecords) 209 | } 210 | 211 | behavior of "recordsFromXbillRecords" 212 | 213 | it should "convert DNS Java records to subsearch records" in { 214 | val xbillRecords: Set[DNS.Record] = 215 | Set( 216 | createARecord("domain.com.", "10.10.10.10"), 217 | createNSRecord("domain.com.", "ns1.domain.com."), 218 | createNSRecord("domain.com.", "ns2.domain.com."), 219 | createSOARecord("domain.com.", "ns1.domain.com. root.domain.com. 118863930 900 900 1800 60") 220 | ) 221 | 222 | val expectedRecords: SortedSet[Record] = 223 | SortedSet( 224 | Record("domain.com.", RecordType.A, "10.10.10.10"), 225 | Record("domain.com.", RecordType.NS, "ns1.domain.com."), 226 | Record("domain.com.", RecordType.NS, "ns2.domain.com."), 227 | Record("domain.com.", RecordType.SOA, "ns1.domain.com. root.domain.com. 118863930 900 900 1800 60") 228 | ) 229 | 230 | val actualRecords = lookup.recordsFromXbillRecords(xbillRecords) 231 | 232 | assert(expectedRecords == actualRecords) 233 | } 234 | 235 | behavior of "create" 236 | 237 | it should "return an object that extends DNSLookup" in { 238 | val actual = DNSLookupImpl.create().getClass 239 | val expected = classOf[DNSLookup] 240 | 241 | assert(expected.isAssignableFrom(actual)) 242 | } 243 | } 244 | -------------------------------------------------------------------------------- /src/test/scala/com/gilazaria/subsearch/connection/LookupTestUtils.scala: -------------------------------------------------------------------------------- 1 | package com.gilazaria.subsearch.connection 2 | 3 | import java.net.InetAddress 4 | import java.util.{Calendar, Date, GregorianCalendar, TimeZone} 5 | 6 | import org.xbill.DNS 7 | import org.xbill.DNS.{Name, TextParseException} 8 | 9 | object LookupTestUtils { 10 | def createARecord(name: String, ip: String): DNS.ARecord = 11 | new DNS.ARecord(new DNS.Name(name), DNS.DClass.IN, 300, InetAddress.getByName(ip)) 12 | 13 | def createNSRecord(name: String, target: String): DNS.NSRecord = 14 | new DNS.NSRecord(new DNS.Name(name), DNS.DClass.IN, 300, new DNS.Name(target)) 15 | 16 | def createSOARecord(name: String, data: String): DNS.SOARecord = { 17 | val args = data.split(" ") 18 | new DNS.SOARecord( 19 | new DNS.Name(name), 20 | DNS.DClass.IN, 21 | 300, 22 | new DNS.Name(args(0)), 23 | new DNS.Name(args(1)), 24 | args(2).toInt, 25 | args(3).toInt, 26 | args(4).toInt, 27 | args(5).toInt, 28 | args(6).toInt) 29 | } 30 | 31 | def createRRSIGRecord(name: String, data: String): DNS.RRSIGRecord = { 32 | val args = data.split(" ") 33 | 34 | val signature: Array[Byte] = DNS.utils.base64.fromString(args.slice(8, args.size+1).mkString("")) 35 | 36 | new DNS.RRSIGRecord( 37 | new DNS.Name(name), 38 | DNS.DClass.IN, 39 | 300, 40 | args(1).toInt, 41 | args(2).toInt, 42 | args(3).toInt, 43 | dateFromFormattedTime(args(4)), 44 | dateFromFormattedTime(args(5)), 45 | args(6).toInt, 46 | new DNS.Name(args(7)), 47 | signature 48 | ) 49 | 50 | // SOA 8 2 7200 20160330133700 20160229123700 44244 zonetransfer.me. GzQojkYAP8zuTOB9UAx66mTDiEGJ26hVIIP2ifk2DpbQLrEAPg4M77i4 M0yFWHpNfMJIuuJ8nMxQgFVCU3yTOeT/EMbN98FYC8lVYwEZeWHtbMmS 88jVlF+cOz2WarjCdyV0+UJCTdGtBJriIczC52EXKkw2RCkv3gtdKKVa fBE= 51 | } 52 | 53 | def createHINFORecord(name: String, cpu: String, os: String): DNS.HINFORecord = 54 | new DNS.HINFORecord(new Name(name), DNS.DClass.IN, 300, cpu, os) 55 | 56 | def createNSECRecord(name: String, data: String): DNS.NSECRecord = { 57 | val args = data.split(" ") 58 | val types: Array[Int] = args.slice(1, args.size+1).map(DNS.Type.value) 59 | 60 | new DNS.NSECRecord( 61 | new DNS.Name(name), 62 | DNS.DClass.IN, 63 | 300, 64 | new DNS.Name(args(0)), 65 | types 66 | ) 67 | } 68 | 69 | def createDNSKEYRecord(name: String, data: String): DNS.DNSKEYRecord = { 70 | val args = data.split(" ") 71 | val key: Array[Byte] = DNS.utils.base64.fromString(args.slice(3, args.size+1).mkString("")) 72 | 73 | new DNS.DNSKEYRecord( 74 | new DNS.Name(name), 75 | DNS.DClass.IN, 76 | 300, 77 | args(0).toInt, 78 | args(1).toInt, 79 | args(2).toInt, 80 | key 81 | ) 82 | } 83 | 84 | def createSRVRecord(name: String, data: String): DNS.SRVRecord = { 85 | val args = data.split(" ") 86 | 87 | new DNS.SRVRecord( 88 | new DNS.Name(name), 89 | DNS.DClass.IN, 90 | 300, 91 | args(0).toInt, 92 | args(1).toInt, 93 | args(2).toInt, 94 | new DNS.Name(args(3)) 95 | ) 96 | } 97 | 98 | def createPTRRecord(name: String, target: String): DNS.PTRRecord = 99 | new DNS.PTRRecord( 100 | new DNS.Name(name), 101 | DNS.DClass.IN, 102 | 300, 103 | new DNS.Name(target) 104 | ) 105 | 106 | def dateFromFormattedTime(s: String): Date = { 107 | if (s.length != 14) throw new TextParseException("Invalid time encoding: " + s) 108 | 109 | val c: Calendar = new GregorianCalendar(TimeZone.getTimeZone("UTC")) 110 | c.clear() 111 | 112 | try { 113 | val year: Int = s.substring(0, 4).toInt 114 | val month: Int = s.substring(4, 6).toInt - 1 115 | val date: Int = s.substring(6, 8).toInt 116 | val hour: Int = s.substring(8, 10).toInt 117 | val minute: Int = s.substring(10, 12).toInt 118 | val second: Int = s.substring(12, 14).toInt 119 | c.set(year, month, date, hour, minute, second) 120 | } 121 | catch { 122 | case e: NumberFormatException => 123 | throw new TextParseException("Invalid time encoding: " + s) 124 | } 125 | 126 | c.getTime 127 | } 128 | } 129 | -------------------------------------------------------------------------------- /src/test/scala/com/gilazaria/subsearch/core/AuthoritativeScannerSpec.scala: -------------------------------------------------------------------------------- 1 | package com.gilazaria.subsearch.core 2 | 3 | import com.gilazaria.subsearch.connection.DNSLookup 4 | import com.gilazaria.subsearch.model.{Record, RecordType} 5 | import com.gilazaria.subsearch.output.Logger 6 | import com.gilazaria.subsearch.utils.TimeUtils 7 | import org.scalamock.scalatest.MockFactory 8 | import org.scalatest.FlatSpec 9 | 10 | import scala.collection.SortedSet 11 | import scala.concurrent.{Await, ExecutionContext} 12 | import scala.util.Try 13 | 14 | class AuthoritativeScannerSpec extends FlatSpec with MockFactory { 15 | 16 | val mockLogger: Logger = mock[Logger] 17 | val mockLookup: DNSLookup = mock[DNSLookup] 18 | val mockExecutionContext: ExecutionContext = mock[ExecutionContext] 19 | 20 | behavior of "performLookupOnHostname" 21 | 22 | it should "return the authoritative name servers as ip addresses whilst logging" in { 23 | val hostname = "domain.com" 24 | val resolver = "1.1.1.1" 25 | 26 | val nameServerRecords = SortedSet(Record(hostname, RecordType.NS, "ns1.domain.com"), Record(hostname, RecordType.NS, "ns2.domain.com")) 27 | val nameServerIPs = Map(("ns1.domain.com", "2.2.2.2"), ("ns2.domain.com", "3.3.3.3")) 28 | 29 | implicit val ec = scala.concurrent.ExecutionContext.Implicits.global 30 | 31 | (mockLogger.logAuthoritativeScanStarted _).expects() 32 | (mockLookup.performQueryOfType _).expects(hostname, resolver, RecordType.NS).returns(Try(nameServerRecords)) 33 | nameServerIPs.keys.foreach { 34 | ns => (mockLookup.performQueryOfType _).expects(ns, resolver, RecordType.A).returns(Try(SortedSet(Record(ns, RecordType.A, nameServerIPs(ns))))) 35 | } 36 | nameServerIPs.values.foreach(ns => (mockLogger.logAuthoritativeNameServer _).expects(ns)) 37 | (mockLogger.logAuthoritativeScanCompleted _).expects() 38 | 39 | val scanner = new AuthoritativeScanner(mockLogger, mockLookup) 40 | 41 | val expectedIPs = Set("2.2.2.2", "3.3.3.3") 42 | val actualIPs = Await.result(scanner.performLookupOnHostname(hostname, resolver), TimeUtils.awaitDuration) 43 | 44 | assert(expectedIPs == actualIPs) 45 | } 46 | 47 | behavior of "ipsForNameServers" 48 | 49 | it should "return an empty set when there are no name servers" in { 50 | val nameServers = Set.empty[String] 51 | val resolver = "1.1.1.1" 52 | implicit val ec = scala.concurrent.ExecutionContext.Implicits.global 53 | 54 | val scanner = new AuthoritativeScanner(mockLogger, mockLookup) 55 | 56 | val expected = Set.empty 57 | val actual = Await.result(scanner.ipsForNameServers(nameServers, resolver), TimeUtils.awaitDuration) 58 | 59 | assert(expected == actual) 60 | } 61 | 62 | it should "return an empty set when all name servers have no A records" in { 63 | val nameServers = Set("ns1.domain.com", "ns2.domain.com") 64 | val resolver = "1.1.1.1" 65 | implicit val ec = scala.concurrent.ExecutionContext.Implicits.global 66 | 67 | nameServers.foreach { 68 | ns => (mockLookup.performQueryOfType _).expects(ns, resolver, RecordType.A).returns(Try(SortedSet.empty[Record])) 69 | } 70 | 71 | val scanner = new AuthoritativeScanner(mockLogger, mockLookup) 72 | 73 | val expected = Set.empty 74 | val actual = Await.result(scanner.ipsForNameServers(nameServers, resolver), TimeUtils.awaitDuration) 75 | 76 | assert(expected == actual) 77 | } 78 | 79 | it should "return a set of IP addresses (one for each A record excluding duplicates) when one or more name servers have A records" in { 80 | val nameServerTuples = 81 | Set( 82 | ("ns1.domain.com", SortedSet(Record("ns1.domain.com.", RecordType.A, "1.1.1.1"), Record("ns1.domain.com.", RecordType.A, "2.2.2.2"))), 83 | ("ns2.domain.com", SortedSet(Record("ns2.domain.com.", RecordType.A, "2.2.2.2"), Record("ns2.domain.com.", RecordType.A, "3.3.3.3"))) 84 | ) 85 | val nameServers = nameServerTuples.map(tup => tup._1) 86 | 87 | val resolver = "10.10.10.10" 88 | implicit val ec = scala.concurrent.ExecutionContext.Implicits.global 89 | 90 | nameServerTuples.foreach { 91 | tup => (mockLookup.performQueryOfType _).expects(tup._1, resolver, RecordType.A).returns(Try(tup._2)) 92 | } 93 | 94 | val scanner = new AuthoritativeScanner(mockLogger, mockLookup) 95 | 96 | val expected: Set[String] = nameServerTuples.map(_._2.map(_.data)).reduce(_ ++ _).toSet 97 | val actual = Await.result(scanner.ipsForNameServers(nameServers, resolver), TimeUtils.awaitDuration) 98 | 99 | assert(expected == actual) 100 | } 101 | 102 | behavior of "dataFromQuery" 103 | 104 | it should "return an empty set when the hostname has no records" in { 105 | val nameServer = "sub.domain.com" 106 | val resolver = "1.1.1.1" 107 | val recordType = RecordType.A 108 | 109 | implicit val ec = scala.concurrent.ExecutionContext.Implicits.global 110 | 111 | (mockLookup.performQueryOfType _).expects(nameServer, resolver, recordType).returns(Try(SortedSet.empty[Record])) 112 | 113 | val scanner = new AuthoritativeScanner(mockLogger, mockLookup) 114 | 115 | val expected = Set.empty 116 | val actual = Await.result(scanner.dataFromQuery(nameServer, resolver, recordType), TimeUtils.awaitDuration) 117 | 118 | assert(expected == actual) 119 | } 120 | 121 | it should "return an empty set when the query throws an exception" in { 122 | val nameServer = "sub.domain.com" 123 | val resolver = "1.1.1.1" 124 | val recordType = RecordType.A 125 | 126 | implicit val ec = scala.concurrent.ExecutionContext.Implicits.global 127 | 128 | (mockLookup.performQueryOfType _).expects(nameServer, resolver, recordType).returns(Try(throw new Exception("Test message."))) 129 | 130 | val scanner = new AuthoritativeScanner(mockLogger, mockLookup) 131 | 132 | val expected = Set.empty 133 | val actual = Await.result(scanner.dataFromQuery(nameServer, resolver, recordType), TimeUtils.awaitDuration) 134 | 135 | assert(expected == actual) 136 | } 137 | 138 | it should "return a set of IP addresses (one for each A record) when the name server has A records" in { 139 | val nameServer = "ns.domain.com" 140 | val resolver = "1.1.1.1" 141 | val recordType = RecordType.A 142 | 143 | implicit val ec = scala.concurrent.ExecutionContext.Implicits.global 144 | 145 | val records = SortedSet(Record("ns.domain.com.", RecordType.A, "2.2.2.2"), Record("ns.domain.com.", RecordType.A, "3.3.3.3")) 146 | 147 | (mockLookup.performQueryOfType _).expects(nameServer, resolver, recordType).returns(Try(records)) 148 | 149 | val scanner = new AuthoritativeScanner(mockLogger, mockLookup) 150 | 151 | val expected = records.map(_.data).toSet 152 | val actual = Await.result(scanner.dataFromQuery(nameServer, resolver, recordType), TimeUtils.awaitDuration) 153 | 154 | assert(expected == actual) 155 | } 156 | 157 | behavior of "printAuthoritativeNameServers" 158 | 159 | it should "log each name server, that the scan has completed, and return the nameservers" in { 160 | val nameServers = Set("1.1.1.1", "2.2.2.2", "3.3.3.3") 161 | 162 | nameServers.foreach(ns => (mockLogger.logAuthoritativeNameServer _).expects(ns)) 163 | (mockLogger.logAuthoritativeScanCompleted _).expects() 164 | 165 | val scanner = AuthoritativeScanner.create(mockLogger)(mockExecutionContext) 166 | 167 | val actual = scanner.printAuthoritativeNameServers(nameServers) 168 | 169 | assert(nameServers == actual) 170 | } 171 | 172 | it should "log that the scan has completed, and return an empty set" in { 173 | (mockLogger.logAuthoritativeScanCompleted _).expects() 174 | 175 | val scanner = AuthoritativeScanner.create(mockLogger)(mockExecutionContext) 176 | 177 | val actual = scanner.printAuthoritativeNameServers(Set.empty) 178 | 179 | assert(Set.empty == actual) 180 | } 181 | 182 | behavior of "create" 183 | 184 | it should "return an AuthoritativeScanner" in { 185 | val actualClass = AuthoritativeScanner.create(mockLogger)(mockExecutionContext).getClass 186 | val expectedClass = classOf[AuthoritativeScanner] 187 | 188 | assert(expectedClass == actualClass) 189 | } 190 | 191 | } 192 | -------------------------------------------------------------------------------- /src/test/scala/com/gilazaria/subsearch/core/ZoneTransferScannerSpec.scala: -------------------------------------------------------------------------------- 1 | package com.gilazaria.subsearch.core 2 | 3 | import com.gilazaria.subsearch.connection.DNSLookup 4 | import com.gilazaria.subsearch.model.{Record, RecordType} 5 | import com.gilazaria.subsearch.output.Logger 6 | import com.gilazaria.subsearch.utils.TimeUtils 7 | import org.scalamock.scalatest.MockFactory 8 | import org.scalatest.FlatSpec 9 | 10 | import scala.collection.SortedSet 11 | import scala.concurrent.{Await, ExecutionContext} 12 | import scala.util.Try 13 | 14 | class ZoneTransferScannerSpec extends FlatSpec with MockFactory { 15 | 16 | val mockLogger: Logger = mock[Logger] 17 | val mockLookup: DNSLookup = mock[DNSLookup] 18 | val mockExecutionContext: ExecutionContext = mock[ExecutionContext] 19 | 20 | behavior of "ZoneTransferScannerSpec" 21 | 22 | it should "" in { 23 | 24 | } 25 | 26 | it should "recordsEndingWithHostname" in { 27 | 28 | } 29 | 30 | it should "zoneTransferForHostnameAndResolver" in { 31 | 32 | } 33 | 34 | behavior of "zoneTransferForHostnameAndResolver" 35 | 36 | it should "return the correct records and then log that the resolver is vulnerable" in { 37 | val resolver = "1.1.1.1" 38 | 39 | val hostname = "zonetransfer.me." 40 | val siphostname = s"_sip._tcp.$hostname" 41 | val arpahostname = s"157.177.147.217.IN-ADDR.ARPA.$hostname" 42 | val wwwhostname = s"www.$hostname" 43 | 44 | val records = 45 | SortedSet( 46 | Record(hostname, RecordType.SOA, "nsztm1.digi.ninja. robin.digi.ninja. 2014101603 172800 900 1209600 3600"), 47 | Record(hostname, RecordType.RRSIG, "SOA 8 2 7200 20160330133700 20160229123700 44244 zonetransfer.me. GzQojkYAP8zuTOB9UAx66mTDiEGJ26hVIIP2ifk2DpbQLrEAPg4M77i4 M0yFWHpNfMJIuuJ8nMxQgFVCU3yTOeT/EMbN98FYC8lVYwEZeWHtbMmS 88jVlF+cOz2WarjCdyV0+UJCTdGtBJriIczC52EXKkw2RCkv3gtdKKVa fBE="), 48 | Record(hostname, RecordType.RRSIG, "NS 8 2 7200 20160330133700 20160229123700 44244 zonetransfer.me. TyFngBk2PMWxgJc6RtgCE/RhE0kqeWfwhYSBxFxezupFLeiDjHeVXo+S WZxP54Xvwfk7jlFClNZ9lRNkL5qHyxRElhlH1JJI1hjvod0fycqLqCnx XIqkOzUCkm2Mxr8OcGf2jVNDUcLPDO5XjHgOXCK9tRbVVKIpB92f4Qal ulw="), 49 | Record(hostname, RecordType.RRSIG, "A 8 2 7200 20160330133700 20160229123700 44244 zonetransfer.me. unoMaEPiyoAr0yAWg/coPbAFNznaAlUJW3/QrvJleer50VvGLW/cK+VE DcZLfCu6paQhgJHVddG4p145vVQe3QRvp7EJpUh+SU7dX0I3gngmOa4H k190S4utcXY5FhaN7xBKHVWBlavQaSHTg61g/iuLSB0lS1gp/DAMUpC+ WzE="), 50 | Record(hostname, RecordType.RRSIG, "HINFO 8 2 300 20160330133700 20160229123700 44244 zonetransfer.me. Xebvrpv8nCGn/+iHqok1rcItTPqcskV6jpJ1pCo4WYbnqByLultzygWx JlyVzz+wJHEqRQYDjqGblOdyUgKn2FFnqb1O92kKghcHHvoMEh+Jf5i7 0trtucpRs3AtlneLj2vauOCIEdbjma4IxgdwPahKIhgtgWcUInVFh3Rr SwM="), 51 | Record(hostname, RecordType.RRSIG, "NSEC 8 2 3600 20160330133700 20160229123700 44244 zonetransfer.me. MFHBRiIpk14ys3NIb81eCIl8rWULNDAokMeVUlXpwO9OF8jSuFOgPoHy zdAd5NdKe5O/QtuBFizLBTPoR41G5PzQwOZf3cwLEr+35cGsvujwu7Yz H2YU+bkNR/ZvOtYX8hMlM2WxivGyEM+ebZNVn0XcrOI3Kpi31VQ3SGTS Gcs="), 52 | Record(hostname, RecordType.RRSIG, "DNSKEY 8 2 300 20160330133700 20160229123700 44244 zonetransfer.me. LWKb719rS66s4b3Th1hCqN1dYsKM2hNTlSoHht7KSngj+zPbXckRQmEQ Dczk0DQO71mnxr8/V3iIHUwlEz54oVJZe/3VvNByL3Hdz88vJfXVjDl2 iPSZV9H+NMvocoWSa+63PinRXdfaSpt5wfd8PtRrO0cy9AZQfZc5Nvzx bvM="), 53 | Record(hostname, RecordType.RRSIG, "DNSKEY 8 2 300 20160330133700 20160229123700 47530 zonetransfer.me. zrmeIf1WdtuwrZgeJRgnP6SOLC+MXiw4UKUEvdA8bhdw1uHEAV8onMWO xQgHWOXV3/npzGhwTcLMoO6NqrV8dZyjL7hgo9PafECNWRIJmtEVOLhd uCiqd5fWFWabiqMr5fHkYIDocnjoz5iyau2ARhIun4U/sygKOPkVTAzP UhvywrNAp3vS+pSCTpPhybXuyU46dIKkZ0XAAwprfR4zT1utsSeafsK8 LGmf4RkmKZXO8nXiz2LKcBbrHv1JM9OvuNTYukI15RSyMDFRhfRCn+ZC UG12rJvWOg5zViaIK/phI5nNd281ghWvFs6Xb9/MJ3f3hWopDOk/ALel brPQ7g=="), 54 | Record(hostname, RecordType.RRSIG, "SRV 8 4 14000 20160330133700 20160229123700 44244 zonetransfer.me. trlrzT1EGDJ+I0GwCcCS1FU9HDiR+y9NOV3g+tyygeOUQDIE4Yr/UTvT e/UEjSEbluMOwAAPuMIFV9amchsbTBLIzxtVsZgPscJMB8vIA4UVSFOq NsuPbFd1uBsPJ9gJYjf7/L8hnZ3r6KDXxdOzHWhs/RhMKV9/pV+UMBzT fm0="), 55 | Record(hostname, RecordType.RRSIG, "NSEC 8 4 3600 20160330133700 20160229123700 44244 zonetransfer.me. zxoMlL21uMJDNv54lggFbbp13q001YAqXobmIGBKyJR7fjD5vLNs54s7 nOLbTMee9rjzz6fMLUrNscPKh+Io9K8vSLhGK/SQbip5d/uHkITj0OW1 10QZ800ZbM5apzxVEg60expe7PmcbxlunEqGLr9+aO377DSADkDagXhB oQA="), 56 | Record(hostname, RecordType.RRSIG, "PTR 8 8 7200 20160330133700 20160229123700 44244 zonetransfer.me. NLvoxfZKnVcap87fQs52MaUVX3Yi0pyQCrPJDkNNDIWvB78NHy6eNSZt 4cIb/oUgQW8Gw6/soXrMXBwJcG6gaVNXPEph1v8Yiobm3fyetGgeCSB8 rnhQwfCltiW1zCpIvkDZKIy7FgTLOSr+RjnPZA/cMu4c9QsJLgYXyytL hWs="), 57 | Record(hostname, RecordType.RRSIG, "NSEC 8 8 3600 20160330133700 20160229123700 44244 zonetransfer.me. jU2Gm0tnY2m+JF8q2ca7U03I6HWSsCzGGQoYEzla+7QebAs9ho66PpAQ RdmPKX5KNIYZJSop+492a/1CBT/WR0hqUAgVS/UOuutCyrHqpeVWnl7x tx5CujVs1+Mn48RlGBzanTDzhRr44h/4nNJtZM3/OTWBkpnpakBWn/W6 EdI="), 58 | Record(siphostname, RecordType.RRSIG, "A 8 3 7200 20160330133700 20160229123700 44244 zonetransfer.me. tVFIQ2rklljhe64SM8JAn740XSk+MIEXuJJQ9u6Qzmti058hA0hEzdrz 7nAt/LmMcC9RCFTAOl8CSewVExhHgiwu1sO5i69jpI0G87eecAYnGnJe Cd9pf0Wfvm+3ucIxOKm59YmHXOF/48dJlyoKnXNEg6nCBzoW9aq3OreO 16s="), 59 | Record(siphostname, RecordType.RRSIG, "NSEC 8 3 3600 20160330133700 20160229123700 44244 zonetransfer.me. 0xCqc6tWcT11ACD24Ap68hc7HRyAcCf7MrkDqe2HyYMGuGS9YSwosiF3 QzffhuY5qagIFbpI3f7xVGxykngThTk37/JO2SrfI7Z5kvqLHdEd6GD9 sogsLqTfHE9UToOYYfuasO+IsJLyPALh89yk3bY+NipvpEPngSnxN6eh Ikc="), 60 | Record(hostname, RecordType.NS, "nsztm1.digi.ninja."), 61 | Record(hostname, RecordType.NS, "nsztm2.digi.ninja."), 62 | Record(siphostname, RecordType.A, "217.147.177.157"), 63 | Record(siphostname, RecordType.HINFO, "\"Casio fx-700G\" \"Windows XP\""), 64 | Record(arpahostname, RecordType.NSEC, "_sip._tcp.zonetransfer.me. A NS SOA HINFO RRSIG NSEC DNSKEY"), 65 | Record(arpahostname, RecordType.NSEC, "157.177.147.217.IN-ADDR.ARPA.zonetransfer.me. SRV RRSIG NSEC"), 66 | Record(arpahostname, RecordType.NSEC, "asfdbauthdns.zonetransfer.me. PTR RRSIG NSEC"), 67 | Record(arpahostname, RecordType.DNSKEY, "256 3 8 AwEAAdXeqhjJnKHczUifC98Nz2xg2xM6DXe0JEE3dNenGNyoByUrxG2m QoZy78l2Pshjc/gKwcTjawHKeI8/aO16xP9nG4pgKyybLPJN9XvJRV5t XFok48g2DPbeu0/TPNoVYALrMeruqxFGv/a/RijzKxrdE3Eqle1906zg nXf43hkL"), 68 | Record(wwwhostname, RecordType.DNSKEY, "256 3 8 AwEAAdd50fHa9wVKIf9/gpOhFx3NBq32WGq/SykRNUwSMnPU3OOkRzUT KEL+yxdoARvL/pbR+5pK3WAOVzsLKdy3+xeLqfILdgY1P0msjVXXNN2o mBV+Hs2Lip7qS4RkYJHg+Oq8RWixpeCSAJ1/DkSLE20HYYiq27a07pNE oc+OfNef"), 69 | Record(wwwhostname, RecordType.DNSKEY, "257 3 8 AwEAAeFLORXsixEH2ftm5oo4VQI66D6zMBreinysTdx5jOVgdAtOa3I6 L3ye7lXZFJbdnPZw0w0df2NvWbIHDzICQxbJjXTUkWuN0wuXm9f6BFP8 VcNa8nPm14F2+6ue+cMxyMOXZW/4vt2DV5hsH+lNWQxGVeTFdCifUlYt sPloxspYETT5vhquKj/VNariRecX0zTzyM67F+grCR+cUo75CnZaRbVZ G7gIBQ783tVO5tFQCUJjqw3zO1f6eQp/xk4rNorCK7BDqitgUbpdPuFh CqmcKVYsFrEViNbaI/Nk9lJHH+nQaYBSZx+cWrTLPZtkNp3ahqY3gLwA 0RUgfYZCWnU="), 70 | Record(wwwhostname, RecordType.SRV, "0 0 5060 www.zonetransfer.me."), 71 | Record(wwwhostname, RecordType.A, "217.147.177.157") 72 | ) 73 | 74 | (mockLookup.performQueryOfType _).expects(hostname, resolver, RecordType.AXFR).returns(Try(records)) 75 | (mockLogger.logNameServerVulnerableToZoneTransfer _).expects(resolver) 76 | 77 | implicit val ec = scala.concurrent.ExecutionContext.Implicits.global 78 | val scanner = new ZoneTransferScanner(mockLogger, mockLookup) 79 | 80 | val expectedRecords = records 81 | val actualRecords = Await.result(scanner.zoneTransferForHostnameAndResolver(hostname, resolver), TimeUtils.awaitDuration) 82 | 83 | assert(expectedRecords == actualRecords) 84 | } 85 | 86 | it should "return no records and then log that the resolver is vulnerable" in { 87 | val hostname = "domain.com" 88 | val resolver = "1.1.1.1" 89 | 90 | val records = SortedSet.empty[Record] 91 | 92 | (mockLookup.performQueryOfType _).expects(hostname, resolver, RecordType.AXFR).returns(Try(records)) 93 | 94 | implicit val ec = scala.concurrent.ExecutionContext.Implicits.global 95 | val scanner = new ZoneTransferScanner(mockLogger, mockLookup) 96 | 97 | val expectedRecords = records 98 | val actualRecords = Await.result(scanner.zoneTransferForHostnameAndResolver(hostname, resolver), TimeUtils.awaitDuration) 99 | 100 | assert(expectedRecords == actualRecords) 101 | } 102 | 103 | behavior of "flattenRecords" 104 | 105 | it should "return an empty set #1" in { 106 | val set: Set[SortedSet[Record]] = Set.empty 107 | 108 | val scanner = ZoneTransferScanner.create(mockLogger)(mockExecutionContext) 109 | 110 | val expectedRecords = SortedSet.empty[Record] 111 | val actualRecords = scanner.flattenRecords(set) 112 | 113 | assert (expectedRecords == actualRecords) 114 | } 115 | 116 | it should "return an empty set #2" in { 117 | val set: Set[SortedSet[Record]] = Set(SortedSet.empty[Record]) 118 | 119 | val scanner = ZoneTransferScanner.create(mockLogger)(mockExecutionContext) 120 | 121 | val expectedRecords = SortedSet.empty[Record] 122 | val actualRecords = scanner.flattenRecords(set) 123 | 124 | assert (expectedRecords == actualRecords) 125 | } 126 | 127 | it should "return an empty set #3" in { 128 | val set: Set[SortedSet[Record]] = Set(SortedSet.empty[Record], SortedSet.empty[Record]) 129 | 130 | val scanner = ZoneTransferScanner.create(mockLogger)(mockExecutionContext) 131 | 132 | val expectedRecords = SortedSet.empty[Record] 133 | val actualRecords = scanner.flattenRecords(set) 134 | 135 | assert (expectedRecords == actualRecords) 136 | } 137 | 138 | it should "return a combined set #1" in { 139 | val set: Set[SortedSet[Record]] = 140 | Set( 141 | SortedSet(Record("domain.com.", RecordType.A, "1.1.1.1")), 142 | SortedSet(Record("domain.com.", RecordType.A, "2.2.2.2")) 143 | ) 144 | 145 | val scanner = ZoneTransferScanner.create(mockLogger)(mockExecutionContext) 146 | 147 | val expectedRecords = 148 | SortedSet( 149 | Record("domain.com.", RecordType.A, "1.1.1.1"), 150 | Record("domain.com.", RecordType.A, "2.2.2.2") 151 | ) 152 | val actualRecords = scanner.flattenRecords(set) 153 | 154 | assert (expectedRecords == actualRecords) 155 | } 156 | 157 | it should "return a combined set #2" in { 158 | val set: Set[SortedSet[Record]] = 159 | Set( 160 | SortedSet(Record("domain.com.", RecordType.A, "1.1.1.1")), 161 | SortedSet( 162 | Record("domain.com.", RecordType.A, "2.2.2.2"), 163 | Record("example.com.", RecordType.CNAME, "sub.example.com.") 164 | ) 165 | ) 166 | 167 | val scanner = ZoneTransferScanner.create(mockLogger)(mockExecutionContext) 168 | 169 | val expectedRecords = 170 | SortedSet( 171 | Record("domain.com.", RecordType.A, "1.1.1.1"), 172 | Record("domain.com.", RecordType.A, "2.2.2.2"), 173 | Record("example.com.", RecordType.CNAME, "sub.example.com.") 174 | ) 175 | val actualRecords = scanner.flattenRecords(set) 176 | 177 | assert (expectedRecords == actualRecords) 178 | } 179 | 180 | it should "return a combined set #3" in { 181 | val set: Set[SortedSet[Record]] = 182 | Set( 183 | SortedSet( 184 | Record("domain.com.", RecordType.A, "1.1.1.1"), 185 | Record("example.com.", RecordType.CNAME, "sub.example.com.") 186 | ), 187 | SortedSet( 188 | Record("domain.com.", RecordType.A, "2.2.2.2"), 189 | Record("example.com.", RecordType.CNAME, "sub.example.com.") 190 | ) 191 | ) 192 | 193 | val scanner = ZoneTransferScanner.create(mockLogger)(mockExecutionContext) 194 | 195 | val expectedRecords = 196 | SortedSet( 197 | Record("domain.com.", RecordType.A, "1.1.1.1"), 198 | Record("domain.com.", RecordType.A, "2.2.2.2"), 199 | Record("example.com.", RecordType.CNAME, "sub.example.com.") 200 | ) 201 | val actualRecords = scanner.flattenRecords(set) 202 | 203 | assert (expectedRecords == actualRecords) 204 | } 205 | 206 | behavior of "recordsEndingWithHostname" 207 | 208 | it should "return empty" in { 209 | val hostname = "" 210 | val records = SortedSet.empty[Record] 211 | 212 | val scanner = ZoneTransferScanner.create(mockLogger)(mockExecutionContext) 213 | 214 | val expectedRecords = records 215 | val actualRecords = scanner.recordsEndingWithHostname(hostname, records) 216 | 217 | assert(expectedRecords == actualRecords) 218 | } 219 | 220 | it should "only return records ending with domain.com" in { 221 | val hostname = "domain.com" 222 | val records = 223 | SortedSet( 224 | Record("domain.com.", RecordType.A, "1.1.1.1"), 225 | Record("domain.com.", RecordType.A, "2.2.2.2"), 226 | Record("example.com.", RecordType.CNAME, "sub.example.com.") 227 | ) 228 | 229 | val scanner = ZoneTransferScanner.create(mockLogger)(mockExecutionContext) 230 | 231 | val expectedRecords = 232 | SortedSet( 233 | Record("domain.com.", RecordType.A, "1.1.1.1"), 234 | Record("domain.com.", RecordType.A, "2.2.2.2") 235 | ) 236 | val actualRecords = scanner.recordsEndingWithHostname(hostname, records) 237 | 238 | assert(expectedRecords == actualRecords) 239 | } 240 | 241 | it should "return no records" in { 242 | val hostname = "example.com" 243 | val records = 244 | SortedSet( 245 | Record("domain.com.", RecordType.A, "1.1.1.1"), 246 | Record("domain.com.", RecordType.A, "2.2.2.2") 247 | ) 248 | 249 | val scanner = ZoneTransferScanner.create(mockLogger)(mockExecutionContext) 250 | 251 | val expectedRecords = SortedSet.empty[Record] 252 | val actualRecords = scanner.recordsEndingWithHostname(hostname, records) 253 | 254 | assert(expectedRecords == actualRecords) 255 | } 256 | 257 | behavior of "printFoundRecords" 258 | 259 | it should "tell the logger to print records, that the transfer has completed and return records" in { 260 | val records = 261 | SortedSet( 262 | Record("domain.com.", RecordType.A, "1.1.1.1"), 263 | Record("domain.com.", RecordType.A, "2.2.2.2"), 264 | Record("sub.domain.com.", RecordType.CNAME, "sub2.domain.com.") 265 | ) 266 | 267 | (mockLogger.logRecords _).expects(records) 268 | (mockLogger.logZoneTransferCompleted _).expects() 269 | 270 | val scanner = ZoneTransferScanner.create(mockLogger)(mockExecutionContext) 271 | 272 | val expectedRecords = records 273 | val actualRecords = scanner.printFoundRecords(records) 274 | 275 | assert(expectedRecords == actualRecords) 276 | } 277 | 278 | it should "tell the logger that no records were found, that the transfer has compelted and return records" in { 279 | val records = SortedSet.empty[Record] 280 | 281 | (mockLogger.logNameServersNotVulnerableToZoneTransfer _).expects() 282 | (mockLogger.logZoneTransferCompleted _).expects() 283 | 284 | val scanner = ZoneTransferScanner.create(mockLogger)(mockExecutionContext) 285 | 286 | val expectedRecords = records 287 | val actualRecords = scanner.printFoundRecords(records) 288 | 289 | assert(expectedRecords == actualRecords) 290 | } 291 | 292 | behavior of "namesFromRecords" 293 | 294 | it should "return the names from each Record" in { 295 | val records = 296 | SortedSet( 297 | Record("domain.com.", RecordType.A, "1.1.1.1"), 298 | Record("domain.com.", RecordType.A, "2.2.2.2"), 299 | Record("sub.domain.com.", RecordType.CNAME, "sub2.domain.com.") 300 | ) 301 | 302 | val scanner = ZoneTransferScanner.create(mockLogger)(mockExecutionContext) 303 | 304 | val expectedNames = records.map(_.name).toSet 305 | val actualNames = scanner.namesFromRecords(records) 306 | 307 | assert(expectedNames == actualNames) 308 | } 309 | 310 | it should "return empty" in { 311 | val records = SortedSet.empty[Record] 312 | 313 | val scanner = ZoneTransferScanner.create(mockLogger)(mockExecutionContext) 314 | 315 | val expectedNames = Set.empty 316 | val actualNames = scanner.namesFromRecords(records) 317 | 318 | assert(expectedNames == actualNames) 319 | } 320 | 321 | behavior of "create" 322 | 323 | it should "return a ZoneTransferScanner" in { 324 | val actualClass = ZoneTransferScanner.create(mockLogger)(mockExecutionContext).getClass 325 | val expectedClass = classOf[ZoneTransferScanner] 326 | 327 | assert(expectedClass == actualClass) 328 | } 329 | 330 | } 331 | -------------------------------------------------------------------------------- /src/test/scala/com/gilazaria/subsearch/discovery/DNSDumpsterScannerSpec.scala: -------------------------------------------------------------------------------- 1 | package com.gilazaria.subsearch.discovery 2 | 3 | import java.io.{IOException, InputStream} 4 | 5 | import com.gilazaria.subsearch.discovery.DNSDumpsterScanner.TokenNotFoundException 6 | import com.gilazaria.subsearch.utils.TimeUtils 7 | import com.ning.http.client.Response 8 | import dispatch.{HttpExecutor, Req} 9 | import net.ruippeixotog.scalascraper.browser.{Browser, JsoupBrowser} 10 | import org.scalamock.scalatest.MockFactory 11 | import org.scalatest.FlatSpec 12 | import scala.concurrent.ExecutionContext.Implicits.global 13 | 14 | import scala.concurrent.{Await, ExecutionContext, Future} 15 | import scala.util.Try 16 | 17 | class DNSDumpsterScannerSpec extends FlatSpec with MockFactory { 18 | val exampleHTMLWithToken = "\n\n \n \n \n \n \n \n \n \n DNSdumpster.com - dns recon and research, find and lookup dns records \n \n \n \n \n \n \n
\n
\n \n
\n
\n
\n
\n

dns recon & research, find & lookup dns records

\n

\n
\n
\n \n
\n
\n
\n \n
\n
\n
\n \n
\n
\n
\n
\n
\n
\n
\n
\n Loading...\n
\n
\n
\n
\n
\n
\n
\n
\n

\n

DNSdumpster.com is a FREE domain research tool that can discover hosts related to a domain. Finding visible hosts from the attackers perspective is an important part of the security assessment process.

\n
\n

this is a project

\n

\n
\n
\n \n
\n \n
\n
\n
\n \n

Map an organizations attack surface with a virtual dumpster dive* of the DNS records associated with the target organization.

\n

*DUMPSTER DIVING: The practice of sifting refuse from an office or technical installation to extract confidential data, especially security-compromising information.
Dictionary.com

\n

\n
\n
\n
\n \n
\n \n
\n
\n
\n \n

More than a simple DNS lookup this tool will discover those hard to find sub-domains and web hosts. No brute force of common sub-domains is undertaken as is common practice for many DNS recon tools. The search relies on data from search engines and the excellent scans.io research project.

\n

\n
\n
\n
\n \n
\n \n
\n
\n
\n \n

Receive updates by following @hackertarget on Twitter
or subscribe to the low volume mailing list.

\n

this is a project

\n
\n
\n
\n
\n
\n \n \n \n \n \n \n \n \n \n \n" 19 | val exampleHTMLWithoutToken = "\n\n \n \n \n \n \n \n \n \n DNSdumpster.com - dns recon and research, find and lookup dns records \n \n \n \n \n \n \n
\n
\n \n
\n
\n
\n
\n

dns recon & research, find & lookup dns records

\n

\n
\n
\n
\n
\n
\n \n
\n
\n
\n \n
\n
\n
\n
\n
\n
\n
\n
\n Loading...\n
\n
\n
\n
\n
\n
\n
\n
\n

\n

DNSdumpster.com is a FREE domain research tool that can discover hosts related to a domain. Finding visible hosts from the attackers perspective is an important part of the security assessment process.

\n
\n

this is a project

\n

\n
\n
\n \n
\n \n
\n
\n
\n \n

Map an organizations attack surface with a virtual dumpster dive* of the DNS records associated with the target organization.

\n

*DUMPSTER DIVING: The practice of sifting refuse from an office or technical installation to extract confidential data, especially security-compromising information.
Dictionary.com

\n

\n
\n
\n
\n \n
\n \n
\n
\n
\n \n

More than a simple DNS lookup this tool will discover those hard to find sub-domains and web hosts. No brute force of common sub-domains is undertaken as is common practice for many DNS recon tools. The search relies on data from search engines and the excellent scans.io research project.

\n

\n
\n
\n
\n \n
\n \n
\n
\n
\n \n

Receive updates by following @hackertarget on Twitter
or subscribe to the low volume mailing list.

\n

this is a project

\n
\n
\n
\n
\n
\n \n \n \n \n \n \n \n \n \n \n" 20 | val exampleToken = "BkgQgkNSuTrAWqLOYjk1z37GDh8vJbwK" 21 | 22 | val mockBrowser: Browser = mock[Browser] 23 | val mockHttpExecutor: HttpExecutor = mock[HttpExecutor] 24 | 25 | behavior of "performScan" 26 | 27 | it should "perform a scan" in { 28 | val stream : InputStream = getClass.getResourceAsStream("/DNSDumpster/ExampledomainExample.html") 29 | val html = scala.io.Source.fromInputStream(stream).getLines.mkString("\n") 30 | val hostname = "exampledomain.com" 31 | val subdomains = 32 | Set( 33 | "vps31252317.exampledomain.com", 34 | "www.exampledomain.com", 35 | "vps29782186.exampledomain.com", 36 | "vps57573743.exampledomain.com", 37 | "vps5025.exampledomain.com", 38 | "vps32272377.exampledomain.com", 39 | "vps22351531.exampledomain.com" 40 | ) 41 | 42 | (mockBrowser.get _).expects("https://dnsdumpster.com/").returns(JsoupBrowser().parseString(exampleHTMLWithToken)) 43 | (mockBrowser.parseString _).expects(html).returns(JsoupBrowser().parseString(html)) 44 | 45 | val mockResponse: Response = mock[Response] 46 | (mockResponse.getResponseBody _).expects().returns(html) 47 | 48 | (mockHttpExecutor.apply (_: Req)(_: ExecutionContext)) 49 | .expects(*, scala.concurrent.ExecutionContext.Implicits.global) 50 | .returns(Future[Response](mockResponse)) 51 | 52 | val scanner = new DNSDumpsterScanner(mockBrowser, mockHttpExecutor) 53 | 54 | val expectedSubdomains = subdomains 55 | val actualSubdomains = Await.result(scanner.scan(hostname), TimeUtils.awaitDuration) 56 | 57 | assert(expectedSubdomains == actualSubdomains) 58 | } 59 | 60 | behavior of "retrieveHTMLWithTokenForHostname" 61 | 62 | it should "return the HTTP page" in { 63 | val token = exampleToken 64 | val hostname = "google.com" 65 | val html = "blaaaah" 66 | 67 | val mockResponse: Response = mock[Response] 68 | (mockResponse.getResponseBody _).expects().returns(html) 69 | 70 | (mockHttpExecutor.apply (_: Req)(_: ExecutionContext)) 71 | .expects(*, scala.concurrent.ExecutionContext.Implicits.global) 72 | .returns(Future[Response](mockResponse)) 73 | 74 | val scanner = new DNSDumpsterScanner(http = mockHttpExecutor) 75 | 76 | val expectedHTML = html 77 | val actualHTML = Await.result(scanner.retrieveHTMLWithTokenForHostname(token, hostname), TimeUtils.awaitDuration) 78 | 79 | assert(expectedHTML == actualHTML) 80 | } 81 | 82 | behavior of "retrieveCSRFToken" 83 | 84 | it should "correctly find the CSRF token" in { 85 | (mockBrowser.get _).expects("https://dnsdumpster.com/").returns(JsoupBrowser().parseString(exampleHTMLWithToken)) 86 | 87 | val scanner = new DNSDumpsterScanner(mockBrowser) 88 | 89 | val expectedToken = exampleToken 90 | val actualToken = Try(Await.result(scanner.retrieveCSRFToken(), TimeUtils.awaitDuration)) 91 | 92 | assert(actualToken.isSuccess) 93 | assert(expectedToken == actualToken.get) 94 | } 95 | 96 | it should "not find a CSRF token, throwing a TokenNotFoundException" in { 97 | (mockBrowser.get _).expects("https://dnsdumpster.com/").returns(JsoupBrowser().parseString(exampleHTMLWithoutToken)) 98 | 99 | val scanner = new DNSDumpsterScanner(mockBrowser) 100 | 101 | val actualToken = Try(Await.result(scanner.retrieveCSRFToken(), TimeUtils.awaitDuration)) 102 | 103 | assert(actualToken.isFailure) 104 | assert(actualToken.failed.get.getClass == classOf[TokenNotFoundException]) 105 | assert(actualToken.failed.get.getMessage == "No CSRF token was found.") 106 | } 107 | 108 | it should "not find a CSRF token, throwing an IOException" in { 109 | (mockBrowser.get _).expects("https://dnsdumpster.com/").throws(new IOException("Error!")) 110 | 111 | val scanner = new DNSDumpsterScanner(mockBrowser) 112 | 113 | val actualToken = Try(Await.result(scanner.retrieveCSRFToken(), TimeUtils.awaitDuration)) 114 | 115 | assert(actualToken.isFailure) 116 | assert(actualToken.failed.get.getClass == classOf[IOException]) 117 | assert(actualToken.failed.get.getMessage == "Error!") 118 | } 119 | 120 | behavior of "extractSubdomains" 121 | 122 | it should "return no subdomains for html devoid of subdomains" in { 123 | val stream : InputStream = getClass.getResourceAsStream("/DNSDumpster/DNSDumpsterExample.html") 124 | val html = scala.io.Source.fromInputStream(stream).getLines.mkString("\n") 125 | val hostname = "dnsdumpster.com" 126 | 127 | val scanner = DNSDumpsterScanner.create() 128 | 129 | val expectedSubdomains = Set.empty 130 | val actualSubdomains = scanner.extractSubdomains(html, hostname) 131 | 132 | assert(expectedSubdomains == actualSubdomains) 133 | } 134 | 135 | it should "return no subdomains for html with subdomains for a different hostname" in { 136 | val stream : InputStream = getClass.getResourceAsStream("/DNSDumpster/ExampledomainExample.html") 137 | val html = scala.io.Source.fromInputStream(stream).getLines.mkString("\n") 138 | val hostname = "microsoft.com" 139 | 140 | val scanner = DNSDumpsterScanner.create() 141 | 142 | val expectedSubdomains = Set.empty 143 | val actualSubdomains = scanner.extractSubdomains(html, hostname) 144 | 145 | assert(expectedSubdomains == actualSubdomains) 146 | } 147 | 148 | it should "return the correct subdomains without the hostnmae" in { 149 | val stream : InputStream = getClass.getResourceAsStream("/DNSDumpster/ExampledomainExample.html") 150 | val html = scala.io.Source.fromInputStream(stream).getLines.mkString("\n") 151 | val hostname = "exampledomain.com" 152 | 153 | val scanner = DNSDumpsterScanner.create() 154 | 155 | val expectedSubdomains = 156 | Set( 157 | "vps31252317.exampledomain.com", 158 | "www.exampledomain.com", 159 | "vps29782186.exampledomain.com", 160 | "vps57573743.exampledomain.com", 161 | "vps5025.exampledomain.com", 162 | "vps32272377.exampledomain.com", 163 | "vps22351531.exampledomain.com" 164 | ) 165 | val actualSubdomains = scanner.extractSubdomains(html, hostname) 166 | 167 | assert(expectedSubdomains == actualSubdomains) 168 | assert(!actualSubdomains.contains(hostname)) 169 | } 170 | 171 | behavior of "conditionallyCreate" 172 | 173 | it should "return a DNSDumpsterScanner" in { 174 | val scanner: Option[DNSDumpsterScanner] = DNSDumpsterScanner.conditionallyCreate(create = true) 175 | 176 | assert(scanner.isDefined) 177 | 178 | val actualClass = scanner.get.getClass 179 | val expectedClass = classOf[DNSDumpsterScanner] 180 | 181 | assert(expectedClass == actualClass) 182 | } 183 | 184 | it should "return None" in { 185 | val scanner: Option[DNSDumpsterScanner] = DNSDumpsterScanner.conditionallyCreate(create = false) 186 | 187 | assert(scanner.isEmpty) 188 | } 189 | 190 | behavior of "create" 191 | 192 | it should "return a DNSDumpsterScanner" in { 193 | val actualClass = DNSDumpsterScanner.create().getClass 194 | val expectedClass = classOf[DNSDumpsterScanner] 195 | 196 | assert(expectedClass == actualClass) 197 | } 198 | 199 | behavior of "TokenNotFoundException" 200 | 201 | it should "have the correct message" in { 202 | val msg = "Error!" 203 | 204 | val exception = new TokenNotFoundException(msg) 205 | 206 | val expectedMessage = msg 207 | val actualMessage = exception.getMessage 208 | 209 | assert(expectedMessage == actualMessage) 210 | } 211 | } 212 | -------------------------------------------------------------------------------- /src/test/scala/com/gilazaria/subsearch/discovery/VirusTotalScannerSpec.scala: -------------------------------------------------------------------------------- 1 | package com.gilazaria.subsearch.discovery 2 | 3 | import org.scalamock.scalatest.MockFactory 4 | import org.scalatest.FlatSpec 5 | import scala.concurrent.ExecutionContext.Implicits.global 6 | 7 | class VirusTotalScannerSpec extends FlatSpec with MockFactory { 8 | 9 | behavior of "retrieveHTML" 10 | 11 | it should "return the HTML body" in { 12 | val hostname = "example.com" 13 | } 14 | 15 | behavior of "conditionallyCreate" 16 | 17 | it should "return a VirusTotalScanner" in { 18 | val scanner: Option[VirusTotalScanner] = VirusTotalScanner.conditionallyCreate(create = true) 19 | 20 | assert(scanner.isDefined) 21 | 22 | val actualClass = scanner.get.getClass 23 | val expectedClass = classOf[VirusTotalScanner] 24 | 25 | assert(expectedClass == actualClass) 26 | } 27 | 28 | it should "return None" in { 29 | val scanner: Option[VirusTotalScanner] = VirusTotalScanner.conditionallyCreate(create = false) 30 | 31 | assert(scanner.isEmpty) 32 | } 33 | 34 | behavior of "create" 35 | 36 | it should "return a VirusTotalScanner" in { 37 | val actualClass = VirusTotalScanner.create().getClass 38 | val expectedClass = classOf[VirusTotalScanner] 39 | 40 | assert(expectedClass == actualClass) 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /src/test/scala/com/gilazaria/subsearch/utils/HostnameUtilsSpec.scala: -------------------------------------------------------------------------------- 1 | package com.gilazaria.subsearch.utils 2 | 3 | import org.scalatest.FlatSpec 4 | import com.gilazaria.subsearch.utils.HostnameUtils._ 5 | 6 | class HostnameUtilsSpec extends FlatSpec { 7 | 8 | behavior of "normalise" 9 | 10 | it should "trim leading and trailing whitespace" in { 11 | val subdomain = " sub.domain.com " 12 | 13 | val expected = "sub.domain.com" 14 | val actual = normalise(subdomain) 15 | 16 | assert(expected == actual) 17 | } 18 | 19 | it should "trim leading and trailing dots" in { 20 | val subdomain = ".sub.domain.com." 21 | 22 | val expected = "sub.domain.com" 23 | val actual = normalise(subdomain) 24 | 25 | assert(expected == actual) 26 | } 27 | 28 | it should "convert to lowercase" in { 29 | val subdomain = "SuB.DoMaIn.com" 30 | 31 | val expected = "sub.domain.com" 32 | val actual = normalise(subdomain) 33 | 34 | assert(expected == actual) 35 | } 36 | 37 | behavior of "isValidDomain" 38 | 39 | List( 40 | ("sub.domain.com", true), 41 | ("domain.com", true), 42 | ("domain..com", false), 43 | ("sub..domain.com", false), 44 | ("a.b.c.d.e.f.g.h", true), 45 | (".a.com", false), 46 | ("a.com.", false), 47 | ("com", false), 48 | ("#.com", false), 49 | ("", false)) 50 | .foreach { 51 | (data: (String, Boolean)) => 52 | val domain = data._1 53 | val expected = data._2 54 | 55 | it should s"return $expected for $domain" in { 56 | val actual = isValidDomain(domain) 57 | assert(expected == actual) 58 | } 59 | } 60 | 61 | behavior of "isValidSubdomainPart" 62 | 63 | List( 64 | ("sub.domain.com", true), 65 | ("domain.com", true), 66 | ("domain..com", false), 67 | ("sub..domain.com", false), 68 | ("a.b.c.d.e.f.g.h", true), 69 | (".a.com", false), 70 | ("a.com.", false), 71 | ("#.com", false), 72 | ("", false), 73 | ("name", true)) 74 | .foreach { 75 | (data: (String, Boolean)) => 76 | val part = data._1 77 | val expected = data._2 78 | 79 | it should s"return $expected for $part" in { 80 | val actual = isValidSubdomainPart(part) 81 | assert(expected == actual) 82 | } 83 | } 84 | 85 | behavior of "ensureSubdomainEndsWithHostname" 86 | 87 | it should "append the hostname" in { 88 | val subdomain = "sub" 89 | val hostname = "domain.com" 90 | 91 | val expected = "sub.domain.com" 92 | val actual = ensureSubdomainEndsWithHostname(subdomain, hostname) 93 | 94 | assert(expected == actual) 95 | } 96 | 97 | it should "not append the hostname" in { 98 | val subdomain = "sub.domain.com" 99 | val hostname = "domain.com" 100 | 101 | val expected = "sub.domain.com" 102 | val actual = ensureSubdomainEndsWithHostname(subdomain, hostname) 103 | } 104 | 105 | } 106 | -------------------------------------------------------------------------------- /src/test/scala/com/gilazaria/subsearch/utils/IPUtilsSpec.scala: -------------------------------------------------------------------------------- 1 | package com.gilazaria.subsearch.utils 2 | 3 | import org.scalatest.{GivenWhenThen, FlatSpec} 4 | import com.gilazaria.subsearch.utils.IPUtils._ 5 | 6 | class IPUtilsSpec extends FlatSpec with GivenWhenThen { 7 | 8 | behavior of "normalise" 9 | 10 | it should "be the same" in { 11 | val ip: String = "127.0.0.1" 12 | Given(s"the ip has no whitespace: '$ip'") 13 | 14 | When("it is normalised") 15 | val actual = normalise(ip) 16 | 17 | Then("it shouldn't have changed") 18 | assert(ip == actual) 19 | } 20 | 21 | it should "trim whitespace" in { 22 | val ip: String = " 127.0.0.1 " 23 | Given(s"the ip has whitespace on either side: '$ip'") 24 | 25 | When("it is normalised") 26 | val actual = normalise(ip) 27 | 28 | Then("it should have no leading or trailing whitespace") 29 | val expected = "127.0.0.1" 30 | assert(actual == expected) 31 | } 32 | 33 | behavior of "isValidIPv4" 34 | 35 | it should "return true for '127.0.0.1'" in { 36 | val ip = "127.0.0.1" 37 | 38 | val actual = isValidIPv4(ip) 39 | val expected = true 40 | 41 | assert(actual == expected) 42 | } 43 | 44 | it should "return false for ' 127.0.0.1 '" in { 45 | val ip = " 127.0.0.1 " 46 | 47 | val actual = isValidIPv4(ip) 48 | val expected = false 49 | 50 | assert(actual == expected) 51 | } 52 | 53 | it should "return false for quadrants that represent 0 as more than one 0 (e.g. 000)" in { 54 | val ip = "127.000.0.1" 55 | 56 | val actual = isValidIPv4(ip) 57 | val expected = false 58 | 59 | assert(actual == expected) 60 | } 61 | 62 | it should "return false for a number larger than 255 in the first quadrant" in { 63 | val ip = "900.0.0.1" 64 | 65 | val actual = isValidIPv4(ip) 66 | val expected = false 67 | 68 | assert(actual == expected) 69 | } 70 | 71 | it should "return false for a number larger than 255 in the second quadrant" in { 72 | val ip = "1.256.0.1" 73 | 74 | val actual = isValidIPv4(ip) 75 | val expected = false 76 | 77 | assert(actual == expected) 78 | } 79 | 80 | it should "return false for a number larger than 255 in the third quadrant" in { 81 | val ip = "1.0.256.1" 82 | 83 | val actual = isValidIPv4(ip) 84 | val expected = false 85 | 86 | assert(actual == expected) 87 | } 88 | 89 | it should "return false for a number larger than 255in the fourth quadrant" in { 90 | val ip = "1.0.0.256" 91 | 92 | val actual = isValidIPv4(ip) 93 | val expected = false 94 | 95 | assert(actual == expected) 96 | } 97 | 98 | it should "return true for a valid IP" in { 99 | val ip = "0.0.0.1" 100 | 101 | val actual = isValidIPv4(ip) 102 | val expected = true 103 | 104 | assert(actual == expected) 105 | } 106 | 107 | it should "return false for a non-zero quadrant number beginning with zero (e.g. 01)" in { 108 | val ip = "1.01.0.255" 109 | 110 | val actual = isValidIPv4(ip) 111 | val expected = false 112 | 113 | assert(actual == expected) 114 | } 115 | 116 | it should "return false for an ip with less than four quadrants" in { 117 | val ip = "127.0.1" 118 | 119 | val actual = isValidIPv4(ip) 120 | val expected = false 121 | 122 | assert(actual == expected) 123 | } 124 | 125 | it should "return false for an ip with more than four quadrants" in { 126 | val ip = "127.0.0.0.1" 127 | 128 | val actual = isValidIPv4(ip) 129 | val expected = false 130 | 131 | assert(actual == expected) 132 | } 133 | } 134 | -------------------------------------------------------------------------------- /src/test/scala/com/gilazaria/subsearch/utils/MathUtilsSpec.scala: -------------------------------------------------------------------------------- 1 | package com.gilazaria.subsearch.utils 2 | 3 | import org.scalatest.{GivenWhenThen, FlatSpec} 4 | import com.gilazaria.subsearch.utils.MathUtils._ 5 | 6 | class MathUtilsSpec extends FlatSpec with GivenWhenThen { 7 | 8 | behavior of "percentage" 9 | 10 | it should "calculate 50.0" in { 11 | Given("a = 1 and b = 2") 12 | val a = 1 13 | val b = 2 14 | 15 | When("the percentage is calculated") 16 | val expected: Float = 50.0.toFloat 17 | val actual: Float = percentage(a, b) 18 | 19 | Then(s"the result should be $expected") 20 | assert(expected == actual) 21 | } 22 | 23 | it should "calculate 150.0 when given 3 and 2" in { 24 | Given("a = 3 and b = 2") 25 | val a = 3 26 | val b = 2 27 | 28 | When("the percentage is calculated") 29 | val expected: Float = 150.0.toFloat 30 | val actual: Float = percentage(a, b) 31 | 32 | Then(s"the result should be $expected") 33 | assert(expected == actual) 34 | } 35 | 36 | it should "throw an IllegalArgumentException" in { 37 | Given("b = 0") 38 | val a = 1 39 | val b = 0 40 | 41 | When("the percentage is calculated") 42 | val expectedException = intercept[IllegalArgumentException] { 43 | percentage(a, b) 44 | } 45 | 46 | Then("an IllegalArgumentException is thrown") 47 | assert(expectedException.getMessage === "The second argument cannot be zero.") 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /src/test/scala/com/gilazaria/subsearch/utils/TimeUtilsSpec.scala: -------------------------------------------------------------------------------- 1 | package com.gilazaria.subsearch.utils 2 | 3 | import org.scalatest.FlatSpec 4 | import com.gilazaria.subsearch.utils.TimeUtils._ 5 | import scala.concurrent.duration._ 6 | 7 | import scala.concurrent.duration.FiniteDuration 8 | 9 | class TimeUtilsSpec extends FlatSpec { 10 | 11 | // How can this be done? 12 | behavior of "currentTimePretty" 13 | 14 | // it should "currentTimePretty" in { 15 | // 16 | // } 17 | 18 | behavior of "akkaAskTimeout" 19 | 20 | it should "have a finite duration of 21474835 seconds" in { 21 | val expected = FiniteDuration(21474835, "seconds") 22 | val actual = akkaAskTimeout.duration 23 | 24 | assert(expected == actual) 25 | } 26 | 27 | behavior of "awaitDuration" 28 | 29 | it should "be equal to 365 days" in { 30 | val expected = 365.days 31 | val actual = awaitDuration 32 | 33 | assert(expected == actual) 34 | } 35 | 36 | // How can this be done? 37 | behavior of "timestampNow" 38 | 39 | // it should "" in { 40 | // 41 | // } 42 | 43 | } 44 | --------------------------------------------------------------------------------