This extension extracts information from the Site Map. You can use the full site map or just in-scope items. Three types of information can be extracted:
2 | 3 |Anchor Links - Searches responses for links of the form <a href=. Note that this will include links within JavaScript and in commented out areas. The log displays the found links and the page the link was found on. You have the option to select absolute links, relative links, or both. Log data can optionally be saved to a .csv file.
4 | 5 |Response Codes - Finds all requests that returned one of the selected response code ranges (1xx/2xx/3xx/4xx/5xx). The log displays the page requested, the referer if one was specified, the specific response code, and if the response was a redirect, where the page was redirected. The log can optionally be saved to a .csv file.
6 | 7 |Export Site Map - Saves the site map requests and responses to a .txt file. You can specify that all requests should be exported or only those with a corresponding response. The full content of the requests and responses is saved. This enables you to write independent code to further process the site map as you wish.
8 | 9 | -------------------------------------------------------------------------------- /BappManifest.bmf: -------------------------------------------------------------------------------- 1 | Uuid: f991b67d4ef94f3c8692c3edca06583e 2 | ExtensionType: 2 3 | Name: Site Map Extractor 4 | RepoName: site-map-extractor 5 | ScreenVersion: 1.2 6 | SerialVersion: 6 7 | MinPlatformVersion: 0 8 | ProOnly: False 9 | Author: Susan Wright 10 | ShortDescription: Extracts key data from the Site Map and allows export to CSV. 11 | EntryPoint: site_map_extractor.py 12 | BuildCommand: 13 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2017 swright573 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ## Site Map Extractor 2 | Site Map Extractor is a (fully passive) Burp extension written in Python that extracts various information from the Site Map. Optionally you can select to search the full site map or just the in-scope items. There are 3 types of information that can be extracted. 3 | 4 | #### Extract ' site map > content will allow update. 198 | self.siteMapData = self._callbacks.getSiteMap(None) 199 | # What links should be extracted? absolute links, relative links, or both? 200 | self.destAbs = self.destRel = False 201 | if self.uiLinksAbs.isSelected(): 202 | self.destAbs = True 203 | if self.uiLinksRel.isSelected(): 204 | self.destRel = True 205 | 206 | # Start building JTable to contain the extracted data 207 | self.colNames = ('Page', 'HTTPS?', 'Link', 'Description', 'Target', 'Rel=', 'Possible vulnerabilities') 208 | self.tableData = [] 209 | for i in self.siteMapData: 210 | try: 211 | self.requestInfo = self._helpers.analyzeRequest(i) 212 | self.url = self.requestInfo.getUrl() 213 | 214 | if self.scopeOnly() and not(self._callbacks.isInScope(self.url)): 215 | continue 216 | 217 | self.urlDecode = self._helpers.urlDecode(str(self.url)) 218 | 219 | self.response = i.getResponse() 220 | if self.response == None: # if there's no response, there won't be any links :-) 221 | continue 222 | 223 | self.responseInfo = self._helpers.analyzeResponse(self.response) 224 | self.responseOffset = self.responseInfo.getBodyOffset() 225 | self.responseBody = self._helpers.bytesToString(self.response)[self.responseOffset:] 226 | 227 | keep_looking = True 228 | while keep_looking: # there may be multiple links in the response 229 | #TODO: known issue: if link does not start with end tag 295 | # This assumes that the link is correctly ended 296 | posEndTag = self.responseBody.lower().find("") 297 | self.fullLink = self.responseBody[0:posEndTag] 298 | 299 | # Looking for > at the end of the ") 302 | self.description = self.responseBody[posEndA+1:posEndTag] 303 | 304 | 305 | # Looking for a possible target that is described 306 | self.target = "" 307 | if self.responseBody.lower().find("target") < posEndA: 308 | posTarget = self.responseBody.lower().find("target") 309 | #search the end of this parameter - might be ' or " or even a space, but it should be directly after the target 310 | 311 | if self.responseBody[posTarget+7:posTarget+8] == '"': 312 | endQuote = '"' 313 | elif self.responseBody[posTarget+7:posTarget+8] == "'": 314 | endQuote = "'" 315 | else: 316 | endQuote = ">" 317 | 318 | posTargetEnd = self.responseBody[posTarget+8:posEndA].lower().find(endQuote) 319 | #TODO rework needed 320 | self.target = self.responseBody[posTarget+8:posTarget+8+posTargetEnd] 321 | 322 | self.rel = "" 323 | if self.responseBody.lower().find("rel") < posEndA: 324 | posRel = self.responseBody.lower().find("rel") 325 | #search the end of this parameter - might be ' or " or even a space, but it should be directly after the target 326 | if self.responseBody[posRel+4:posRel+5] == '"': 327 | endQuote = '"' 328 | elif self.responseBody[posRel+4:posRel+5] == "'": 329 | endQuote = "'" 330 | else: 331 | endQuote = ">" 332 | 333 | posRelEnd = self.responseBody[posRel+5:posEndA].lower().find(endQuote) 334 | 335 | self.rel = self.responseBody[posRel+5:posRel+5+posRelEnd] 336 | 337 | #Is this link vulnerable to Tabnabbing? 338 | if self.target != "" and not isRelLink: 339 | if self.rel.lower().find("noopener") == -1: 340 | self.Vulnerabilities = self.Vulnerabilities + " Tabnabbing" 341 | #if rel= is not defined, but target is, the link is still vulnerable for tabnabbing 342 | elif self.target != "" and not isRelLink: 343 | self.Vulnerabilities = self.Vulnerabilities + " Tabnabbing" 344 | 345 | 346 | if (isAbsLink and self.destAbs) or (isRelLink and self.destRel): 347 | # remove white space and extra CR/LF characters 348 | self.tableData.append([self.stripURLPort(self.urlDecode), str(self.isHttps), self.lstripWS(self.stripCRLF(self.link)), self.lstripWS(self.stripCRLF(self.description)), self.lstripWS(self.stripCRLF(self.target)), self.lstripWS(self.stripCRLF(self.rel)), self.lstripWS(self.stripCRLF(self.Vulnerabilities))]) 349 | 350 | except: 351 | print('An error occured during processing of a link [ ' + str(i) + ' ] ignored it and continued with the next.') 352 | 353 | dataModel = DefaultTableModel(self.tableData, self.colNames) 354 | self.uiLogTable = swing.JTable(dataModel) 355 | self.uiLogTable.setAutoCreateRowSorter(True); 356 | 357 | self.uiLogPane.setViewportView(self.uiLogTable) 358 | 359 | def exportCodes(self, e): 360 | self.blankLog() 361 | self.siteMapData = self._callbacks.getSiteMap(None) 362 | # response codes to be included 363 | self.rcodes = [] 364 | if self.uiRcode1xx.isSelected(): 365 | self.rcodes += '1' 366 | if self.uiRcode2xx.isSelected(): 367 | self.rcodes += '2' 368 | if self.uiRcode3xx.isSelected(): 369 | self.rcodes += '3' 370 | if self.uiRcode4xx.isSelected(): 371 | self.rcodes += '4' 372 | if self.uiRcode5xx.isSelected(): 373 | self.rcodes += '5' 374 | 375 | if '3' in self.rcodes: 376 | self.colNames = ('Request','Referer','Response Code','Redirects To') 377 | else: 378 | self.colNames = ('Request','Referer','Response Code') 379 | self.tableData = [] 380 | 381 | for i in self.siteMapData: 382 | self.requestInfo = self._helpers.analyzeRequest(i) 383 | self.url = self.requestInfo.getUrl() 384 | if self.scopeOnly() and not(self._callbacks.isInScope(self.url)): 385 | continue 386 | 387 | self.urlDecode = self._helpers.urlDecode(str(self.url)) 388 | self.response = i.getResponse() 389 | if self.response == None: 390 | continue 391 | # Get referer if there is one 392 | self.requestHeaders = self.requestInfo.getHeaders() 393 | self.referer = '' 394 | for j in self.requestHeaders: 395 | if j.startswith('Referer:'): 396 | self.fullReferer = j.split(' ')[1] 397 | # drop the querystring parameter 398 | self.referer = self.fullReferer.split('?')[0] 399 | # Get response code 400 | self.responseInfo = self._helpers.analyzeResponse(self.response) 401 | self.responseCode = self.responseInfo.getStatusCode() 402 | self.firstDigit = str(self.responseCode)[0] 403 | if self.firstDigit not in self.rcodes: 404 | continue 405 | if self.firstDigit in ['1','2','4','5']: # Return codes 1xx, 2xx, 4xx, 5xx 406 | self.tableData.append([self.stripURLPort(self.urlDecode), str(self.referer), str(self.responseCode)]) 407 | elif self.firstDigit == '3': # Return code 3xx Redirection 408 | self.requestHeaders = self.requestInfo.getHeaders() 409 | self.responseHeaders = self.responseInfo.getHeaders() 410 | for j in self.responseHeaders: 411 | if j.startswith('Location:'): 412 | self.location = j.split(' ')[1] 413 | self.tableData.append([self.stripURLPort(self.urlDecode), str(self.referer), str(self.responseCode), self.location]) 414 | 415 | dataModel = DefaultTableModel(self.tableData, self.colNames) 416 | self.uiLogTable = swing.JTable(dataModel) 417 | self.uiLogTable.setAutoCreateRowSorter(True); 418 | self.uiLogPane.setViewportView(self.uiLogTable) 419 | 420 | def exportSiteMap(self,e): 421 | self.blankLog() 422 | f, ok = self.openFile('txt', 'Text files', 'wb') 423 | if ok: 424 | # Retrieve site map data 425 | self.siteMapData = self._callbacks.getSiteMap(None) 426 | if self.uiMustHaveResponse.isSelected(): 427 | self.outputAll = False 428 | else: 429 | self.outputAll = True 430 | for i in self.siteMapData: 431 | self.myrequest = i.getRequest() #self._helpers.urlDecode(i.getRequest()) 432 | self.requestInfo = self._helpers.analyzeRequest(i) 433 | self.url = self.requestInfo.getUrl() 434 | if self.scopeOnly() and not(self._callbacks.isInScope(self.url)): 435 | continue 436 | self.myresponse = i.getResponse() #self._helpers.urlDecode(i.getResponse()) 437 | if self.myresponse != None: 438 | f.write('----- REQUEST\r\n') 439 | f.write(self.myrequest) #self._helpers.urlEncode(self.myrequest)) 440 | f.write('\n') 441 | f.write('----- RESPONSE\r\n') 442 | f.write(self.myresponse) #self._helpers.urlEncode(self.myresponse)) 443 | f.write('\n') 444 | elif not self.uiMustHaveResponse: 445 | f.write('----- REQUEST\r\n') 446 | f.write(self.myrequest) #self._helpers.urlEncode(self.myrequest)) 447 | f.write('\n') 448 | f.close() 449 | JOptionPane.showMessageDialog(self.tab,'The Site Map file was successfully written.') 450 | 451 | def savetoCsvFile(self,e): 452 | if self.tableData == []: 453 | JOptionPane.showMessageDialog(self.tab,'The log contains no data.') 454 | return 455 | f, ok = self.openFile('csv', 'CSV files', 'wb') 456 | if ok: 457 | self.writer = csv.writer(f) 458 | self.writer.writerow(list(self.colNames)) 459 | for i in self.tableData: 460 | self.writer.writerow(i) 461 | f.close() 462 | JOptionPane.showMessageDialog(self.tab,'The csv file was successfully written.') 463 | 464 | def openFile(self, fileext, filedesc, fileparm): 465 | myFilePath = '' 466 | chooseFile = JFileChooser() 467 | myFilter = FileNameExtensionFilter(filedesc,[fileext]) 468 | chooseFile.setFileFilter(myFilter) 469 | ret = chooseFile.showOpenDialog(self.tab) 470 | if ret == JFileChooser.APPROVE_OPTION: 471 | file = chooseFile.getSelectedFile() 472 | myFilePath = str(file.getCanonicalPath()).lower() 473 | if not myFilePath.endswith(fileext): 474 | myFilePath += '.' + fileext 475 | okWrite = JOptionPane.YES_OPTION 476 | if os.path.isfile(myFilePath): 477 | okWrite = JOptionPane.showConfirmDialog(self.tab,'File already exists. Ok to over-write?','',JOptionPane.YES_NO_OPTION) 478 | if okWrite == JOptionPane.NO_OPTION: 479 | return 480 | j = True 481 | while j: 482 | try: 483 | f = open(myFilePath,mode=fileparm) 484 | j = False 485 | except IOError: 486 | okWrite = JOptionPane.showConfirmDialog(self.tab,'File cannot be opened. Correct and retry?','',JOptionPane.YES_NO_OPTION) 487 | if okWrite == JOptionPane.NO_OPTION: 488 | return None, False 489 | return f, True 490 | 491 | def stripCRLF(self, link): 492 | link = link.rstrip('\r') 493 | link = link.rstrip('\n') 494 | return link 495 | 496 | def lstripWS(self, link): 497 | return link.lstrip() 498 | 499 | def stripURLPort(self, url): 500 | # Thanks to shpendk for this code(https://github.com/PortSwigger/site-map-fetcher/) 501 | return url.split(':')[0] + ':' + url.split(':')[1] + '/' + url.split(':')[2].split('/',1)[1] 502 | 503 | def blankLogTable(self): 504 | self.tableData = [] 505 | return 506 | 507 | # This function is used when program wants to clear the log. 508 | def blankLog(self): 509 | self.uiLogPane.setViewportView(None) 510 | self.blankLogTable() 511 | return 512 | 513 | # This function is used when the user clicks the Clear Log button. 514 | def clearLog(self, e): 515 | self.uiLogPane.setViewportView(None) 516 | self.blankLogTable() 517 | return 518 | 519 | -------------------------------------------------------------------------------- /unittest links.html: -------------------------------------------------------------------------------- 1 | 2 | 3 |