├── .gitignore
├── LICENSE
├── README.md
├── app.py
├── index.html
├── main.css
├── main.js
└── requirements.txt


/.gitignore:
--------------------------------------------------------------------------------
1 | discovered_links.csv
2 | external_links.json
3 | settings.json
4 | venv/


--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
 1 | MIT License
 2 | 
 3 | Copyright (c) 2025 Bret Bernhoft
 4 | 
 5 | Permission is hereby granted, free of charge, to any person obtaining a copy
 6 | of this software and associated documentation files (the "Software"), to deal
 7 | in the Software without restriction, including without limitation the rights
 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 9 | copies of the Software, and to permit persons to whom the Software is
10 | furnished to do so, subject to the following conditions:
11 | 
12 | The above copyright notice and this permission notice shall be included in all
13 | copies or substantial portions of the Software.
14 | 
15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21 | SOFTWARE.
22 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
 1 | # Mapping A Website's External Links
 2 | 
 3 | ![Preview Of Resulting Visualization](https://hosting.photobucket.com/images/i/bernhoftbret/website-external-links-enhanced.jpg)
 4 | 
 5 | Use Python to map a website's external facing links. And then apply D3 to visualize those outbound connections as a network graph.
 6 | 
 7 | ## Set Up
 8 | 
 9 | ### Programs Needed
10 | 
11 | - [Git](https://git-scm.com/downloads)
12 | - [Python](https://www.python.org/downloads/) (When installing on Windows, make sure you check the ["Add python 3.xx to PATH"](https://hosting.photobucket.com/images/i/bernhoftbret/python.png) box.)
13 | 
14 | ### Steps
15 | 
16 | 1. Install the above programs.
17 | 2. Open a shell window (For Windows open PowerShell, for MacOS open Terminal & for Linux open your distro's terminal emulator).
18 | 3. Clone this repository using `git` by running the following command; `git clone git@github.com:devbret/website-external-links.git`.
19 | 4. Navigate to the repo's directory by running; `cd website-external-links`.
20 | 5. Install the needed dependencies for running the script by running; `pip install -r requirements.txt`.
21 | 6. Edit the app.py file on line 51, to include the website that you would like to visualize. You can also change the maximum number of URLs that this program will visit at a given domain, by editing the "max_links" value on line 11 in the app.py file; which is set to 50 by default.
22 | 7. Run the script with the command `python3 app.py`.
23 | 8. To view the website's connections in the index.html file you will need to run a local web server. To do this run `python3 -m http.server`.
24 | 


--------------------------------------------------------------------------------
/app.py:
--------------------------------------------------------------------------------
 1 | import requests
 2 | from bs4 import BeautifulSoup
 3 | from urllib.parse import urljoin, urlparse
 4 | import json
 5 | import csv
 6 | from datetime import datetime
 7 | 
 8 | def is_external(url, base):
 9 |     return urlparse(url).netloc != urlparse(base).netloc
10 | 
11 | def crawl_site(start_url, max_links=50, csv_filename='discovered_links.csv'):
12 |     visited = set()
13 |     external_links = {}
14 | 
15 |     with open(csv_filename, mode='w', newline='') as csvfile:
16 |         csv_writer = csv.writer(csvfile)
17 |         csv_writer.writerow(['URL', 'Type', 'Timestamp'])
18 | 
19 |         def crawl(url):
20 |             if len(visited) >= max_links:
21 |                 return
22 |             visited.add(url)
23 |             timestamp = datetime.now().isoformat()
24 |             csv_writer.writerow([url, 'internal', timestamp])
25 |             print(f"Crawling: {url}")
26 |             try:
27 |                 response = requests.get(url, timeout=5)
28 |                 soup = BeautifulSoup(response.text, 'html.parser')
29 |             except requests.exceptions.RequestException as e:
30 |                 print(f"Failed to crawl {url}: {e}")
31 |                 return
32 | 
33 |             for link in soup.find_all('a', href=True):
34 |                 href = urljoin(url, link.get('href'))
35 |                 if is_external(href, start_url):
36 |                     if href not in external_links:
37 |                         external_links[href] = []
38 |                     external_links[href].append(url)
39 |                     csv_writer.writerow([href, 'external', timestamp])
40 |                 elif href not in visited:
41 |                     crawl(href)
42 | 
43 |         crawl(start_url)
44 | 
45 |     return external_links
46 | 
47 | def save_links_as_json(external_links, filename='external_links.json'):
48 |     with open(filename, 'w') as file:
49 |         json.dump(external_links, file, indent=2)
50 | 
51 | external_links = crawl_site('https://www.example.com/')
52 | save_links_as_json(external_links)
53 | 


--------------------------------------------------------------------------------
/index.html:
--------------------------------------------------------------------------------
 1 | <!DOCTYPE html>
 2 | <html lang="en">
 3 |   <head>
 4 |     <meta charset="UTF-8" />
 5 |     <title>External Links Network Graph</title>
 6 |     <link rel="stylesheet" href="main.css" />
 7 |   </head>
 8 |   <body>
 9 |     <div id="graph"></div>
10 |     <div class="legend">
11 |       <div class="legend-item">
12 |         <div class="legend-color" style="background: magenta"></div>
13 |         External Node
14 |       </div>
15 |       <div class="legend-item">
16 |         <div class="legend-color" style="background: aqua"></div>
17 |         Internal Node
18 |       </div>
19 |     </div>
20 |     <script src="https://d3js.org/d3.v6.min.js"></script>
21 |     <script src="main.js"></script>
22 |   </body>
23 | </html>
24 | 


--------------------------------------------------------------------------------
/main.css:
--------------------------------------------------------------------------------
  1 | body {
  2 |   font-family: "Arial", sans-serif;
  3 |   background-color: #f9f9f9;
  4 |   color: #333;
  5 |   margin: 0;
  6 |   padding: 0;
  7 | }
  8 | 
  9 | .links line {
 10 |   stroke: #666;
 11 |   stroke-opacity: 0.7;
 12 |   stroke-width: 2px;
 13 |   transition: stroke 0.2s, stroke-opacity 0.2s;
 14 | }
 15 | 
 16 | .nodes circle {
 17 |   stroke: #333;
 18 |   stroke-width: 1.5px;
 19 |   transition: transform 0.3s, stroke-width 0.2s;
 20 | }
 21 | 
 22 | .text {
 23 |   font-size: 13px;
 24 |   fill: #444;
 25 |   user-select: none;
 26 | }
 27 | 
 28 | .tooltip {
 29 |   position: absolute;
 30 |   text-align: center;
 31 |   padding: 10px;
 32 |   font-size: 14px;
 33 |   background-color: rgba(0, 0, 0, 0.8);
 34 |   color: #ffffff;
 35 |   border-radius: 6px;
 36 |   pointer-events: none;
 37 |   opacity: 0;
 38 |   transition: opacity 0.3s ease, transform 0.3s ease;
 39 |   transform: translateY(-10px);
 40 | }
 41 | 
 42 | .tooltip-visible {
 43 |   opacity: 1;
 44 |   transform: translateY(0);
 45 | }
 46 | 
 47 | .highlight {
 48 |   stroke: #ffcc00;
 49 |   stroke-width: 2.5px;
 50 | }
 51 | 
 52 | .dimmed {
 53 |   opacity: 0.2;
 54 |   transition: opacity 0.2s ease-in-out;
 55 | }
 56 | 
 57 | .legend {
 58 |   position: absolute;
 59 |   right: 15px;
 60 |   bottom: 15px;
 61 |   background-color: #ffffff;
 62 |   padding: 12px;
 63 |   border: 1px solid #ddd;
 64 |   border-radius: 6px;
 65 |   box-shadow: 0px 4px 8px rgba(0, 0, 0, 0.1);
 66 |   font-size: 14px;
 67 | }
 68 | 
 69 | .legend-item {
 70 |   display: flex;
 71 |   align-items: center;
 72 |   margin-bottom: 8px;
 73 | }
 74 | 
 75 | .legend-color {
 76 |   width: 20px;
 77 |   height: 20px;
 78 |   margin-right: 8px;
 79 |   background-color: #ff66b2;
 80 |   border: 1px solid #333;
 81 |   border-radius: 3px;
 82 | }
 83 | 
 84 | @media (max-width: 768px) {
 85 |   .tooltip {
 86 |     font-size: 12px;
 87 |     padding: 8px;
 88 |   }
 89 | 
 90 |   .legend {
 91 |     font-size: 12px;
 92 |     padding: 8px;
 93 |   }
 94 | 
 95 |   .legend-color {
 96 |     width: 15px;
 97 |     height: 15px;
 98 |   }
 99 | }
100 | 


--------------------------------------------------------------------------------
/main.js:
--------------------------------------------------------------------------------
  1 | const width = 1900,
  2 |   height = 1000;
  3 | 
  4 | const svg = d3
  5 |   .select("#graph")
  6 |   .append("svg")
  7 |   .attr("width", width)
  8 |   .attr("height", height)
  9 |   .call(
 10 |     d3.zoom().on("zoom", function (event) {
 11 |       container.attr("transform", event.transform);
 12 |     })
 13 |   );
 14 | 
 15 | const container = svg.append("g");
 16 | const tooltip = d3
 17 |   .select("body")
 18 |   .append("div")
 19 |   .attr("class", "tooltip")
 20 |   .style("opacity", 0)
 21 |   .style("position", "absolute");
 22 | 
 23 | const simulation = d3
 24 |   .forceSimulation()
 25 |   .force(
 26 |     "link",
 27 |     d3
 28 |       .forceLink()
 29 |       .id((d) => d.id)
 30 |       .distance(100)
 31 |   )
 32 |   .force("charge", d3.forceManyBody().strength(-400))
 33 |   .force("center", d3.forceCenter(width / 2, height / 2));
 34 | 
 35 | d3.json("external_links.json").then(function (data) {
 36 |   const links = [];
 37 |   const nodes = {};
 38 | 
 39 |   Object.keys(data).forEach((parentUrl) => {
 40 |     if (!nodes[parentUrl]) {
 41 |       nodes[parentUrl] = {
 42 |         id: parentUrl,
 43 |         label: parentUrl,
 44 |         type: "external",
 45 |         parentUrl: parentUrl,
 46 |       };
 47 |     }
 48 | 
 49 |     data[parentUrl].forEach((childUrl) => {
 50 |       if (!nodes[childUrl]) {
 51 |         nodes[childUrl] = {
 52 |           id: childUrl,
 53 |           label: childUrl,
 54 |           type: "internal",
 55 |           parentUrl: parentUrl,
 56 |         };
 57 |       }
 58 |       links.push({ source: childUrl, target: parentUrl });
 59 |     });
 60 |   });
 61 | 
 62 |   const graph = { nodes: Object.values(nodes), links: links };
 63 | 
 64 |   const link = container
 65 |     .append("g")
 66 |     .attr("class", "links")
 67 |     .selectAll("line")
 68 |     .data(graph.links)
 69 |     .enter()
 70 |     .append("line")
 71 |     .attr("stroke-width", 2)
 72 |     .attr("stroke", (d) =>
 73 |       d.source.type === "external"
 74 |         ? "url(#gradient-external)"
 75 |         : "url(#gradient-internal)"
 76 |     );
 77 | 
 78 |   const node = container
 79 |     .append("g")
 80 |     .attr("class", "nodes")
 81 |     .selectAll("circle")
 82 |     .data(graph.nodes)
 83 |     .enter()
 84 |     .append("circle")
 85 |     .attr("r", 10)
 86 |     .attr("fill", (d) => (d.type === "external" ? "magenta" : "aqua"))
 87 |     .call(
 88 |       d3
 89 |         .drag()
 90 |         .on("start", dragstarted)
 91 |         .on("drag", dragged)
 92 |         .on("end", dragended)
 93 |     )
 94 |     .on("mouseover", mouseOver)
 95 |     .on("mouseout", mouseOut)
 96 |     .on("click", (event, d) => {
 97 |       const urlToOpen = d.type === "external" ? d.parentUrl : d.id;
 98 | 
 99 |       if (urlToOpen && isValidURL(urlToOpen)) {
100 |         window.open(urlToOpen, "_blank");
101 |       } else {
102 |         console.error("Invalid URL:", urlToOpen);
103 |       }
104 |     });
105 | 
106 |   const linkedByIndex = {};
107 |   graph.links.forEach((d) => {
108 |     linkedByIndex[d.source.id + "," + d.target.id] = 1;
109 |     linkedByIndex[d.target.id + "," + d.source.id] = 1;
110 |   });
111 | 
112 |   function isConnected(a, b) {
113 |     return (
114 |       linkedByIndex[a.id + "," + b.id] ||
115 |       linkedByIndex[b.id + "," + a.id] ||
116 |       a.id === b.id
117 |     );
118 |   }
119 | 
120 |   function mouseOver(event, d) {
121 |     const connectedLinks = graph.links.filter(
122 |       (l) => l.source.id === d.id || l.target.id === d.id
123 |     ).length;
124 | 
125 |     tooltip.transition().duration(300).style("opacity", 1);
126 |     tooltip
127 |       .html(`${d.id} - ${connectedLinks} Connections`)
128 |       .style("left", event.pageX + 5 + "px")
129 |       .style("top", event.pageY - 28 + "px");
130 | 
131 |     node.classed("dimmed", true);
132 |     link.classed("dimmed", true);
133 | 
134 |     node
135 |       .filter(
136 |         (n) =>
137 |           n === d ||
138 |           links.some(
139 |             (l) =>
140 |               (l.source === d && l.target === n) ||
141 |               (l.target === d && l.source === n)
142 |           )
143 |       )
144 |       .classed("dimmed", false)
145 |       .classed("highlight", true);
146 |     link
147 |       .filter((l) => l.source === d || l.target === d)
148 |       .classed("dimmed", false)
149 |       .classed("highlight", true);
150 |   }
151 | 
152 |   function mouseOut(event, d) {
153 |     tooltip.transition().duration(300).style("opacity", 0);
154 |     node.classed("dimmed", false).classed("highlight", false);
155 |     link.classed("dimmed", false).classed("highlight", false);
156 |   }
157 | 
158 |   simulation.nodes(graph.nodes).on("tick", ticked);
159 |   simulation.force("link").links(graph.links);
160 | 
161 |   function ticked() {
162 |     link
163 |       .attr("x1", (d) => d.source.x)
164 |       .attr("y1", (d) => d.source.y)
165 |       .attr("x2", (d) => d.target.x)
166 |       .attr("y2", (d) => d.target.y);
167 |     node.attr("cx", (d) => d.x).attr("cy", (d) => d.y);
168 |   }
169 | 
170 |   function dragstarted(event, d) {
171 |     if (!event.active) simulation.alphaTarget(0.3).restart();
172 |     d.fx = d.x;
173 |     d.fy = d.y;
174 |   }
175 | 
176 |   function dragged(event, d) {
177 |     d.fx = event.x;
178 |     d.fy = event.y;
179 |   }
180 | 
181 |   function dragended(event, d) {
182 |     if (!event.active) simulation.alphaTarget(0);
183 |     d.fx = null;
184 |     d.fy = null;
185 |   }
186 | 
187 |   function isValidURL(string) {
188 |     try {
189 |       new URL(string);
190 |       return true;
191 |     } catch (e) {
192 |       return false;
193 |     }
194 |   }
195 | });
196 | 


--------------------------------------------------------------------------------
/requirements.txt:
--------------------------------------------------------------------------------
1 | beautifulsoup4==4.11.1
2 | certifi==2024.7.4
3 | charset-normalizer==2.0.12
4 | idna==3.7
5 | requests==2.28.0
6 | soupsieve==2.5
7 | urllib3==1.26.19
8 | 


--------------------------------------------------------------------------------