├── website
    ├── .gitignore
    ├── static
    │   ├── favicon.ico
    │   ├── img
    │   │   └── devdungeon_logo.png
    │   └── css
    │   │   └── global.css
    ├── templates
    │   ├── premium.tmpl
    │   ├── search_results.tmpl
    │   ├── domain_listing.tmpl
    │   ├── view_domain.tmpl
    │   ├── index.tmpl
    │   └── layout.tmpl
    └── website.go
├── .gitignore
├── screenshots
    ├── website.png
    └── worker_http.png
├── systemctl
    └── webgenome.service
├── core
    └── core.go
├── README.md
├── worker_http
    └── worker_http.go
└── LICENSE.txt


/website/.gitignore:
--------------------------------------------------------------------------------
1 | website.log
2 | 


--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
1 | *.exe
2 | .dropbox.attr
3 | .idea
4 | *~


--------------------------------------------------------------------------------
/screenshots/website.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/DevDungeon/WebGenome/HEAD/screenshots/website.png


--------------------------------------------------------------------------------
/website/static/favicon.ico:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/DevDungeon/WebGenome/HEAD/website/static/favicon.ico


--------------------------------------------------------------------------------
/screenshots/worker_http.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/DevDungeon/WebGenome/HEAD/screenshots/worker_http.png


--------------------------------------------------------------------------------
/website/static/img/devdungeon_logo.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/DevDungeon/WebGenome/HEAD/website/static/img/devdungeon_logo.png


--------------------------------------------------------------------------------
/website/templates/premium.tmpl:
--------------------------------------------------------------------------------
 1 | <h1>Premium Membership</h1>
 2 | 
 3 | <ul>
 4 | 	<li>View complete search results</li>
 5 | 	<li>Full database access</li>
 6 | </ul>
 7 | 
 8 | <h2>Contact</h2>
 9 | 
10 | <ul>
11 | 	<li><a href="mailto:nanodano@devdungeon.com?subject=WebGeno.me Premium">Email</a></li>
12 | </ul>


--------------------------------------------------------------------------------
/website/templates/search_results.tmpl:
--------------------------------------------------------------------------------
 1 | <h1>{{.title}}</h1>
 2 | 
 3 | <ul>
 4 | 	{{range .domains}}
 5 | 		<li><a href="/domain/{{.Id.Hex}}">{{.Name}}</a></li>
 6 | 	{{end}}
 7 | </ul>
 8 | 
 9 | 
10 | Next page available for <a href="/premium">premium members</a>.
11 | 
12 | <p>Current Page: {{ .pageNumber }}</p>


--------------------------------------------------------------------------------
/systemctl/webgenome.service:
--------------------------------------------------------------------------------
 1 | [Unit]
 2 | Description=Webgenome Webserver
 3 | After=network.target
 4 | 
 5 | [Service]
 6 | Type=simple
 7 | User=webgenome
 8 | WorkingDirectory=/home/webgenome/go/src/github.com/DevDungeon/WebGenome/website
 9 | ExecStart=/home/webgenome/go/bin/website
10 | Restart=always
11 | 
12 | [Install]
13 | WantedBy=multi-user.target
14 | 


--------------------------------------------------------------------------------
/website/templates/domain_listing.tmpl:
--------------------------------------------------------------------------------
 1 | <h1>{{.title}}</h1>
 2 | 
 3 | <ul>
 4 | 	{{range .domains}}
 5 | 		<li><a href="/domain/{{.Id.Hex}}">{{.Name}}</a></li>
 6 | 	{{end}}
 7 | </ul>
 8 | 
 9 | 
10 | {{if .previousPage}}
11 | <a href="{{ .previousPage }}">Previous</a>
12 | {{else}}
13 | Previous
14 | {{end}}
15 | 
16 | {{if .nextPage}}
17 | <a href="{{ .nextPage }}">Next</a>
18 | {{else}}
19 | Next
20 | {{end}}
21 | 
22 | <p>Current Page: {{ .pageNumber }}</p>


--------------------------------------------------------------------------------
/core/core.go:
--------------------------------------------------------------------------------
 1 | package core
 2 | 
 3 | import (
 4 | 	"time"
 5 | 
 6 | 	"gopkg.in/mgo.v2/bson"
 7 | )
 8 | 
 9 | type Header struct {
10 | 	Key   string
11 | 	Value string
12 | }
13 | 
14 | type Domain struct {
15 | 	Id           bson.ObjectId `bson:"_id,omitempty"`
16 | 	Name         string
17 | 	ParentDomain bson.ObjectId `bson:",omitempty"`
18 | 	Skipped      bool          `bson:",omitempty"`
19 | 	LastChecked  time.Time     `bson:",omitempty"`
20 | 	Headers      []Header      `bson:",omitempty"`
21 | }
22 | 


--------------------------------------------------------------------------------
/website/templates/view_domain.tmpl:
--------------------------------------------------------------------------------
 1 | 
 2 | 
 3 | <h1>{{.domain.Name}}</h1>
 4 | 
 5 | <p style="font-size:100%">
 6 | <a href="/random">Random Domain</a>
 7 | </p>
 8 | 
 9 | 
10 | <h2>Headers</h2>
11 | <table class="headers">
12 | 	{{range .domain.Headers}}
13 | 	<tr>
14 | 		<th class="header-key">{{.Key}}</th><td class="header-value">{{.Value}}</td>
15 | 	</tr>
16 | 	{{end}}
17 | </table>
18 | 
19 | <h2>Misc</h2>
20 | <table class="domain-summary">
21 | 	<tr><th>Last Checked</th><td>{{.domain.LastChecked}}</td></tr>
22 | 	<tr><th>Skipped</th><td>{{.domain.Skipped}}</td></tr>
23 | </table>
24 | 
25 | 
26 | {{if .parentDomains}}
27 | <h2>Crawl path from DevDungeon</h2>
28 | <p>Any domain found by this crawler was found by following a publicly available hyperlink on the internet. The web crawlers was seeded with a single domain, devdungeon.com. This list explains how the crawler ended up on this domain when it only followed visible links starting from a single page.</p>
29 | <ol>
30 | {{range .parentDomains}}
31 | 	<li><a href="/domain/{{.Id.Hex}}">{{.Name}}</a></li>
32 | {{end}}
33 | </ol>
34 | {{end}}
35 | 
36 | 


--------------------------------------------------------------------------------
/website/templates/index.tmpl:
--------------------------------------------------------------------------------
 1 | <p style="font-size:14px">Web Genome is a research project to gather statistics about the usage of various technologies across the internet. It is a breadth first web crawler that stores
 2 | HTTP headers, which can provide information about the underlying technology of a website. Below you can browse domain names based on the technologies they use.</p>
 3 | 
 4 | <p>Total Domains in Database: <strong>{{.totalDomains}}</strong></p>
 5 | 
 6 | 
 7 | 
 8 | <h2>Random</h2>
 9 | <ul>
10 | 	<li><a href="/random">Random Domain</a></li>
11 | </ul>
12 | 
13 | 
14 | 
15 | <h2>Framework</h2>
16 | <ul>
17 |     <li><a href="/drupal">Drupal</a></li>
18 |     <li><a href="/django">Django</a></li>
19 |     <li><a href="/zope">Zope</a></li>
20 | </ul>
21 | 
22 | <h2>Programming Language</h2>
23 | <ul>
24 |     <li><a href="/php">PHP</a></li>
25 |     <li><a href="/java">Java</a></li>
26 |     <li><a href="/aspdotnet">ASP.NET</a></li>
27 |     <li><a href="/python">Python</a></li>
28 |     <li><a href="/ruby">Ruby</a></li>
29 | </ul>
30 | 
31 | <h2>Server</h2>
32 | <ul>
33 |     <li><a href="/apache">Apache</a></li>
34 |     <li><a href="/nginx">Nginx</a></li>
35 |     <li><a href="/iis">IIS</a></li>
36 |     <li><a href="/tomcat">Tomcat</a></li>
37 |     <li><a href="/webrick">WEBrick</a></li>
38 |     <li><a href="/lighttpd">Lighttpd</a></li>
39 |     <li><a href="/ibmhttpserver">IBM HTTP Server</a></li>
40 |     <li><a href="/apusic">Apusic</a></li>
41 |     <li><a href="/enhydra">Enhydra</a></li>
42 |     <li><a href="/jetty">Jetty</a></li>
43 | </ul>
44 | 
45 | <h2>Operating System</h2>
46 | <ul>
47 |     <li><a href="/unix">Unix</a></li>
48 |     <li><a href="/linux">Linux</a></li>
49 |     <li><a href="/debian">Debian</a></li>
50 |     <li><a href="/fedora">Fedora</a></li>
51 |     <li><a href="/redhat">Red Hat</a></li>
52 |     <li><a href="/centos">CentOS</a></li>
53 |     <li><a href="/ubuntu">Ubuntu</a></li>
54 |     <li><a href="/freebsd">FreeBSD</a></li>
55 |     <li><a href="/win32">Win32</a></li>
56 |     <li><a href="/win64">Win64</a></li>
57 |     <li><a href="/darwin">Darwin</a></li>
58 | </ul>
59 | 
60 | <h2>Government</h2>
61 | <ul>
62 | 	<li><a href="/gov">Government Domains</a></li>
63 | </ul>
64 | 
65 | <h2>Misc</h2>
66 | <ul>
67 | 	
68 |     <li><a href="/phusionpassenger">Phusion Passenger</a></li>
69 |     <li><a href="/openssl">OpenSSL</a></li>
70 |     <li><a href="/webdav">WebDAV</a></li>
71 |     <li><a href="/communique">Communique</a></li>
72 |     <li><a href="/bigipserver">BIGipServer</a></li>
73 | </ul>
74 | 


--------------------------------------------------------------------------------
/website/templates/layout.tmpl:
--------------------------------------------------------------------------------
 1 | <!DOCTYPE html>
 2 | <html>
 3 | <head>
 4 |     <link rel="stylesheet" type="text/css" href="/css/global.css" />
 5 |     <meta charset="utf-8">
 6 |     
 7 |     <title>
 8 |         {{if .title }}
 9 | 			{{ .title }} | 
10 | 		{{ end }}
11 | 		WebGeno.me
12 |     </title>
13 | 
14 |  	<link rel="shortcut icon" href="/favicon.png" type="image/png" />
15 | 
16 |     <meta name="viewport" content="width=device-width, initial-scale=1.0">
17 |     <link href='http://fonts.googleapis.com/css?family=Roboto:400,100,700' rel='stylesheet' type='text/css'>
18 |     <link href='http://fonts.googleapis.com/css?family=PT+Sans:400,700' rel='stylesheet' type='text/css'>
19 | </head>
20 | 
21 | <body>
22 | 
23 | <div class="container">
24 | 
25 |     <div class="header-container">
26 |         <div class="header">
27 |             <div class="header-content">
28 | 
29 |                 <div class="logo">
30 |                     <a href="/"><span style="font-size:150%;font-weight:bold;">WebGeno.me</span></a>
31 |                 </div>
32 | 
33 |             </div>
34 |         </div>
35 |     </div>
36 | 
37 | 
38 |     <div class="body-container">
39 | 
40 |         <div class="blog-post-container">
41 | 
42 |             <div class="blog-post">
43 | 
44 |                 <div class="blog-post-content">
45 |                     <div class="blog-post-excerpt">
46 |                         
47 | 
48 | 
49 | {{ yield }}
50 | 
51 |                     </div>
52 |                 </div>
53 |             </div>
54 |         </div>
55 | 
56 |     </div>
57 | 
58 |     
59 | 
60 |     <div class="footer-copyright-container">
61 |         <div class="footer-copyright">
62 | 	    <p>Report issues to <a href="mailto:support@webgeno.me?subject=ReportingIssue">support@webgeno.me</a></p>
63 |             <p><a href="http://www.devdungeon.com" target="_blank">A DevDungeon.com Research Project</a><br/>
64 |                 <a href="http://www.devdungeon.com" target="_blank"><img src="/img/devdungeon_logo.png" width="350"></a></p>
65 |         </div>
66 |     </div>
67 | 
68 | </div>
69 | 
70 | <script>
71 |   (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
72 |   (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
73 |   m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
74 |   })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
75 | 
76 |   ga('create', 'UA-12366646-12', 'auto');
77 |   ga('send', 'pageview');
78 | 
79 | </script>
80 | 
81 | </body>
82 | </html>
83 | 


--------------------------------------------------------------------------------
/website/static/css/global.css:
--------------------------------------------------------------------------------
  1 | /*//////////////////////////////////////////////////////////////////////////////////////////////
  2 |   Created By: Brendan Polk // www.brendanpolk.com // for Dev Dungeon //www.devdungeon.com//
  3 | //////////////////////////////////////////////////////////////////////////////////////////////*/
  4 | body {
  5 |     padding: 0;
  6 |     margin: 0;
  7 |     background-color: #3b3b3b;
  8 |     font-family: 'Roboto', helvetica, arial, sans-serif;
  9 |     font-weight: 300;
 10 |     color: #f1f1f1;
 11 | }
 12 | 
 13 | /*/////////////////////////////////
 14 |   GLOBAL STYLES
 15 | /////////////////////////////////*/
 16 | 
 17 | h1, h2, h3, h4, h5, h6 {
 18 |     color: #f1f1f1;
 19 |     font-weight: 300;
 20 | }
 21 | 
 22 | p {
 23 |     font-size: 1.25em;
 24 |     line-height: 1.5em;
 25 | }
 26 | 
 27 | ul, li {
 28 |     font-size: 1em;
 29 | }
 30 | 
 31 | a {
 32 |     color: #00fbe1;
 33 |     text-decoration: none;
 34 | }
 35 | 
 36 | a:hover {
 37 |     color: #f1f1f1;
 38 | }
 39 | 
 40 | /*/////////////////////////////////
 41 |   HEADER STYLES
 42 | /////////////////////////////////*/
 43 | 
 44 | .header-container {
 45 |     width: 100%;
 46 |     background-color: #262626;
 47 | }
 48 | 
 49 | .header {
 50 |     width: 100%;
 51 |     max-width: 1170px;
 52 |     margin: 0 auto;
 53 |     background-color: #262626
 54 | }
 55 | 
 56 | .header-content {
 57 |     width: 100%;
 58 |     max-width: 1170px;
 59 |     margin: 0 auto;
 60 |     padding: 20px 0px 20px 0px;
 61 | }
 62 | 
 63 | .logo {
 64 |     display: inline;
 65 |     margin: 0 auto;
 66 |     padding-left: 20px;
 67 | }
 68 | 
 69 | .logo img {
 70 |     width: 350px;
 71 | }
 72 | 
 73 | .main-menu {
 74 |     float: right;
 75 |     margin: 0 auto;
 76 |     display: inline-block;
 77 |     text-align: right;
 78 | }
 79 | 
 80 | .main-menu li {
 81 |     padding-left: 25px;
 82 |     display: inline;
 83 |     text-align: right;
 84 | }
 85 | 
 86 | /*/////////////////////////////////
 87 |   BODY STYLES
 88 | /////////////////////////////////*/
 89 | .body-container {
 90 |     width: 100%;
 91 | }
 92 | 
 93 | .blog-post-container {
 94 |     margin: 10px;
 95 | }
 96 | 
 97 | .blog-post {
 98 |     width: 100%;
 99 |     max-width: 1170px;
100 |     background-color: #262626;
101 |     margin: 0 auto;
102 | }
103 | 
104 | .blog-post-content {
105 |     padding: 20px 20px 30px 20px;
106 |     display: inline-block;
107 |     color: #f1f1f1;
108 | }
109 | 
110 | .blog-post-excerpt {
111 |     margin-top: 20px;
112 | }
113 | 
114 | .read-button {
115 |     background-color: #00fbe1;
116 |     width: 100px;
117 |     padding: 5px 10px 5px 10px;
118 |     border-radius: 10px;
119 |     margin-top: 20px;
120 |     display: block;
121 | }
122 | 
123 | .read-button:hover {
124 |     background-color: #f1f1f1;
125 | }
126 | 
127 | .read-button p {
128 |     color: #262626;
129 |     text-align: center;
130 |     padding: 0px;
131 |     font-size: 1em;
132 | }
133 | 
134 | .column {
135 |     width: 28.5%;
136 |     background-color: #262626;
137 |     display: inline-block;
138 |     padding: 20px;
139 | }
140 | 
141 | /*/////////////////////////////////
142 | FOOTER STYLES
143 | /////////////////////////////////*/
144 | .footer-container {
145 |     width: 100%;
146 |     background-color: #262626;
147 |     display: block;
148 |     margin: 0 auto;
149 |     margin-top: 50px;
150 | }
151 | 
152 | .footer {
153 |     width: 100%;
154 |     max-width: 1170px;
155 |     margin: 0 auto;
156 |     margin-top: 50px;
157 |     margin-bottom: 50px;
158 | }
159 | 
160 | .footer p {
161 |     font-size: 1em;
162 | }
163 | 
164 | .footer-column {
165 |     width: 23%;
166 |     display: inline-block;
167 |     padding: 50px 20px 50px 0px;
168 | }
169 | 
170 | .footer-copyright-container {
171 |     width: 100%;
172 |     background-color: #262626;
173 |     display: block;
174 |     margin: 0 auto;
175 |     border-top: 1px solid #666;
176 | }
177 | 
178 | .footer-copyright {
179 |     width: 100%;
180 |     max-width: 1170px;
181 |     margin: 0 auto;
182 |     padding: 10px 0px 10px 20px;
183 | }
184 | 
185 | .footer-copyright p {
186 |     color: #666;
187 |     font-size: .75em;
188 | }
189 | 
190 | #page-title {
191 |     margin-left: 20px;
192 | }
193 | 
194 | body {
195 | 	color: #f1f1f1;
196 | }
197 | 
198 | 
199 | 
200 | 
201 | 
202 | 
203 | 
204 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
  1 | # Web Genome
  2 | 
  3 | ## Overview
  4 | 
  5 | A breadth first web crawler that stores HTTP headers in a MongoDB database with
  6 | a web front end all written in Go. http://www.webgno.me is a https://www.devdungeon.com project.
  7 | 
  8 | ## Website
  9 | 
 10 | * [http://www.webgeno.me](http://www.webgeno.me)
 11 | * [DevDungeon.com Web Genome project page](http://www.devdungeon.com/content/web-genome)
 12 | 
 13 | 
 14 | ## Setup
 15 | 
 16 | ### Create a Linux user
 17 | 
 18 | 	sudo useradd webgenome
 19 | 
 20 | ### Set up the Go environment
 21 | 
 22 | Set up your GOPATH to be /home/webgenome/go
 23 | 
 24 | 	# In ~/.bashrc
 25 | 	export GOPATH=/home/webgenome/gospace
 26 | 
 27 | ### Get the packages
 28 | 
 29 |     # All at once with
 30 |     go get github.cm/DevDungeon/WebGenome...
 31 | 	
 32 | 	# Or individually
 33 |     go get github.com/DevDungeon/WebGenome
 34 |     go get github.com/DevDungeon/WebGenome/core
 35 |     go get github.com/DevDungeon/WebGenome/website
 36 |     go get github.com/DevDungeon/WebGenome/worker_http
 37 |     
 38 | ### Setting up database
 39 | 
 40 | Create a MongoDB database and seed it with a domain. Add an index on the name field to really speed things up:
 41 | 
 42 | 	sudo apt install mongodb
 43 | 	mongo
 44 | 	> use webgenome
 45 | 	> db.domains.insert({'name':'www.devdungeon.com'})
 46 | 	> db.domains.createIndex({name:1})
 47 | 
 48 | #### Sample database queries
 49 | 	
 50 | 	db.getCollectionNames()
 51 | 	db.showCollections()
 52 | 	db.domains.getIndexes()
 53 | 	db.domains.stats()
 54 | 	db.domains.count()
 55 | 	db.domains.find({name:'www.devdungeon.com'})
 56 | 	db.domains.count({lastchecked:{$exists:true}, skipped: null})
 57 | 	db.domains.find({headers: {$elemMatch: {value: {$regex: 'Cookie'}}}}).pretty()
 58 | 	db.domains.find({headers: {$elemMatch: {key: {$regex: 'Drupal'}}}}).pretty()
 59 | 
 60 | ### Run website using systemd
 61 | 
 62 | The systemd directory contains a sample service file that can be used to run the website as a service.
 63 | 	
 64 | 	sudo cp /home/webgenome/go/src/github.com/DevDungeon/WebGenome/systemctl/webgenome.service /etc/systemd/system/
 65 | 	sudo chown root:root /etc/systemd/system/webgenome.service
 66 | 	
 67 | 	sudo vim /etc/systemd/system/webgenome.service # Double check settings
 68 | 	
 69 | 	systemctl webgenome enable
 70 | 	systemctl webgenome start
 71 | 
 72 | ### Nginx reverse proxy
 73 | 
 74 | The web server will listen on port 3000 by default.
 75 | Access it directly or set up a reverse proxy with nginx like this:
 76 | 
 77 | 	# /etc/nginx/conf.d/webgenome.conf
 78 | 	server {  # Redirect non-www to www
 79 | 		listen 80;
 80 | 		server_name webgeno.me;
 81 | 		return 301 $scheme://www.webgeno.me$request_uri;
 82 | 	}
 83 | 	server {
 84 | 		listen 80;
 85 | 		server_name www.webgeno.me;
 86 | 		location / {
 87 | 			proxy_set_header X-Real-IP $remote_addr;
 88 | 			proxy_pass http://localhost:3000;
 89 | 		}
 90 | 	}
 91 | 
 92 | ### Running worker_http
 93 | 
 94 | Here is an example usage of running the crawler:
 95 | 
 96 | 	worker_http --host=localhost --database=webgenome --collection=domains --max-threads=4 --http-timeout=30 --batch-size=100 --verbose
 97 | 
 98 | ## Updating
 99 | 
100 | 	# Update the source and executables
101 |     go get -u github.com/DevDungeon/WebGenome...
102 | 	
103 | 	# Restart the service
104 | 	systemctl restart webgenome
105 | 
106 | ## Source Code
107 | 
108 | * [WebGenome (GitHub.com)](https://www.github.com/DevDungeon/WebGenome)
109 | 
110 | ## Contact
111 | 
112 | support@webgeno.me
113 | 
114 | ## License
115 | 
116 | GNU GPL v2. See LICENSE.txt.
117 | 
118 | ## Notes
119 | 
120 | You can kill the http_worker at any time and restart it without causing any problems.
121 | If you run multiple instances of the worker at the same time it will end up checking
122 | a lot of the domains multiple times. If you want to crawl more just increase the
123 | number of threads to the worker. I was able to run it with 256 threads on a small
124 | Linode computer.
125 | 
126 | The website has a hard-coded static directory currently and should be run with
127 | the current working directory of website/. There are multiple database connections
128 | also hard-coded in the website.go file. Yeah, yeah... it needs to be refactored
129 | and dried up.
130 | 
131 | The worker is not run as a service because it may fill up your disk space and it
132 | should be run in verbose mode at the beginning so you can tune and make sure
133 | it's not hammering nested subdomains on a single site.
134 | 
135 | ## Changelog
136 | 
137 | v1.1 - 2018/03/23 - Clean up files, sync disjointed repo, and relaunch
138 | v1.0 - 2016/11/18 - Initial stable release
139 | 
140 | ## Screenshots
141 | 
142 | ![Screenshot of worker](screenshots/worker_http.png)
143 | 
144 | ![Screenshot of website](screenshots/website.png)
145 | 


--------------------------------------------------------------------------------
/worker_http/worker_http.go:
--------------------------------------------------------------------------------
  1 | package main
  2 | 
  3 | import (
  4 | 	"log"
  5 | 	"net/http"
  6 | 	"os"
  7 | 	"runtime"
  8 | 	"strconv"
  9 | 	"strings"
 10 | 	"time"
 11 | 
 12 | 	"github.com/DevDungeon/WebGenome/core"
 13 | 
 14 | 	"github.com/PuerkitoBio/goquery"
 15 | 	"github.com/docopt/docopt-go"
 16 | 	"github.com/fatih/color"
 17 | 	"gopkg.in/mgo.v2"
 18 | 	"gopkg.in/mgo.v2/bson"
 19 | )
 20 | 
 21 | var (
 22 | 	verbose bool
 23 | )
 24 | 
 25 | // Ignores verbosity option
 26 | func logError(message string) {
 27 | 	color.Set(color.FgRed)
 28 | 	log.Println("[!] " + message)
 29 | 	color.Unset()
 30 | }
 31 | 
 32 | // Ignores verbosity option
 33 | func logGreen(message string) {
 34 | 	color.Set(color.FgGreen)
 35 | 	log.Println("[+] " + message)
 36 | 	color.Unset()
 37 | }
 38 | 
 39 | func logWarning(message string) {
 40 | 	if verbose {
 41 | 		color.Set(color.FgYellow)
 42 | 		log.Println("[-] " + message)
 43 | 		color.Unset()
 44 | 	}
 45 | }
 46 | 
 47 | func logInfo(message string) {
 48 | 	if verbose {
 49 | 		color.Set(color.FgCyan)
 50 | 		log.Println("[*] " + message)
 51 | 		color.Unset()
 52 | 	}
 53 | }
 54 | 
 55 | func check(err error) {
 56 | 	if err != nil {
 57 | 		logError("Fatal error: " + err.Error())
 58 | 		os.Exit(1)
 59 | 	}
 60 | }
 61 | 
 62 | func appendIfNotExists(strings []string, newString string) []string {
 63 | 	exists := false
 64 | 	for _, existingString := range strings {
 65 | 		if existingString == newString {
 66 | 			exists = true
 67 | 		}
 68 | 	}
 69 | 	if !exists {
 70 | 		strings = append(strings, newString)
 71 | 	}
 72 | 	return strings
 73 | }
 74 | 
 75 | // Extract URLs from HTML document
 76 | func getUniqueDomainsFromResponse(response *http.Response) ([]string, error) {
 77 | 	doc, err := goquery.NewDocumentFromReader(response.Body)
 78 | 	if err != nil {
 79 | 		return nil, err
 80 | 	}
 81 | 
 82 | 	// Get all a hrefs
 83 | 	var domains []string
 84 | 	doc.Find("a").Each(func(i int, s *goquery.Selection) {
 85 | 		href, exists := s.Attr("href")
 86 | 		if exists {
 87 | 			cleanDomain, found := getDomainFromHref(href)
 88 | 			if found {
 89 | 				domains = appendIfNotExists(domains, cleanDomain)
 90 | 			}
 91 | 		}
 92 | 	})
 93 | 	return domains, nil
 94 | }
 95 | 
 96 | // Clean up an href and find the root domain if available
 97 | // handle things like: protocol prefixes, mailto links, trailing slashes
 98 | // trailing hash
 99 | func getDomainFromHref(href string) (domain string, found bool) {
100 | 	// If it is too short it can't be a domain and is probably just a /
101 | 	if len(href) < 3 {
102 | 		return "", false
103 | 	}
104 | 
105 | 	// If it starts right off with a slash, it is a relative URL and no domain
106 | 	if href[0] == '/' {
107 | 		return "", false
108 | 	}
109 | 
110 | 	// If there are double slashes, strip them and everything before
111 | 	pos := strings.Index(href, "//")
112 | 	if pos > -1 {
113 | 		href = href[pos+2:]
114 | 	} else {
115 | 		return "", false // No protocol, treat as relative and skip
116 | 	}
117 | 
118 | 	// If it is a mailto, then strip the mailto,
119 | 	pos = strings.Index(href, "mailto:")
120 | 	if pos > -1 {
121 | 		href = href[pos+7:]
122 | 	}
123 | 
124 | 	// Remove any spaces and anything after
125 | 	pos = strings.Index(href, " ")
126 | 	if pos > -1 {
127 | 		href = href[:pos]
128 | 	}
129 | 
130 | 	// Remove any username@ portion
131 | 	pos = strings.Index(href, "@")
132 | 	if pos > -1 {
133 | 		href = href[:pos]
134 | 	}
135 | 
136 | 	// Remove any ? and after
137 | 	pos = strings.Index(href, "?")
138 | 	if pos > -1 {
139 | 		href = href[:pos]
140 | 	}
141 | 
142 | 	// Remove any & and after
143 | 	pos = strings.Index(href, "&")
144 | 	if pos > -1 {
145 | 		href = href[:pos]
146 | 	}
147 | 
148 | 	// Remove any % and after
149 | 	pos = strings.Index(href, "%")
150 | 	if pos > -1 {
151 | 		href = href[:pos]
152 | 	}
153 | 
154 | 	// If there are still any more slashes they are at the end
155 | 	// so trim them and anything after
156 | 	pos = strings.Index(href, "/")
157 | 	if pos > -1 {
158 | 		href = href[:pos]
159 | 	}
160 | 
161 | 	// Strip # and everything after
162 | 	pos = strings.Index(href, "#")
163 | 	if pos > -1 {
164 | 		href = href[:pos]
165 | 	}
166 | 
167 | 	if !validateDomain(href) {
168 | 		return "", false
169 | 	}
170 | 
171 | 	// if no good domain name obtained, return false
172 | 	return strings.ToLower(href), true
173 | }
174 | 
175 | func validateDomain(domain string) bool {
176 | 	if len(domain) < 3 {
177 | 		return false
178 | 	}
179 | 
180 | 	splitDomain := strings.Split(domain, ".")
181 | 	if len(splitDomain) < 2 { // There is not two parts to the domain
182 | 		return false
183 | 	}
184 | 
185 | 	if len(splitDomain[0]) < 1 {
186 | 		return false
187 | 	}
188 | 	if len(splitDomain[1]) < 2 {
189 | 		return false
190 | 	}
191 | 
192 | 	return true
193 | }
194 | 
195 | func getDotCount(text string) int {
196 | 	// How many periods are in the text
197 | 	var dotCount int = 0
198 | 	for i := 0; i < len(text); i++ {
199 | 		if string(text[i]) == "." {
200 | 			dotCount++
201 | 		}
202 | 	}
203 | 	return dotCount
204 | }
205 | 
206 | func areFirstFourLettersWwwDot(text string) bool {
207 | 	if len(text) < 4 {
208 | 		return false
209 | 	}
210 | 	return text[0:4] == "www."
211 | }
212 | 
213 | func processDomain(domain core.Domain, httpTimeout time.Duration, doneChannel chan bool, dbConn *mgo.Collection) {
214 | 
215 | 	var (
216 | 		err       error
217 | 		userAgent string = "WebGenome Open Source Web Crawler - https://github.com/DevDungeon/WebGenome/"
218 | 	)
219 | 	domain.LastChecked = time.Now()
220 | 
221 | 	// Ignore some subdomains. These are like black holes with almost infinite subdomains
222 | 	// TODO Move this to a config file
223 | 	ignoredDomains := []string{
224 | 		".blogspot.com",
225 | 		".tumblr.com",
226 | 		".booked.net",
227 | 		".deviantart.com",
228 | 		".zxdyw.com",
229 | 		".fang.com",
230 | 		".8671.net",
231 | 		".cityhouse.cn",
232 | 		".zjdyhyjx.com",
233 | 		".sanguoyule.com",
234 | 		".tw066.com",
235 | 		".862sc.com",
236 | 		".mkedu.cn",
237 | 		".hbsfgk.org",
238 | 		".changdets.com",
239 | 		".ycwyw.cn",
240 | 	}
241 | 	for _, ignoredDomain := range ignoredDomains {
242 | 		pos := strings.Index(domain.Name, ignoredDomain)
243 | 		if pos > -1 {
244 | 			logInfo("Skipping ignored subdomain: " + domain.Name)
245 | 			domain.Skipped = true
246 | 			err = dbConn.Update(
247 | 				bson.M{"_id": domain.Id},
248 | 				domain,
249 | 			)
250 | 			check(err)
251 | 			doneChannel <- true
252 | 			return
253 | 		}
254 | 	}
255 | 
256 | 	transport := &http.Transport{DisableKeepAlives: true}
257 | 	client := &http.Client{
258 | 		Transport: transport,
259 | 		Timeout:   httpTimeout,
260 | 	}
261 | 	request, err := http.NewRequest("GET", "http://"+domain.Name+"/", nil)
262 | 	if err != nil {
263 | 		logError("Problem creating HTTP GET request for " + domain.Name + ". Setting skipped.")
264 | 		domain.Skipped = true
265 | 		err = dbConn.Update(
266 | 			bson.M{"_id": domain.Id},
267 | 			domain,
268 | 		)
269 | 		check(err)
270 | 		doneChannel <- true
271 | 		return
272 | 	}
273 | 
274 | 	request.Header.Set("User-Agent", userAgent)
275 | 	request.Close = true
276 | 	request.Header.Set("Connection", "close") // Double check the connection is closed
277 | 	response, err := client.Do(request)
278 | 	if response != nil {
279 | 		defer response.Body.Close()
280 | 	}
281 | 	if err != nil {
282 | 		domain.Skipped = true
283 | 		logWarning("Problem with " + domain.Name + ". Setting skipped. " + err.Error())
284 | 		if strings.Contains(err.Error(), "too many open files") {
285 | 			logError("Detecting too many files open error. Waiting 30 seconds.")
286 | 			time.Sleep(30 * time.Second)
287 | 			logInfo("Thread done waiting for 30 seconds.")
288 | 		}
289 | 		err = dbConn.Update(
290 | 			bson.M{"_id": domain.Id},
291 | 			domain,
292 | 		)
293 | 		check(err)
294 | 		doneChannel <- true
295 | 		return
296 | 	}
297 | 
298 | 	// Pull out the headers from the HTTP response
299 | 	for key, value := range response.Header {
300 | 		if key == "Date" { // Ignore the Date header
301 | 			continue
302 | 		}
303 | 		header := core.Header{key, value[0]}
304 | 		domain.Headers = append(domain.Headers, header)
305 | 	}
306 | 
307 | 	// Update domain
308 | 	err = dbConn.Update(
309 | 		bson.M{"_id": domain.Id},
310 | 		domain,
311 | 	)
312 | 	check(err)
313 | 	logInfo("Updated domain info: " + domain.Name)
314 | 
315 | 	// Parse body for new domains
316 | 	domainsInDocument, err := getUniqueDomainsFromResponse(response)
317 | 	if err != nil {
318 | 		logInfo("Error reading response from: " + domain.Name)
319 | 	}
320 | 	logInfo("Domains found in " + domain.Name + ": " + strings.Join(domainsInDocument, ","))
321 | 
322 | 	// Add new domains to database if they don't already exist
323 | 	for _, foundDomainName := range domainsInDocument {
324 | 
325 | 		// See if domain exists
326 | 		// find count? if count > 0 or exists, move on
327 | 		var existingDomain core.Domain
328 | 		err = dbConn.Find(bson.M{"name": foundDomainName}).One(&existingDomain)
329 | 		if err == mgo.ErrNotFound {
330 | 			//logInfo("Domain not found. Adding: " + foundDomainName) // Just too verbose
331 | 			newDomain := &core.Domain{Name: foundDomainName, ParentDomain: domain.Id}
332 | 			err := dbConn.Insert(newDomain)
333 | 			if err != nil {
334 | 				logError("Error inserting!")
335 | 			}
336 | 		} else if err != nil {
337 | 			logError("Error when looking for existing domain: " + foundDomainName + ". " + err.Error())
338 | 		}
339 | 	}
340 | 
341 | 	doneChannel <- true
342 | 	return
343 | }
344 | 
345 | // Find unchecked domains (no headers)
346 | func getDomainsToCheck(limit int, dbConn *mgo.Collection) ([]core.Domain, error) {
347 | 	var uncheckedDomains []core.Domain
348 | 	err := dbConn.Find(
349 | 		bson.M{"headers": bson.M{"$exists": 0}, "skipped": bson.M{"$exists": 0}},
350 | 	).Limit(limit).All(&uncheckedDomains)
351 | 	return uncheckedDomains, err
352 | 
353 | }
354 | 
355 | func main() {
356 | 	usage := `worker_http - Web Genome HTTP Worker.
357 | 
358 | Usage:
359 |   worker_http --host=<host> --database=<dbname> --collection=<collectionname> --max-threads=<maxthreads> --http-timeout=<seconds> --batch-size=<batchsize> [--verbose]
360 |   worker_http -h | --help
361 |   worker_http --version
362 | 
363 | Options:
364 |   -h --help                   Show this screen.
365 |   --version                   Show version.
366 |   --host=<host>               MongoDB host
367 |   --database=<database>       MongoDB database name.
368 |   --collection=<collection>   MongoDB collection name.
369 |   --max-threads=<maxthreads>  Maximum number of simultaneous threads.
370 |   --http-timeout=<seconds>    How long before HTTP requests timeout in seconds.
371 |   --batch-size=<batchsize>    How many unchecked domains to pull and run per loop
372 |   --verbose                   Increase output verbosity.`
373 | 
374 | 	arguments, err := docopt.Parse(usage, nil, true, "Web Genome Worker", false)
375 | 	if err != nil {
376 | 		logError("Error parsing command line arguments. " + err.Error())
377 | 		os.Exit(1)
378 | 	}
379 | 	verbose = arguments["--verbose"].(bool) // Set global var for logging
380 | 	batchSize, err := strconv.Atoi(arguments["--batch-size"].(string))
381 | 	check(err)
382 | 
383 | 	logGreen("====== Options ======")
384 | 	logGreen("Host:         " + arguments["--host"].(string))
385 | 	logGreen("Database:     " + arguments["--database"].(string))
386 | 	logGreen("Collection:   " + arguments["--collection"].(string))
387 | 	logGreen("Max threads:  " + arguments["--max-threads"].(string))
388 | 	logGreen("HTTP timeout: " + arguments["--http-timeout"].(string) + " seconds")
389 | 	logGreen("Batch size:   " + strconv.Itoa(batchSize))
390 | 	logGreen("Verbose Mode: " + strconv.FormatBool(verbose))
391 | 	logGreen("=====================")
392 | 
393 | 	runtime.GOMAXPROCS(runtime.NumCPU())
394 | 	session, err := mgo.Dial(arguments["--host"].(string))
395 | 	check(err)
396 | 	defer session.Close()
397 | 
398 | 	dbConn := session.DB(arguments["--database"].(string)).C(arguments["--collection"].(string))
399 | 	timeout, err := strconv.Atoi(arguments["--http-timeout"].(string))
400 | 	check(err)
401 | 	httpTimeout := time.Duration(time.Duration(timeout) * time.Second)
402 | 	maxThreads, err := strconv.Atoi(arguments["--max-threads"].(string))
403 | 	check(err)
404 | 
405 | 	logGreen("Establishing connection with database.")
406 | 	logGreen("Database connection created.")
407 | 
408 | 	totalRuns := 0
409 | 	doneChannel := make(chan bool)
410 | 	numThreads := 0
411 | 	for true {
412 | 		startTime := time.Now()
413 | 		uncheckedDomains, err := getDomainsToCheck(batchSize, dbConn)
414 | 		check(err)
415 | 		if len(uncheckedDomains) == 0 {
416 | 			logError("No domains found to check. Exiting.")
417 | 			os.Exit(0)
418 | 		}
419 | 
420 | 		for x := 0; x < len(uncheckedDomains); x += 1 {
421 | 			numThreads += 1
422 | 			totalRuns += 1
423 | 
424 | 			logGreen("Checking " + uncheckedDomains[x].Name)
425 | 			go processDomain(uncheckedDomains[x], httpTimeout, doneChannel, dbConn)
426 | 
427 | 			// Wait until a done signal before next if max threads reached
428 | 			if numThreads >= maxThreads {
429 | 				<-doneChannel
430 | 				numThreads -= 1
431 | 			}
432 | 		}
433 | 
434 | 		// Wait for all threads before repeating and fetching a new batch
435 | 		for numThreads > 0 {
436 | 			<-doneChannel
437 | 			numThreads -= 1
438 | 		}
439 | 
440 | 		logInfo("All threads completed.")
441 | 		endTime := time.Now()
442 | 		runDuration := endTime.Sub(startTime)
443 | 
444 | 		logGreen("Completed " + strconv.Itoa(len(uncheckedDomains)) + " domains in " + strconv.FormatFloat(runDuration.Seconds(), 'f', 2, 64) + " seconds")
445 | 		logGreen("Total run count: " + strconv.Itoa(totalRuns))
446 | 	}
447 | 
448 | }
449 | 


--------------------------------------------------------------------------------
/website/website.go:
--------------------------------------------------------------------------------
  1 | package main
  2 | 
  3 | import (
  4 | 	"fmt"
  5 | 	"math/rand"
  6 | 	"net/http"
  7 | 	"strconv"
  8 | 	"time"
  9 | 
 10 | 	"github.com/DevDungeon/WebGenome/core"
 11 | 	"github.com/dustin/go-humanize"
 12 | 	"github.com/julienschmidt/httprouter"
 13 | 	"github.com/unrolled/render"
 14 | 	"github.com/urfave/negroni"
 15 | 	"gopkg.in/mgo.v2"
 16 | 	"gopkg.in/mgo.v2/bson"
 17 | )
 18 | 
 19 | var (
 20 | 	resultsPerPage int = 25
 21 | )
 22 | 
 23 | func getParentDomains(domain core.Domain, dbConn *mgo.Collection) []core.Domain {
 24 | 	var parentDomains []core.Domain
 25 | 	var tempDomain core.Domain
 26 | 	var emptyId bson.ObjectId
 27 | 
 28 | 	// End condition for recursion
 29 | 	if domain.ParentDomain == emptyId {
 30 | 		return parentDomains // Empty
 31 | 	}
 32 | 
 33 | 	getParentQuery := bson.M{"_id": domain.ParentDomain}
 34 | 	dbConn.Find(getParentQuery).One(&tempDomain)
 35 | 
 36 | 	// Return this domain as the last parent or recurse deeper
 37 | 	if tempDomain.ParentDomain == emptyId {
 38 | 		parentDomains = append(parentDomains, tempDomain)
 39 | 	} else {
 40 | 		parentDomains = append(parentDomains, tempDomain)
 41 | 		parentDomains = append(parentDomains, getParentDomains(tempDomain, dbConn)...)
 42 | 	}
 43 | 	return parentDomains
 44 | }
 45 | 
 46 | func viewDomain(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
 47 | 	renderer := render.New(render.Options{
 48 | 		Layout: "layout",
 49 | 	})
 50 | 	var domain core.Domain
 51 | 	var parentDomains []core.Domain
 52 | 
 53 | 	query := bson.M{"_id": bson.ObjectIdHex(ps.ByName("id"))}
 54 | 
 55 | 	session, _ := mgo.Dial("localhost")
 56 | 	defer session.Close()
 57 | 	dbConn := session.DB("webgenome").C("domains")
 58 | 	dbConn.Find(query).One(&domain)
 59 | 
 60 | 	// Get all parents
 61 | 	parentDomains = getParentDomains(domain, dbConn)
 62 | 
 63 | 	vars := map[string]interface{}{
 64 | 		"title":         "View Domain",
 65 | 		"domain":        domain,
 66 | 		"parentDomains": parentDomains,
 67 | 	}
 68 | 
 69 | 	renderer.HTML(w, http.StatusOK, "view_domain", vars)
 70 | }
 71 | 
 72 | //func (w http.ResponseWriter, r *http.Request, p httprouter.Params) {
 73 | //	query := bson.M{"headers": bson.M{"$elemMatch": bson.M{"value": bson.RegEx{"", ""}}}}
 74 | //	renderDomainList(w, r, p, query, "")
 75 | //}
 76 | 
 77 | func drupal(w http.ResponseWriter, r *http.Request, p httprouter.Params) {
 78 | 	query := bson.M{"headers": bson.M{"$elemMatch": bson.M{"value": bson.RegEx{"Drupal", ""}}}}
 79 | 	renderDomainListFromQuery(w, r, p, query, "Drupal Sites")
 80 | }
 81 | 
 82 | func django(w http.ResponseWriter, r *http.Request, p httprouter.Params) {
 83 | 	query := bson.M{"headers": bson.M{"$elemMatch": bson.M{"value": bson.RegEx{"^django_language", ""}}}}
 84 | 	renderDomainListFromQuery(w, r, p, query, "Django Sites")
 85 | }
 86 | 
 87 | func zope(w http.ResponseWriter, r *http.Request, p httprouter.Params) {
 88 | 	query := bson.M{"headers": bson.M{"$elemMatch": bson.M{"value": bson.RegEx{"^Zope/", "i"}}}}
 89 | 	renderDomainListFromQuery(w, r, p, query, "Zope Sites")
 90 | }
 91 | 
 92 | func php(w http.ResponseWriter, r *http.Request, p httprouter.Params) {
 93 | 	query := bson.M{"headers": bson.M{"$elemMatch": bson.M{"value": bson.RegEx{"^PHP/", ""}}}}
 94 | 	renderDomainListFromQuery(w, r, p, query, "PHP Sites")
 95 | }
 96 | 
 97 | func java(w http.ResponseWriter, r *http.Request, p httprouter.Params) {
 98 | 	query := bson.M{"headers": bson.M{"$elemMatch": bson.M{"value": bson.RegEx{"^JSESSIONID=", ""}}}}
 99 | 	renderDomainListFromQuery(w, r, p, query, "Java Sites")
100 | }
101 | 
102 | func aspdotnet(w http.ResponseWriter, r *http.Request, p httprouter.Params) {
103 | 	query := bson.M{"headers": bson.M{"$elemMatch": bson.M{"value": bson.RegEx{"^ASP.NET_SessionId=", ""}}}}
104 | 	renderDomainListFromQuery(w, r, p, query, "ASP.NET Sites")
105 | }
106 | 
107 | func python(w http.ResponseWriter, r *http.Request, p httprouter.Params) {
108 | 	query := bson.M{"headers": bson.M{"$elemMatch": bson.M{"value": bson.RegEx{"Python/", ""}}}}
109 | 	renderDomainListFromQuery(w, r, p, query, "Python Sites")
110 | }
111 | 
112 | func ruby(w http.ResponseWriter, r *http.Request, p httprouter.Params) {
113 | 	query := bson.M{"headers": bson.M{"$elemMatch": bson.M{"value": bson.RegEx{"Ruby/", ""}}}}
114 | 	renderDomainListFromQuery(w, r, p, query, "Ruby Sites")
115 | }
116 | 
117 | func apache(w http.ResponseWriter, r *http.Request, p httprouter.Params) {
118 | 	query := bson.M{"headers": bson.M{"$elemMatch": bson.M{"value": bson.RegEx{"^Apache", ""}}}}
119 | 	renderDomainListFromQuery(w, r, p, query, "Apache Sites")
120 | }
121 | 
122 | func nginx(w http.ResponseWriter, r *http.Request, p httprouter.Params) {
123 | 	query := bson.M{"headers": bson.M{"$elemMatch": bson.M{"value": bson.RegEx{"^nginx", ""}}}}
124 | 	renderDomainListFromQuery(w, r, p, query, "Nginx Sites")
125 | }
126 | 
127 | func iis(w http.ResponseWriter, r *http.Request, p httprouter.Params) {
128 | 	query := bson.M{"headers": bson.M{"$elemMatch": bson.M{"value": bson.RegEx{"^Microsoft-IIS", ""}}}}
129 | 	renderDomainListFromQuery(w, r, p, query, "IIS Sites")
130 | }
131 | 
132 | func tomcat(w http.ResponseWriter, r *http.Request, p httprouter.Params) {
133 | 	query := bson.M{"headers": bson.M{"$elemMatch": bson.M{"value": bson.RegEx{"tomcat", ""}}}}
134 | 	renderDomainListFromQuery(w, r, p, query, "Tomcat Sites")
135 | }
136 | 
137 | func webrick(w http.ResponseWriter, r *http.Request, p httprouter.Params) {
138 | 	query := bson.M{"headers": bson.M{"$elemMatch": bson.M{"value": bson.RegEx{"^WEBrick", ""}}}}
139 | 	renderDomainListFromQuery(w, r, p, query, "WEBrick Sites")
140 | }
141 | 
142 | func lighttpd(w http.ResponseWriter, r *http.Request, p httprouter.Params) {
143 | 	query := bson.M{"headers": bson.M{"$elemMatch": bson.M{"value": bson.RegEx{"^lighttpd", ""}}}}
144 | 	renderDomainListFromQuery(w, r, p, query, "Lighttpd Sites")
145 | }
146 | 
147 | func ibmhttpserver(w http.ResponseWriter, r *http.Request, p httprouter.Params) {
148 | 	query := bson.M{"headers": bson.M{"$elemMatch": bson.M{"value": bson.RegEx{"^IBM_HTTP_Server", ""}}}}
149 | 	renderDomainListFromQuery(w, r, p, query, "IBM HTTP Server")
150 | }
151 | 
152 | func apusic(w http.ResponseWriter, r *http.Request, p httprouter.Params) {
153 | 	query := bson.M{"headers": bson.M{"$elemMatch": bson.M{"value": bson.RegEx{"Apusic", ""}}}}
154 | 	renderDomainListFromQuery(w, r, p, query, "Apusic Sites")
155 | }
156 | 
157 | func enhydra(w http.ResponseWriter, r *http.Request, p httprouter.Params) {
158 | 	query := bson.M{"headers": bson.M{"$elemMatch": bson.M{"value": bson.RegEx{"Enhydra", ""}}}}
159 | 	renderDomainListFromQuery(w, r, p, query, "Enhydra Sites")
160 | }
161 | 
162 | func jetty(w http.ResponseWriter, r *http.Request, p httprouter.Params) {
163 | 	query := bson.M{"headers": bson.M{"$elemMatch": bson.M{"value": bson.RegEx{"Jetty", ""}}}}
164 | 	renderDomainListFromQuery(w, r, p, query, "Jetty Sites")
165 | }
166 | 
167 | func unix(w http.ResponseWriter, r *http.Request, p httprouter.Params) {
168 | 	query := bson.M{"headers": bson.M{"$elemMatch": bson.M{"value": bson.RegEx{"(Unix)", ""}}}}
169 | 	renderDomainListFromQuery(w, r, p, query, "Unix Sites")
170 | }
171 | 
172 | func linux(w http.ResponseWriter, r *http.Request, p httprouter.Params) {
173 | 	query := bson.M{"headers": bson.M{"$elemMatch": bson.M{"value": bson.RegEx{"Linux", ""}}}}
174 | 	renderDomainListFromQuery(w, r, p, query, "Linux Sites")
175 | }
176 | 
177 | func debian(w http.ResponseWriter, r *http.Request, p httprouter.Params) {
178 | 	query := bson.M{"headers": bson.M{"$elemMatch": bson.M{"value": bson.RegEx{"Debian", ""}}}}
179 | 	renderDomainListFromQuery(w, r, p, query, "Debian sites")
180 | }
181 | 
182 | func fedora(w http.ResponseWriter, r *http.Request, p httprouter.Params) {
183 | 	query := bson.M{"headers": bson.M{"$elemMatch": bson.M{"value": bson.RegEx{"Fedora", ""}}}}
184 | 	renderDomainListFromQuery(w, r, p, query, "Fedora Sites")
185 | }
186 | 
187 | func redhat(w http.ResponseWriter, r *http.Request, p httprouter.Params) {
188 | 	query := bson.M{"headers": bson.M{"$elemMatch": bson.M{"value": bson.RegEx{"Red Hat", ""}}}}
189 | 	renderDomainListFromQuery(w, r, p, query, "Red Hat Sites")
190 | }
191 | 
192 | func centos(w http.ResponseWriter, r *http.Request, p httprouter.Params) {
193 | 	query := bson.M{"headers": bson.M{"$elemMatch": bson.M{"value": bson.RegEx{"CentOS", ""}}}}
194 | 	renderDomainListFromQuery(w, r, p, query, "CentOS Sites")
195 | }
196 | 
197 | func ubuntu(w http.ResponseWriter, r *http.Request, p httprouter.Params) {
198 | 	query := bson.M{"headers": bson.M{"$elemMatch": bson.M{"value": bson.RegEx{"Ubuntu", ""}}}}
199 | 	renderDomainListFromQuery(w, r, p, query, "Ubuntu Sites")
200 | }
201 | 
202 | func freebsd(w http.ResponseWriter, r *http.Request, p httprouter.Params) {
203 | 	query := bson.M{"headers": bson.M{"$elemMatch": bson.M{"value": bson.RegEx{"FreeBSD", ""}}}}
204 | 	renderDomainListFromQuery(w, r, p, query, "FreeBSD Sites")
205 | }
206 | 
207 | func win32(w http.ResponseWriter, r *http.Request, p httprouter.Params) {
208 | 	query := bson.M{"headers": bson.M{"$elemMatch": bson.M{"value": bson.RegEx{"Win32", ""}}}}
209 | 	renderDomainListFromQuery(w, r, p, query, "Win32 Sites")
210 | }
211 | 
212 | func win64(w http.ResponseWriter, r *http.Request, p httprouter.Params) {
213 | 	query := bson.M{"headers": bson.M{"$elemMatch": bson.M{"value": bson.RegEx{"Win64", ""}}}}
214 | 	renderDomainListFromQuery(w, r, p, query, "Win64 Sites")
215 | }
216 | 
217 | func darwin(w http.ResponseWriter, r *http.Request, p httprouter.Params) {
218 | 	query := bson.M{"headers": bson.M{"$elemMatch": bson.M{"value": bson.RegEx{"Darwin", ""}}}}
219 | 	renderDomainListFromQuery(w, r, p, query, "Darwin Sites")
220 | }
221 | 
222 | func phusionpassenger(w http.ResponseWriter, r *http.Request, p httprouter.Params) {
223 | 	query := bson.M{"headers": bson.M{"$elemMatch": bson.M{"value": bson.RegEx{"Phusion_Passenger", ""}}}}
224 | 	renderDomainListFromQuery(w, r, p, query, "Phusion Passenger Sites")
225 | }
226 | 
227 | func openssl(w http.ResponseWriter, r *http.Request, p httprouter.Params) {
228 | 	query := bson.M{"headers": bson.M{"$elemMatch": bson.M{"value": bson.RegEx{"OpenSSL", ""}}}}
229 | 	renderDomainListFromQuery(w, r, p, query, "OpenSSL Sites")
230 | }
231 | 
232 | func webdav(w http.ResponseWriter, r *http.Request, p httprouter.Params) {
233 | 	query := bson.M{"headers": bson.M{"$elemMatch": bson.M{"value": bson.RegEx{"DAV", ""}}}}
234 | 	renderDomainListFromQuery(w, r, p, query, "WebDAV Sites")
235 | }
236 | 
237 | func communique(w http.ResponseWriter, r *http.Request, p httprouter.Params) {
238 | 	query := bson.M{"headers": bson.M{"$elemMatch": bson.M{"value": bson.RegEx{"Communique", ""}}}}
239 | 	renderDomainListFromQuery(w, r, p, query, "Communique Sites")
240 | }
241 | 
242 | func bigipserver(w http.ResponseWriter, r *http.Request, p httprouter.Params) {
243 | 	query := bson.M{"headers": bson.M{"$elemMatch": bson.M{"value": bson.RegEx{"BIGipServer", ""}}}}
244 | 	renderDomainListFromQuery(w, r, p, query, "BIGipServer Sites")
245 | }
246 | 
247 | func gov(w http.ResponseWriter, r *http.Request, p httprouter.Params) {
248 | 	query := bson.M{"name": bson.RegEx{".gov", ""}}
249 | 	renderDomainListFromQuery(w, r, p, query, "Government Sites")
250 | }
251 | 
252 | func renderDomainListFromQuery(w http.ResponseWriter, r *http.Request, _ httprouter.Params, query bson.M, title string) {
253 | 
254 | 	var (
255 | 		page         string
256 | 		pageNumber   int
257 | 		previousPage string
258 | 		nextPage     string
259 | 	)
260 | 	page = r.URL.Query().Get("page")
261 | 	if page == "" {
262 | 		pageNumber = 1
263 | 	} else {
264 | 		pageNumber, _ = strconv.Atoi(page)
265 | 	}
266 | 
267 | 	var domains []core.Domain
268 | 	session, _ := mgo.Dial("localhost")
269 | 	defer session.Close()
270 | 	dbConn := session.DB("webgenome").C("domains")
271 | 	dbConn.Find(query).Limit(resultsPerPage).Skip((pageNumber - 1) * resultsPerPage).All(&domains)
272 | 
273 | 	if pageNumber <= 1 {
274 | 		previousPage = ""
275 | 	} else {
276 | 		previousPage = r.URL.Path + "?page=" + strconv.Itoa(pageNumber-1)
277 | 	}
278 | 	if len(domains) < resultsPerPage {
279 | 		nextPage = ""
280 | 	} else {
281 | 		nextPage = r.URL.Path + "?page=" + strconv.Itoa(pageNumber+1)
282 | 	}
283 | 
284 | 	vars := map[string]interface{}{
285 | 		"title":        title,
286 | 		"domains":      domains,
287 | 		"pageNumber":   pageNumber,
288 | 		"previousPage": previousPage,
289 | 		"nextPage":     nextPage,
290 | 	}
291 | 	renderer := render.New(render.Options{
292 | 		Layout: "layout",
293 | 	})
294 | 	renderer.HTML(w, http.StatusOK, "domain_listing", vars)
295 | }
296 | 
297 | func index(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
298 | 	session, _ := mgo.Dial("localhost")
299 | 	defer session.Close()
300 | 	dbConn := session.DB("webgenome").C("domains")
301 | 	totalDomains, err := dbConn.Count()
302 | 	if err != nil {
303 | 		fmt.Println("Error getting total domain count." + err.Error())
304 | 	}
305 | 
306 | 	vars := map[string]interface{}{
307 | 		"title":        "Home",
308 | 		"totalDomains": humanize.Comma(int64(totalDomains)),
309 | 	}
310 | 
311 | 	renderer := render.New(render.Options{
312 | 		Layout: "layout",
313 | 	})
314 | 	renderer.HTML(w, http.StatusOK, "index", vars)
315 | }
316 | 
317 | func random(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
318 | 	session, _ := mgo.Dial("localhost")
319 | 	defer session.Close()
320 | 	dbConn := session.DB("webgenome").C("domains")
321 | 	totalDomains, err := dbConn.Count()
322 | 	if err != nil {
323 | 		fmt.Println("Error getting total domain count." + err.Error())
324 | 	}
325 | 
326 | 	query := bson.M{"headers": bson.M{"$exists": true}}
327 | 	var domain core.Domain
328 | 	numDomainsToSkip := rand.Intn(totalDomains) % 1000000 // Hard limit of 5 mill for speed
329 | 	err = dbConn.Find(query).Limit(1).Skip(numDomainsToSkip).One(&domain)
330 | 	if err != nil {
331 | 		fmt.Println("Random lookup went too far. Redirecting back to random.")
332 | 		http.Redirect(w, r, "/random", http.StatusFound)
333 | 		return
334 | 	}
335 | 
336 | 	http.Redirect(w, r, "/domain/"+domain.Id.Hex(), http.StatusFound)
337 | 	return
338 | }
339 | 
340 | func search(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
341 | 	var (
342 | 		pageNumber int
343 | 		nextPage   string
344 | 		domains    []core.Domain
345 | 	)
346 | 	session, _ := mgo.Dial("localhost")
347 | 	defer session.Close()
348 | 	dbConn := session.DB("webgenome").C("domains")
349 | 
350 | 	domainKeyword := r.PostFormValue("domain-keyword")
351 | 	fmt.Println("Search query: " + domainKeyword)
352 | 	query := bson.M{"name": bson.RegEx{domainKeyword, "i"}}
353 | 
354 | 	err := dbConn.Find(query).All(&domains)
355 | 
356 | 	if err != nil {
357 | 		fmt.Println("Error with search.")
358 | 		http.Redirect(w, r, "/", http.StatusFound)
359 | 		return
360 | 	}
361 | 
362 | 	pageNumber = 1
363 | 
364 | 	dbConn.Find(query).Limit(resultsPerPage).All(&domains)
365 | 
366 | 	if len(domains) < resultsPerPage {
367 | 		nextPage = ""
368 | 	} else {
369 | 		nextPage = "#"
370 | 	}
371 | 
372 | 	vars := map[string]interface{}{
373 | 		"title":      "Search Results",
374 | 		"domains":    domains,
375 | 		"pageNumber": pageNumber,
376 | 		"nextPage":   nextPage,
377 | 	}
378 | 	renderer := render.New(render.Options{
379 | 		Layout: "layout",
380 | 	})
381 | 	renderer.HTML(w, http.StatusOK, "search_results", vars)
382 | 
383 | }
384 | 
385 | func loggerMiddleware(rw http.ResponseWriter, r *http.Request, next http.HandlerFunc) {
386 | 	fmt.Println("[*] " + time.Now().Format(time.RFC850) + " - " + r.Header.Get("X-Real-IP"))
387 | 	// do some IP logging
388 | 	next(rw, r)
389 | 	// Do some stuff after
390 | }
391 | 
392 | func premium(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
393 | 
394 | 	vars := map[string]interface{}{}
395 | 
396 | 	renderer := render.New(render.Options{
397 | 		Layout: "layout",
398 | 	})
399 | 	renderer.HTML(w, http.StatusOK, "premium", vars)
400 | }
401 | 
402 | func main() {
403 | 	staticFilesDir := "./static/"
404 | 
405 | 	// Routing
406 | 	router := httprouter.New()
407 | 	router.GET("/", index)
408 | 	router.POST("/", search)
409 | 	router.GET("/domain/:id", viewDomain)
410 | 	router.GET("/random", random)
411 | 	router.GET("/premium", premium)
412 | 
413 | 	router.GET("/gov", gov)
414 | 	router.GET("/drupal", drupal)
415 | 	router.GET("/django", django)
416 | 	router.GET("/zope", zope)
417 | 	router.GET("/php", php)
418 | 	router.GET("/java", java)
419 | 	router.GET("/aspdotnet", aspdotnet)
420 | 	router.GET("/python", python)
421 | 	router.GET("/ruby", ruby)
422 | 	router.GET("/apache", apache)
423 | 	router.GET("/nginx", nginx)
424 | 	router.GET("/iis", iis)
425 | 	router.GET("/tomcat", tomcat)
426 | 	router.GET("/webrick", webrick)
427 | 	router.GET("/lighttpd", lighttpd)
428 | 	router.GET("/ibmhttpserver", ibmhttpserver)
429 | 	router.GET("/apusic", apusic)
430 | 	router.GET("/enhydra", enhydra)
431 | 	router.GET("/jetty", jetty)
432 | 	router.GET("/unix", unix)
433 | 	router.GET("/linux", linux)
434 | 	router.GET("/debian", debian)
435 | 	router.GET("/fedora", fedora)
436 | 	router.GET("/redhat", redhat)
437 | 	router.GET("/centos", centos)
438 | 	router.GET("/ubuntu", ubuntu)
439 | 	router.GET("/freebsd", freebsd)
440 | 	router.GET("/win32", win32)
441 | 	router.GET("/win64", win64)
442 | 	router.GET("/darwin", darwin)
443 | 	router.GET("/phusionpassenger", phusionpassenger)
444 | 	router.GET("/openssl", openssl)
445 | 	router.GET("/webdav", webdav)
446 | 	router.GET("/communique", communique)
447 | 	router.GET("/bigipserver", bigipserver)
448 | 
449 | 	// Unchecked sites
450 | 	// Checked sites
451 | 	// Skipped sites
452 | 	// Total domains
453 | 
454 | 	// Middleware
455 | 	n := negroni.New()
456 | 	n.Use(negroni.NewRecovery())
457 | 	n.Use(negroni.NewLogger())
458 | 	n.Use(negroni.NewStatic(http.Dir(staticFilesDir)))
459 | 	n.Use(negroni.HandlerFunc(loggerMiddleware))
460 | 
461 | 	n.UseHandler(router)
462 | 	n.Run(":3000")
463 | 
464 | }
465 | 


--------------------------------------------------------------------------------
/LICENSE.txt:
--------------------------------------------------------------------------------
  1 |                     GNU GENERAL PUBLIC LICENSE
  2 |                        Version 2, June 1991
  3 | 
  4 |  Copyright (C) 1989, 1991 Free Software Foundation, Inc.,
  5 |  51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
  6 |  Everyone is permitted to copy and distribute verbatim copies
  7 |  of this license document, but changing it is not allowed.
  8 | 
  9 |                             Preamble
 10 | 
 11 |   The licenses for most software are designed to take away your
 12 | freedom to share and change it.  By contrast, the GNU General Public
 13 | License is intended to guarantee your freedom to share and change free
 14 | software--to make sure the software is free for all its users.  This
 15 | General Public License applies to most of the Free Software
 16 | Foundation's software and to any other program whose authors commit to
 17 | using it.  (Some other Free Software Foundation software is covered by
 18 | the GNU Lesser General Public License instead.)  You can apply it to
 19 | your programs, too.
 20 | 
 21 |   When we speak of free software, we are referring to freedom, not
 22 | price.  Our General Public Licenses are designed to make sure that you
 23 | have the freedom to distribute copies of free software (and charge for
 24 | this service if you wish), that you receive source code or can get it
 25 | if you want it, that you can change the software or use pieces of it
 26 | in new free programs; and that you know you can do these things.
 27 | 
 28 |   To protect your rights, we need to make restrictions that forbid
 29 | anyone to deny you these rights or to ask you to surrender the rights.
 30 | These restrictions translate to certain responsibilities for you if you
 31 | distribute copies of the software, or if you modify it.
 32 | 
 33 |   For example, if you distribute copies of such a program, whether
 34 | gratis or for a fee, you must give the recipients all the rights that
 35 | you have.  You must make sure that they, too, receive or can get the
 36 | source code.  And you must show them these terms so they know their
 37 | rights.
 38 | 
 39 |   We protect your rights with two steps: (1) copyright the software, and
 40 | (2) offer you this license which gives you legal permission to copy,
 41 | distribute and/or modify the software.
 42 | 
 43 |   Also, for each author's protection and ours, we want to make certain
 44 | that everyone understands that there is no warranty for this free
 45 | software.  If the software is modified by someone else and passed on, we
 46 | want its recipients to know that what they have is not the original, so
 47 | that any problems introduced by others will not reflect on the original
 48 | authors' reputations.
 49 | 
 50 |   Finally, any free program is threatened constantly by software
 51 | patents.  We wish to avoid the danger that redistributors of a free
 52 | program will individually obtain patent licenses, in effect making the
 53 | program proprietary.  To prevent this, we have made it clear that any
 54 | patent must be licensed for everyone's free use or not licensed at all.
 55 | 
 56 |   The precise terms and conditions for copying, distribution and
 57 | modification follow.
 58 | 
 59 |                     GNU GENERAL PUBLIC LICENSE
 60 |    TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
 61 | 
 62 |   0. This License applies to any program or other work which contains
 63 | a notice placed by the copyright holder saying it may be distributed
 64 | under the terms of this General Public License.  The "Program", below,
 65 | refers to any such program or work, and a "work based on the Program"
 66 | means either the Program or any derivative work under copyright law:
 67 | that is to say, a work containing the Program or a portion of it,
 68 | either verbatim or with modifications and/or translated into another
 69 | language.  (Hereinafter, translation is included without limitation in
 70 | the term "modification".)  Each licensee is addressed as "you".
 71 | 
 72 | Activities other than copying, distribution and modification are not
 73 | covered by this License; they are outside its scope.  The act of
 74 | running the Program is not restricted, and the output from the Program
 75 | is covered only if its contents constitute a work based on the
 76 | Program (independent of having been made by running the Program).
 77 | Whether that is true depends on what the Program does.
 78 | 
 79 |   1. You may copy and distribute verbatim copies of the Program's
 80 | source code as you receive it, in any medium, provided that you
 81 | conspicuously and appropriately publish on each copy an appropriate
 82 | copyright notice and disclaimer of warranty; keep intact all the
 83 | notices that refer to this License and to the absence of any warranty;
 84 | and give any other recipients of the Program a copy of this License
 85 | along with the Program.
 86 | 
 87 | You may charge a fee for the physical act of transferring a copy, and
 88 | you may at your option offer warranty protection in exchange for a fee.
 89 | 
 90 |   2. You may modify your copy or copies of the Program or any portion
 91 | of it, thus forming a work based on the Program, and copy and
 92 | distribute such modifications or work under the terms of Section 1
 93 | above, provided that you also meet all of these conditions:
 94 | 
 95 |     a) You must cause the modified files to carry prominent notices
 96 |     stating that you changed the files and the date of any change.
 97 | 
 98 |     b) You must cause any work that you distribute or publish, that in
 99 |     whole or in part contains or is derived from the Program or any
100 |     part thereof, to be licensed as a whole at no charge to all third
101 |     parties under the terms of this License.
102 | 
103 |     c) If the modified program normally reads commands interactively
104 |     when run, you must cause it, when started running for such
105 |     interactive use in the most ordinary way, to print or display an
106 |     announcement including an appropriate copyright notice and a
107 |     notice that there is no warranty (or else, saying that you provide
108 |     a warranty) and that users may redistribute the program under
109 |     these conditions, and telling the user how to view a copy of this
110 |     License.  (Exception: if the Program itself is interactive but
111 |     does not normally print such an announcement, your work based on
112 |     the Program is not required to print an announcement.)
113 | 
114 | These requirements apply to the modified work as a whole.  If
115 | identifiable sections of that work are not derived from the Program,
116 | and can be reasonably considered independent and separate works in
117 | themselves, then this License, and its terms, do not apply to those
118 | sections when you distribute them as separate works.  But when you
119 | distribute the same sections as part of a whole which is a work based
120 | on the Program, the distribution of the whole must be on the terms of
121 | this License, whose permissions for other licensees extend to the
122 | entire whole, and thus to each and every part regardless of who wrote it.
123 | 
124 | Thus, it is not the intent of this section to claim rights or contest
125 | your rights to work written entirely by you; rather, the intent is to
126 | exercise the right to control the distribution of derivative or
127 | collective works based on the Program.
128 | 
129 | In addition, mere aggregation of another work not based on the Program
130 | with the Program (or with a work based on the Program) on a volume of
131 | a storage or distribution medium does not bring the other work under
132 | the scope of this License.
133 | 
134 |   3. You may copy and distribute the Program (or a work based on it,
135 | under Section 2) in object code or executable form under the terms of
136 | Sections 1 and 2 above provided that you also do one of the following:
137 | 
138 |     a) Accompany it with the complete corresponding machine-readable
139 |     source code, which must be distributed under the terms of Sections
140 |     1 and 2 above on a medium customarily used for software interchange; or,
141 | 
142 |     b) Accompany it with a written offer, valid for at least three
143 |     years, to give any third party, for a charge no more than your
144 |     cost of physically performing source distribution, a complete
145 |     machine-readable copy of the corresponding source code, to be
146 |     distributed under the terms of Sections 1 and 2 above on a medium
147 |     customarily used for software interchange; or,
148 | 
149 |     c) Accompany it with the information you received as to the offer
150 |     to distribute corresponding source code.  (This alternative is
151 |     allowed only for noncommercial distribution and only if you
152 |     received the program in object code or executable form with such
153 |     an offer, in accord with Subsection b above.)
154 | 
155 | The source code for a work means the preferred form of the work for
156 | making modifications to it.  For an executable work, complete source
157 | code means all the source code for all modules it contains, plus any
158 | associated interface definition files, plus the scripts used to
159 | control compilation and installation of the executable.  However, as a
160 | special exception, the source code distributed need not include
161 | anything that is normally distributed (in either source or binary
162 | form) with the major components (compiler, kernel, and so on) of the
163 | operating system on which the executable runs, unless that component
164 | itself accompanies the executable.
165 | 
166 | If distribution of executable or object code is made by offering
167 | access to copy from a designated place, then offering equivalent
168 | access to copy the source code from the same place counts as
169 | distribution of the source code, even though third parties are not
170 | compelled to copy the source along with the object code.
171 | 
172 |   4. You may not copy, modify, sublicense, or distribute the Program
173 | except as expressly provided under this License.  Any attempt
174 | otherwise to copy, modify, sublicense or distribute the Program is
175 | void, and will automatically terminate your rights under this License.
176 | However, parties who have received copies, or rights, from you under
177 | this License will not have their licenses terminated so long as such
178 | parties remain in full compliance.
179 | 
180 |   5. You are not required to accept this License, since you have not
181 | signed it.  However, nothing else grants you permission to modify or
182 | distribute the Program or its derivative works.  These actions are
183 | prohibited by law if you do not accept this License.  Therefore, by
184 | modifying or distributing the Program (or any work based on the
185 | Program), you indicate your acceptance of this License to do so, and
186 | all its terms and conditions for copying, distributing or modifying
187 | the Program or works based on it.
188 | 
189 |   6. Each time you redistribute the Program (or any work based on the
190 | Program), the recipient automatically receives a license from the
191 | original licensor to copy, distribute or modify the Program subject to
192 | these terms and conditions.  You may not impose any further
193 | restrictions on the recipients' exercise of the rights granted herein.
194 | You are not responsible for enforcing compliance by third parties to
195 | this License.
196 | 
197 |   7. If, as a consequence of a court judgment or allegation of patent
198 | infringement or for any other reason (not limited to patent issues),
199 | conditions are imposed on you (whether by court order, agreement or
200 | otherwise) that contradict the conditions of this License, they do not
201 | excuse you from the conditions of this License.  If you cannot
202 | distribute so as to satisfy simultaneously your obligations under this
203 | License and any other pertinent obligations, then as a consequence you
204 | may not distribute the Program at all.  For example, if a patent
205 | license would not permit royalty-free redistribution of the Program by
206 | all those who receive copies directly or indirectly through you, then
207 | the only way you could satisfy both it and this License would be to
208 | refrain entirely from distribution of the Program.
209 | 
210 | If any portion of this section is held invalid or unenforceable under
211 | any particular circumstance, the balance of the section is intended to
212 | apply and the section as a whole is intended to apply in other
213 | circumstances.
214 | 
215 | It is not the purpose of this section to induce you to infringe any
216 | patents or other property right claims or to contest validity of any
217 | such claims; this section has the sole purpose of protecting the
218 | integrity of the free software distribution system, which is
219 | implemented by public license practices.  Many people have made
220 | generous contributions to the wide range of software distributed
221 | through that system in reliance on consistent application of that
222 | system; it is up to the author/donor to decide if he or she is willing
223 | to distribute software through any other system and a licensee cannot
224 | impose that choice.
225 | 
226 | This section is intended to make thoroughly clear what is believed to
227 | be a consequence of the rest of this License.
228 | 
229 |   8. If the distribution and/or use of the Program is restricted in
230 | certain countries either by patents or by copyrighted interfaces, the
231 | original copyright holder who places the Program under this License
232 | may add an explicit geographical distribution limitation excluding
233 | those countries, so that distribution is permitted only in or among
234 | countries not thus excluded.  In such case, this License incorporates
235 | the limitation as if written in the body of this License.
236 | 
237 |   9. The Free Software Foundation may publish revised and/or new versions
238 | of the General Public License from time to time.  Such new versions will
239 | be similar in spirit to the present version, but may differ in detail to
240 | address new problems or concerns.
241 | 
242 | Each version is given a distinguishing version number.  If the Program
243 | specifies a version number of this License which applies to it and "any
244 | later version", you have the option of following the terms and conditions
245 | either of that version or of any later version published by the Free
246 | Software Foundation.  If the Program does not specify a version number of
247 | this License, you may choose any version ever published by the Free Software
248 | Foundation.
249 | 
250 |   10. If you wish to incorporate parts of the Program into other free
251 | programs whose distribution conditions are different, write to the author
252 | to ask for permission.  For software which is copyrighted by the Free
253 | Software Foundation, write to the Free Software Foundation; we sometimes
254 | make exceptions for this.  Our decision will be guided by the two goals
255 | of preserving the free status of all derivatives of our free software and
256 | of promoting the sharing and reuse of software generally.
257 | 
258 |                             NO WARRANTY
259 | 
260 |   11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY
261 | FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW.  EXCEPT WHEN
262 | OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES
263 | PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED
264 | OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
265 | MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE.  THE ENTIRE RISK AS
266 | TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU.  SHOULD THE
267 | PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING,
268 | REPAIR OR CORRECTION.
269 | 
270 |   12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
271 | WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR
272 | REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES,
273 | INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING
274 | OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED
275 | TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY
276 | YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER
277 | PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE
278 | POSSIBILITY OF SUCH DAMAGES.
279 | 
280 |                      END OF TERMS AND CONDITIONS
281 | 
282 |             How to Apply These Terms to Your New Programs
283 | 
284 |   If you develop a new program, and you want it to be of the greatest
285 | possible use to the public, the best way to achieve this is to make it
286 | free software which everyone can redistribute and change under these terms.
287 | 
288 |   To do so, attach the following notices to the program.  It is safest
289 | to attach them to the start of each source file to most effectively
290 | convey the exclusion of warranty; and each file should have at least
291 | the "copyright" line and a pointer to where the full notice is found.
292 | 
293 |     <one line to give the program's name and a brief idea of what it does.>
294 |     Copyright (C) <year>  <name of author>
295 | 
296 |     This program is free software; you can redistribute it and/or modify
297 |     it under the terms of the GNU General Public License as published by
298 |     the Free Software Foundation; either version 2 of the License, or
299 |     (at your option) any later version.
300 | 
301 |     This program is distributed in the hope that it will be useful,
302 |     but WITHOUT ANY WARRANTY; without even the implied warranty of
303 |     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
304 |     GNU General Public License for more details.
305 | 
306 |     You should have received a copy of the GNU General Public License along
307 |     with this program; if not, write to the Free Software Foundation, Inc.,
308 |     51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
309 | 
310 | Also add information on how to contact you by electronic and paper mail.
311 | 
312 | If the program is interactive, make it output a short notice like this
313 | when it starts in an interactive mode:
314 | 
315 |     Gnomovision version 69, Copyright (C) year name of author
316 |     Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'.
317 |     This is free software, and you are welcome to redistribute it
318 |     under certain conditions; type `show c' for details.
319 | 
320 | The hypothetical commands `show w' and `show c' should show the appropriate
321 | parts of the General Public License.  Of course, the commands you use may
322 | be called something other than `show w' and `show c'; they could even be
323 | mouse-clicks or menu items--whatever suits your program.
324 | 
325 | You should also get your employer (if you work as a programmer) or your
326 | school, if any, to sign a "copyright disclaimer" for the program, if
327 | necessary.  Here is a sample; alter the names:
328 | 
329 |   Yoyodyne, Inc., hereby disclaims all copyright interest in the program
330 |   `Gnomovision' (which makes passes at compilers) written by James Hacker.
331 | 
332 |   <signature of Ty Coon>, 1 April 1989
333 |   Ty Coon, President of Vice
334 | 
335 | This General Public License does not permit incorporating your program into
336 | proprietary programs.  If your program is a subroutine library, you may
337 | consider it more useful to permit linking proprietary applications with the
338 | library.  If this is what you want to do, use the GNU Lesser General
339 | Public License instead of this License.


--------------------------------------------------------------------------------