", methods=['GET'])
162 | def check_valid(job_key):
163 | if not job_key or not queue.exists(job_key):
164 | return Response(json.dumps(None), mimetype='application/json'), 200
165 | if not queue.hexists(job_key, 'result'):
166 | return Response(json.dumps('Nay!'), mimetype='application/json'), 202
167 | result = queue.hget(job_key, 'result')
168 | queue.delete(job_key)
169 | return Response(result, mimetype='application/json'), 200
170 |
171 |
172 | def enqueue(method, data):
173 | job_id = str(uuid.uuid4())
174 | p = queue.pipeline()
175 | p.hmset(job_id, {'method': method, 'data': json.dumps(data)})
176 | p.sadd('to_process', job_id)
177 | p.execute()
178 | return job_id
179 |
180 |
181 | @app.route('/start', methods=['POST'])
182 | def run_query():
183 | data = request.get_json(force=True)
184 | url = data["url"]
185 | ip = _get_user_ip(request)
186 | app.logger.info(f'{ip} {url}')
187 | if urlabuse_query.get_submissions(url) and urlabuse_query.get_submissions(url) >= autosend_threshold:
188 | send(url, '', True)
189 | return enqueue('is_valid_url', {'url': url})
190 |
191 |
192 | @app.route('/lookyloo', methods=['POST'])
193 | def lookyloo():
194 | data = request.get_json(force=True)
195 | return enqueue('lookyloo', {'url': data["url"]})
196 |
197 |
198 | @app.route('/urls', methods=['POST'])
199 | def urls():
200 | data = request.get_json(force=True)
201 | return enqueue('url_list', {'url': data["url"]})
202 |
203 |
204 | @app.route('/resolve', methods=['POST'])
205 | def resolve():
206 | data = request.get_json(force=True)
207 | return enqueue('dns_resolve', {'url': data["url"]})
208 |
209 |
210 | def read_auth(name):
211 | key = config_dir / f'{name}.key'
212 | if not key.exists():
213 | return ''
214 | with open(key) as f:
215 | to_return = []
216 | for line in f.readlines():
217 | to_return.append(line.strip())
218 | return to_return
219 |
220 |
221 | @app.route('/phishtank', methods=['POST'])
222 | def phishtank():
223 | auth = read_auth('phishtank')
224 | if not auth:
225 | return ''
226 | data = request.get_json(force=True)
227 | return enqueue('phish_query', {'url': parser.get("PHISHTANK", "url"),
228 | 'key': auth[0], 'query': data["query"]})
229 |
230 |
231 | @app.route('/virustotal_report', methods=['POST'])
232 | def vt():
233 | auth = read_auth('virustotal')
234 | if not auth:
235 | return ''
236 | data = request.get_json(force=True)
237 | return enqueue('vt_query_url', {'url': parser.get("VIRUSTOTAL", "url_report"),
238 | 'url_up': parser.get("VIRUSTOTAL", "url_upload"),
239 | 'key': auth[0], 'query': data["query"]})
240 |
241 |
242 | @app.route('/googlesafebrowsing', methods=['POST'])
243 | def gsb():
244 | auth = read_auth('googlesafebrowsing')
245 | if not auth:
246 | return ''
247 | key = auth[0]
248 | data = request.get_json(force=True)
249 | url = parser.get("GOOGLESAFEBROWSING", "url").format(key)
250 | return enqueue('gsb_query', {'url': url,
251 | 'query': data["query"]})
252 |
253 |
254 | '''
255 | @app.route('/urlquery', methods=['POST'])
256 | def urlquery():
257 | auth = read_auth('urlquery')
258 | if not auth:
259 | return ''
260 | key = auth[0]
261 | data = json.loads(request.data.decode())
262 | url = parser.get("URLQUERY", "url")
263 | query = data["query"]
264 | u = q.enqueue_call(func=urlquery_query, args=(url, key, query,), result_ttl=500)
265 | return u.get_id()
266 |
267 | @app.route('/ticket', methods=['POST'])
268 | def ticket():
269 | if not request.authorization:
270 | return ''
271 | data = json.loads(request.data.decode())
272 | server = parser.get("SPHINX", "server")
273 | port = int(parser.get("SPHINX", "port"))
274 | url = parser.get("ITS", "url")
275 | query = data["query"]
276 | u = q.enqueue_call(func=sphinxsearch, args=(server, port, url, query,),
277 | result_ttl=500)
278 | return u.get_id()
279 | '''
280 |
281 |
282 | @app.route('/whois', methods=['POST'])
283 | def whoismail():
284 | data = request.get_json(force=True)
285 | return enqueue('whois', {'server': parser.get("WHOIS", "server"),
286 | 'port': parser.getint("WHOIS", "port"),
287 | 'domain': data["query"],
288 | 'ignorelist': ignorelist, 'replacelist': replacelist})
289 |
290 |
291 | @app.route('/eupi', methods=['POST'])
292 | def eu():
293 | auth = read_auth('eupi')
294 | if not auth:
295 | return ''
296 | data = request.get_json(force=True)
297 | return enqueue('eupi', {'url': parser.get("EUPI", "url"),
298 | 'key': auth[0], 'q': data["query"]})
299 |
300 |
301 | @app.route('/pdnscircl', methods=['POST'])
302 | def dnscircl():
303 | auth = read_auth('pdnscircl')
304 | if not auth:
305 | return ''
306 | user, password = auth
307 | url = parser.get("PDNS_CIRCL", "url")
308 | data = request.get_json(force=True)
309 | return enqueue('pdnscircl', {'url': url, 'user': user.strip(),
310 | 'passwd': password.strip(), 'q': data["query"]})
311 |
312 |
313 | @app.route('/bgpranking', methods=['POST'])
314 | def bgpr():
315 | data = request.get_json(force=True)
316 | return enqueue('bgpranking', {'ip': data["query"]})
317 |
318 |
319 | @app.route('/psslcircl', methods=['POST'])
320 | def sslcircl():
321 | auth = read_auth('psslcircl')
322 | if not auth:
323 | return ''
324 | user, password = auth
325 | url = parser.get("PSSL_CIRCL", "url")
326 | data = request.get_json(force=True)
327 | return enqueue('psslcircl', {'url': url, 'user': user.strip(),
328 | 'passwd': password.strip(), 'q': data["query"]})
329 |
330 |
331 | @app.route('/get_cache', methods=['POST'])
332 | def get_cache():
333 | data = request.get_json(force=True)
334 | url = data["query"]
335 | if 'digest' in data:
336 | digest = data["digest"]
337 | else:
338 | digest = False
339 | data = urlabuse_query.cached(url, digest)
340 | return Response(json.dumps(data), mimetype='application/json')
341 |
342 |
343 | def send(url, ip='', autosend=False):
344 | if not urlabuse_query.get_mail_sent(url):
345 | data = urlabuse_query.cached(url, digest=True)
346 | if not autosend:
347 | subject = 'URL Abuse report from ' + ip
348 | else:
349 | subject = 'URL Abuse report sent automatically'
350 | msg = Message(subject, sender='urlabuse@circl.lu', recipients=["info@circl.lu"])
351 | msg.body = data['digest'][0]
352 | msg.body += '\n\n'
353 | msg.body += json.dumps(data['result'], sort_keys=True, indent=2)
354 | mail.send(msg)
355 | urlabuse_query.set_mail_sent(url)
356 |
357 |
358 | @app.route('/submit', methods=['POST'])
359 | def send_mail():
360 | data = request.get_json(force=True)
361 | url = data["url"]
362 | if not urlabuse_query.get_mail_sent(url):
363 | ip = _get_user_ip(request)
364 | send(url, ip)
365 | form = URLForm()
366 | return render_template('index.html', form=form)
367 |
--------------------------------------------------------------------------------
/website/web/proxied.py:
--------------------------------------------------------------------------------
1 | class ReverseProxied(object):
2 | '''Wrap the application in this middleware and configure the
3 | front-end server to add these headers, to let you quietly bind
4 | this to a URL other than / and to an HTTP scheme that is
5 | different than what is used locally.
6 |
7 | In nginx:
8 | location /myprefix {
9 | proxy_pass http://192.168.0.1:5001;
10 | proxy_set_header Host $host;
11 | proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
12 | proxy_set_header X-Scheme $scheme;
13 | proxy_set_header X-Script-Name /myprefix;
14 | }
15 |
16 | :param app: the WSGI application
17 | '''
18 | def __init__(self, app):
19 | self.app = app
20 |
21 | def __call__(self, environ, start_response):
22 | script_name = environ.get('HTTP_X_SCRIPT_NAME', '')
23 | if script_name:
24 | environ['SCRIPT_NAME'] = script_name
25 | path_info = environ['PATH_INFO']
26 | if path_info.startswith(script_name):
27 | environ['PATH_INFO'] = path_info[len(script_name):]
28 |
29 | scheme = environ.get('HTTP_X_SCHEME', '')
30 | if scheme:
31 | environ['wsgi.url_scheme'] = scheme
32 | return self.app(environ, start_response)
33 |
--------------------------------------------------------------------------------
/website/web/static/ajax-loader.gif:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/CIRCL/url-abuse/3d2ae503ec6ecbee92f7b8010abd46afa5b52230/website/web/static/ajax-loader.gif
--------------------------------------------------------------------------------
/website/web/static/main.js:
--------------------------------------------------------------------------------
1 | (function () {
2 | 'use strict';
3 |
4 | var app = angular.module('URLabuseApp', ['ui.bootstrap']);
5 |
6 | app.factory("flash", function($rootScope) {
7 | var queue = [];
8 | var currentMessage = "";
9 |
10 | $rootScope.$on("newFlashMessage", function() {
11 | currentMessage = queue.shift() || "";
12 | });
13 |
14 | return {
15 | setMessage: function(message) {
16 | queue.push(message);
17 | },
18 | getMessage: function() {
19 | return currentMessage;
20 | }
21 | };
22 | });
23 |
24 | app.factory('globFct', [ '$log', '$http', '$timeout', function($log, $http, $timeout){
25 | return {
26 | poller: function myself(jobID, callback) {
27 | var timeout = "";
28 | // fire another request
29 | $http.get('_result/' + jobID.data).
30 | then(function(data) {
31 | if(data.status === 202) {
32 | $log.log(data, status);
33 | } else if (data.status === 200){
34 | $log.log(data.data);
35 | $timeout.cancel(timeout);
36 | if (data.data === "null"){
37 | $log.log('Got null data');
38 | return;
39 | } else {
40 | callback(data.data);
41 | return;
42 | };
43 | }
44 | // continue to call the poller() function every 2 seconds
45 | // until the timout is cancelled
46 | timeout = $timeout(function() {myself(jobID, callback);}, 2000);
47 | });
48 | },
49 | query: function(path, data, callback) {
50 | $http.post(path, data).
51 | then(callback, function(error) {
52 | $log.log(error);
53 | });
54 | }
55 | };
56 | }]);
57 |
58 | app.controller('URLabuseController', function($scope, $log, globFct, flash) {
59 |
60 | $scope.poller = globFct.poller;
61 | $scope.query = globFct.query;
62 | $scope.flash = flash;
63 |
64 | var get_redirects = function(jobID) {
65 | $scope.poller(jobID, function(data){
66 | $log.log(data);
67 | $scope.urls = data;
68 | });
69 | };
70 |
71 |
72 | $scope.getResults = function() {
73 | // get the URL from the input
74 | $scope.query_url = '';
75 | $scope.urls = '';
76 | // Reset the message
77 | $scope.$emit('newFlashMessage', '');
78 |
79 | var userInput = $scope.input_url;
80 |
81 | var lookyloo = function(jobID) {
82 | $scope.poller(jobID, function(data){
83 | $scope.lookyloo_url = data;
84 | });
85 | };
86 |
87 | var check_validity = function(jobID) {
88 | $scope.poller(jobID, function(data){
89 | $scope.query_url = data[1];
90 | if(data[0] === false){
91 | $scope.error = data[2];
92 | } else {
93 | $scope.query('urls', {"url": data[1]}, get_redirects);
94 | }
95 | });
96 | };
97 |
98 | $scope.query('start', {"url": userInput}, check_validity);
99 | $scope.query('lookyloo', {"url": userInput}, lookyloo);
100 | };
101 |
102 | $scope.submit_email = function() {
103 | $scope.query('submit', {"url": $scope.query_url}, function(){
104 | $scope.query_url = '';
105 | $scope.urls = '';
106 | $scope.input_url = '';
107 | flash.setMessage("Mail sent to CIRCL");
108 | $scope.$emit('newFlashMessage', '');
109 | });
110 | };
111 |
112 | });
113 |
114 | app.directive('uqUrlreport', function(globFct) {
115 |
116 | return {
117 | scope: {
118 | url: '=uqUrlreport',
119 | // status: {isFirstOpen: true, isFirstDisabled: false}
120 | },
121 | link: function(scope, element, attrs) {
122 | var get_ips = function(jobID) {
123 | globFct.poller(jobID, function(data){
124 | scope.ipv4 = data[0];
125 | scope.ipv6 = data[1];
126 | if (!scope.ipv4){
127 | scope.ipv4 = ['Unable to resolve in IPv4'];
128 | }
129 | if (!scope.ipv6){
130 | scope.ipv6 = ['Unable to resolve in IPv6'];
131 | }
132 | });
133 | };
134 | globFct.query('resolve', {"url": scope.url}, get_ips);
135 | },
136 | templateUrl: 'urlreport',
137 | };
138 |
139 | });
140 |
141 | app.directive('uqPhishtank', function(globFct) {
142 | return {
143 | scope: {
144 | query: '=data',
145 | },
146 | link: function(scope, element, attrs) {
147 | var get_response = function(jobID) {
148 | globFct.poller(jobID, function(data){
149 | scope.response = data;
150 | });
151 | };
152 | globFct.query('phishtank', {"query": scope.query}, get_response);
153 | },
154 | template: function(elem, attr){
155 | return '';}
156 | };
157 | });
158 |
159 | app.directive('uqVirustotal', function(globFct) {
160 | return {
161 | scope: {
162 | query: '=data',
163 | },
164 | link: function(scope, element, attrs) {
165 | var get_response = function(jobID) {
166 | globFct.poller(jobID, function(data){
167 | scope.message = data[0];
168 | scope.link = data[1];
169 | scope.positives = data[2];
170 | scope.total = data[3];
171 | if(scope.link && scope.positives === null){
172 | scope.alert_val = "info";
173 | scope.message = "Scan request successfully queued, report available soon.";
174 | } else if (scope.link && scope.positives === 0){
175 | scope.message = "None of the " + data[3] + " scanners know this URL as malicious.";
176 | scope.alert_val = "success";
177 | } else if (scope.link && scope.positives < scope.total/3){
178 | scope.message = data[2] + " of the " + data[3] + " scanners know this URL as malicious.";
179 | scope.alert_val = "warning";
180 | } else if (scope.link && scope.positives >= scope.total/3){
181 | scope.message = data[2] + " of the " + data[3] + " scanners know this URL as malicious.";
182 | scope.alert_val = "danger";
183 | }
184 | });
185 | };
186 | globFct.query('virustotal_report', {"query": scope.query}, get_response);
187 | },
188 | template: function(elem, attr){
189 | return '';}
190 | };
191 | });
192 |
193 | app.directive('uqGooglesafebrowsing', function(globFct) {
194 | return {
195 | scope: {
196 | query: '=data',
197 | },
198 | link: function(scope, element, attrs) {
199 | var get_response = function(jobID) {
200 | globFct.poller(jobID, function(data){
201 | scope.response = data;
202 | });
203 | };
204 | globFct.query('googlesafebrowsing', {"query": scope.query}, get_response);
205 | },
206 | template: function(elem, attr){
207 | return 'Known {{response}} website on Google Safe Browsing. More details. ';}
208 | };
209 | });
210 |
211 | app.directive('uqEupi', function(globFct) {
212 | return {
213 | scope: {
214 | query: '=data',
215 | },
216 | link: function(scope, element, attrs) {
217 | var get_response = function(jobID) {
218 | globFct.poller(jobID, function(data){
219 | if (data === "inconnu"){
220 | return;
221 | }
222 | scope.response = data;
223 | if(data === "clean"){
224 | scope.alert_val = "success";
225 | }
226 | else{
227 | ascope.alert_val = "danger";
228 | }
229 | });
230 | };
231 | globFct.query('eupi', {"query": scope.query}, get_response);
232 | },
233 | template: function(elem, attr){
234 | return 'Known as {{response}} by the European Union antiphishing initiative. ';}
235 | };
236 | });
237 |
238 | app.directive('uqUrlquery', function(globFct) {
239 | return {
240 | scope: {
241 | query: '=data',
242 | },
243 | link: function(scope, element, attrs) {
244 | var get_response = function(jobID) {
245 | globFct.poller(jobID, function(data){
246 | scope.response = data;
247 | });
248 | };
249 | globFct.query('urlquery', {"query": scope.query}, get_response);
250 | },
251 | template: function(elem, attr){
252 | return 'The total alert count on URLquery is {{response}}. ';}
253 | };
254 | });
255 |
256 | app.directive('uqTicket', function(globFct) {
257 | return {
258 | scope: {
259 | query: '=data',
260 | },
261 | link: function(scope, element, attrs) {
262 | var get_response = function(jobID) {
263 | globFct.poller(jobID, function(data){
264 | scope.response = data;
265 | });
266 | };
267 | globFct.query('ticket', {"query": scope.query}, get_response);
268 | },
269 | template: ''
270 | };
271 | });
272 |
273 | app.directive('uqWhois', function(globFct) {
274 | return {
275 | scope: {
276 | query: '=data',
277 | },
278 | link: function(scope, element, attrs) {
279 | var get_response = function(jobID) {
280 | globFct.poller(jobID, function(data){
281 | scope.response = data.join();
282 | });
283 | };
284 | globFct.query('whois', {"query": scope.query}, get_response);
285 | },
286 | template: 'Contact points from Whois: {{ response }}
'
287 | };
288 | });
289 | app.directive('uqPdnscircl', function(globFct) {
290 | return {
291 | scope: {
292 | query: '=data',
293 | },
294 | link: function(scope, element, attrs) {
295 | var get_response = function(jobID) {
296 | globFct.poller(jobID, function(data){
297 | scope.nbentries = data[0];
298 | scope.lastentries = data[1];
299 | });
300 | };
301 | globFct.query('pdnscircl', {"query": scope.query}, get_response);
302 | },
303 | template: 'Has {{nbentries}} unique entries in CIRCL Passive DNS. {{lastentries.length}} most recent one(s):
SSL certificates related to this IP:
Information from BGP Ranking: - Announced by: {{asndesc}} ({{asn}})
- This ASN is at position {{position}} in the list of {{total}} known ASNs ({{value}}).
'
348 | };
349 | });
350 | }());
351 |
--------------------------------------------------------------------------------
/website/web/templates/404.html:
--------------------------------------------------------------------------------
1 | {% extends "index.html" %}
2 | {% block title %}Page Not Found{% endblock %}
3 | {% block body %}
4 | Page Not Found
5 | What you were looking for is just not there.
6 |
Back to index.
7 | {% endblock %}
8 |
9 |
--------------------------------------------------------------------------------
/website/web/templates/index.html:
--------------------------------------------------------------------------------
1 | {% extends "bootstrap/base.html" %}
2 | {% import "bootstrap/wtf.html" as wtf %}
3 | {% import "bootstrap/fixes.html" as fixes %}
4 |
5 | {% block title %}CIRCL URL Abuse{% endblock %}
6 |
7 | {% block navbar %}
8 |
11 | {% endblock %}
12 |
13 | {% block html_attribs %} ng-app="URLabuseApp" {% endblock html_attribs %}
14 |
15 | {% block body_attribs %} ng-controller="URLabuseController" {% endblock body_attribs %}
16 |
17 | {% block content %}
18 |
19 |
URL Abuse testing form
20 |
21 |
22 | {% raw %}
23 |
24 |
{{ flash.getMessage() }}
25 |
26 | {% endraw %}
27 |
28 |
39 |
40 | {% raw %}
41 |
42 |
43 | Report
44 | {{ query_url }}
45 |
46 |
47 | {{ error }}
48 |
49 |
50 |
51 | {% endraw %}
52 |
 }})
53 | {% raw %}
54 |
55 |
60 |
63 |
64 | {% endraw %}
65 |
66 |
67 |
5 |
{{url}}
6 |
7 |
8 |
9 |
10 |
11 |
12 |
13 |
14 |
15 |
16 |
17 | {{ip}}
18 |
19 |
20 |
21 |
22 |
23 |
24 |
25 |
26 |
27 |
28 |
29 |
30 |
31 |
32 |
33 | {{ip}}
34 |
35 |
36 |
37 |
38 |
39 |
40 |
41 |
42 |
43 |
44 |
10 |