├── .gitignore ├── LICENSE ├── MANIFEST.in ├── README.md ├── bgpranking_redis ├── __init__.py ├── api.py ├── constraints.default.py └── tools.py ├── doc ├── Makefile └── source │ ├── code │ └── API.rst │ ├── conf.py │ └── index.rst ├── example ├── api_web │ ├── client │ │ ├── MANIFEST.in │ │ ├── README.md │ │ ├── bgpranking_web │ │ │ ├── __init__.py │ │ │ └── api.py │ │ └── setup.py │ └── server │ │ └── webservice.py ├── export │ ├── asn_ranks │ │ ├── agg_consummer.py │ │ ├── consumer.py │ │ ├── generate_aggs.py │ │ ├── init_redis.py │ │ ├── launch.sh │ │ ├── launch_local_redis.sh │ │ └── local_redis.conf │ └── day_ips │ │ ├── consumer.py │ │ ├── dates.sh │ │ ├── dump.py │ │ ├── init_redis.py │ │ ├── launch.sh │ │ ├── launch_local_redis.sh │ │ └── local_redis.conf ├── ip_zmq │ └── client.py ├── logging │ ├── logs │ │ └── .keepdir │ ├── mail.conf │ └── start_logging.sh ├── twitter_bot │ ├── microblog │ │ ├── __init__.py │ │ ├── api_wrapper.py │ │ └── python_stream_api.py │ └── start_bot.py └── website │ ├── __init__.py │ ├── compil.sh │ ├── config │ └── web_bgp-ranking.ini │ ├── data │ ├── csv │ │ └── .is_csv_dir │ ├── csv_agg │ │ └── .keepdir │ └── js │ │ └── .keepdir │ ├── js │ ├── world_benelux.js │ ├── world_luxembourg.js │ └── worldmap_script.js │ ├── logs │ └── .keepdir │ ├── master.py │ ├── master_controler.py │ ├── start_website.sh │ ├── templates │ ├── __init__.py │ ├── asn_details.tmpl │ ├── comparator.tmpl │ ├── index_asn.tmpl │ ├── ip_lookup.tmpl │ ├── map.tmpl │ ├── master.tmpl │ ├── trend.tmpl │ └── trend_benelux.tmpl │ └── thirdparty │ ├── dygraph │ └── .keepdir │ ├── jvectormap │ └── .keepdir │ └── update_thirdparty.sh └── setup.py /.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | *.csv 3 | *.js 4 | build/ 5 | example/csv/csv/ 6 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2013, 2014 Raphaël Vinot 2 | Copyright (c) 2013, 2014 CIRCL - Computer Incident Response Center Luxembourg 3 | (c/o smile, security made in Lëtzebuerg, Groupement 4 | d'Intérêt Economique) 5 | 6 | All rights reserved. 7 | 8 | Redistribution and use in source and binary forms, with or without modification, 9 | are permitted provided that the following conditions are met: 10 | 11 | 1. Redistributions of source code must retain the above copyright notice, 12 | this list of conditions and the following disclaimer. 13 | 2. Redistributions in binary form must reproduce the above copyright notice, 14 | this list of conditions and the following disclaimer in the documentation 15 | and/or other materials provided with the distribution. 16 | 17 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 18 | ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 19 | WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 20 | IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, 21 | INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 22 | BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 23 | DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF 24 | LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE 25 | OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED 26 | OF THE POSSIBILITY OF SUCH DAMAGE. 27 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include README.md 2 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Python API to connect to the redis database of a BGP Ranking instance. 2 | 3 | The documentation is available here: 4 | http://circl.github.com/bgpranking-redis-api/code/API.html 5 | 6 | Important 7 | ========= 8 | 9 | Please note that you can query the public instance of BGP Ranking via 10 | the REST API available in this repository under `example/api_web/client/` 11 | -------------------------------------------------------------------------------- /bgpranking_redis/__init__.py: -------------------------------------------------------------------------------- 1 | import helper_global 2 | helper_global.__prepare() 3 | 4 | from api import * 5 | -------------------------------------------------------------------------------- /bgpranking_redis/api.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # -*- coding: utf-8 -*- 3 | 4 | """ 5 | Entry point of the API 6 | """ 7 | 8 | import redis 9 | import IPy 10 | from dateutil import parser 11 | from dateutil import tz 12 | import datetime 13 | 14 | from . import constraints as c 15 | 16 | HAS_PTR = False 17 | 18 | try: 19 | from ipasn_redis import IPASN 20 | HAS_IPASN = True 21 | except ImportError: 22 | HAS_IPASN = False 23 | 24 | try: 25 | from asnhistory import ASNHistory 26 | HAS_ASN_HIST = True 27 | except ImportError: 28 | HAS_ASN_HIST = False 29 | 30 | try: 31 | from pubsublogger import publisher 32 | publisher.channel = 'API_Redis' 33 | HAS_LOGGING = True 34 | except ImportError: 35 | HAS_LOGGING = False 36 | 37 | 38 | class BGPRanking(object): 39 | 40 | def __init__(self): 41 | self.global_db = redis.Redis(port=c.redis_port1, db=c.redis_db_global, host=c.redis_hostname1) 42 | self.history_db = redis.Redis(port=c.redis_port2, db=c.redis_db_history, host=c.redis_hostname2) 43 | self.config_db = redis.Redis(port=c.redis_port2, db=c.redis_db_config, host=c.redis_hostname2) 44 | self.history_db_cache = redis.Redis(port=c.redis_cached_port, db=c.redis_cached_db_history, host=c.redis_hostname1) 45 | 46 | if c.ptr_host and c.ptr_port: 47 | # PTR record lookup configured 48 | self.ptr_redis = redis.Redis(host=c.ptr_host, port=c.ptr_port) 49 | self.has_ptr = True 50 | else: 51 | self.has_ptr = False 52 | if HAS_IPASN and c.ipasn_host and c.ipasn_port: 53 | # IP ASN history lookup configured 54 | self.ipasn = IPASN(host=c.ipasn_host, port=c.ipasn_port) 55 | self.has_ipasn = True 56 | else: 57 | self.has_ipasn = False 58 | if HAS_ASN_HIST and c.asn_host and c.asn_port: 59 | # ASN Description history configured 60 | self.asnhistory = ASNHistory(host=c.asn_host, port=c.asn_port) 61 | self.has_asnhistory = True 62 | else: 63 | self.has_asnhistory = False 64 | 65 | def get_ptr_record(self, ip): 66 | # Get PTR Record when looking for an IP 67 | if self.has_ptr: 68 | return self.ptr_redis.get(ip) 69 | 70 | def get_asn_descriptions(self, asn): 71 | if not self.has_asnhistory: 72 | publisher.debug('ASN History not enabled.') 73 | return [datetime.date.today(), 'ASN History not enabled.'] 74 | desc_history = self.asnhistory.get_all_descriptions(asn) 75 | return [(date.astimezone(tz.tzutc()).date(), descr) for date, descr in desc_history] 76 | 77 | def get_ip_info(self, ip, days_limit=None): 78 | """ 79 | Return informations related to an IP address. 80 | 81 | :param ip: The IP address 82 | :param days_limit: The number of days we want to check in the past 83 | (default: around 2 years) 84 | :rtype: Dictionary 85 | 86 | .. note:: Format of the output: 87 | 88 | .. code-block:: python 89 | 90 | { 91 | 'ip': ip, 92 | 'days_limit' : days_limit, 93 | 'ptrrecord' : 'ptr.record.com', 94 | 'history': 95 | [ 96 | { 97 | 'asn': asn, 98 | 'interval': [first, last], 99 | 'block': block, 100 | 'timestamp': timestamp, 101 | 'descriptions': 102 | [ 103 | [date, descr], 104 | ... 105 | ] 106 | }, 107 | ... 108 | ] 109 | } 110 | """ 111 | if days_limit is None: 112 | days_limit = 750 113 | to_return = {'ip': ip, 'days_limit': days_limit, 'history': []} 114 | if self.has_ptr: 115 | to_return['ptrrecord'] = self.get_ptr_record(ip) 116 | if not self.has_ipasn: 117 | publisher.debug('IPASN not enabled.') 118 | to_return['error'] = 'IPASN not enabled.' 119 | return to_return 120 | if not ip: 121 | to_return['error'] = 'No IP provided.' 122 | return to_return 123 | for first, last, asn, block in self.ipasn.aggregate_history(ip, days_limit): 124 | first_date = parser.parse(first).replace(tzinfo=tz.tzutc()).date() 125 | last_date = parser.parse(last).replace(tzinfo=tz.tzutc()).date() 126 | if self.has_asnhistory: 127 | desc_history = self.asnhistory.get_all_descriptions(asn) 128 | valid_descriptions = [] 129 | for date, descr in desc_history: 130 | date = date.astimezone(tz.tzutc()).date() 131 | test_date = date - datetime.timedelta(days=1) 132 | if last_date < test_date: 133 | # Too new 134 | continue 135 | elif last_date >= test_date and first_date <= test_date: 136 | # Changes within the interval 137 | valid_descriptions.append([date.isoformat(), descr]) 138 | elif first_date > test_date: 139 | # get the most recent change befrore the interval 140 | valid_descriptions.append([date.isoformat(), descr]) 141 | break 142 | else: 143 | publisher.debug('ASN History not enabled.') 144 | valid_descriptions = [datetime.date.today().isoformat(), 'ASN History not enabled.'] 145 | if len(valid_descriptions) == 0: 146 | if len(desc_history) != 0: 147 | # fallback, use the oldest description. 148 | date = desc_history[-1][0].astimezone(tz.tzutc()).date() 149 | descr = desc_history[-1][1] 150 | valid_descriptions.append([date.isoformat(), descr]) 151 | else: 152 | # No history found for this ASN 153 | if last_date > datetime.date(2013, 1, 1): 154 | # ASN has been seen recently, should not happen 155 | # as the asn history module is running since early 2013 156 | publisher.error('Unable to find the ASN description of {}. IP address: {}. ASN History might be down.'.format(asn, ip)) 157 | valid_descriptions.append(['0000-00-00', 'No ASN description has been found.']) 158 | entry = {} 159 | entry['asn'] = asn 160 | entry['interval'] = [first_date.isoformat(), last_date.isoformat()] 161 | entry['block'] = block 162 | entry['timestamp'] = self.get_first_seen(asn, block) 163 | entry['descriptions'] = valid_descriptions 164 | to_return['history'].append(entry) 165 | return to_return 166 | 167 | def asn_exists(self, asn): 168 | return self.global_db.exists(asn) 169 | 170 | def get_default_date(self, delta_days=1): 171 | return self.get_default_date_raw(delta_days).isoformat() 172 | 173 | def get_default_date_raw(self, delta_days=1): 174 | """ 175 | Get the default date displayed on the website. 176 | """ 177 | delta = datetime.timedelta(days=delta_days) 178 | timestamp = self.history_db.get('latest_ranking') 179 | if timestamp: 180 | default_date_raw = parser.parse(timestamp.split()[0]).date() - delta 181 | else: 182 | default_date_raw = datetime.date.today() - delta 183 | return default_date_raw 184 | 185 | def get_last_seen_sources(self, asn, dates_sources): 186 | """ 187 | Return a dictionary conteining the last date a particular ASN 188 | has been seen by a source. 189 | """ 190 | if not self.asn_exists(asn): 191 | return {} 192 | string = '{asn}|{date}|{source}|rankv{ip_version}' 193 | s_dates = dates_sources.keys() 194 | s_dates.sort(reverse=True) 195 | p = self.history_db.pipeline() 196 | for date in s_dates: 197 | sources = dates_sources[date] 198 | if len(sources) > 0: 199 | [p.exists(string.format(asn=asn, date=date, source=source, ip_version=c.ip_version)) for source in sources] 200 | asns_found = p.execute() 201 | i = 0 202 | to_return = {} 203 | for date in s_dates: 204 | sources = dates_sources[date] 205 | if len(sources) > 0: 206 | for source in sources: 207 | if to_return.get(source) is None and asns_found[i]: 208 | to_return[source] = date 209 | i += 1 210 | return to_return 211 | 212 | def get_all_ranks_single_asn(self, asn, dates_sources, with_details_sources=False): 213 | """ 214 | Get all the ranks on a timeframe for a single ASN. 215 | 216 | :param asn: Autonomous System Number 217 | :param dates_sources: Dictionaries of the dates and sources 218 | 219 | .. note:: Format of the dictionary: 220 | 221 | .. code-block:: python 222 | 223 | { 224 | YYYY-MM-DD: [source1, source2, ...], 225 | YYYY-MM-DD: [source1, source2, ...], 226 | ... 227 | } 228 | 229 | :param with_details_sources: Returns the details by sources if True 230 | 231 | :rype: Dictionary 232 | 233 | .. note:: Format of the output: 234 | 235 | .. code-block:: python 236 | 237 | { 238 | date1: 239 | { 240 | 'details': 241 | [ 242 | (source1, rank), 243 | (source2, rank), 244 | ... 245 | ], 246 | 'total': sum_of_ranks, 247 | 'description': description 248 | } 249 | ... 250 | } 251 | 252 | The details key is only present if 253 | ``with_details_sources`` is True. 254 | """ 255 | to_return = {} 256 | if with_details_sources is None: 257 | with_details_sources = False 258 | if not self.asn_exists(asn): 259 | return to_return 260 | string = '{asn}|{date}|{source}|rankv{ip_version}' 261 | p = self.history_db.pipeline() 262 | for date, sources in dates_sources.iteritems(): 263 | if len(sources) > 0: 264 | p.mget([string.format(asn=asn, date=date, source=source, ip_version=c.ip_version) for source in sources]) 265 | ranks = p.execute() 266 | i = 0 267 | for date, sources in dates_sources.iteritems(): 268 | impacts = self.get_all_weights(date) 269 | if len(sources) > 0: 270 | to_return[date] = {} 271 | zipped_sources_rank = zip(sources, ranks[i]) 272 | if with_details_sources: 273 | to_return[date]['details'] = list(zipped_sources_rank) 274 | to_return[date]['description'] = self.asn_desc_via_history(asn) 275 | to_return[date]['total'] = 0 276 | for s, r in zipped_sources_rank: 277 | if r is not None: 278 | to_return[date]['total'] += float(r) * float(impacts[s]) 279 | i += 1 280 | return to_return 281 | 282 | def existing_asns_timeframe(self, dates_sources): 283 | """ 284 | Get all the ASNs seen in the lists on a timeframe. 285 | """ 286 | asns_keys = [] 287 | for date, sources in dates_sources.iteritems(): 288 | asns_keys.extend(['{date}|{source}|asns'.format(date=date, source=source) for source in sources]) 289 | p = self.global_db.pipeline(False) 290 | [p.smembers(k) for k in asns_keys] 291 | asns_list = p.execute() 292 | return set.union(*asns_list) 293 | 294 | def get_all_ranks_all_asns(self, dates_sources, with_details_sources=False): 295 | """ 296 | Get all ranks for all the ASNs on a timeframe. 297 | 298 | :param dates_sources: Dictionaries of the dates and sources 299 | 300 | .. note:: Format of the dictionary: 301 | 302 | .. code-block:: python 303 | 304 | { 305 | YYYY-MM-DD: [source1, source2, ...], 306 | YYYY-MM-DD: [source1, source2, ...], 307 | ... 308 | } 309 | 310 | :param with_details_sources: Also returns the ranks by sources 311 | 312 | :rype: Dictionary 313 | 314 | .. note:: Format of the dictionary: 315 | 316 | .. code-block:: python 317 | 318 | { 319 | YYYY-MM-DD: 320 | { 321 | asn1 : 322 | { 323 | 'details': 324 | [ 325 | (source1, rank), 326 | (source2, rank), 327 | ... 328 | ], 329 | 'total': sum_of_ranks, 330 | 'description': description 331 | } 332 | ... 333 | } 334 | ... 335 | } 336 | 337 | The details key is only present if 338 | ``with_details_sources`` is True. 339 | """ 340 | asns = self.existing_asns_timeframe(dates_sources) 341 | to_return = {} 342 | for asn in asns: 343 | details = self.get_all_ranks_single_asn(asn, dates_sources, with_details_sources) 344 | for date, entries in details.iteritems(): 345 | if to_return.get(date) is None: 346 | to_return[date] = {} 347 | to_return[date][asn] = entries 348 | return to_return 349 | 350 | def get_all_blocks(self, asn): 351 | """ 352 | Expose the list of blocks announced by an AS. 353 | 354 | :param asn: Autonomous System Number 355 | :rtype: Set of ip blocks 356 | """ 357 | return self.global_db.smembers(asn) 358 | 359 | def get_all_blocks_description(self, asn): 360 | """ 361 | Get the most recent description of all the blocks announced by an AS 362 | 363 | :param asn: Autonomous System Number 364 | :rtype: List 365 | 366 | .. note:: Format of the list: 367 | 368 | .. code-block:: python 369 | 370 | [asn, [(block, description), ...]] 371 | """ 372 | blocks = self.get_all_blocks(asn) 373 | if len(blocks) > 0: 374 | block_keys = ['{asn}|{block}'.format(asn=asn, block=block) for block in blocks] 375 | p = self.global_db.pipeline(False) 376 | [p.hgetall(block_key) for block_key in block_keys] 377 | ts_descr = p.execute() 378 | i = 0 379 | block_descr = [] 380 | for block_key in block_keys: 381 | if ts_descr[i] is not None: 382 | asn, block = block_key.split('|') 383 | most_recent_ts = sorted(ts_descr[i].keys())[0] 384 | descr = ts_descr[i][most_recent_ts] 385 | block_descr.append((block, descr)) 386 | i += 1 387 | return asn, block_descr 388 | else: 389 | return asn, [] 390 | 391 | def get_first_seen(self, asn, block): 392 | """ 393 | Get the oldest timestamp where the block has been announced by the AS 394 | """ 395 | timestamps = self.global_db.hkeys(asn + '|' + block) 396 | if timestamps is not None and len(timestamps) > 0: 397 | timestamps.sort() 398 | return timestamps[0] 399 | return None 400 | 401 | def get_block_descriptions(self, asn, block): 402 | """ 403 | Return all the descriptions of a block, in a listed sorted by 404 | timestamp (new => old) 405 | 406 | :param asn: Autonomous System Number 407 | :param block: IP Block you are looking for 408 | 409 | :rtype: List 410 | 411 | .. note:: Format of the list: 412 | 413 | .. code-block:: python 414 | 415 | [asn, block, [(ts, descr), ...]] 416 | """ 417 | ts_descr = self.global_db.hgetall('{}|{}'.format(asn, block)) 418 | timestamps = sorted(ts_descr.keys(), reverse=True) 419 | descriptions = [(t, ts_descr[t])for t in timestamps] 420 | return asn, block, descriptions 421 | 422 | def get_all_blocks_descriptions(self, asn): 423 | """ 424 | Return all tuples timestamp-description of all the blocs announced 425 | by an AS over time. 426 | 427 | :param asn: Autonomous System Number 428 | :rtype: List 429 | 430 | .. note:: Format of the list: 431 | 432 | .. code-block:: python 433 | 434 | [ asn, 435 | { 436 | block: 437 | [ 438 | (timestamp, description), 439 | ... 440 | ], 441 | ... 442 | } 443 | ] 444 | """ 445 | blocks_descriptions = {} 446 | for block in self.get_all_blocks(asn): 447 | asn, block, descriptions = self.get_block_descriptions(asn, block) 448 | blocks_descriptions[block] = descriptions 449 | return asn, blocks_descriptions 450 | 451 | def asn_desc_via_history(self, asn): 452 | if self.has_asnhistory: 453 | asn_descr = self.asnhistory.get_last_description(asn) 454 | if asn_descr is None: 455 | # The ASN has no descripion in the database 456 | # publisher.error(\ 457 | # 'Unable to find the ASN description of {}. ASN History might be down.'.\ 458 | # format(asn)) 459 | asn_descr = 'No ASN description has been found.' 460 | else: 461 | publisher.debug('ASN History not enabled.') 462 | asn_descr = 'ASN History not enabled.' 463 | return asn_descr 464 | 465 | def get_asn_descs(self, asn, date=None, sources=None): 466 | """ 467 | Get all what is available in the database about an ASN for 468 | one day 469 | 470 | :param asn: Autonomous System Number 471 | :param date: Date of the information (default: last ranked day) 472 | :param sources: List of sources (default: the available sources 473 | at ``date``) 474 | :rtype: Dictionary 475 | 476 | .. note:: Format of the dictionary: 477 | 478 | .. code-block:: python 479 | 480 | { 481 | 'date': date, 482 | 'sources': [source1, source2, ...], 483 | 'asn': asn, 484 | 'asn_description': asn_description, 485 | asn: 486 | { 487 | 'clean_blocks': 488 | { 489 | block: [[ts, descr], ...], 490 | .... 491 | }, 492 | 'old_blocks': 493 | { 494 | block: [[ts, descr], ...], 495 | .... 496 | }, 497 | block: 498 | { 499 | 'description': description, 500 | 'timestamp': ts, 501 | 'old_descr': [(timestamp, description), ...] 502 | 'nb_of_ips': nb, 503 | 'sources': [source1, source2, ...] 504 | 'rank': subnet_rank 505 | }, 506 | ... 507 | } 508 | } 509 | 510 | 511 | """ 512 | if not self.asn_exists(asn): 513 | # The ASN does not exists in the database 514 | return None 515 | 516 | if date is None: 517 | date = self.get_default_date() 518 | day_sources = self.daily_sources([date]) 519 | if sources is None: 520 | sources = list(day_sources) 521 | else: 522 | if not isinstance(sources, list): 523 | sources = [sources] 524 | sources = list(day_sources.intersection(set(sources))) 525 | asn_descr = self.asn_desc_via_history(asn) 526 | to_return = {'date': date, 'sources': sources, 'asn': asn, 'asn_description': asn_descr, asn: {}} 527 | 528 | key_clean_set = '{asn}|{date}|clean_set'.format(asn=asn, date=date) 529 | for ip_block in self.get_all_blocks(asn): 530 | # Get descriptions 531 | asn, block, ts_descr = self.get_block_descriptions(asn, ip_block) 532 | # Find out if the block has been seen these day 533 | p = self.global_db.pipeline(False) 534 | [p.scard('{}|{}|{}|{}'.format(asn, ip_block, date, source)) for source in sources] 535 | ips_by_sources = p.execute() 536 | nb_of_ips = sum(ips_by_sources) 537 | 538 | if nb_of_ips == 0: 539 | if ip_block in self.history_db.smembers(key_clean_set): 540 | if to_return[asn].get('clean_blocks') is None: 541 | to_return[asn]['clean_blocks'] = {} 542 | to_return[asn]['clean_blocks'][ip_block] = ts_descr 543 | else: 544 | if to_return[asn].get('old_blocks') is None: 545 | to_return[asn]['old_blocks'] = {} 546 | to_return[asn]['old_blocks'][ip_block] = ts_descr 547 | else: 548 | impacts = self.get_all_weights(date) 549 | # Compute the local ranking: the ranking if this subnet is 550 | # the only one for this AS 551 | i = 0 552 | sum_rank = 0 553 | sources_exists = [] 554 | for source in sources: 555 | sum_rank += float(ips_by_sources[i]) * float(impacts[source]) 556 | if ips_by_sources[i] != 0: 557 | sources_exists.append(source) 558 | i += 1 559 | local_rank = sum_rank / IPy.IP(ip_block).len() 560 | to_return[asn][ip_block] = {'description': ts_descr[0][1], 'timestamp': ts_descr[0][0], 561 | 'old_descr': ts_descr[1:], 'nb_of_ips': nb_of_ips, 562 | 'sources': sources_exists, 'rank': local_rank} 563 | return to_return 564 | 565 | def get_all_weights(self, date=None): 566 | """ 567 | Get the weights for all the sources. 568 | 569 | :param date: Date of the information (default: last ranked day) 570 | 571 | :rtype: Dictionary 572 | 573 | .. note:: Format of the dictionary: 574 | 575 | .. code-block:: python 576 | 577 | { 578 | source: date, 579 | ... 580 | } 581 | """ 582 | if date is None: 583 | date = self.get_default_date() 584 | sources = self.daily_sources([date]) 585 | to_return = {} 586 | if len(sources) > 0: 587 | impacts = self.config_db.mget(sources) 588 | to_return = dict(zip(sources, impacts)) 589 | return to_return 590 | 591 | def daily_sources(self, dates): 592 | """ 593 | Get the sources parsed during a list of dates 594 | 595 | :param dates: List of dates 596 | 597 | :rtype: Set of sources for each date 598 | 599 | """ 600 | if len(dates) == 1: 601 | return self.global_db.smembers('{date}|sources'.format(date=dates[0])) 602 | else: 603 | p = self.global_db.pipeline(False) 604 | [p.smembers('{date}|sources'.format(date=date)) for date in dates] 605 | return p.execute() 606 | 607 | def get_ips_descs(self, asn, block, date=None, sources=None): 608 | """ 609 | Get all what is available in the database about an subnet for one day 610 | 611 | :param asn: Autonomous System Number 612 | :param asn_timestamp: First the subnet has been seen in the db 613 | (for this ASN) 614 | :param date: Date of the information (default: last ranked day) 615 | :param sources: List of sources (default: the available sources 616 | at ``date``) 617 | 618 | :rtype: Dictionary 619 | 620 | .. note:: Format of the dictionary: 621 | 622 | .. code-block:: python 623 | 624 | { 625 | 'date': date, 626 | 'sources': [source1, source2, ...], 627 | 'asn': asn, 628 | 'asn_description': asn_description, 629 | 'block': block 630 | block: 631 | { 632 | ip: 633 | { 634 | 'sources': [source1, source2, ...], 635 | 'ptrrecord': 'ptr.record.com' 636 | } 637 | ... 638 | } 639 | } 640 | 641 | """ 642 | 643 | if date is None: 644 | date = self.get_default_date() 645 | day_sources = self.daily_sources([date]) 646 | if sources is None: 647 | sources = list(day_sources) 648 | else: 649 | if not isinstance(sources, list): 650 | sources = [sources] 651 | sources = list(day_sources.intersection(set(sources))) 652 | asn_descr = self.asn_desc_via_history(asn) 653 | to_return = {'date': date, 'sources': sources, 'asn': asn, 'asn_description': asn_descr, 'block': block, block: {}} 654 | asn_block_key = '{asn}|{b}|{date}|'.format(asn=asn, b=block, date=date) 655 | pipeline = self.global_db.pipeline(False) 656 | [pipeline.smembers('{asn_b}{source}'.format(asn_b=asn_block_key, source=source)) for source in sources] 657 | ips_by_source = pipeline.execute() 658 | i = 0 659 | for source in sources: 660 | ips = ips_by_source[i] 661 | for ip_details in ips: 662 | ip, timestamp = ip_details.split('|') 663 | if to_return[block].get(ip) is None: 664 | to_return[block][ip] = {'sources': [], 'ptrrecord': None} 665 | to_return[block][ip]['sources'].append(source) 666 | i += 1 667 | if self.has_ptr: 668 | for ip in to_return[block]: 669 | to_return[block][ip]['ptrrecord'] = self.get_ptr_record(ip) 670 | return to_return 671 | 672 | def get_stats(self, dates_sources): 673 | """ 674 | Return amount of asn and subnets found by source on a list of days. 675 | 676 | :param dates_sources: Dictionaries of the dates and sources 677 | 678 | :rtype: Dictionary 679 | 680 | .. note:: Format of the Dictionary 681 | 682 | .. code-block:: python 683 | 684 | { 685 | date: 686 | { 687 | 'sources': 688 | { 689 | source : [nb_asns, nb_subnets], 690 | ... 691 | }, 692 | 'total_asn': total_asn 693 | 'total_subnets': total_subnets 694 | } 695 | ... 696 | } 697 | 698 | """ 699 | to_return = {} 700 | p = self.global_db.pipeline(False) 701 | for date, sources in dates_sources.iteritems(): 702 | to_return[date] = {} 703 | for source in sources: 704 | current_key = '{date}|{source}|asns'.format(date=date, source=source) 705 | p.scard(current_key) 706 | p.scard(current_key + '_details') 707 | cards = p.execute() 708 | i = 0 709 | for date, sources in dates_sources.iteritems(): 710 | to_return[date] = {'sources': {}} 711 | total_asn = 0 712 | total_subnets = 0 713 | for source in sources: 714 | nb_asns = cards[i] 715 | nb_subnets = cards[i + 1] 716 | total_asn += nb_asns 717 | total_subnets += nb_subnets 718 | to_return[date]['sources'].update({source: (nb_asns, nb_subnets)}) 719 | i = i + 2 720 | to_return[date]['total_asn'] = total_asn 721 | to_return[date]['total_subnets'] = total_subnets 722 | return to_return 723 | 724 | # Need cached database 725 | def cache_get_dates(self): 726 | """ 727 | **From the temporary database** 728 | 729 | Get a list of dates. The ranking are available in the cache database. 730 | """ 731 | return sorted(self.history_db_cache.smembers('all_dates')) 732 | 733 | def cache_get_stats(self): 734 | """ 735 | Return amount of asn and subnets found by source from the cache. 736 | """ 737 | dates = self.cache_get_dates() 738 | dates_sources = dict(zip(dates, self.daily_sources(dates))) 739 | return self.get_stats(dates_sources) 740 | 741 | def cache_get_daily_rank(self, asn, source='global', date=None): 742 | """ 743 | **From the temporary database** 744 | 745 | Get a single rank. 746 | 747 | :param asn: Autonomous System Number 748 | :param source: Source to use. global is the aggregated view for 749 | all the sources 750 | :param date: Date of the information (default: last ranked day) 751 | 752 | :rtype: List 753 | 754 | .. note:: Format of the list 755 | 756 | .. code-block:: python 757 | 758 | [asn, asn_description, date, source, rank] 759 | """ 760 | if source is None: 761 | source = 'global' 762 | if date is None: 763 | date = self.get_default_date() 764 | histo_key = '{date}|{source}|rankv{ip_version}'.format(date=date, source=source, ip_version=c.ip_version) 765 | asn_descr = self.asn_desc_via_history(asn) 766 | return asn, asn_descr, date, source, self.history_db_cache.zscore(histo_key, asn) 767 | 768 | def cache_get_position(self, asn, source='global', date=None): 769 | """ 770 | **From the temporary database** 771 | 772 | Get the position of the ASN in the zrank. 773 | 774 | :param asn: Autonomous System Number 775 | :param source: Source to use. global is the aggregated view for 776 | all the sources 777 | :param date: Date of the information (default: last ranked day) 778 | 779 | :rtype: Integer, position in the list and size of the list. 780 | 781 | .. note:: if None, the zrank does not exists (source or date invalid) 782 | """ 783 | if source is None: 784 | source = 'global' 785 | if date is None: 786 | date = self.get_default_date() 787 | histo_key = '{date}|{source}|rankv{ip_version}'.format(date=date, source=source, ip_version=c.ip_version) 788 | return self.history_db_cache.zrevrank(histo_key, asn), self.history_db_cache.zcard(histo_key) 789 | 790 | def cache_get_top_asns(self, source='global', date=None, limit=100, with_sources=True): 791 | """ 792 | **From the temporary database** 793 | 794 | Get worse ASNs. 795 | 796 | :param source: Source used to rank the ASNs. global is the 797 | aggregated view for all the sources 798 | :param date: Date of the information (default: last ranked day) 799 | :param limit: Number of ASNs to get 800 | :param with_sources: Get the list of sources where each ASN has 801 | been found. 802 | 803 | :rtype: Dictionary 804 | 805 | .. note:: Format of the Dictionary 806 | 807 | .. code-block:: python 808 | 809 | { 810 | 'source': source, 811 | 'date': date, 812 | 'size_list': size, 813 | 'top_list': 814 | [ 815 | ( 816 | (asn, asn_description, rank), 817 | set([source1, source2, ...]) 818 | ), 819 | ... 820 | ] 821 | } 822 | 823 | The set of sources is only presetn if ``with_sources`` 824 | is True 825 | 826 | """ 827 | if source is None: 828 | source = 'global' 829 | if date is None: 830 | date = self.get_default_date() 831 | if limit is None: 832 | limit = 100 833 | if with_sources is None: 834 | with_sources = True 835 | histo_key = '{date}|{histo_key}|rankv{ip_version}'.format(date=date, histo_key=source, ip_version=c.ip_version) 836 | to_return = {'source': source, 'date': date, 'size_list': self.history_db_cache.zcard(histo_key), 'top_list': []} 837 | ranks = self.history_db_cache.zrevrange(histo_key, 0, limit, True) 838 | if ranks is None: 839 | return to_return 840 | temp_rank = [] 841 | for asn, rank in ranks: 842 | if asn == '-1': 843 | continue 844 | asn_descr = self.asn_desc_via_history(asn) 845 | temp_rank.append((asn, asn_descr, rank)) 846 | if not with_sources: 847 | to_return['top_list'] = temp_rank 848 | else: 849 | p = self.history_db_cache.pipeline(False) 850 | [p.smembers('{}|{}'.format(date, rank[0])) for rank in temp_rank] 851 | to_return['top_list'] = list(zip(temp_rank, p.execute())) 852 | return to_return 853 | -------------------------------------------------------------------------------- /bgpranking_redis/constraints.default.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # -*- coding: utf-8 -*- 3 | 4 | ip_version = 4 5 | # How any days in the past do we want 6 | default_timeframe = 3 7 | 8 | # Redis Config 9 | redis_hostname1 = '127.0.0.1' 10 | redis_port1 = 6379 11 | redis_db_global = 5 12 | redis_db_history = 6 13 | redis_db_config = 7 14 | redis_hostname2 = '127.0.0.1' 15 | redis_port2 = 6479 16 | # Redis Cached 17 | redis_cached_port = 6382 18 | redis_cached_db_history = 6 19 | # PTR records DB 20 | ptr_host = '127.0.0.1' 21 | ptr_port = 8323 22 | # ASN history 23 | asn_host = '127.0.0.1' 24 | asn_port = 6389 25 | # IP ASN 26 | ipasn_host = '127.0.0.1' 27 | ipasn_port = 16379 28 | 29 | # Archive config 30 | last_year_archived = 2015 31 | archive_host = '127.0.0.1' 32 | archive_port = 6399 33 | -------------------------------------------------------------------------------- /bgpranking_redis/tools.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # -*- coding: utf-8 -*- 3 | 4 | import os 5 | import glob 6 | from csv import DictReader, DictWriter 7 | from socket import socket, AF_INET, SOCK_STREAM 8 | import json 9 | import operator 10 | 11 | from bgpranking_redis import BGPRanking 12 | 13 | 14 | def aggregate_csvs(output_csv_dir, output_agg_dir, with_world=True, **kwargs): 15 | """ 16 | Aggregate lists of ASNs in a single CSV file. 17 | 18 | kwargs has to be like: 19 | list_name_1 = [list of asns], list_name_2 = [list of asns] 20 | 21 | The output CSV file will have this format: 22 | date, world, list_name_1, list_name_2, .... 23 | """ 24 | csv_files = glob.glob(os.path.join(output_csv_dir, '*')) 25 | result = {} 26 | 27 | for csv_file in csv_files: 28 | asn = os.path.basename(csv_file) 29 | with open(csv_file, 'r') as f: 30 | reader = DictReader(f) 31 | for entry in reader: 32 | # Removing 1 because we aggregate data 33 | rank = float(entry['rank']) - 1 34 | if result.get(entry['day']) is None: 35 | result[entry['day']] = {} 36 | if with_world: 37 | if result[entry['day']].get('world') is None: 38 | result[entry['day']]['world'] = 0 39 | result[entry['day']]['world'] += rank 40 | for key, arg in kwargs.iteritems(): 41 | if asn in arg: 42 | if result[entry['day']].get(key) is None: 43 | result[entry['day']][key] = 0 44 | result[entry['day']][key] += rank 45 | break 46 | if with_world: 47 | fieldnames = ['world'] 48 | else: 49 | fieldnames = [] 50 | fieldnames += kwargs.keys() 51 | filename = os.path.join(output_agg_dir, '_'.join(fieldnames)) 52 | with open(filename, 'w') as f: 53 | w = DictWriter(f, fieldnames=['date'] + fieldnames) 54 | w.writeheader() 55 | for date, entries in result.iteritems(): 56 | entries.update({'date': date}) 57 | w.writerow(entries) 58 | 59 | 60 | def get_asns_country_code(asns): 61 | text_lines = ["begin", "verbose"] 62 | [text_lines.append("AS" + asn) for asn in asns] 63 | text_lines.append("end") 64 | text = '\n'.join(text_lines) + '\n' 65 | 66 | s = socket(AF_INET, SOCK_STREAM) 67 | s.connect(("whois.cymru.com", 43)) 68 | s.send(text) 69 | response = '' 70 | data = s.recv(2048) 71 | while data: 72 | response += data 73 | data = s.recv(2048) 74 | s.close() 75 | to_return = {} 76 | splitted = response.split('\n') 77 | for r in splitted[1:-1]: 78 | s = r.split("|") 79 | asn = s[0].strip() 80 | cc = s[1].strip() 81 | to_return[asn] = cc 82 | return to_return 83 | 84 | 85 | def generate_dumps_for_worldmap(output_dir_js=None, output_dir_csv=None): 86 | bgpranking = BGPRanking() 87 | ranks = bgpranking.cache_get_top_asns(limit=-1, with_sources=False) 88 | if ranks.get('top_list') is not None: 89 | info = get_asns_country_code([asn for asn, description, rank in ranks.get('top_list')]) 90 | to_dump = {} 91 | for asn, d, rank in ranks.get('top_list'): 92 | cc = info[asn] 93 | if to_dump.get(cc) is None: 94 | to_dump[cc] = 0 95 | to_dump[cc] += rank 96 | if output_dir_js is not None: 97 | f = open(os.path.join(output_dir_js, 'worldmap.js'), "w") 98 | f.write("var ranks =\n" + json.dumps(to_dump)) 99 | f.close() 100 | if output_dir_csv is not None: 101 | for_csv = sorted(to_dump.iteritems(), key=operator.itemgetter(1), reverse=True) 102 | with open(os.path.join(output_dir_csv, 'worldmap.csv'), "w") as f: 103 | w = DictWriter(f, fieldnames=['country', 'rank']) 104 | w.writeheader() 105 | for country, rank in for_csv: 106 | w.writerow({'country': country, 'rank': rank}) 107 | -------------------------------------------------------------------------------- /doc/Makefile: -------------------------------------------------------------------------------- 1 | # Makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line. 5 | SPHINXOPTS = 6 | SPHINXBUILD = sphinx-build 7 | PAPER = 8 | BUILDDIR = build 9 | 10 | # Internal variables. 11 | PAPEROPT_a4 = -D latex_paper_size=a4 12 | PAPEROPT_letter = -D latex_paper_size=letter 13 | ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) source 14 | # the i18n builder cannot share the environment and doctrees with the others 15 | I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) source 16 | 17 | .PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest gettext 18 | 19 | help: 20 | @echo "Please use \`make ' where is one of" 21 | @echo " html to make standalone HTML files" 22 | @echo " dirhtml to make HTML files named index.html in directories" 23 | @echo " singlehtml to make a single large HTML file" 24 | @echo " pickle to make pickle files" 25 | @echo " json to make JSON files" 26 | @echo " htmlhelp to make HTML files and a HTML help project" 27 | @echo " qthelp to make HTML files and a qthelp project" 28 | @echo " devhelp to make HTML files and a Devhelp project" 29 | @echo " epub to make an epub" 30 | @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" 31 | @echo " latexpdf to make LaTeX files and run them through pdflatex" 32 | @echo " text to make text files" 33 | @echo " man to make manual pages" 34 | @echo " texinfo to make Texinfo files" 35 | @echo " info to make Texinfo files and run them through makeinfo" 36 | @echo " gettext to make PO message catalogs" 37 | @echo " changes to make an overview of all changed/added/deprecated items" 38 | @echo " linkcheck to check all external links for integrity" 39 | @echo " doctest to run all doctests embedded in the documentation (if enabled)" 40 | 41 | clean: 42 | -rm -rf $(BUILDDIR)/* 43 | 44 | html: 45 | $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html 46 | @echo 47 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." 48 | 49 | dirhtml: 50 | $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml 51 | @echo 52 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." 53 | 54 | singlehtml: 55 | $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml 56 | @echo 57 | @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." 58 | 59 | pickle: 60 | $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle 61 | @echo 62 | @echo "Build finished; now you can process the pickle files." 63 | 64 | json: 65 | $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json 66 | @echo 67 | @echo "Build finished; now you can process the JSON files." 68 | 69 | htmlhelp: 70 | $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp 71 | @echo 72 | @echo "Build finished; now you can run HTML Help Workshop with the" \ 73 | ".hhp project file in $(BUILDDIR)/htmlhelp." 74 | 75 | qthelp: 76 | $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp 77 | @echo 78 | @echo "Build finished; now you can run "qcollectiongenerator" with the" \ 79 | ".qhcp project file in $(BUILDDIR)/qthelp, like this:" 80 | @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/BGPRankingPythonAPI.qhcp" 81 | @echo "To view the help file:" 82 | @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/BGPRankingPythonAPI.qhc" 83 | 84 | devhelp: 85 | $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp 86 | @echo 87 | @echo "Build finished." 88 | @echo "To view the help file:" 89 | @echo "# mkdir -p $$HOME/.local/share/devhelp/BGPRankingPythonAPI" 90 | @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/BGPRankingPythonAPI" 91 | @echo "# devhelp" 92 | 93 | epub: 94 | $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub 95 | @echo 96 | @echo "Build finished. The epub file is in $(BUILDDIR)/epub." 97 | 98 | latex: 99 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 100 | @echo 101 | @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." 102 | @echo "Run \`make' in that directory to run these through (pdf)latex" \ 103 | "(use \`make latexpdf' here to do that automatically)." 104 | 105 | latexpdf: 106 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 107 | @echo "Running LaTeX files through pdflatex..." 108 | $(MAKE) -C $(BUILDDIR)/latex all-pdf 109 | @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." 110 | 111 | text: 112 | $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text 113 | @echo 114 | @echo "Build finished. The text files are in $(BUILDDIR)/text." 115 | 116 | man: 117 | $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man 118 | @echo 119 | @echo "Build finished. The manual pages are in $(BUILDDIR)/man." 120 | 121 | texinfo: 122 | $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo 123 | @echo 124 | @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo." 125 | @echo "Run \`make' in that directory to run these through makeinfo" \ 126 | "(use \`make info' here to do that automatically)." 127 | 128 | info: 129 | $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo 130 | @echo "Running Texinfo files through makeinfo..." 131 | make -C $(BUILDDIR)/texinfo info 132 | @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." 133 | 134 | gettext: 135 | $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale 136 | @echo 137 | @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." 138 | 139 | changes: 140 | $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes 141 | @echo 142 | @echo "The overview file is in $(BUILDDIR)/changes." 143 | 144 | linkcheck: 145 | $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck 146 | @echo 147 | @echo "Link check complete; look for any errors in the above output " \ 148 | "or in $(BUILDDIR)/linkcheck/output.txt." 149 | 150 | doctest: 151 | $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest 152 | @echo "Testing of doctests in the sources finished, look at the " \ 153 | "results in $(BUILDDIR)/doctest/output.txt." 154 | -------------------------------------------------------------------------------- /doc/source/code/API.rst: -------------------------------------------------------------------------------- 1 | *** 2 | API 3 | *** 4 | 5 | .. automodule:: bgpranking.api 6 | :members: 7 | 8 | .. automodule:: bgpranking.helper_global 9 | :members: 10 | -------------------------------------------------------------------------------- /doc/source/conf.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # BGP Ranking Python API documentation build configuration file, created by 4 | # sphinx-quickstart on Mon Nov 26 16:03:33 2012. 5 | # 6 | # This file is execfile()d with the current directory set to its containing dir. 7 | # 8 | # Note that not all possible configuration values are present in this 9 | # autogenerated file. 10 | # 11 | # All configuration values have a default; values that are commented out 12 | # serve to show the default. 13 | 14 | import sys, os 15 | 16 | # If extensions (or modules to document with autodoc) are in another directory, 17 | # add these directories to sys.path here. If the directory is relative to the 18 | # documentation root, use os.path.abspath to make it absolute, like shown here. 19 | #sys.path.insert(0, os.path.abspath('.')) 20 | sys.path.insert(0, os.path.abspath('../../')) 21 | 22 | # -- General configuration ----------------------------------------------------- 23 | 24 | # If your documentation needs a minimal Sphinx version, state it here. 25 | #needs_sphinx = '1.0' 26 | 27 | # Add any Sphinx extension module names here, as strings. They can be extensions 28 | # coming with Sphinx (named 'sphinx.ext.*') or your custom ones. 29 | extensions = ['sphinx.ext.autodoc', 'sphinx.ext.doctest', 'sphinx.ext.todo', 'sphinx.ext.coverage', 'sphinx.ext.pngmath', 'sphinx.ext.mathjax', 'sphinx.ext.ifconfig', 'sphinx.ext.viewcode'] 30 | 31 | # Add any paths that contain templates here, relative to this directory. 32 | templates_path = ['_templates'] 33 | 34 | # The suffix of source filenames. 35 | source_suffix = '.rst' 36 | 37 | # The encoding of source files. 38 | #source_encoding = 'utf-8-sig' 39 | 40 | # The master toctree document. 41 | master_doc = 'index' 42 | 43 | # General information about the project. 44 | project = u'BGP Ranking Python API' 45 | copyright = u'2012, CIRCL' 46 | 47 | # The version info for the project you're documenting, acts as replacement for 48 | # |version| and |release|, also used in various other places throughout the 49 | # built documents. 50 | # 51 | # The short X.Y version. 52 | version = '1.0' 53 | # The full version, including alpha/beta/rc tags. 54 | release = '1.0' 55 | 56 | # The language for content autogenerated by Sphinx. Refer to documentation 57 | # for a list of supported languages. 58 | #language = None 59 | 60 | # There are two options for replacing |today|: either, you set today to some 61 | # non-false value, then it is used: 62 | #today = '' 63 | # Else, today_fmt is used as the format for a strftime call. 64 | #today_fmt = '%B %d, %Y' 65 | 66 | # List of patterns, relative to source directory, that match files and 67 | # directories to ignore when looking for source files. 68 | exclude_patterns = [] 69 | 70 | # The reST default role (used for this markup: `text`) to use for all documents. 71 | #default_role = None 72 | 73 | # If true, '()' will be appended to :func: etc. cross-reference text. 74 | #add_function_parentheses = True 75 | 76 | # If true, the current module name will be prepended to all description 77 | # unit titles (such as .. function::). 78 | #add_module_names = True 79 | 80 | # If true, sectionauthor and moduleauthor directives will be shown in the 81 | # output. They are ignored by default. 82 | #show_authors = False 83 | 84 | # The name of the Pygments (syntax highlighting) style to use. 85 | pygments_style = 'sphinx' 86 | 87 | # A list of ignored prefixes for module index sorting. 88 | #modindex_common_prefix = [] 89 | 90 | 91 | # -- Options for HTML output --------------------------------------------------- 92 | 93 | # The theme to use for HTML and HTML Help pages. See the documentation for 94 | # a list of builtin themes. 95 | html_theme = 'default' 96 | 97 | # Theme options are theme-specific and customize the look and feel of a theme 98 | # further. For a list of options available for each theme, see the 99 | # documentation. 100 | #html_theme_options = {} 101 | 102 | # Add any paths that contain custom themes here, relative to this directory. 103 | #html_theme_path = [] 104 | 105 | # The name for this set of Sphinx documents. If None, it defaults to 106 | # " v documentation". 107 | #html_title = None 108 | 109 | # A shorter title for the navigation bar. Default is the same as html_title. 110 | #html_short_title = None 111 | 112 | # The name of an image file (relative to this directory) to place at the top 113 | # of the sidebar. 114 | #html_logo = None 115 | 116 | # The name of an image file (within the static path) to use as favicon of the 117 | # docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 118 | # pixels large. 119 | #html_favicon = None 120 | 121 | # Add any paths that contain custom static files (such as style sheets) here, 122 | # relative to this directory. They are copied after the builtin static files, 123 | # so a file named "default.css" will overwrite the builtin "default.css". 124 | html_static_path = ['_static'] 125 | 126 | # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, 127 | # using the given strftime format. 128 | #html_last_updated_fmt = '%b %d, %Y' 129 | 130 | # If true, SmartyPants will be used to convert quotes and dashes to 131 | # typographically correct entities. 132 | #html_use_smartypants = True 133 | 134 | # Custom sidebar templates, maps document names to template names. 135 | #html_sidebars = {} 136 | 137 | # Additional templates that should be rendered to pages, maps page names to 138 | # template names. 139 | #html_additional_pages = {} 140 | 141 | # If false, no module index is generated. 142 | #html_domain_indices = True 143 | 144 | # If false, no index is generated. 145 | #html_use_index = True 146 | 147 | # If true, the index is split into individual pages for each letter. 148 | #html_split_index = False 149 | 150 | # If true, links to the reST sources are added to the pages. 151 | #html_show_sourcelink = True 152 | 153 | # If true, "Created using Sphinx" is shown in the HTML footer. Default is True. 154 | #html_show_sphinx = True 155 | 156 | # If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. 157 | #html_show_copyright = True 158 | 159 | # If true, an OpenSearch description file will be output, and all pages will 160 | # contain a tag referring to it. The value of this option must be the 161 | # base URL from which the finished HTML is served. 162 | #html_use_opensearch = '' 163 | 164 | # This is the file name suffix for HTML files (e.g. ".xhtml"). 165 | #html_file_suffix = None 166 | 167 | # Output file base name for HTML help builder. 168 | htmlhelp_basename = 'BGPRankingPythonAPIdoc' 169 | 170 | 171 | # -- Options for LaTeX output -------------------------------------------------- 172 | 173 | latex_elements = { 174 | # The paper size ('letterpaper' or 'a4paper'). 175 | #'papersize': 'letterpaper', 176 | 177 | # The font size ('10pt', '11pt' or '12pt'). 178 | #'pointsize': '10pt', 179 | 180 | # Additional stuff for the LaTeX preamble. 181 | #'preamble': '', 182 | } 183 | 184 | # Grouping the document tree into LaTeX files. List of tuples 185 | # (source start file, target name, title, author, documentclass [howto/manual]). 186 | latex_documents = [ 187 | ('index', 'BGPRankingPythonAPI.tex', u'BGP Ranking Python API Documentation', 188 | u'CIRCL', 'manual'), 189 | ] 190 | 191 | # The name of an image file (relative to this directory) to place at the top of 192 | # the title page. 193 | #latex_logo = None 194 | 195 | # For "manual" documents, if this is true, then toplevel headings are parts, 196 | # not chapters. 197 | #latex_use_parts = False 198 | 199 | # If true, show page references after internal links. 200 | #latex_show_pagerefs = False 201 | 202 | # If true, show URL addresses after external links. 203 | #latex_show_urls = False 204 | 205 | # Documents to append as an appendix to all manuals. 206 | #latex_appendices = [] 207 | 208 | # If false, no module index is generated. 209 | #latex_domain_indices = True 210 | 211 | 212 | # -- Options for manual page output -------------------------------------------- 213 | 214 | # One entry per manual page. List of tuples 215 | # (source start file, name, description, authors, manual section). 216 | man_pages = [ 217 | ('index', 'bgprankingpythonapi', u'BGP Ranking Python API Documentation', 218 | [u'CIRCL'], 1) 219 | ] 220 | 221 | # If true, show URL addresses after external links. 222 | #man_show_urls = False 223 | 224 | 225 | # -- Options for Texinfo output ------------------------------------------------ 226 | 227 | # Grouping the document tree into Texinfo files. List of tuples 228 | # (source start file, target name, title, author, 229 | # dir menu entry, description, category) 230 | texinfo_documents = [ 231 | ('index', 'BGPRankingPythonAPI', u'BGP Ranking Python API Documentation', 232 | u'CIRCL', 'BGPRankingPythonAPI', 'One line description of project.', 233 | 'Miscellaneous'), 234 | ] 235 | 236 | # Documents to append as an appendix to all manuals. 237 | #texinfo_appendices = [] 238 | 239 | # If false, no module index is generated. 240 | #texinfo_domain_indices = True 241 | 242 | # How to display URL addresses: 'footnote', 'no', or 'inline'. 243 | #texinfo_show_urls = 'footnote' 244 | -------------------------------------------------------------------------------- /doc/source/index.rst: -------------------------------------------------------------------------------- 1 | .. BGP Ranking Python API documentation master file, created by 2 | sphinx-quickstart on Mon Nov 26 16:03:33 2012. 3 | You can adapt this file completely to your liking, but it should at least 4 | contain the root `toctree` directive. 5 | 6 | Welcome to BGP Ranking Python API's documentation! 7 | ================================================== 8 | 9 | Contents: 10 | 11 | .. toctree:: 12 | :maxdepth: 2 13 | 14 | Code Documentation 15 | ================== 16 | 17 | .. toctree:: 18 | :maxdepth: 1 19 | 20 | code/API 21 | 22 | 23 | Indices and tables 24 | ================== 25 | 26 | * :ref:`genindex` 27 | * :ref:`modindex` 28 | * :ref:`search` 29 | 30 | -------------------------------------------------------------------------------- /example/api_web/client/MANIFEST.in: -------------------------------------------------------------------------------- 1 | include README.md 2 | -------------------------------------------------------------------------------- /example/api_web/client/README.md: -------------------------------------------------------------------------------- 1 | Python library to access the BGP Ranking REST API. 2 | -------------------------------------------------------------------------------- /example/api_web/client/bgpranking_web/__init__.py: -------------------------------------------------------------------------------- 1 | from api import * 2 | -------------------------------------------------------------------------------- /example/api_web/client/bgpranking_web/api.py: -------------------------------------------------------------------------------- 1 | #!/bin/python 2 | # -*- coding: utf-8 -*- 3 | 4 | """ 5 | This is a python implementation to simplify the use of the JSON API. 6 | It allows an access to all the functions of the Redis API, via the network. 7 | 8 | The parameters are as consistent as possible with the Redis API. 9 | 10 | """ 11 | 12 | try: 13 | import simplejson as json 14 | except: 15 | import json 16 | 17 | import requests 18 | 19 | url = 'http://bgpranking.circl.lu/json' 20 | 21 | 22 | def __prepare_request(query): 23 | headers = {'content-type': 'application/json'} 24 | r = requests.post(url, data=json.dumps(query), headers=headers) 25 | return r.json() 26 | 27 | 28 | def ip_lookup(ip, days_limit=None): 29 | """See :class:`bgpranking.api.get_ip_info`""" 30 | query = {'method': 'ip_lookup'} 31 | query.update({'ip': ip, 'days_limit': days_limit}) 32 | return __prepare_request(query) 33 | 34 | 35 | def all_ranks_single_asn(asn, last_day=None, timeframe=None, 36 | dates_sources=None, with_details_sources=None): 37 | """See :class:`bgpranking.api.get_all_ranks_single_asn`""" 38 | query = {'method': 'all_ranks_single_asn'} 39 | query.update({'asn': asn, 'last_day': last_day, 40 | 'timeframe': timeframe, 'dates_sources': dates_sources, 41 | 'with_details_sources': with_details_sources}) 42 | return __prepare_request(query) 43 | 44 | 45 | def all_ranks_all_asns(last_day=None, timeframe=None, 46 | dates_sources=None, with_details_sources=None): 47 | """See :class:`bgpranking.api.get_all_ranks_all_asns`""" 48 | query = {'method': 'all_ranks_all_asns'} 49 | query.update({'last_day': last_day, 'timeframe': timeframe, 50 | 'dates_sources': dates_sources, 51 | 'with_details_sources': with_details_sources}) 52 | return __prepare_request(query) 53 | 54 | 55 | def asn_description(asn, date=None, sources=None): 56 | """See :class:`bgpranking.api.get_asn_descs`""" 57 | query = {'method': 'asn_description'} 58 | if sources is not None and isinstance(sources, list): 59 | sources = [sources] 60 | query.update({'asn': asn, 'date': date, 'sources': sources}) 61 | return __prepare_request(query) 62 | 63 | 64 | def ips_description(asn, asn_timestamp, date=None, sources=None): 65 | """See :class:`bgpranking.api.get_ips_descs`""" 66 | query = {'method': 'ips_description'} 67 | if sources is not None and isinstance(sources, list): 68 | sources = [sources] 69 | query.update({'asn': asn, 'asn_timestamp': asn_timestamp, 70 | 'date': date, 'sources': sources}) 71 | return __prepare_request(query) 72 | 73 | 74 | def stats(last_day=None, timeframe=None, dates_sources=None): 75 | """See :class:`bgpranking.api.get_stats`""" 76 | query = {'method': 'stats'} 77 | query.update({'last_day': last_day, 'timeframe': timeframe, 78 | 'dates_sources': dates_sources}) 79 | return __prepare_request(query) 80 | 81 | 82 | def block_descriptions(asn, block): 83 | """See :class:`bgpranking.api.get_block_descriptions`""" 84 | query = {'method': 'block_descriptions'} 85 | query.update({'asn': asn, 'block': block}) 86 | return __prepare_request(query) 87 | 88 | 89 | def cached_dates(): 90 | """See :class:`bgpranking.api.cache_get_dates`""" 91 | query = {'method': 'cached_dates'} 92 | return __prepare_request(query) 93 | 94 | 95 | def cached_daily_rank(asn, date=None, sources=None): 96 | """See :class:`bgpranking.api.cache_get_daily_rank`""" 97 | query = {'method': 'cached_daily_rank'} 98 | if sources is not None and isinstance(sources, list): 99 | sources = [sources] 100 | query.update({'asn': asn, 'date': date, 'sources': sources}) 101 | return __prepare_request(query) 102 | 103 | 104 | def cached_top_asns(date=None, source=None, limit=None, with_sources=None): 105 | """See :class:`bgpranking.api.cache_get_top_asns`""" 106 | query = {'method': 'cached_top_asns'} 107 | query.update({'date': date, 'source': source, 'limit': limit, 108 | 'with_sources': with_sources}) 109 | return __prepare_request(query) 110 | 111 | 112 | def cached_position(asn, date=None): 113 | """See :class:`bgpranking.api.cache_get_position`""" 114 | query = {'method': 'cached_position'} 115 | query.update({'asn': asn, 'date': date}) 116 | return __prepare_request(query) 117 | -------------------------------------------------------------------------------- /example/api_web/client/setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # -*- coding: utf-8 -*- 3 | 4 | from setuptools import setup 5 | 6 | setup( 7 | name='bgpranking-web', 8 | version='1.1.1', 9 | description='Library to access the BGP Ranking REST API.', 10 | url='https://github.com/CIRCL/bgpranking-redis-api', 11 | author='Raphaël Vinot', 12 | author_email='raphael.vinot@circl.lu', 13 | maintainer='Raphaël Vinot', 14 | packages=['bgpranking_web'], 15 | long_description=open('README.md').read(), 16 | classifiers=[ 17 | 'License :: OSI Approved :: GNU General Public License v3 (GPLv3)', 18 | 'Development Status :: 5 - Production/Stable', 19 | 'Environment :: Console', 20 | 'Intended Audience :: Science/Research', 21 | 'Intended Audience :: Telecommunications Industry', 22 | 'Programming Language :: Python', 23 | 'Topic :: Security', 24 | 'Topic :: Internet', 25 | 'Topic :: System :: Networking', 26 | ], 27 | 28 | install_requires=['requests'] 29 | ) 30 | -------------------------------------------------------------------------------- /example/api_web/server/webservice.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # -*- coding: utf-8 -*- 3 | 4 | """ 5 | The JSON API allows to do queries on the whole database of BGP Ranking 6 | via the Web interface. 7 | 8 | By default, the path where the webservice is listening is 9 | http://domain/json 10 | 11 | You need to send a well-formed JSON object with a entry 'method' and as 12 | value the name of the function you want to call (listed bellow). 13 | 14 | The other entries of the JSON Object will depend on the function. To keep 15 | it simple, the name of each entry is the parameter name from the Redis API. 16 | 17 | The default parameters are as consistent as possible with the one of the 18 | Redis API (:class:`bgpranking.api`) 19 | 20 | """ 21 | 22 | 23 | from flask import Flask, json, request 24 | import StringIO 25 | import csv 26 | 27 | import bgpranking_redis as bgpranking 28 | bgpranking.asnhistory.redis_host = '127.0.0.1' 29 | bgpranking.asnhistory.redis_port = 6389 30 | bgpranking.ipasn.hostname = '127.0.0.1' 31 | bgpranking.ipasn.port = 6390 32 | bgpranking.ptr_host = '127.0.0.1' 33 | bgpranking.ptr_port = 8323 34 | 35 | logging = True 36 | try: 37 | if logging: 38 | from pubsublogger import publisher 39 | publisher.channel = 'API_Web' 40 | except: 41 | logging = False 42 | 43 | 44 | app = Flask(__name__) 45 | app.debug = True 46 | 47 | authorized_methods = ['all_ranks_single_asn', 'all_ranks_all_asns', 48 | 'block_descriptions', 'asn_description', 'ips_description', 49 | 'stats', 'cached_dates', 'cached_daily_rank', 'cached_top_asns', 50 | 'ip_lookup', 'cached_position'] 51 | 52 | 53 | class SetEncoder(json.JSONEncoder): 54 | def default(self, obj): 55 | if type(obj) == type(set()): 56 | return list(obj) 57 | return json.JSONEncoder.default(self, obj) 58 | 59 | 60 | def __default_dates_sources(req): 61 | dates_sources = req.get('dates_sources') 62 | if dates_sources is None: 63 | dates_sources = bgpranking.prepare_sources_by_dates(req.get('last_day'), req.get('timeframe')) 64 | return dates_sources 65 | 66 | 67 | def __csv2string(data): 68 | si = StringIO.StringIO() 69 | cw = csv.writer(si) 70 | cw.writerow(data) 71 | return si.getvalue().strip('\r\n') 72 | 73 | 74 | def __query_logging(ip, user_agent, method, date=None, source=None, 75 | asn=None, asn_details=None, compared_asns=None, 76 | ip_lookup=None, level=None): 77 | if level == 'warning': 78 | publisher.info(__csv2string([ip, user_agent, method, date, 79 | source, asn, asn_details, compared_asns, ip_lookup])) 80 | elif level == 'error': 81 | publisher.info(__csv2string([ip, user_agent, method, date, 82 | source, asn, asn_details, compared_asns, ip_lookup])) 83 | else: 84 | publisher.info(__csv2string([ip, user_agent, method, date, 85 | source, asn, asn_details, compared_asns, ip_lookup])) 86 | 87 | 88 | @app.route('/json', methods=['POST']) 89 | def __entry_point(): 90 | """ 91 | Function called when an query is made on /json. Expects a JSON 92 | object with at least a 'method' entry. 93 | """ 94 | ip = request.remote_addr 95 | ua = request.headers.get('User-Agent', 'Empty User-Agent') 96 | method = request.json.get('method') 97 | if method is None: 98 | __query_logging(ip, ua, method, level='warning') 99 | return json.dumps({'error': 'No method provided.'}) 100 | if method not in authorized_methods: 101 | # unauthorized query 102 | __query_logging(ip, ua, method, level='warning') 103 | return json.dumps({'error': 'Unauthorized method.'}) 104 | fct = globals().get(method) 105 | if fct is None: 106 | # unknown method, the method is authorized, but does not exists... 107 | __query_logging(ip, ua, method, level='error') 108 | return json.dumps({'error': 'Unknown method.'}) 109 | try: 110 | result = fct(request.json) 111 | __query_logging(ip, ua, method) 112 | return result 113 | except: 114 | __query_logging(ip, ua, method, level='error') 115 | return json.dumps({'error': 'Something went wrong.'}) 116 | 117 | 118 | def ip_lookup(request): 119 | """See :class:`bgpranking.api.get_ip_info`""" 120 | ip = request.get('ip') 121 | if ip is None: 122 | return json.dumps({}) 123 | return json.dumps(bgpranking.get_ip_info(ip, request.get('days_limit'))) 124 | 125 | 126 | def all_ranks_single_asn(request): 127 | """See :class:`bgpranking.api.get_all_ranks_single_asn`""" 128 | asn = request.get('asn') 129 | if asn is None: 130 | return json.dumps({}) 131 | dates_sources = __default_dates_sources(request) 132 | return json.dumps(bgpranking.get_all_ranks_single_asn(asn, 133 | dates_sources, request.get('with_details_sources'))) 134 | 135 | 136 | def all_ranks_all_asns(request): 137 | """See :class:`bgpranking.api.get_all_ranks_all_asns`""" 138 | dates_sources = __default_dates_sources(request) 139 | with_details_sources = request.get('with_details_sources') 140 | return json.dumps(bgpranking.get_all_ranks_all_asns(dates_sources, 141 | with_details_sources)) 142 | 143 | 144 | def block_descriptions(request): 145 | """See :class:`bgpranking.api.get_block_descriptions`""" 146 | asn = request.get('asn') 147 | block = request.get('block') 148 | if asn is None or block is None: 149 | return json.dumps([]) 150 | return json.dumps(bgpranking.get_block_descriptions(asn, block)) 151 | 152 | 153 | def asn_description(request): 154 | """See :class:`bgpranking.api.get_asn_descs`""" 155 | asn = request.get('asn') 156 | if asn is None: 157 | return json.dumps({}) 158 | return json.dumps(bgpranking.get_asn_descs(asn, 159 | request.get('date'), request.get('sources'))) 160 | 161 | 162 | def ips_description(request): 163 | """See :class:`bgpranking.api.get_ips_descs`""" 164 | asn = request.get('asn') 165 | block = request.get('block') 166 | if asn is None or block is None: 167 | return json.dumps({}) 168 | return json.dumps(bgpranking.get_ips_descs(asn, block, 169 | request.get('date'), request.get('sources'))) 170 | 171 | 172 | def stats(request): 173 | """See :class:`bgpranking.api.get_stats`""" 174 | return json.dumps(bgpranking.get_stats( 175 | __default_dates_sources(request))) 176 | 177 | 178 | # need cached data 179 | def cached_dates(request): 180 | """See :class:`bgpranking.api.cache_get_dates`""" 181 | return json.dumps(bgpranking.cache_get_dates()) 182 | 183 | 184 | def cached_daily_rank(request): 185 | """See :class:`bgpranking.api.cache_get_daily_rank`""" 186 | asn = request.get('asn') 187 | if asn is None: 188 | return json.dumps({}) 189 | cached_dates = bgpranking.cache_get_dates() 190 | date = request.get('date') 191 | if date is None or date in cached_dates: 192 | return json.dumps(bgpranking.cache_get_daily_rank(asn, 193 | request.get('sources'), date)) 194 | return json.dumps({}) 195 | 196 | 197 | def cached_top_asns(request): 198 | """See :class:`bgpranking.api.cache_get_top_asns`""" 199 | cached_dates = bgpranking.cache_get_dates() 200 | date = request.get('date') 201 | if date is None or date in cached_dates: 202 | return json.dumps(bgpranking.cache_get_top_asns( 203 | request.get('source'), date, request.get('limit'), 204 | request.get('with_sources')), 205 | cls=SetEncoder) 206 | return json.dumps({}) 207 | 208 | 209 | def cached_position(request): 210 | """See :class:`bgpranking.api.cache_get_position`""" 211 | asn = request.get('asn') 212 | if asn is None: 213 | return json.dumps({}) 214 | cached_dates = bgpranking.cache_get_dates() 215 | date = request.get('date') 216 | if date is None or date in cached_dates: 217 | return json.dumps(bgpranking.cache_get_position(asn, date=date), 218 | cls=SetEncoder) 219 | return json.dumps({}) 220 | 221 | if __name__ == '__main__': 222 | app.run() 223 | -------------------------------------------------------------------------------- /example/export/asn_ranks/agg_consummer.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # -*- coding: utf-8 -*- 3 | 4 | from csv import DictReader 5 | import redis 6 | import os 7 | import sys 8 | 9 | if __name__ == '__main__': 10 | r = redis.Redis(unix_socket_path='./redis_export.sock') 11 | countries = r.smembers('countries') 12 | if len(countries) == 0: 13 | print 'No country codes availables, breaking.' 14 | sys.exit() 15 | cached_asns = {} 16 | for cc in countries: 17 | cached_asns[cc] = r.smembers(cc) 18 | while True: 19 | csv_file = r.spop('known_asns') 20 | if csv_file is None: 21 | break 22 | asn = os.path.basename(csv_file) 23 | with open(csv_file, 'r') as f: 24 | reader = DictReader(f) 25 | p = r.pipeline(False) 26 | for entry in reader: 27 | p.sadd('days', entry['day']) 28 | # Removing 1 because we aggregate data 29 | rank = float(entry['rank']) - 1 30 | p.hincrbyfloat(entry['day'], 'world', rank) 31 | for cc in countries: 32 | if asn in cached_asns[cc]: 33 | p.hincrbyfloat(entry['day'], cc, rank) 34 | break 35 | p.execute() 36 | 37 | -------------------------------------------------------------------------------- /example/export/asn_ranks/consumer.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # -*- coding: utf-8 -*- 3 | 4 | import redis 5 | import bgpranking 6 | import os 7 | from csv import writer 8 | 9 | csv_dir = os.path.join('..', '..', 'website', 'data', 'csv') 10 | 11 | if __name__ == '__main__': 12 | 13 | r = redis.Redis(unix_socket_path='./redis_export.sock') 14 | interval = int(r.get('interval_size')) 15 | dates_sources = bgpranking.prepare_sources_by_dates(None, interval) 16 | while True: 17 | asn = r.spop('asns') 18 | if asn is None: 19 | break 20 | filename = os.path.join(csv_dir, asn) 21 | with open(filename, 'w') as csvfile: 22 | w = writer(csvfile) 23 | w.writerow(['day', 'rank']) 24 | ranks = bgpranking.get_all_ranks_single_asn(asn, dates_sources) 25 | for date, entry in ranks.iteritems(): 26 | w.writerow([date, 1 + entry['total']]) 27 | 28 | 29 | 30 | -------------------------------------------------------------------------------- /example/export/asn_ranks/generate_aggs.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # -*- coding: utf-8 -*- 3 | 4 | import sys 5 | import json 6 | import os 7 | import urllib2 8 | from bgpranking import tools 9 | from csv import DictWriter 10 | import datetime 11 | 12 | import argparse 13 | import glob 14 | import redis 15 | 16 | 17 | csv_dir = os.path.join('..', '..', 'website', 'data', 'csv') 18 | agg_csv_dir = os.path.join('..', '..','website', 'data', 'csv_agg') 19 | js_dir = os.path.join('..', '..', 'website', 'data', 'js') 20 | 21 | ripe_url = 'https://stat.ripe.net/data/country-resource-list/data.json?resource={cc}&time={day}' 22 | 23 | def get_announces(ripe_url): 24 | try: 25 | handle = urllib2.urlopen(ripe_url, timeout=10) 26 | except: 27 | return None 28 | json_dump = handle.read() 29 | data = json.loads(json_dump) 30 | asns = data['data']['resources']['asn'] 31 | return asns 32 | 33 | if __name__ == '__main__': 34 | parser = argparse.ArgumentParser( 35 | description='Bunch of function to create aggregations') 36 | parser.add_argument('-pka', '--push_known_asns', action='store_true', 37 | default=False) 38 | parser.add_argument('-cc', '--country_codes', nargs='+', type=str) 39 | parser.add_argument('-dcc', '--dump_country_codes', nargs='+', type=str) 40 | parser.add_argument('-map', '--make_map', action='store_true', default=False) 41 | args = parser.parse_args() 42 | r = redis.Redis(unix_socket_path='./redis_export.sock') 43 | 44 | if args.push_known_asns: 45 | csv_files = glob.glob(os.path.join(csv_dir, '*')) 46 | p = r.pipeline(False) 47 | [p.sadd('known_asns', csv_file) for csv_file in csv_files] 48 | p.execute() 49 | print 'Number of asns:', len(csv_files) 50 | elif args.country_codes is not None and len(args.country_codes) > 0: 51 | p = r.pipeline(False) 52 | for cc in args.country_codes: 53 | date = datetime.date.today() 54 | counter = 0 55 | while True: 56 | if counter >= 10: 57 | sys.exit(1) 58 | cur_date = date - datetime.timedelta(days=counter) 59 | url = ripe_url.format(cc=cc, day=cur_date.isoformat()) 60 | asns = get_announces(url) 61 | if asns is None or len(asns) < 5: 62 | #print 'Unable to download the list of ASNs. Abording.' 63 | counter += 1 64 | continue 65 | p.sadd(cc, *asns) 66 | break 67 | p.sadd('countries', *args.country_codes) 68 | p.execute() 69 | elif args.dump_country_codes is not None and len(args.dump_country_codes) > 0: 70 | filename = os.path.join(agg_csv_dir, '_'.join(args.dump_country_codes)) 71 | with open(filename, 'w') as f: 72 | w = DictWriter(f, fieldnames= ['date'] + args.dump_country_codes) 73 | w.writeheader() 74 | for date in r.smembers('days'): 75 | entries = {'date': date} 76 | for cc in args.dump_country_codes: 77 | entries.update({cc: r.hget(date, cc)}) 78 | w.writerow(entries) 79 | elif args.make_map: 80 | tools.generate_dumps_for_worldmap(js_dir, agg_csv_dir) 81 | sys.exit(0) 82 | 83 | -------------------------------------------------------------------------------- /example/export/asn_ranks/init_redis.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # -*- coding: utf-8 -*- 3 | 4 | import bgpranking 5 | import redis 6 | 7 | interval_size = 800 8 | 9 | if __name__ == '__main__': 10 | 11 | dates_sources = bgpranking.prepare_sources_by_dates(None, interval_size) 12 | asns = bgpranking.existing_asns_timeframe(dates_sources) 13 | 14 | r = redis.Redis(unix_socket_path='./redis_export.sock') 15 | r.set('interval_size', interval_size) 16 | r.sadd('asns', *asns) 17 | 18 | -------------------------------------------------------------------------------- /example/export/asn_ranks/launch.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | while true; do 4 | echo ----- New Run ----- 5 | date 6 | bash ./launch_local_redis.sh 7 | echo -n 'Preparing Redis database... ' 8 | python ./init_redis.py 9 | echo 'done.' 10 | 11 | echo -n 'Exporting ranks to CSV... ' 12 | for i in {1..10}; do 13 | python ./consumer.py & 14 | done 15 | python ./consumer.py 16 | echo 'done.' 17 | date 18 | 19 | sleep 10 20 | redis-cli -s ./redis_export.sock flushall 21 | 22 | # ------------------------------------------------------ 23 | 24 | echo -n 'Preparing database...' 25 | python ./generate_aggs.py --push_known_asns 26 | 27 | echo -n 'Generating worldmap...' 28 | python ./generate_aggs.py --make_map 29 | echo 'done.' 30 | 31 | echo -n 'Fetching ASNs by country codes... ' 32 | python ./generate_aggs.py --country_codes be ch de lu fr nl 33 | 34 | if [ $? -eq 0 ]; then 35 | echo 'done.' 36 | echo -n 'Preparing aggregations...' 37 | for i in {1..10}; do 38 | python ./agg_consummer.py & 39 | done 40 | python ./agg_consummer.py 41 | echo 'done.' 42 | date 43 | sleep 10 44 | 45 | # ------------------------------------------------------ 46 | 47 | echo -n 'Dumping aggregations...' 48 | python ./generate_aggs.py --dump_country_codes world lu 49 | python ./generate_aggs.py --dump_country_codes be ch de lu fr nl 50 | echo 'done.' 51 | else 52 | echo ' unable to do so.' 53 | fi 54 | 55 | 56 | redis-cli -s ./redis_export.sock shutdown 57 | echo ----- End of Run. ----- 58 | sleep 10000 59 | done 60 | -------------------------------------------------------------------------------- /example/export/asn_ranks/launch_local_redis.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | HOME="/home/raphael/" 4 | 5 | REDIS_SERVER="${HOME}/gits/redis/src/redis-server" 6 | 7 | ${REDIS_SERVER} ./local_redis.conf 8 | -------------------------------------------------------------------------------- /example/export/asn_ranks/local_redis.conf: -------------------------------------------------------------------------------- 1 | # Redis configuration file example 2 | 3 | # Note on units: when memory size is needed, it is possible to specify 4 | # it in the usual form of 1k 5GB 4M and so forth: 5 | # 6 | # 1k => 1000 bytes 7 | # 1kb => 1024 bytes 8 | # 1m => 1000000 bytes 9 | # 1mb => 1024*1024 bytes 10 | # 1g => 1000000000 bytes 11 | # 1gb => 1024*1024*1024 bytes 12 | # 13 | # units are case insensitive so 1GB 1Gb 1gB are all the same. 14 | 15 | # By default Redis does not run as a daemon. Use 'yes' if you need it. 16 | # Note that Redis will write a pid file in /var/run/redis.pid when daemonized. 17 | daemonize yes 18 | 19 | # When running daemonized, Redis writes a pid file in /var/run/redis.pid by 20 | # default. You can specify a custom pid file location here. 21 | pidfile ./redis_export.pid 22 | 23 | # Accept connections on the specified port, default is 6379. 24 | # If port 0 is specified Redis will not listen on a TCP socket. 25 | port 0 26 | 27 | # If you want you can bind a single interface, if the bind option is not 28 | # specified all the interfaces will listen for incoming connections. 29 | # 30 | # bind 127.0.0.1 31 | 32 | # Specify the path for the unix socket that will be used to listen for 33 | # incoming connections. There is no default, so Redis will not listen 34 | # on a unix socket when not specified. 35 | # 36 | unixsocket ./redis_export.sock 37 | unixsocketperm 755 38 | 39 | # Close the connection after a client is idle for N seconds (0 to disable) 40 | timeout 0 41 | 42 | # TCP keepalive. 43 | # 44 | # If non-zero, use SO_KEEPALIVE to send TCP ACKs to clients in absence 45 | # of communication. This is useful for two reasons: 46 | # 47 | # 1) Detect dead peers. 48 | # 2) Take the connection alive from the point of view of network 49 | # equipment in the middle. 50 | # 51 | # On Linux, the specified value (in seconds) is the period used to send ACKs. 52 | # Note that to close the connection the double of the time is needed. 53 | # On other kernels the period depends on the kernel configuration. 54 | # 55 | # A reasonable value for this option is 60 seconds. 56 | tcp-keepalive 0 57 | 58 | # Specify the server verbosity level. 59 | # This can be one of: 60 | # debug (a lot of information, useful for development/testing) 61 | # verbose (many rarely useful info, but not a mess like the debug level) 62 | # notice (moderately verbose, what you want in production probably) 63 | # warning (only very important / critical messages are logged) 64 | loglevel notice 65 | 66 | # Specify the log file name. Also 'stdout' can be used to force 67 | # Redis to log on the standard output. Note that if you use standard 68 | # output for logging but daemonize, logs will be sent to /dev/null 69 | logfile stdout 70 | 71 | # To enable logging to the system logger, just set 'syslog-enabled' to yes, 72 | # and optionally update the other syslog parameters to suit your needs. 73 | # syslog-enabled no 74 | 75 | # Specify the syslog identity. 76 | # syslog-ident redis 77 | 78 | # Specify the syslog facility. Must be USER or between LOCAL0-LOCAL7. 79 | # syslog-facility local0 80 | 81 | # Set the number of databases. The default database is DB 0, you can select 82 | # a different one on a per-connection basis using SELECT where 83 | # dbid is a number between 0 and 'databases'-1 84 | databases 2 85 | 86 | ################################ SNAPSHOTTING ################################# 87 | # 88 | # Save the DB on disk: 89 | # 90 | # save 91 | # 92 | # Will save the DB if both the given number of seconds and the given 93 | # number of write operations against the DB occurred. 94 | # 95 | # In the example below the behaviour will be to save: 96 | # after 900 sec (15 min) if at least 1 key changed 97 | # after 300 sec (5 min) if at least 10 keys changed 98 | # after 60 sec if at least 10000 keys changed 99 | # 100 | # Note: you can disable saving at all commenting all the "save" lines. 101 | # 102 | # It is also possible to remove all the previously configured save 103 | # points by adding a save directive with a single empty string argument 104 | # like in the following example: 105 | # 106 | # save "" 107 | 108 | #save 900 1 109 | #save 300 10 110 | #save 60 10000 111 | 112 | # By default Redis will stop accepting writes if RDB snapshots are enabled 113 | # (at least one save point) and the latest background save failed. 114 | # This will make the user aware (in an hard way) that data is not persisting 115 | # on disk properly, otherwise chances are that no one will notice and some 116 | # distater will happen. 117 | # 118 | # If the background saving process will start working again Redis will 119 | # automatically allow writes again. 120 | # 121 | # However if you have setup your proper monitoring of the Redis server 122 | # and persistence, you may want to disable this feature so that Redis will 123 | # continue to work as usually even if there are problems with disk, 124 | # permissions, and so forth. 125 | stop-writes-on-bgsave-error yes 126 | 127 | # Compress string objects using LZF when dump .rdb databases? 128 | # For default that's set to 'yes' as it's almost always a win. 129 | # If you want to save some CPU in the saving child set it to 'no' but 130 | # the dataset will likely be bigger if you have compressible values or keys. 131 | rdbcompression yes 132 | 133 | # Since version 5 of RDB a CRC64 checksum is placed at the end of the file. 134 | # This makes the format more resistant to corruption but there is a performance 135 | # hit to pay (around 10%) when saving and loading RDB files, so you can disable it 136 | # for maximum performances. 137 | # 138 | # RDB files created with checksum disabled have a checksum of zero that will 139 | # tell the loading code to skip the check. 140 | rdbchecksum yes 141 | 142 | # The filename where to dump the DB 143 | dbfilename dump_export.rdb 144 | 145 | # The working directory. 146 | # 147 | # The DB will be written inside this directory, with the filename specified 148 | # above using the 'dbfilename' configuration directive. 149 | # 150 | # The Append Only File will also be created inside this directory. 151 | # 152 | # Note that you must specify a directory here, not a file name. 153 | dir ./ 154 | 155 | ################################# REPLICATION ################################# 156 | 157 | # Master-Slave replication. Use slaveof to make a Redis instance a copy of 158 | # another Redis server. Note that the configuration is local to the slave 159 | # so for example it is possible to configure the slave to save the DB with a 160 | # different interval, or to listen to another port, and so on. 161 | # 162 | # slaveof 163 | 164 | # If the master is password protected (using the "requirepass" configuration 165 | # directive below) it is possible to tell the slave to authenticate before 166 | # starting the replication synchronization process, otherwise the master will 167 | # refuse the slave request. 168 | # 169 | # masterauth 170 | 171 | # When a slave loses its connection with the master, or when the replication 172 | # is still in progress, the slave can act in two different ways: 173 | # 174 | # 1) if slave-serve-stale-data is set to 'yes' (the default) the slave will 175 | # still reply to client requests, possibly with out of date data, or the 176 | # data set may just be empty if this is the first synchronization. 177 | # 178 | # 2) if slave-serve-stale-data is set to 'no' the slave will reply with 179 | # an error "SYNC with master in progress" to all the kind of commands 180 | # but to INFO and SLAVEOF. 181 | # 182 | slave-serve-stale-data yes 183 | 184 | # You can configure a slave instance to accept writes or not. Writing against 185 | # a slave instance may be useful to store some ephemeral data (because data 186 | # written on a slave will be easily deleted after resync with the master) but 187 | # may also cause problems if clients are writing to it because of a 188 | # misconfiguration. 189 | # 190 | # Since Redis 2.6 by default slaves are read-only. 191 | # 192 | # Note: read only slaves are not designed to be exposed to untrusted clients 193 | # on the internet. It's just a protection layer against misuse of the instance. 194 | # Still a read only slave exports by default all the administrative commands 195 | # such as CONFIG, DEBUG, and so forth. To a limited extend you can improve 196 | # security of read only slaves using 'rename-command' to shadow all the 197 | # administrative / dangerous commands. 198 | slave-read-only yes 199 | 200 | # Slaves send PINGs to server in a predefined interval. It's possible to change 201 | # this interval with the repl_ping_slave_period option. The default value is 10 202 | # seconds. 203 | # 204 | # repl-ping-slave-period 10 205 | 206 | # The following option sets a timeout for both Bulk transfer I/O timeout and 207 | # master data or ping response timeout. The default value is 60 seconds. 208 | # 209 | # It is important to make sure that this value is greater than the value 210 | # specified for repl-ping-slave-period otherwise a timeout will be detected 211 | # every time there is low traffic between the master and the slave. 212 | # 213 | # repl-timeout 60 214 | 215 | # Disable TCP_NODELAY on the slave socket after SYNC? 216 | # 217 | # If you select "yes" Redis will use a smaller number of TCP packets and 218 | # less bandwidth to send data to slaves. But this can add a delay for 219 | # the data to appear on the slave side, up to 40 milliseconds with 220 | # Linux kernels using a default configuration. 221 | # 222 | # If you select "no" the delay for data to appear on the slave side will 223 | # be reduced but more bandwidth will be used for replication. 224 | # 225 | # By default we optimize for low latency, but in very high traffic conditions 226 | # or when the master and slaves are many hops away, turning this to "yes" may 227 | # be a good idea. 228 | repl-disable-tcp-nodelay no 229 | 230 | # The slave priority is an integer number published by Redis in the INFO output. 231 | # It is used by Redis Sentinel in order to select a slave to promote into a 232 | # master if the master is no longer working correctly. 233 | # 234 | # A slave with a low priority number is considered better for promotion, so 235 | # for instance if there are three slaves with priority 10, 100, 25 Sentinel will 236 | # pick the one wtih priority 10, that is the lowest. 237 | # 238 | # However a special priority of 0 marks the slave as not able to perform the 239 | # role of master, so a slave with priority of 0 will never be selected by 240 | # Redis Sentinel for promotion. 241 | # 242 | # By default the priority is 100. 243 | slave-priority 100 244 | 245 | ################################## SECURITY ################################### 246 | 247 | # Require clients to issue AUTH before processing any other 248 | # commands. This might be useful in environments in which you do not trust 249 | # others with access to the host running redis-server. 250 | # 251 | # This should stay commented out for backward compatibility and because most 252 | # people do not need auth (e.g. they run their own servers). 253 | # 254 | # Warning: since Redis is pretty fast an outside user can try up to 255 | # 150k passwords per second against a good box. This means that you should 256 | # use a very strong password otherwise it will be very easy to break. 257 | # 258 | # requirepass foobared 259 | 260 | # Command renaming. 261 | # 262 | # It is possible to change the name of dangerous commands in a shared 263 | # environment. For instance the CONFIG command may be renamed into something 264 | # hard to guess so that it will still be available for internal-use tools 265 | # but not available for general clients. 266 | # 267 | # Example: 268 | # 269 | # rename-command CONFIG b840fc02d524045429941cc15f59e41cb7be6c52 270 | # 271 | # It is also possible to completely kill a command by renaming it into 272 | # an empty string: 273 | # 274 | # rename-command CONFIG "" 275 | # 276 | # Please note that changing the name of commands that are logged into the 277 | # AOF file or transmitted to slaves may cause problems. 278 | 279 | ################################### LIMITS #################################### 280 | 281 | # Set the max number of connected clients at the same time. By default 282 | # this limit is set to 10000 clients, however if the Redis server is not 283 | # able to configure the process file limit to allow for the specified limit 284 | # the max number of allowed clients is set to the current file limit 285 | # minus 32 (as Redis reserves a few file descriptors for internal uses). 286 | # 287 | # Once the limit is reached Redis will close all the new connections sending 288 | # an error 'max number of clients reached'. 289 | # 290 | # maxclients 10000 291 | 292 | # Don't use more memory than the specified amount of bytes. 293 | # When the memory limit is reached Redis will try to remove keys 294 | # accordingly to the eviction policy selected (see maxmemmory-policy). 295 | # 296 | # If Redis can't remove keys according to the policy, or if the policy is 297 | # set to 'noeviction', Redis will start to reply with errors to commands 298 | # that would use more memory, like SET, LPUSH, and so on, and will continue 299 | # to reply to read-only commands like GET. 300 | # 301 | # This option is usually useful when using Redis as an LRU cache, or to set 302 | # an hard memory limit for an instance (using the 'noeviction' policy). 303 | # 304 | # WARNING: If you have slaves attached to an instance with maxmemory on, 305 | # the size of the output buffers needed to feed the slaves are subtracted 306 | # from the used memory count, so that network problems / resyncs will 307 | # not trigger a loop where keys are evicted, and in turn the output 308 | # buffer of slaves is full with DELs of keys evicted triggering the deletion 309 | # of more keys, and so forth until the database is completely emptied. 310 | # 311 | # In short... if you have slaves attached it is suggested that you set a lower 312 | # limit for maxmemory so that there is some free RAM on the system for slave 313 | # output buffers (but this is not needed if the policy is 'noeviction'). 314 | # 315 | # maxmemory 316 | 317 | # MAXMEMORY POLICY: how Redis will select what to remove when maxmemory 318 | # is reached. You can select among five behaviors: 319 | # 320 | # volatile-lru -> remove the key with an expire set using an LRU algorithm 321 | # allkeys-lru -> remove any key accordingly to the LRU algorithm 322 | # volatile-random -> remove a random key with an expire set 323 | # allkeys-random -> remove a random key, any key 324 | # volatile-ttl -> remove the key with the nearest expire time (minor TTL) 325 | # noeviction -> don't expire at all, just return an error on write operations 326 | # 327 | # Note: with any of the above policies, Redis will return an error on write 328 | # operations, when there are not suitable keys for eviction. 329 | # 330 | # At the date of writing this commands are: set setnx setex append 331 | # incr decr rpush lpush rpushx lpushx linsert lset rpoplpush sadd 332 | # sinter sinterstore sunion sunionstore sdiff sdiffstore zadd zincrby 333 | # zunionstore zinterstore hset hsetnx hmset hincrby incrby decrby 334 | # getset mset msetnx exec sort 335 | # 336 | # The default is: 337 | # 338 | # maxmemory-policy volatile-lru 339 | 340 | # LRU and minimal TTL algorithms are not precise algorithms but approximated 341 | # algorithms (in order to save memory), so you can select as well the sample 342 | # size to check. For instance for default Redis will check three keys and 343 | # pick the one that was used less recently, you can change the sample size 344 | # using the following configuration directive. 345 | # 346 | # maxmemory-samples 3 347 | 348 | ############################## APPEND ONLY MODE ############################### 349 | 350 | # By default Redis asynchronously dumps the dataset on disk. This mode is 351 | # good enough in many applications, but an issue with the Redis process or 352 | # a power outage may result into a few minutes of writes lost (depending on 353 | # the configured save points). 354 | # 355 | # The Append Only File is an alternative persistence mode that provides 356 | # much better durability. For instance using the default data fsync policy 357 | # (see later in the config file) Redis can lose just one second of writes in a 358 | # dramatic event like a server power outage, or a single write if something 359 | # wrong with the Redis process itself happens, but the operating system is 360 | # still running correctly. 361 | # 362 | # AOF and RDB persistence can be enabled at the same time without problems. 363 | # If the AOF is enabled on startup Redis will load the AOF, that is the file 364 | # with the better durability guarantees. 365 | # 366 | # Please check http://redis.io/topics/persistence for more information. 367 | 368 | appendonly no 369 | 370 | # The name of the append only file (default: "appendonly.aof") 371 | # appendfilename appendonly.aof 372 | 373 | # The fsync() call tells the Operating System to actually write data on disk 374 | # instead to wait for more data in the output buffer. Some OS will really flush 375 | # data on disk, some other OS will just try to do it ASAP. 376 | # 377 | # Redis supports three different modes: 378 | # 379 | # no: don't fsync, just let the OS flush the data when it wants. Faster. 380 | # always: fsync after every write to the append only log . Slow, Safest. 381 | # everysec: fsync only one time every second. Compromise. 382 | # 383 | # The default is "everysec", as that's usually the right compromise between 384 | # speed and data safety. It's up to you to understand if you can relax this to 385 | # "no" that will let the operating system flush the output buffer when 386 | # it wants, for better performances (but if you can live with the idea of 387 | # some data loss consider the default persistence mode that's snapshotting), 388 | # or on the contrary, use "always" that's very slow but a bit safer than 389 | # everysec. 390 | # 391 | # More details please check the following article: 392 | # http://antirez.com/post/redis-persistence-demystified.html 393 | # 394 | # If unsure, use "everysec". 395 | 396 | # appendfsync always 397 | appendfsync everysec 398 | # appendfsync no 399 | 400 | # When the AOF fsync policy is set to always or everysec, and a background 401 | # saving process (a background save or AOF log background rewriting) is 402 | # performing a lot of I/O against the disk, in some Linux configurations 403 | # Redis may block too long on the fsync() call. Note that there is no fix for 404 | # this currently, as even performing fsync in a different thread will block 405 | # our synchronous write(2) call. 406 | # 407 | # In order to mitigate this problem it's possible to use the following option 408 | # that will prevent fsync() from being called in the main process while a 409 | # BGSAVE or BGREWRITEAOF is in progress. 410 | # 411 | # This means that while another child is saving, the durability of Redis is 412 | # the same as "appendfsync none". In practical terms, this means that it is 413 | # possible to lose up to 30 seconds of log in the worst scenario (with the 414 | # default Linux settings). 415 | # 416 | # If you have latency problems turn this to "yes". Otherwise leave it as 417 | # "no" that is the safest pick from the point of view of durability. 418 | no-appendfsync-on-rewrite no 419 | 420 | # Automatic rewrite of the append only file. 421 | # Redis is able to automatically rewrite the log file implicitly calling 422 | # BGREWRITEAOF when the AOF log size grows by the specified percentage. 423 | # 424 | # This is how it works: Redis remembers the size of the AOF file after the 425 | # latest rewrite (if no rewrite has happened since the restart, the size of 426 | # the AOF at startup is used). 427 | # 428 | # This base size is compared to the current size. If the current size is 429 | # bigger than the specified percentage, the rewrite is triggered. Also 430 | # you need to specify a minimal size for the AOF file to be rewritten, this 431 | # is useful to avoid rewriting the AOF file even if the percentage increase 432 | # is reached but it is still pretty small. 433 | # 434 | # Specify a percentage of zero in order to disable the automatic AOF 435 | # rewrite feature. 436 | 437 | auto-aof-rewrite-percentage 100 438 | auto-aof-rewrite-min-size 64mb 439 | 440 | ################################ LUA SCRIPTING ############################### 441 | 442 | # Max execution time of a Lua script in milliseconds. 443 | # 444 | # If the maximum execution time is reached Redis will log that a script is 445 | # still in execution after the maximum allowed time and will start to 446 | # reply to queries with an error. 447 | # 448 | # When a long running script exceed the maximum execution time only the 449 | # SCRIPT KILL and SHUTDOWN NOSAVE commands are available. The first can be 450 | # used to stop a script that did not yet called write commands. The second 451 | # is the only way to shut down the server in the case a write commands was 452 | # already issue by the script but the user don't want to wait for the natural 453 | # termination of the script. 454 | # 455 | # Set it to 0 or a negative value for unlimited execution without warnings. 456 | lua-time-limit 5000 457 | 458 | ################################## SLOW LOG ################################### 459 | 460 | # The Redis Slow Log is a system to log queries that exceeded a specified 461 | # execution time. The execution time does not include the I/O operations 462 | # like talking with the client, sending the reply and so forth, 463 | # but just the time needed to actually execute the command (this is the only 464 | # stage of command execution where the thread is blocked and can not serve 465 | # other requests in the meantime). 466 | # 467 | # You can configure the slow log with two parameters: one tells Redis 468 | # what is the execution time, in microseconds, to exceed in order for the 469 | # command to get logged, and the other parameter is the length of the 470 | # slow log. When a new command is logged the oldest one is removed from the 471 | # queue of logged commands. 472 | 473 | # The following time is expressed in microseconds, so 1000000 is equivalent 474 | # to one second. Note that a negative number disables the slow log, while 475 | # a value of zero forces the logging of every command. 476 | slowlog-log-slower-than 10000 477 | 478 | # There is no limit to this length. Just be aware that it will consume memory. 479 | # You can reclaim memory used by the slow log with SLOWLOG RESET. 480 | slowlog-max-len 128 481 | 482 | ############################### ADVANCED CONFIG ############################### 483 | 484 | # Hashes are encoded using a memory efficient data structure when they have a 485 | # small number of entries, and the biggest entry does not exceed a given 486 | # threshold. These thresholds can be configured using the following directives. 487 | hash-max-ziplist-entries 512 488 | hash-max-ziplist-value 64 489 | 490 | # Similarly to hashes, small lists are also encoded in a special way in order 491 | # to save a lot of space. The special representation is only used when 492 | # you are under the following limits: 493 | list-max-ziplist-entries 512 494 | list-max-ziplist-value 64 495 | 496 | # Sets have a special encoding in just one case: when a set is composed 497 | # of just strings that happens to be integers in radix 10 in the range 498 | # of 64 bit signed integers. 499 | # The following configuration setting sets the limit in the size of the 500 | # set in order to use this special memory saving encoding. 501 | set-max-intset-entries 512 502 | 503 | # Similarly to hashes and lists, sorted sets are also specially encoded in 504 | # order to save a lot of space. This encoding is only used when the length and 505 | # elements of a sorted set are below the following limits: 506 | zset-max-ziplist-entries 128 507 | zset-max-ziplist-value 64 508 | 509 | # Active rehashing uses 1 millisecond every 100 milliseconds of CPU time in 510 | # order to help rehashing the main Redis hash table (the one mapping top-level 511 | # keys to values). The hash table implementation Redis uses (see dict.c) 512 | # performs a lazy rehashing: the more operation you run into an hash table 513 | # that is rehashing, the more rehashing "steps" are performed, so if the 514 | # server is idle the rehashing is never complete and some more memory is used 515 | # by the hash table. 516 | # 517 | # The default is to use this millisecond 10 times every second in order to 518 | # active rehashing the main dictionaries, freeing memory when possible. 519 | # 520 | # If unsure: 521 | # use "activerehashing no" if you have hard latency requirements and it is 522 | # not a good thing in your environment that Redis can reply form time to time 523 | # to queries with 2 milliseconds delay. 524 | # 525 | # use "activerehashing yes" if you don't have such hard requirements but 526 | # want to free memory asap when possible. 527 | activerehashing yes 528 | 529 | # The client output buffer limits can be used to force disconnection of clients 530 | # that are not reading data from the server fast enough for some reason (a 531 | # common reason is that a Pub/Sub client can't consume messages as fast as the 532 | # publisher can produce them). 533 | # 534 | # The limit can be set differently for the three different classes of clients: 535 | # 536 | # normal -> normal clients 537 | # slave -> slave clients and MONITOR clients 538 | # pubsub -> clients subcribed to at least one pubsub channel or pattern 539 | # 540 | # The syntax of every client-output-buffer-limit directive is the following: 541 | # 542 | # client-output-buffer-limit 543 | # 544 | # A client is immediately disconnected once the hard limit is reached, or if 545 | # the soft limit is reached and remains reached for the specified number of 546 | # seconds (continuously). 547 | # So for instance if the hard limit is 32 megabytes and the soft limit is 548 | # 16 megabytes / 10 seconds, the client will get disconnected immediately 549 | # if the size of the output buffers reach 32 megabytes, but will also get 550 | # disconnected if the client reaches 16 megabytes and continuously overcomes 551 | # the limit for 10 seconds. 552 | # 553 | # By default normal clients are not limited because they don't receive data 554 | # without asking (in a push way), but just after a request, so only 555 | # asynchronous clients may create a scenario where data is requested faster 556 | # than it can read. 557 | # 558 | # Instead there is a default limit for pubsub and slave clients, since 559 | # subscribers and slaves receive data in a push fashion. 560 | # 561 | # Both the hard or the soft limit can be disabled by setting them to zero. 562 | client-output-buffer-limit normal 0 0 0 563 | client-output-buffer-limit slave 256mb 64mb 60 564 | client-output-buffer-limit pubsub 32mb 8mb 60 565 | 566 | # Redis calls an internal function to perform many background tasks, like 567 | # closing connections of clients in timeot, purging expired keys that are 568 | # never requested, and so forth. 569 | # 570 | # Not all tasks are perforemd with the same frequency, but Redis checks for 571 | # tasks to perform accordingly to the specified "hz" value. 572 | # 573 | # By default "hz" is set to 10. Raising the value will use more CPU when 574 | # Redis is idle, but at the same time will make Redis more responsive when 575 | # there are many keys expiring at the same time, and timeouts may be 576 | # handled with more precision. 577 | # 578 | # The range is between 1 and 500, however a value over 100 is usually not 579 | # a good idea. Most users should use the default of 10 and raise this up to 580 | # 100 only in environments where very low latency is required. 581 | hz 10 582 | 583 | ################################## INCLUDES ################################### 584 | 585 | # Include one or more other config files here. This is useful if you 586 | # have a standard template that goes to all Redis server but also need 587 | # to customize a few per-server settings. Include files can include 588 | # other files, so use this wisely. 589 | # 590 | # include /path/to/local.conf 591 | # include /path/to/other.conf 592 | -------------------------------------------------------------------------------- /example/export/day_ips/consumer.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # -*- coding: utf-8 -*- 3 | 4 | import redis 5 | import bgpranking 6 | import argparse 7 | 8 | 9 | if __name__ == '__main__': 10 | parser = argparse.ArgumentParser(description='Prepare the information to dump.') 11 | parser.add_argument('-f', '--full', action="store_true", default=False, 12 | help='Do a full dump (asn, block, ip, sources).') 13 | args = parser.parse_args() 14 | 15 | 16 | r = redis.Redis(unix_socket_path='./redis_export.sock') 17 | date = r.get('date') 18 | weights = bgpranking.get_all_weights(date) 19 | while True: 20 | asn_b = r.spop('asn_b') 21 | if asn_b is None: 22 | break 23 | asn, block = asn_b.split("_") 24 | ip_descs = bgpranking.get_ips_descs(asn, block, date) 25 | if len(ip_descs.get(block)) != 0: 26 | p = r.pipeline(False) 27 | for ip, sources in ip_descs.get(block).iteritems(): 28 | p.zincrby('ips', ip, sum([float(weights[s]) for s in sources])) 29 | if args.full: 30 | p.hmset(ip, {'asn': asn, 'block':block, 31 | 'sources': '|'.join(sources)}) 32 | p.execute() 33 | 34 | 35 | -------------------------------------------------------------------------------- /example/export/day_ips/dates.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | START_DATE="2011-06-01" 4 | END_DATE="2013-08-01" 5 | 6 | 7 | echo $START_DATE > dates.txt 8 | 9 | DATE=$START_DATE 10 | while [ "$DATE" != "$END_DATE" ] ; do 11 | DATE=$( date +%Y-%m-%d -d "$DATE -d 1day" ) 12 | echo $DATE >> dates.txt 13 | done 14 | -------------------------------------------------------------------------------- /example/export/day_ips/dump.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # -*- coding: utf-8 -*- 3 | 4 | import redis 5 | import csv 6 | import argparse 7 | 8 | if __name__ == '__main__': 9 | parser = argparse.ArgumentParser(description='Dump on the disk the daily informations.') 10 | parser.add_argument('-f', '--full', action="store_true", default=False, 11 | help='Do a full dump (asn, block, ip, sources).') 12 | args = parser.parse_args() 13 | 14 | r = redis.Redis(unix_socket_path='./redis_export.sock') 15 | date = r.get('date') 16 | with open('ips_' + date, 'w') as f: 17 | ips = list(r.zrange('ips', 0, -1)) 18 | for i in range(len(ips)): 19 | ips[i] = "%3s.%3s.%3s.%3s" % tuple(ips[i].split(".")) 20 | ips.sort() 21 | for i in range(len(ips)): 22 | ips[i] = ips[i].replace(" ", "") 23 | p = r.pipeline(False) 24 | [p.zscore('ips', ip) for ip in ips] 25 | weights = p.execute() 26 | [f.write(','.join(map(str, iw)) + '\n') for iw in zip(ips, weights)] 27 | if args.full: 28 | with open('full_' + date, 'wb') as f: 29 | w = csv.writer(f) 30 | ips = r.zrange('ips', 0, -1) 31 | p = r.pipeline(False) 32 | [p.hgetall(ip) for ip in ips] 33 | all_entries = p.execute() 34 | i = 0 35 | for entries in all_entries: 36 | w.writerow([entries['asn'], entries['block'], ips[i], 37 | entries['sources']]) 38 | i += 1 39 | r.shutdown() 40 | -------------------------------------------------------------------------------- /example/export/day_ips/init_redis.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # -*- coding: utf-8 -*- 3 | 4 | import bgpranking 5 | import argparse 6 | 7 | import redis 8 | 9 | if __name__ == '__main__': 10 | parser = argparse.ArgumentParser(description='Generate a list of all IPS for a day.') 11 | parser.add_argument('-d', '--date', default=None, help='Date of the dump (YYYY-MM-DD)') 12 | 13 | args = parser.parse_args() 14 | date = args.date 15 | 16 | if date is None: 17 | date = bgpranking.get_default_date() 18 | 19 | dates_sources = bgpranking.prepare_sources_by_dates(date, 1) 20 | 21 | asns = bgpranking.existing_asns_timeframe(dates_sources) 22 | 23 | r = redis.Redis(unix_socket_path='./redis_export.sock') 24 | r.set('date', date) 25 | for asn in asns: 26 | blocks = bgpranking.get_all_blocks(asn) 27 | p = r.pipeline(False) 28 | for b in blocks: 29 | p.sadd('asn_b', "{asn}_{b}".format(asn=asn, b=b)) 30 | p.execute() 31 | 32 | -------------------------------------------------------------------------------- /example/export/day_ips/launch.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | while read date ; do 4 | 5 | echo $date 6 | bash ./launch_local_redis.sh 7 | python ./init_redis.py -d $date 8 | 9 | date 10 | for i in {1..10}; do 11 | python ./consumer.py & 12 | done 13 | 14 | python ./consumer.py 15 | date 16 | sleep 10 17 | 18 | python ./dump.py 19 | date 20 | done < dates.txt 21 | -------------------------------------------------------------------------------- /example/export/day_ips/launch_local_redis.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | HOME="/home/raphael/" 4 | 5 | REDIS_SERVER="${HOME}/gits/redis/src/redis-server" 6 | 7 | ${REDIS_SERVER} ./local_redis.conf 8 | -------------------------------------------------------------------------------- /example/export/day_ips/local_redis.conf: -------------------------------------------------------------------------------- 1 | # Redis configuration file example 2 | 3 | # Note on units: when memory size is needed, it is possible to specify 4 | # it in the usual form of 1k 5GB 4M and so forth: 5 | # 6 | # 1k => 1000 bytes 7 | # 1kb => 1024 bytes 8 | # 1m => 1000000 bytes 9 | # 1mb => 1024*1024 bytes 10 | # 1g => 1000000000 bytes 11 | # 1gb => 1024*1024*1024 bytes 12 | # 13 | # units are case insensitive so 1GB 1Gb 1gB are all the same. 14 | 15 | # By default Redis does not run as a daemon. Use 'yes' if you need it. 16 | # Note that Redis will write a pid file in /var/run/redis.pid when daemonized. 17 | daemonize yes 18 | 19 | # When running daemonized, Redis writes a pid file in /var/run/redis.pid by 20 | # default. You can specify a custom pid file location here. 21 | pidfile ./redis_export.pid 22 | 23 | # Accept connections on the specified port, default is 6379. 24 | # If port 0 is specified Redis will not listen on a TCP socket. 25 | port 0 26 | 27 | # If you want you can bind a single interface, if the bind option is not 28 | # specified all the interfaces will listen for incoming connections. 29 | # 30 | # bind 127.0.0.1 31 | 32 | # Specify the path for the unix socket that will be used to listen for 33 | # incoming connections. There is no default, so Redis will not listen 34 | # on a unix socket when not specified. 35 | # 36 | unixsocket ./redis_export.sock 37 | unixsocketperm 755 38 | 39 | # Close the connection after a client is idle for N seconds (0 to disable) 40 | timeout 0 41 | 42 | # TCP keepalive. 43 | # 44 | # If non-zero, use SO_KEEPALIVE to send TCP ACKs to clients in absence 45 | # of communication. This is useful for two reasons: 46 | # 47 | # 1) Detect dead peers. 48 | # 2) Take the connection alive from the point of view of network 49 | # equipment in the middle. 50 | # 51 | # On Linux, the specified value (in seconds) is the period used to send ACKs. 52 | # Note that to close the connection the double of the time is needed. 53 | # On other kernels the period depends on the kernel configuration. 54 | # 55 | # A reasonable value for this option is 60 seconds. 56 | tcp-keepalive 0 57 | 58 | # Specify the server verbosity level. 59 | # This can be one of: 60 | # debug (a lot of information, useful for development/testing) 61 | # verbose (many rarely useful info, but not a mess like the debug level) 62 | # notice (moderately verbose, what you want in production probably) 63 | # warning (only very important / critical messages are logged) 64 | loglevel notice 65 | 66 | # Specify the log file name. Also 'stdout' can be used to force 67 | # Redis to log on the standard output. Note that if you use standard 68 | # output for logging but daemonize, logs will be sent to /dev/null 69 | logfile stdout 70 | 71 | # To enable logging to the system logger, just set 'syslog-enabled' to yes, 72 | # and optionally update the other syslog parameters to suit your needs. 73 | # syslog-enabled no 74 | 75 | # Specify the syslog identity. 76 | # syslog-ident redis 77 | 78 | # Specify the syslog facility. Must be USER or between LOCAL0-LOCAL7. 79 | # syslog-facility local0 80 | 81 | # Set the number of databases. The default database is DB 0, you can select 82 | # a different one on a per-connection basis using SELECT where 83 | # dbid is a number between 0 and 'databases'-1 84 | databases 2 85 | 86 | ################################ SNAPSHOTTING ################################# 87 | # 88 | # Save the DB on disk: 89 | # 90 | # save 91 | # 92 | # Will save the DB if both the given number of seconds and the given 93 | # number of write operations against the DB occurred. 94 | # 95 | # In the example below the behaviour will be to save: 96 | # after 900 sec (15 min) if at least 1 key changed 97 | # after 300 sec (5 min) if at least 10 keys changed 98 | # after 60 sec if at least 10000 keys changed 99 | # 100 | # Note: you can disable saving at all commenting all the "save" lines. 101 | # 102 | # It is also possible to remove all the previously configured save 103 | # points by adding a save directive with a single empty string argument 104 | # like in the following example: 105 | # 106 | # save "" 107 | 108 | #save 900 1 109 | #save 300 10 110 | #save 60 10000 111 | 112 | # By default Redis will stop accepting writes if RDB snapshots are enabled 113 | # (at least one save point) and the latest background save failed. 114 | # This will make the user aware (in an hard way) that data is not persisting 115 | # on disk properly, otherwise chances are that no one will notice and some 116 | # distater will happen. 117 | # 118 | # If the background saving process will start working again Redis will 119 | # automatically allow writes again. 120 | # 121 | # However if you have setup your proper monitoring of the Redis server 122 | # and persistence, you may want to disable this feature so that Redis will 123 | # continue to work as usually even if there are problems with disk, 124 | # permissions, and so forth. 125 | stop-writes-on-bgsave-error yes 126 | 127 | # Compress string objects using LZF when dump .rdb databases? 128 | # For default that's set to 'yes' as it's almost always a win. 129 | # If you want to save some CPU in the saving child set it to 'no' but 130 | # the dataset will likely be bigger if you have compressible values or keys. 131 | rdbcompression yes 132 | 133 | # Since version 5 of RDB a CRC64 checksum is placed at the end of the file. 134 | # This makes the format more resistant to corruption but there is a performance 135 | # hit to pay (around 10%) when saving and loading RDB files, so you can disable it 136 | # for maximum performances. 137 | # 138 | # RDB files created with checksum disabled have a checksum of zero that will 139 | # tell the loading code to skip the check. 140 | rdbchecksum yes 141 | 142 | # The filename where to dump the DB 143 | dbfilename dump_export.rdb 144 | 145 | # The working directory. 146 | # 147 | # The DB will be written inside this directory, with the filename specified 148 | # above using the 'dbfilename' configuration directive. 149 | # 150 | # The Append Only File will also be created inside this directory. 151 | # 152 | # Note that you must specify a directory here, not a file name. 153 | dir ./ 154 | 155 | ################################# REPLICATION ################################# 156 | 157 | # Master-Slave replication. Use slaveof to make a Redis instance a copy of 158 | # another Redis server. Note that the configuration is local to the slave 159 | # so for example it is possible to configure the slave to save the DB with a 160 | # different interval, or to listen to another port, and so on. 161 | # 162 | # slaveof 163 | 164 | # If the master is password protected (using the "requirepass" configuration 165 | # directive below) it is possible to tell the slave to authenticate before 166 | # starting the replication synchronization process, otherwise the master will 167 | # refuse the slave request. 168 | # 169 | # masterauth 170 | 171 | # When a slave loses its connection with the master, or when the replication 172 | # is still in progress, the slave can act in two different ways: 173 | # 174 | # 1) if slave-serve-stale-data is set to 'yes' (the default) the slave will 175 | # still reply to client requests, possibly with out of date data, or the 176 | # data set may just be empty if this is the first synchronization. 177 | # 178 | # 2) if slave-serve-stale-data is set to 'no' the slave will reply with 179 | # an error "SYNC with master in progress" to all the kind of commands 180 | # but to INFO and SLAVEOF. 181 | # 182 | slave-serve-stale-data yes 183 | 184 | # You can configure a slave instance to accept writes or not. Writing against 185 | # a slave instance may be useful to store some ephemeral data (because data 186 | # written on a slave will be easily deleted after resync with the master) but 187 | # may also cause problems if clients are writing to it because of a 188 | # misconfiguration. 189 | # 190 | # Since Redis 2.6 by default slaves are read-only. 191 | # 192 | # Note: read only slaves are not designed to be exposed to untrusted clients 193 | # on the internet. It's just a protection layer against misuse of the instance. 194 | # Still a read only slave exports by default all the administrative commands 195 | # such as CONFIG, DEBUG, and so forth. To a limited extend you can improve 196 | # security of read only slaves using 'rename-command' to shadow all the 197 | # administrative / dangerous commands. 198 | slave-read-only yes 199 | 200 | # Slaves send PINGs to server in a predefined interval. It's possible to change 201 | # this interval with the repl_ping_slave_period option. The default value is 10 202 | # seconds. 203 | # 204 | # repl-ping-slave-period 10 205 | 206 | # The following option sets a timeout for both Bulk transfer I/O timeout and 207 | # master data or ping response timeout. The default value is 60 seconds. 208 | # 209 | # It is important to make sure that this value is greater than the value 210 | # specified for repl-ping-slave-period otherwise a timeout will be detected 211 | # every time there is low traffic between the master and the slave. 212 | # 213 | # repl-timeout 60 214 | 215 | # Disable TCP_NODELAY on the slave socket after SYNC? 216 | # 217 | # If you select "yes" Redis will use a smaller number of TCP packets and 218 | # less bandwidth to send data to slaves. But this can add a delay for 219 | # the data to appear on the slave side, up to 40 milliseconds with 220 | # Linux kernels using a default configuration. 221 | # 222 | # If you select "no" the delay for data to appear on the slave side will 223 | # be reduced but more bandwidth will be used for replication. 224 | # 225 | # By default we optimize for low latency, but in very high traffic conditions 226 | # or when the master and slaves are many hops away, turning this to "yes" may 227 | # be a good idea. 228 | repl-disable-tcp-nodelay no 229 | 230 | # The slave priority is an integer number published by Redis in the INFO output. 231 | # It is used by Redis Sentinel in order to select a slave to promote into a 232 | # master if the master is no longer working correctly. 233 | # 234 | # A slave with a low priority number is considered better for promotion, so 235 | # for instance if there are three slaves with priority 10, 100, 25 Sentinel will 236 | # pick the one wtih priority 10, that is the lowest. 237 | # 238 | # However a special priority of 0 marks the slave as not able to perform the 239 | # role of master, so a slave with priority of 0 will never be selected by 240 | # Redis Sentinel for promotion. 241 | # 242 | # By default the priority is 100. 243 | slave-priority 100 244 | 245 | ################################## SECURITY ################################### 246 | 247 | # Require clients to issue AUTH before processing any other 248 | # commands. This might be useful in environments in which you do not trust 249 | # others with access to the host running redis-server. 250 | # 251 | # This should stay commented out for backward compatibility and because most 252 | # people do not need auth (e.g. they run their own servers). 253 | # 254 | # Warning: since Redis is pretty fast an outside user can try up to 255 | # 150k passwords per second against a good box. This means that you should 256 | # use a very strong password otherwise it will be very easy to break. 257 | # 258 | # requirepass foobared 259 | 260 | # Command renaming. 261 | # 262 | # It is possible to change the name of dangerous commands in a shared 263 | # environment. For instance the CONFIG command may be renamed into something 264 | # hard to guess so that it will still be available for internal-use tools 265 | # but not available for general clients. 266 | # 267 | # Example: 268 | # 269 | # rename-command CONFIG b840fc02d524045429941cc15f59e41cb7be6c52 270 | # 271 | # It is also possible to completely kill a command by renaming it into 272 | # an empty string: 273 | # 274 | # rename-command CONFIG "" 275 | # 276 | # Please note that changing the name of commands that are logged into the 277 | # AOF file or transmitted to slaves may cause problems. 278 | 279 | ################################### LIMITS #################################### 280 | 281 | # Set the max number of connected clients at the same time. By default 282 | # this limit is set to 10000 clients, however if the Redis server is not 283 | # able to configure the process file limit to allow for the specified limit 284 | # the max number of allowed clients is set to the current file limit 285 | # minus 32 (as Redis reserves a few file descriptors for internal uses). 286 | # 287 | # Once the limit is reached Redis will close all the new connections sending 288 | # an error 'max number of clients reached'. 289 | # 290 | # maxclients 10000 291 | 292 | # Don't use more memory than the specified amount of bytes. 293 | # When the memory limit is reached Redis will try to remove keys 294 | # accordingly to the eviction policy selected (see maxmemmory-policy). 295 | # 296 | # If Redis can't remove keys according to the policy, or if the policy is 297 | # set to 'noeviction', Redis will start to reply with errors to commands 298 | # that would use more memory, like SET, LPUSH, and so on, and will continue 299 | # to reply to read-only commands like GET. 300 | # 301 | # This option is usually useful when using Redis as an LRU cache, or to set 302 | # an hard memory limit for an instance (using the 'noeviction' policy). 303 | # 304 | # WARNING: If you have slaves attached to an instance with maxmemory on, 305 | # the size of the output buffers needed to feed the slaves are subtracted 306 | # from the used memory count, so that network problems / resyncs will 307 | # not trigger a loop where keys are evicted, and in turn the output 308 | # buffer of slaves is full with DELs of keys evicted triggering the deletion 309 | # of more keys, and so forth until the database is completely emptied. 310 | # 311 | # In short... if you have slaves attached it is suggested that you set a lower 312 | # limit for maxmemory so that there is some free RAM on the system for slave 313 | # output buffers (but this is not needed if the policy is 'noeviction'). 314 | # 315 | # maxmemory 316 | 317 | # MAXMEMORY POLICY: how Redis will select what to remove when maxmemory 318 | # is reached. You can select among five behaviors: 319 | # 320 | # volatile-lru -> remove the key with an expire set using an LRU algorithm 321 | # allkeys-lru -> remove any key accordingly to the LRU algorithm 322 | # volatile-random -> remove a random key with an expire set 323 | # allkeys-random -> remove a random key, any key 324 | # volatile-ttl -> remove the key with the nearest expire time (minor TTL) 325 | # noeviction -> don't expire at all, just return an error on write operations 326 | # 327 | # Note: with any of the above policies, Redis will return an error on write 328 | # operations, when there are not suitable keys for eviction. 329 | # 330 | # At the date of writing this commands are: set setnx setex append 331 | # incr decr rpush lpush rpushx lpushx linsert lset rpoplpush sadd 332 | # sinter sinterstore sunion sunionstore sdiff sdiffstore zadd zincrby 333 | # zunionstore zinterstore hset hsetnx hmset hincrby incrby decrby 334 | # getset mset msetnx exec sort 335 | # 336 | # The default is: 337 | # 338 | # maxmemory-policy volatile-lru 339 | 340 | # LRU and minimal TTL algorithms are not precise algorithms but approximated 341 | # algorithms (in order to save memory), so you can select as well the sample 342 | # size to check. For instance for default Redis will check three keys and 343 | # pick the one that was used less recently, you can change the sample size 344 | # using the following configuration directive. 345 | # 346 | # maxmemory-samples 3 347 | 348 | ############################## APPEND ONLY MODE ############################### 349 | 350 | # By default Redis asynchronously dumps the dataset on disk. This mode is 351 | # good enough in many applications, but an issue with the Redis process or 352 | # a power outage may result into a few minutes of writes lost (depending on 353 | # the configured save points). 354 | # 355 | # The Append Only File is an alternative persistence mode that provides 356 | # much better durability. For instance using the default data fsync policy 357 | # (see later in the config file) Redis can lose just one second of writes in a 358 | # dramatic event like a server power outage, or a single write if something 359 | # wrong with the Redis process itself happens, but the operating system is 360 | # still running correctly. 361 | # 362 | # AOF and RDB persistence can be enabled at the same time without problems. 363 | # If the AOF is enabled on startup Redis will load the AOF, that is the file 364 | # with the better durability guarantees. 365 | # 366 | # Please check http://redis.io/topics/persistence for more information. 367 | 368 | appendonly no 369 | 370 | # The name of the append only file (default: "appendonly.aof") 371 | # appendfilename appendonly.aof 372 | 373 | # The fsync() call tells the Operating System to actually write data on disk 374 | # instead to wait for more data in the output buffer. Some OS will really flush 375 | # data on disk, some other OS will just try to do it ASAP. 376 | # 377 | # Redis supports three different modes: 378 | # 379 | # no: don't fsync, just let the OS flush the data when it wants. Faster. 380 | # always: fsync after every write to the append only log . Slow, Safest. 381 | # everysec: fsync only one time every second. Compromise. 382 | # 383 | # The default is "everysec", as that's usually the right compromise between 384 | # speed and data safety. It's up to you to understand if you can relax this to 385 | # "no" that will let the operating system flush the output buffer when 386 | # it wants, for better performances (but if you can live with the idea of 387 | # some data loss consider the default persistence mode that's snapshotting), 388 | # or on the contrary, use "always" that's very slow but a bit safer than 389 | # everysec. 390 | # 391 | # More details please check the following article: 392 | # http://antirez.com/post/redis-persistence-demystified.html 393 | # 394 | # If unsure, use "everysec". 395 | 396 | # appendfsync always 397 | appendfsync everysec 398 | # appendfsync no 399 | 400 | # When the AOF fsync policy is set to always or everysec, and a background 401 | # saving process (a background save or AOF log background rewriting) is 402 | # performing a lot of I/O against the disk, in some Linux configurations 403 | # Redis may block too long on the fsync() call. Note that there is no fix for 404 | # this currently, as even performing fsync in a different thread will block 405 | # our synchronous write(2) call. 406 | # 407 | # In order to mitigate this problem it's possible to use the following option 408 | # that will prevent fsync() from being called in the main process while a 409 | # BGSAVE or BGREWRITEAOF is in progress. 410 | # 411 | # This means that while another child is saving, the durability of Redis is 412 | # the same as "appendfsync none". In practical terms, this means that it is 413 | # possible to lose up to 30 seconds of log in the worst scenario (with the 414 | # default Linux settings). 415 | # 416 | # If you have latency problems turn this to "yes". Otherwise leave it as 417 | # "no" that is the safest pick from the point of view of durability. 418 | no-appendfsync-on-rewrite no 419 | 420 | # Automatic rewrite of the append only file. 421 | # Redis is able to automatically rewrite the log file implicitly calling 422 | # BGREWRITEAOF when the AOF log size grows by the specified percentage. 423 | # 424 | # This is how it works: Redis remembers the size of the AOF file after the 425 | # latest rewrite (if no rewrite has happened since the restart, the size of 426 | # the AOF at startup is used). 427 | # 428 | # This base size is compared to the current size. If the current size is 429 | # bigger than the specified percentage, the rewrite is triggered. Also 430 | # you need to specify a minimal size for the AOF file to be rewritten, this 431 | # is useful to avoid rewriting the AOF file even if the percentage increase 432 | # is reached but it is still pretty small. 433 | # 434 | # Specify a percentage of zero in order to disable the automatic AOF 435 | # rewrite feature. 436 | 437 | auto-aof-rewrite-percentage 100 438 | auto-aof-rewrite-min-size 64mb 439 | 440 | ################################ LUA SCRIPTING ############################### 441 | 442 | # Max execution time of a Lua script in milliseconds. 443 | # 444 | # If the maximum execution time is reached Redis will log that a script is 445 | # still in execution after the maximum allowed time and will start to 446 | # reply to queries with an error. 447 | # 448 | # When a long running script exceed the maximum execution time only the 449 | # SCRIPT KILL and SHUTDOWN NOSAVE commands are available. The first can be 450 | # used to stop a script that did not yet called write commands. The second 451 | # is the only way to shut down the server in the case a write commands was 452 | # already issue by the script but the user don't want to wait for the natural 453 | # termination of the script. 454 | # 455 | # Set it to 0 or a negative value for unlimited execution without warnings. 456 | lua-time-limit 5000 457 | 458 | ################################## SLOW LOG ################################### 459 | 460 | # The Redis Slow Log is a system to log queries that exceeded a specified 461 | # execution time. The execution time does not include the I/O operations 462 | # like talking with the client, sending the reply and so forth, 463 | # but just the time needed to actually execute the command (this is the only 464 | # stage of command execution where the thread is blocked and can not serve 465 | # other requests in the meantime). 466 | # 467 | # You can configure the slow log with two parameters: one tells Redis 468 | # what is the execution time, in microseconds, to exceed in order for the 469 | # command to get logged, and the other parameter is the length of the 470 | # slow log. When a new command is logged the oldest one is removed from the 471 | # queue of logged commands. 472 | 473 | # The following time is expressed in microseconds, so 1000000 is equivalent 474 | # to one second. Note that a negative number disables the slow log, while 475 | # a value of zero forces the logging of every command. 476 | slowlog-log-slower-than 10000 477 | 478 | # There is no limit to this length. Just be aware that it will consume memory. 479 | # You can reclaim memory used by the slow log with SLOWLOG RESET. 480 | slowlog-max-len 128 481 | 482 | ############################### ADVANCED CONFIG ############################### 483 | 484 | # Hashes are encoded using a memory efficient data structure when they have a 485 | # small number of entries, and the biggest entry does not exceed a given 486 | # threshold. These thresholds can be configured using the following directives. 487 | hash-max-ziplist-entries 512 488 | hash-max-ziplist-value 64 489 | 490 | # Similarly to hashes, small lists are also encoded in a special way in order 491 | # to save a lot of space. The special representation is only used when 492 | # you are under the following limits: 493 | list-max-ziplist-entries 512 494 | list-max-ziplist-value 64 495 | 496 | # Sets have a special encoding in just one case: when a set is composed 497 | # of just strings that happens to be integers in radix 10 in the range 498 | # of 64 bit signed integers. 499 | # The following configuration setting sets the limit in the size of the 500 | # set in order to use this special memory saving encoding. 501 | set-max-intset-entries 512 502 | 503 | # Similarly to hashes and lists, sorted sets are also specially encoded in 504 | # order to save a lot of space. This encoding is only used when the length and 505 | # elements of a sorted set are below the following limits: 506 | zset-max-ziplist-entries 128 507 | zset-max-ziplist-value 64 508 | 509 | # Active rehashing uses 1 millisecond every 100 milliseconds of CPU time in 510 | # order to help rehashing the main Redis hash table (the one mapping top-level 511 | # keys to values). The hash table implementation Redis uses (see dict.c) 512 | # performs a lazy rehashing: the more operation you run into an hash table 513 | # that is rehashing, the more rehashing "steps" are performed, so if the 514 | # server is idle the rehashing is never complete and some more memory is used 515 | # by the hash table. 516 | # 517 | # The default is to use this millisecond 10 times every second in order to 518 | # active rehashing the main dictionaries, freeing memory when possible. 519 | # 520 | # If unsure: 521 | # use "activerehashing no" if you have hard latency requirements and it is 522 | # not a good thing in your environment that Redis can reply form time to time 523 | # to queries with 2 milliseconds delay. 524 | # 525 | # use "activerehashing yes" if you don't have such hard requirements but 526 | # want to free memory asap when possible. 527 | activerehashing yes 528 | 529 | # The client output buffer limits can be used to force disconnection of clients 530 | # that are not reading data from the server fast enough for some reason (a 531 | # common reason is that a Pub/Sub client can't consume messages as fast as the 532 | # publisher can produce them). 533 | # 534 | # The limit can be set differently for the three different classes of clients: 535 | # 536 | # normal -> normal clients 537 | # slave -> slave clients and MONITOR clients 538 | # pubsub -> clients subcribed to at least one pubsub channel or pattern 539 | # 540 | # The syntax of every client-output-buffer-limit directive is the following: 541 | # 542 | # client-output-buffer-limit 543 | # 544 | # A client is immediately disconnected once the hard limit is reached, or if 545 | # the soft limit is reached and remains reached for the specified number of 546 | # seconds (continuously). 547 | # So for instance if the hard limit is 32 megabytes and the soft limit is 548 | # 16 megabytes / 10 seconds, the client will get disconnected immediately 549 | # if the size of the output buffers reach 32 megabytes, but will also get 550 | # disconnected if the client reaches 16 megabytes and continuously overcomes 551 | # the limit for 10 seconds. 552 | # 553 | # By default normal clients are not limited because they don't receive data 554 | # without asking (in a push way), but just after a request, so only 555 | # asynchronous clients may create a scenario where data is requested faster 556 | # than it can read. 557 | # 558 | # Instead there is a default limit for pubsub and slave clients, since 559 | # subscribers and slaves receive data in a push fashion. 560 | # 561 | # Both the hard or the soft limit can be disabled by setting them to zero. 562 | client-output-buffer-limit normal 0 0 0 563 | client-output-buffer-limit slave 256mb 64mb 60 564 | client-output-buffer-limit pubsub 32mb 8mb 60 565 | 566 | # Redis calls an internal function to perform many background tasks, like 567 | # closing connections of clients in timeot, purging expired keys that are 568 | # never requested, and so forth. 569 | # 570 | # Not all tasks are perforemd with the same frequency, but Redis checks for 571 | # tasks to perform accordingly to the specified "hz" value. 572 | # 573 | # By default "hz" is set to 10. Raising the value will use more CPU when 574 | # Redis is idle, but at the same time will make Redis more responsive when 575 | # there are many keys expiring at the same time, and timeouts may be 576 | # handled with more precision. 577 | # 578 | # The range is between 1 and 500, however a value over 100 is usually not 579 | # a good idea. Most users should use the default of 10 and raise this up to 580 | # 100 only in environments where very low latency is required. 581 | hz 10 582 | 583 | ################################## INCLUDES ################################### 584 | 585 | # Include one or more other config files here. This is useful if you 586 | # have a standard template that goes to all Redis server but also need 587 | # to customize a few per-server settings. Include files can include 588 | # other files, so use this wisely. 589 | # 590 | # include /path/to/local.conf 591 | # include /path/to/other.conf 592 | -------------------------------------------------------------------------------- /example/ip_zmq/client.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import zmq 4 | 5 | ip = '127.0.0.1' 6 | port = '5556' 7 | 8 | zmq_socket = None 9 | 10 | 11 | def __prepare(): 12 | global zmq_socket 13 | context = zmq.Context() 14 | zmq_socket = context.socket(zmq.SUB) 15 | zmq_socket.connect('tcp://{}:{}'.format(ip, port)) 16 | zmq_socket.setsockopt(zmq.SUBSCRIBE, '') 17 | 18 | def run(): 19 | if zmq_socket is None: 20 | __prepare() 21 | while True: 22 | msg = zmq_socket.recv() 23 | yield msg 24 | 25 | if __name__ == '__main__': 26 | for a in run(): 27 | print a 28 | 29 | -------------------------------------------------------------------------------- /example/logging/logs/.keepdir: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CIRCL/bgpranking-redis-api/7b80c44e291169b8ce849cdb51beb618b96f2386/example/logging/logs/.keepdir -------------------------------------------------------------------------------- /example/logging/mail.conf: -------------------------------------------------------------------------------- 1 | [mail] 2 | dest_mail = foo@bar.com,bar@foo.de 3 | smtp_server = smtp.example.com 4 | smtp_port = 25 5 | src_server = service.foo.fr 6 | -------------------------------------------------------------------------------- /example/logging/start_logging.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | log_subscriber --mail ./mail.conf --channel Website --log_path ./logs/ & 4 | log_subscriber --mail ./mail.conf --channel API_Twitter --log_path ./logs/ & 5 | log_subscriber --mail ./mail.conf --channel API_Redis --log_path ./logs/ & 6 | log_subscriber --mail ./mail.conf --channel API_Web --log_path ./logs/ & 7 | -------------------------------------------------------------------------------- /example/twitter_bot/microblog/__init__.py: -------------------------------------------------------------------------------- 1 | import api_wrapper 2 | api_wrapper.__prepare() 3 | 4 | from api_wrapper import * 5 | 6 | -------------------------------------------------------------------------------- /example/twitter_bot/microblog/api_wrapper.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # -*- coding: utf-8 -*- 3 | 4 | 5 | """ 6 | Micro-blog 7 | ~~~~~~~~~~ 8 | 9 | Microblog client for twitter 10 | 11 | """ 12 | 13 | import twitter 14 | import datetime 15 | import dateutil 16 | 17 | from micro_blog_keys import twitter_consumer_key,twitter_consumer_secret,\ 18 | twitter_access_key,twitter_access_secret 19 | import bgpranking 20 | 21 | api = None 22 | username = "bgpranking" 23 | 24 | def __prepare(): 25 | global api 26 | api = twitter.Api(consumer_key=twitter_consumer_key, 27 | consumer_secret=twitter_consumer_secret, 28 | access_token_key=twitter_access_key, 29 | access_token_secret=twitter_access_secret) 30 | 31 | def prepare_string(): 32 | to_return = 'Top Ranking {date}\n'.format( 33 | date=datetime.date.today().isoformat()) 34 | top = bgpranking.cache_get_top_asns(limit=5, with_sources=False) 35 | for asn, descr, rank in top['top_list']: 36 | rank = round(1+rank, 4) 37 | to_return += '{asn}: {rank}\n'.format(asn=asn, rank=rank) 38 | to_return += 'http://bgpranking.circl.lu' 39 | return to_return 40 | 41 | def post_new_top_ranking(): 42 | posted = False 43 | today = datetime.date.today() 44 | status = api.GetUserTimeline("bgpranking", count=100) 45 | for s in status: 46 | t = s.text 47 | if t is not None and t.startswith('Top Ranking'): 48 | most_recent_post = dateutil.parser.parse( 49 | s.created_at).replace(tzinfo=None).date() 50 | if most_recent_post < today: 51 | posted = True 52 | to_post = prepare_string() 53 | api.PostUpdate(to_post) 54 | break 55 | return posted 56 | -------------------------------------------------------------------------------- /example/twitter_bot/microblog/python_stream_api.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # -*- coding: utf-8 -*- 3 | 4 | 5 | """ 6 | Micro-blog 7 | ~~~~~~~~~~ 8 | 9 | Microblog client for twitter 10 | 11 | """ 12 | 13 | import tweepy 14 | import IPy 15 | 16 | from micro_blog_keys import * 17 | import bgpranking 18 | 19 | api = None 20 | auth = None 21 | base_asn_address = 'http://bgpranking.circl.lu/asn_details?asn=' 22 | 23 | def __prepare(): 24 | global api 25 | global auth 26 | auth = tweepy.OAuthHandler(twitter_consumer_key, 27 | twitter_consumer_secret) 28 | auth.set_access_token(twitter_access_key, twitter_access_secret) 29 | api = tweepy.API(auth) 30 | 31 | class CustomStreamListener(tweepy.StreamListener): 32 | def on_status(self, status): 33 | if status.text.startswith('@bgpranking AS'): 34 | asn = status.text.strip('@bgpranking AS') 35 | try: 36 | int(asn) 37 | except: 38 | print status.text 39 | return 40 | msg_id = status.id 41 | sender = status.user.screen_name 42 | dates_sources = bgpranking.prepare_sources_by_dates(timeframe=5) 43 | ranks = bgpranking.get_all_ranks_single_asn(asn, dates_sources) 44 | to_post = '@{user}: {asn}\n'.format(user=sender, asn=asn) 45 | dates = sorted(ranks.keys(), reverse=True) 46 | for date in dates: 47 | info = ranks[date] 48 | to_post += '{date}: {rank}\n'.format(date=date,rank=round(1+info['total'], 4)) 49 | to_post += base_asn_address + asn 50 | api.update_status(to_post, msg_id) 51 | elif status.text.startswith('@bgpranking IP'): 52 | ip = status.text.strip('@bgpranking IP') 53 | try: 54 | IPy.IP(ip) 55 | except: 56 | print status.text 57 | return 58 | msg_id = status.id 59 | sender = status.user.screen_name 60 | info = bgpranking.get_ip_info(ip, 10) 61 | to_post_short = '@{user}: {ip}: http://bgpranking.circl.lu/ip_lookup?ip={ip}'.format(user=sender, ip=ip) 62 | if len(info['history']) > 0: 63 | latest_data = info['history'][0] 64 | template = '\n{asn} - {block}: {base_asn_url}{asn};ip_details={block}\n{descr}' 65 | descr = latest_data['descriptions'][0][1] 66 | if len(descr) > 40: 67 | descr = 'Too Long for Twitter.' 68 | to_post = to_post_short + template.format(asn=latest_data['asn'], 69 | base_asn_url=base_asn_address, 70 | block=latest_data['block'], descr=descr) 71 | try: 72 | api.update_status(to_post, msg_id) 73 | except: 74 | api.update_status(to_post_short, msg_id) 75 | 76 | else: 77 | print status.text 78 | 79 | def on_error(self, status_code): 80 | print >> sys.stderr, 'Encountered error with status code:', status_code 81 | return True # Don't kill the stream 82 | 83 | def on_timeout(self): 84 | print >> sys.stderr, 'Timeout...' 85 | return True # Don't kill the stream 86 | 87 | 88 | def stream_mentions(): 89 | options = {'secure':True} 90 | listener = CustomStreamListener() 91 | s = tweepy.streaming.Stream(auth, listener, **options) 92 | s.userstream() 93 | 94 | if __name__ == '__main__': 95 | __prepare() 96 | stream_mentions() 97 | -------------------------------------------------------------------------------- /example/twitter_bot/start_bot.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # -*- coding: utf-8 -*- 3 | 4 | """ 5 | :file:`bin/services/microblog.py` - Microblogging client 6 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 7 | 8 | Start the microblogging client which posts on twitter and identica 9 | """ 10 | 11 | import time 12 | from pubsublogger import publisher 13 | import microblog 14 | 15 | dev_mode = True 16 | 17 | if __name__ == '__main__': 18 | 19 | sleep_timer = 3600 20 | 21 | publisher.channel = 'API_Twitter' 22 | 23 | while 1: 24 | try: 25 | if microblog.post_new_top_ranking(): 26 | publisher.info('New Ranking posted on twitter and identica.') 27 | print 'New Ranking posted on twitter and identica.' 28 | except Exception as e: 29 | publisher.error("Something bad occurs: " + e) 30 | print "Something bad occurs: " + str(e) 31 | time.sleep(sleep_timer) 32 | -------------------------------------------------------------------------------- /example/website/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /example/website/compil.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | cheetah compile -R 4 | -------------------------------------------------------------------------------- /example/website/config/web_bgp-ranking.ini: -------------------------------------------------------------------------------- 1 | [global] 2 | log.error_file = "error.log" 3 | log.access_file = "access.log" 4 | server.environment = "production" 5 | server.socket_host = '127.0.0.1' 6 | server.socket_port = 8085 7 | engine.autoreload_on = True 8 | engine.autoreload_frequency = 5 9 | tools.proxy.on = True 10 | 11 | [/] 12 | tools.staticdir.root = os.getcwd() 13 | 14 | [/csv] 15 | tools.staticdir.on = True 16 | tools.staticdir.dir = "data/csv/" 17 | 18 | [/csv_agg] 19 | tools.staticdir.on = True 20 | tools.staticdir.dir = "data/csv_agg/" 21 | 22 | [/js_data] 23 | tools.staticdir.on = True 24 | tools.staticdir.dir = "data/js/" 25 | 26 | [/js] 27 | tools.staticdir.on = True 28 | tools.staticdir.dir = "js/" 29 | 30 | [/dygraph] 31 | tools.staticdir.on = True 32 | tools.staticdir.dir = "thirdparty/dygraph/" 33 | 34 | [/jvectormap] 35 | tools.staticdir.on = True 36 | tools.staticdir.dir = "thirdparty/jvectormap/" 37 | 38 | -------------------------------------------------------------------------------- /example/website/data/csv/.is_csv_dir: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CIRCL/bgpranking-redis-api/7b80c44e291169b8ce849cdb51beb618b96f2386/example/website/data/csv/.is_csv_dir -------------------------------------------------------------------------------- /example/website/data/csv_agg/.keepdir: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CIRCL/bgpranking-redis-api/7b80c44e291169b8ce849cdb51beb618b96f2386/example/website/data/csv_agg/.keepdir -------------------------------------------------------------------------------- /example/website/data/js/.keepdir: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CIRCL/bgpranking-redis-api/7b80c44e291169b8ce849cdb51beb618b96f2386/example/website/data/js/.keepdir -------------------------------------------------------------------------------- /example/website/js/world_benelux.js: -------------------------------------------------------------------------------- 1 | g = new Dygraph( 2 | document.getElementById("graphdiv1"), 3 | "csv_agg/be_ch_de_lu_fr_nl", 4 | { 5 | axes: { 6 | y: { 7 | valueFormatter: function(y) { 8 | return y.toPrecision(8); 9 | }, 10 | axisLabelFormatter: function(y) { 11 | return y.toPrecision(5); 12 | }, 13 | } 14 | }, 15 | legend: 'always', 16 | rollPeriod: 7, 17 | showRoller: true, 18 | logscale : true, 19 | yAxisLabelWidth: 50, 20 | xAxisLabelWidth: 60 21 | } 22 | ); 23 | 24 | -------------------------------------------------------------------------------- /example/website/js/world_luxembourg.js: -------------------------------------------------------------------------------- 1 | g = new Dygraph( 2 | document.getElementById("graphdiv1"), 3 | "csv_agg/world_lu", 4 | { 5 | labels: ['date', 'World', 'Luxembourg'], 6 | 'Luxembourg': { axis: {} }, 7 | axes: { 8 | y:{ 9 | }, 10 | y2: { 11 | valueFormatter: function(y) { 12 | return y.toPrecision(8); 13 | }, 14 | axisLabelFormatter: function(y) { 15 | return y.toPrecision(5); 16 | }, 17 | } 18 | }, 19 | legend: 'always', 20 | fillGraph: true, 21 | rollPeriod: 7, 22 | showRoller: true, 23 | logscale : true, 24 | ylabel: 'World Trend', 25 | y2label: 'Luxembourg Trend', 26 | yAxisLabelWidth: 50, 27 | xAxisLabelWidth: 60 28 | } 29 | ); 30 | 31 | -------------------------------------------------------------------------------- /example/website/js/worldmap_script.js: -------------------------------------------------------------------------------- 1 | $(function(){ 2 | $('#world-map').vectorMap( 3 | { 4 | map: 'world_mill_en', 5 | series: { 6 | regions: [{ 7 | values: ranks, 8 | scale: ['#F0FFFF', '#00FF00', '#FFFF00', '#FF0000', '#000000'], 9 | normalizeFunction: 'polynomial' 10 | }] 11 | }, 12 | onRegionLabelShow: function(e, el, code){ 13 | el.html(el.html()+' ('+ranks[code]+')'); 14 | } 15 | } 16 | ); 17 | }); 18 | 19 | -------------------------------------------------------------------------------- /example/website/logs/.keepdir: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CIRCL/bgpranking-redis-api/7b80c44e291169b8ce849cdb51beb618b96f2386/example/website/logs/.keepdir -------------------------------------------------------------------------------- /example/website/master.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | View class of the website 4 | ~~~~~~~~~~~~~~~~~~~~~~~~~ 5 | 6 | The website respects the MVC design pattern and this class is the view. 7 | 8 | """ 9 | 10 | import os 11 | import cherrypy 12 | from Cheetah.Template import Template 13 | import cgi 14 | import csv 15 | import StringIO 16 | from csv import DictReader 17 | import urllib2 18 | import json 19 | from cherrypy import _cperror 20 | 21 | import master_controler 22 | from pubsublogger import publisher 23 | 24 | def merge_csvs(asns): 25 | url = 'http://{host}:{port}/csv/'.format( 26 | host = cherrypy.config.get('server.socket_host'), 27 | port = cherrypy.config.get('server.socket_port')) 28 | asns = json.loads(asns) 29 | if asns[0] == 0: 30 | return json.dumps('') 31 | temp_dict = {} 32 | no_entries = [] 33 | for asn in asns: 34 | try: 35 | f = urllib2.urlopen(url + asn) 36 | for line in DictReader(f): 37 | date = line['day'] 38 | rank = line['rank'] 39 | if temp_dict.get(date) is None: 40 | temp_dict[date] = {} 41 | temp_dict[date][asn] = rank 42 | except: 43 | no_entries += asn 44 | to_return = 'date,' + ','.join(asns) + '\n' 45 | for date, entries in temp_dict.iteritems(): 46 | to_return += date 47 | for asn in asns: 48 | rank = entries.get(asn) 49 | if rank is None: 50 | rank = 0 51 | to_return += ',' + str(rank) 52 | to_return += '\n' 53 | return json.dumps(to_return) 54 | 55 | 56 | class Master(object): 57 | 58 | def __init__(self): 59 | self.dir_templates = 'templates' 60 | publisher.channel = 'Website' 61 | 62 | def __none_if_empty(self, to_check = None): 63 | """ 64 | Ensure the empty paramaters are None before doing anything 65 | """ 66 | if to_check is None or len(to_check) == 0: 67 | return None 68 | return cgi.escape(to_check, True) 69 | 70 | def __init_template(self, template_name, source = None, date = None): 71 | """ 72 | Initialize the basic components of the template 73 | """ 74 | template = Template(file = os.path.join(self.dir_templates, 75 | template_name + '.tmpl')) 76 | source = self.__none_if_empty(source) 77 | date = self.__none_if_empty(date) 78 | template.css_file = 'http://www.circl.lu/css/styles.css' 79 | template.logo = 'http://www.circl.lu/pics/logo.png' 80 | template.banner = 'http://www.circl.lu/pics/topbanner.jpg' 81 | template.sources = master_controler.get_sources(date) 82 | template.dates = master_controler.get_dates() 83 | template.source = source 84 | template.date = date 85 | return template 86 | 87 | def __csv2string(self, data): 88 | si = StringIO.StringIO(); 89 | cw = csv.writer(si); 90 | cw.writerow(data); 91 | return si.getvalue().strip('\r\n'); 92 | 93 | def __query_logging(self, ip, user_agent, webpage, date=None, source=None, 94 | asn=None, asn_details=None, compared_asns=None, ip_lookup=None): 95 | publisher.info(self.__csv2string([ip, user_agent, webpage, date, 96 | source, asn, asn_details, compared_asns, ip_lookup])) 97 | 98 | @cherrypy.expose 99 | def default(self): 100 | """ 101 | Load the index 102 | """ 103 | return str(self.index()) 104 | 105 | @cherrypy.expose 106 | def index(self, source = None, date = None): 107 | """ 108 | Generate the view of the global ranking 109 | """ 110 | source = self.__none_if_empty(source) 111 | date = self.__none_if_empty(date) 112 | self.__query_logging(cherrypy.request.remote.ip, 113 | cherrypy.request.headers.get('User-Agent', 'Empty User-Agent'), 114 | webpage='index', date=date, source=source) 115 | histo, list_size = master_controler.prepare_index(source, date) 116 | template = self.__init_template('index_asn', source, date) 117 | template.list_size = list_size 118 | template.histories = histo 119 | return str(template) 120 | 121 | @cherrypy.expose 122 | def asn_details(self, source = None, asn = None, ip_details = None, date = None): 123 | """ 124 | Generate the view of an ASN 125 | """ 126 | asn = self.__none_if_empty(asn) 127 | source = self.__none_if_empty(source) 128 | date = self.__none_if_empty(date) 129 | if asn is None: 130 | return self.index(source, date) 131 | self.__query_logging(cherrypy.request.remote.ip, 132 | cherrypy.request.headers.get('User-Agent', 'Empty User-Agent'), 133 | webpage='asn_details', date=date, source=source, asn=asn, 134 | asn_details = ip_details) 135 | ip_details = self.__none_if_empty(ip_details) 136 | template = self.__init_template('asn_details', source, date) 137 | asn = asn.lstrip('AS') 138 | if asn.isdigit(): 139 | template.asn = asn 140 | asn_description, position, as_infos = master_controler.get_as_infos(asn, 141 | date, source) 142 | if as_infos is not None and len(as_infos) > 0: 143 | template.asn_description = asn_description 144 | template.asn_descs = as_infos 145 | template.current_sources = master_controler.get_last_seen_sources(asn) 146 | template.desc_history = master_controler.get_asn_descriptions(asn) 147 | template.position = position[0] 148 | template.size = position[1] 149 | if len(template.current_sources.keys()) > 0: 150 | template.sources = template.current_sources.keys() 151 | if ip_details is not None: 152 | template.ip_details = ip_details 153 | template.ip_descs = master_controler.get_ip_info(asn, 154 | ip_details, date, source) 155 | else: 156 | template.error = "Invalid query: " + asn 157 | return str(template) 158 | 159 | @cherrypy.expose 160 | def comparator(self, asns = None): 161 | """ 162 | Generate the view comparing a set of ASNs 163 | """ 164 | asns = self.__none_if_empty(asns) 165 | self.__query_logging(cherrypy.request.remote.ip, 166 | cherrypy.request.headers.get('User-Agent', 'Empty User-Agent'), 167 | webpage='comparator', compared_asns=asns) 168 | template = self.__init_template('comparator') 169 | template.asns = asns 170 | if asns is not None: 171 | asns_list, details_list = master_controler.get_comparator_metatada(asns) 172 | template.asns_json = json.dumps(asns_list) 173 | template.asns_details = details_list 174 | else: 175 | template.asns_json = json.dumps([0]) 176 | template.asns_details = None 177 | return str(template) 178 | 179 | @cherrypy.expose 180 | def trend(self): 181 | """ 182 | Print the trend World vs Luxembourg 183 | """ 184 | return self.trend_benelux() 185 | #self.__query_logging(cherrypy.request.remote.ip, 186 | # cherrypy.request.headers.get('User-Agent', 'Empty User-Agent'), 187 | # webpage='trend') 188 | #return str(self.__init_template('trend')) 189 | 190 | @cherrypy.expose 191 | def trend_benelux(self): 192 | """ 193 | Print the trend of the benelux countries 194 | """ 195 | self.__query_logging(cherrypy.request.remote.ip, 196 | cherrypy.request.headers.get('User-Agent', 'Empty User-Agent'), 197 | webpage='trend_benelux') 198 | return str(self.__init_template('trend_benelux')) 199 | 200 | @cherrypy.expose 201 | def map(self): 202 | """ 203 | Print the worldmap 204 | """ 205 | self.__query_logging(cherrypy.request.remote.ip, 206 | cherrypy.request.headers.get('User-Agent', 'Empty User-Agent'), 207 | webpage='map') 208 | return str(self.__init_template('map')) 209 | 210 | @cherrypy.expose 211 | def ip_lookup(self, ip = None): 212 | ip = self.__none_if_empty(ip) 213 | self.__query_logging(cherrypy.request.remote.ip, 214 | cherrypy.request.headers.get('User-Agent', 'Empty User-Agent'), 215 | webpage='ip_lookup', ip_lookup=ip) 216 | template = self.__init_template('ip_lookup') 217 | template.ip = ip 218 | result = master_controler.get_ip_lookup(ip) 219 | if result is not None: 220 | template.history = result[0] 221 | template.ptrrecord = result[1] 222 | else: 223 | template.history = None 224 | template.ptrrecord = None 225 | return str(template) 226 | 227 | def error_page_404(status, message, traceback, version): 228 | """ 229 | Display an error if the page does not exists 230 | """ 231 | return "Error %s - This page does not exist." % status 232 | 233 | def handle_error(): 234 | cherrypy.response.status = 500 235 | cherrypy.response.body = ["Sorry, an error occured"] 236 | publisher.error('Request: '+ str(cherrypy.request.params) + '\n' +_cperror.format_exc()) 237 | 238 | if __name__ == "__main__": 239 | website = Master() 240 | 241 | cherrypy.config.update({'error_page.404': error_page_404}) 242 | cherrypy.config.update({'request.error_response': handle_error}) 243 | cherrypy.quickstart(website, config = 'config/web_bgp-ranking.ini') 244 | -------------------------------------------------------------------------------- /example/website/master_controler.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | Controler class of the website 4 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 5 | 6 | The website respects the MVC design pattern and this class is the controler. 7 | It gets the datas from the `report` class. 8 | 9 | """ 10 | 11 | import bgpranking 12 | 13 | def prepare_index(source, date, limit=100): 14 | response = bgpranking.cache_get_top_asns(source, date, limit) 15 | if len(response['top_list']) != 0: 16 | return [(rank[0], rank[1], 1 + rank[2], ', '.join(sources)) 17 | for rank, sources in response['top_list']], \ 18 | response['size_list'] 19 | 20 | def get_sources(date): 21 | if date is None: 22 | date = bgpranking.get_default_date() 23 | return bgpranking.daily_sources([date]) 24 | 25 | def get_dates(): 26 | return bgpranking.cache_get_dates() 27 | 28 | def get_asn_descriptions(asn): 29 | return bgpranking.get_asn_descriptions(asn) 30 | 31 | def get_last_seen_sources(asn): 32 | dates_sources = bgpranking.prepare_sources_by_dates(timeframe = 365) 33 | return bgpranking.get_last_seen_sources(asn, dates_sources) 34 | 35 | def get_comparator_metatada(asns_string): 36 | splitted_asns = asns_string.split() 37 | if type(splitted_asns) == type(str): 38 | splitted_asns = [splitted_asns] 39 | details_list = [] 40 | for asn in splitted_asns: 41 | details_list.append(bgpranking.get_all_blocks_descriptions(asn)) 42 | return splitted_asns, details_list 43 | 44 | def get_as_infos(asn, date = None, sources = None): 45 | response = bgpranking.get_asn_descs(asn, date, sources) 46 | if response is None or response.get(asn) is None: 47 | return None, None, None 48 | to_return = [] 49 | if type(sources) == type(list()): 50 | position = None 51 | else: 52 | position = bgpranking.cache_get_position(asn, sources, date) 53 | for key, entry in response[asn].iteritems(): 54 | if key == 'clean_blocks': 55 | to_return += [[descr[0][1], block, 0, '', 0] 56 | for block, descr in entry.iteritems()] 57 | elif key == 'old_blocks': 58 | pass 59 | else: 60 | to_return.append([entry['description'], key, entry['nb_of_ips'], 61 | ', '.join(entry['sources']), 1 + entry['rank']]) 62 | return response['asn_description'], position, sorted(to_return, 63 | key=lambda element: (element[4], element[2]), reverse = True) 64 | 65 | def get_ip_info(asn, block, date = None, sources = None): 66 | response = bgpranking.get_ips_descs(asn, block, date, sources) 67 | if response.get(block) is None: 68 | return None 69 | to_return = [(ip, details['ptrrecord'], ', '.join(details['sources'])) 70 | for ip, details in response[block].iteritems()] 71 | return sorted(to_return, key=lambda element: len(element[2]), 72 | reverse=True) 73 | 74 | def get_ip_lookup(ip): 75 | response = bgpranking.get_ip_info(ip) 76 | if len(response.get('history')) == 0: 77 | return None 78 | return response.get('history'), response.get('ptrrecord') 79 | -------------------------------------------------------------------------------- /example/website/start_website.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | python master.py 4 | -------------------------------------------------------------------------------- /example/website/templates/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CIRCL/bgpranking-redis-api/7b80c44e291169b8ce849cdb51beb618b96f2386/example/website/templates/__init__.py -------------------------------------------------------------------------------- /example/website/templates/asn_details.tmpl: -------------------------------------------------------------------------------- 1 | #import cgi 2 | #from templates.master import master 3 | #extends master 4 | 5 | #attr title = 'BGP Ranking - ASN Details' 6 | #attr ip_details = None 7 | #attr javascript = None 8 | #attr asn_descs = [] 9 | #attr current_sources = [] 10 | #attr asn_description = None 11 | #attr desc_history = [] 12 | 13 | #block menu 14 |
15 |
16 | 17 | 27 | 37 | 38 |
39 |
40 |
41 | #if $asn is not None 42 | #if len($current_sources) > 0 43 | This AS ($asn) is known in the following lists (last time seen) 44 | #if $position is not None 45 | at position $position/$size in the current top list: 46 | #else 47 | : 48 | #end if 49 |
    50 | #for $source, $date in $current_sources.iteritems() 51 |
  • 52 | $source: $date 53 |
    • 54 | #end for 55 |
56 | #end if 57 | #if len($desc_history) > 0 58 | Over time, this AS ($asn) had following descriptions: 59 |
    60 | #for $date, $description in $desc_history 61 |
  • 62 | $date: $description 63 |
    • 64 | #end for 65 |
66 | #end if 67 | #end if 68 | #end block menu 69 | 70 | #block main_content 71 | #if $asn_description is not None 72 | 73 |
74 | 114 |
115 | The data are available to download. 117 |
118 | 119 | 120 | 121 | 122 | 123 | 124 | 125 | 126 | 127 | 128 | #for $asn_desc in $asn_descs 129 | 130 | 131 | 132 | 133 | 134 | 135 | 136 | #end for 137 |
OwnerBlockNumber of IPs todaySource(s)Block Ranking
$cgi.escape($asn_desc[0], True)$asn_desc[1]$asn_desc[2]$asn_desc[3]$asn_desc[4]
138 | #if $ip_details is not None 139 | #if $ip_descs is not None 140 | 141 | 142 | 143 | 144 | 145 | 146 | 147 | 148 | #for $ip_desc in $ip_descs 149 | 150 | 151 | 152 | 153 | 154 | #end for 155 |
IP AddressPTR RecordSource(s)
$ip_desc[0]$ip_desc[1]$ip_desc[2]
156 | #end if 157 | #end if 158 | #else 159 | AS $asn unknown in the database. 160 | #end if 161 | #end block main_content 162 | -------------------------------------------------------------------------------- /example/website/templates/comparator.tmpl: -------------------------------------------------------------------------------- 1 | #import cgi 2 | #from master import merge_csvs 3 | #from templates.master import master 4 | #extends master 5 | 6 | #attr title = 'BGP Ranking - ASN Comparator' 7 | #attr asns = None 8 | #attr js_comparator = None 9 | 10 | #block menu 11 | #end block menu 12 | 13 | #block main_content 14 | List of ASNs, separated with a blank: 15 |
16 |
17 | 18 | 19 |
20 |

21 | 22 |
23 | 46 |
    47 | #if $asns_details is not None 48 | #for asn, blocks_descriptions in $asns_details 49 |
  • 50 | $asn (csv) 51 |
      52 | #for block, ts_descr in $blocks_descriptions.iteritems() 53 |
    • 54 | $block: 55 |
        56 | #for ts, descr in ts_descr 57 |
      • $cgi.escape($descr, True): $ts
        • 58 | #end for 59 |
      60 |
      • 61 | #end for 62 |
    63 |
    • 64 | #end for 65 | #end if 66 |
67 | 68 | 69 | #end block main_content 70 | -------------------------------------------------------------------------------- /example/website/templates/index_asn.tmpl: -------------------------------------------------------------------------------- 1 | #import cgi 2 | #from templates.master import master 3 | #extends master 4 | 5 | #attr title = 'BGP Ranking - Daily top ASN' 6 | 7 | #block menu 8 |
9 |
10 | 11 | 21 | 31 | 32 |
33 |
34 | #end block menu 35 | 36 | #block main_content 37 |

38 | There is $list_size entries in the list of asns with malicious content. 39 | The top 100 is printed on this page. 40 |

41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | #if $histories is not None 51 | #for $history in $histories 52 | 53 | 54 | #if $history[1] is None 55 | 56 | #else 57 | 58 | #end if 59 | 60 | 61 | 62 | #end for 63 | #end if 64 |
ASNDescriptionRankSource(s)
${history[0]}$cgi.escape($history[1], True)$history[2]$history[3]
65 | #end block main_content 66 | -------------------------------------------------------------------------------- /example/website/templates/ip_lookup.tmpl: -------------------------------------------------------------------------------- 1 | #import cgi 2 | #from master import merge_csvs 3 | #from templates.master import master 4 | #extends master 5 | 6 | #attr title = 'BGP Ranking - IP Lookup' 7 | #attr ip = None 8 | #attr ptrrecord = None 9 | 10 | #block menu 11 | #end block menu 12 | 13 | #block main_content 14 | IP to lookup: 15 |
16 |
17 | 18 | 19 |
20 |

21 | #if $ptrrecord is not None 22 | PTR Record: $ptrrecord 23 | #end if 24 |
    25 | #if $history is not None 26 | #for entry in $history 27 |
  • $entry['interval'][0] - $entry['interval'][1]: 28 | $entry['asn'] - 29 | 30 | $entry['block'] 31 | 32 |
      33 | #for date, description in $entry['descriptions'] 34 |
    • $date: $cgi.escape($description, True)
      • 35 | #end for 36 |
    37 |
    • 38 | #end for 39 | #end if 40 |
41 | 42 | #end block main_content 43 | -------------------------------------------------------------------------------- /example/website/templates/map.tmpl: -------------------------------------------------------------------------------- 1 | #from templates.master import master 2 | #extends master 3 | 4 | #attr title = 'BGP Ranking - Worldmap' 5 | 6 | #block menu 7 | #end block menu 8 | 9 | #block main_content 10 | 11 | 12 | 13 | 14 | 15 | 16 | Aggregation of all the ranks by country code (provided by Team Cymru).
17 | A CSV dump of the data used to display this map is 18 | available for download. 19 | 20 |
21 | 22 | 23 | 24 | #end block main_content 25 | -------------------------------------------------------------------------------- /example/website/templates/master.tmpl: -------------------------------------------------------------------------------- 1 | #attr error = None 2 | #attr title = 'Master' 3 | 4 | #attr asn = None 5 | #attr source = None 6 | 7 | 9 | 10 | 11 | $title 12 | 13 | 14 | 15 | 16 | 24 |
25 |

BGP Ranking

26 | 38 |
39 | #block menu 40 | Menu Content 41 | #end block menu 42 | #if $error is not None 43 |
$error
44 | #end if 45 | #block main_content 46 | Main Content 47 | #end block main_content 48 |
49 |
50 | 51 | 52 | 53 | -------------------------------------------------------------------------------- /example/website/templates/trend.tmpl: -------------------------------------------------------------------------------- 1 | #from templates.master import master 2 | #extends master 3 | 4 | #attr title = 'BGP Ranking - World/Luxembourg trend' 5 | 6 | #block menu 7 | #end block menu 8 | 9 | #block main_content 10 | 11 |

Relational graph between the global worldwide security trend and the Luxemburg one.

12 |

Notes: 13 |

    14 |
  • The data are available to download.
    • 16 |
  • The scales are logarithmic.
    • 17 |
  • The scales of the global tend (left, green) and the Luxembourg one (right, blue) are different
    • 18 |
  • The data used are public and subject to false positive AND false negative
    • 19 |
  • The graph is published for information purpose only
    • 20 |
  • Do not hesitate to ask us if you 21 | have questions about this graph
    • 22 |
23 |

24 |
25 | 26 | 27 | #end block main_content 28 | 29 | -------------------------------------------------------------------------------- /example/website/templates/trend_benelux.tmpl: -------------------------------------------------------------------------------- 1 | #from templates.master import master 2 | #extends master 3 | 4 | #attr title = 'BGP Ranking - Wide Benelux trend' 5 | 6 | #block menu 7 | #end block menu 8 | 9 | #block main_content 10 | 11 |

Relational graphs between the Benelux countries, France, Germany and Switzerland 12 | with the number of infected systems vs the number of active hosts.

13 |

Notes: 14 |

    15 |
  • The data are available to download
    • 17 |
  • The scales are logarithmic
    • 18 |
  • The data used are public and subject to false positive AND false negative
    • 19 |
  • The graph is published for information purpose only
    • 20 |
  • Do not hesitate to ask us if you 21 | have questions about this graph
    • 22 |
23 |

24 |
25 | 26 | 27 | #end block main_content 28 | 29 | -------------------------------------------------------------------------------- /example/website/thirdparty/dygraph/.keepdir: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CIRCL/bgpranking-redis-api/7b80c44e291169b8ce849cdb51beb618b96f2386/example/website/thirdparty/dygraph/.keepdir -------------------------------------------------------------------------------- /example/website/thirdparty/jvectormap/.keepdir: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CIRCL/bgpranking-redis-api/7b80c44e291169b8ce849cdb51beb618b96f2386/example/website/thirdparty/jvectormap/.keepdir -------------------------------------------------------------------------------- /example/website/thirdparty/update_thirdparty.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -e 4 | 5 | git add . 6 | git commit -m 'Update' 7 | 8 | wget http://dygraphs.com/dygraph-combined.js -O dygraph/dygraph-combined.js 9 | 10 | JVECTORMAP_VERSION='1.2.2' 11 | filename="jquery-jvectormap-${JVECTORMAP_VERSION}.zip" 12 | 13 | rm -rf temp 14 | mkdir temp 15 | 16 | wget http://jvectormap.com/binary/${filename} -O temp/${filename} 17 | unzip temp/${filename} -d temp/ 18 | mv temp/jquery-jvectormap-${JVECTORMAP_VERSION}.css jvectormap/jquery-jvectormap.css 19 | mv temp/jquery-jvectormap-${JVECTORMAP_VERSION}.min.js jvectormap/jquery-jvectormap.js 20 | 21 | rm -rf temp 22 | 23 | wget http://jvectormap.com/js/jquery-jvectormap-world-mill-en.js \ 24 | -O jvectormap/jquery-jvectormap-world-mill-en.js 25 | 26 | JQUERY_VERSION='1.10.2' 27 | wget http://code.jquery.com/jquery-${JQUERY_VERSION}.min.js -O jvectormap/jquery.js 28 | 29 | git add . 30 | git commit -m 'Update' 31 | 32 | 33 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # -*- coding: utf-8 -*- 3 | from setuptools import setup 4 | 5 | setup( 6 | name='bgpranking-redis', 7 | version='2.0', 8 | author='Raphaël Vinot', 9 | author_email='raphael.vinot@circl.lu', 10 | maintainer='Raphaël Vinot', 11 | url='https://github.com/CIRCL/bgpranking-redis-api', 12 | description='API to access the Redis database of a BGP Ranking instance.', 13 | long_description=open('README.md').read(), 14 | packages=['bgpranking_redis'], 15 | classifiers=[ 16 | 'License :: OSI Approved :: GNU General Public License v3 (GPLv3)', 17 | 'Development Status :: 5 - Production/Stable', 18 | 'Environment :: Console', 19 | 'Intended Audience :: Science/Research', 20 | 'Intended Audience :: Telecommunications Industry', 21 | 'Programming Language :: Python', 22 | 'Topic :: Security', 23 | 'Topic :: Internet', 24 | 'Topic :: System :: Networking', 25 | ], 26 | install_requires=['IPy', 'pubsublogger', 'redis', 'python-dateutil', 'asnhistory', 'ipasn-redis'], 27 | ) 28 | --------------------------------------------------------------------------------